<commit_before>import logging
import signal
import time
import uuid
from threading import Thread

import os
from bd2k.util.files import mkdir_p
from toil.job import Job
from toil.leader import FailedJobsException
from toil.lib.docker import dockerCall, dockerCheckOutput, _containerIsRunning, _dockerKill, STOP, FORGO, RM
from toil.test import ToilTest

_log = logging.getLogger(__name__)

class DockerTest(ToilTest):
    """
    Tests dockerCall and ensures no containers are left around.

    When running tests you may optionally set the TOIL_TEST_TEMP environment variable to the path
    of a directory where you want temporary test files be placed. The directory will be created
    if it doesn't exist. The path may be relative in which case it will be assumed to be relative
    to the project root. If TOIL_TEST_TEMP is not defined, temporary files and directories will
    be created in the system's default location for such files and any temporary files or
    directories left over from tests will be removed automatically removed during tear down.
    Otherwise, left-over files will not be removed.
    """
    def setUp(self):
        self.tempDir = self._createTempDir(purpose='tempDir')

    def testDockerClean(self):
        """
        Run the test container that creates a file in the work dir, and sleeps for 5 minutes.  Ensure
        that the calling job gets SIGKILLed after a minute, leaving behind the spooky/ghost/zombie
        container. Ensure that the container is killed on batch system shutdown (through the defer
        mechanism).
        This inherently also tests _docker
        :returns: None
        """
        # We need to test the behaviour of `defer` with `rm` and `detached`. We do not look at the case
        # where `rm` and `detached` are both True.  This is the truth table for the different
        # combinations at the end of the test. R = Running, X = Does not exist, E = Exists but not
        # running.
        #              None     FORGO     STOP    RM
        #    rm        X         R         X      X
        # detached     R         R         E      X
        #  Neither     R         R         E      X
        assert os.getuid() != 0, "Cannot test this if the user is root."
        data_dir = os.path.join(self.tempDir, 'data')
        work_dir = os.path.join(self.tempDir, 'working')
        test_file = os.path.join(data_dir, 'test.txt')
        mkdir_p(data_dir)
        mkdir_p(work_dir)
        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir, 'jobstore'))
        options.logLevel = 'INFO'
        options.workDir = work_dir
        options.clean = 'always'
        for rm in (True, False):
            for detached in (True, False):
                if detached and rm:
                    continue
                for defer in (FORGO, STOP, RM, None):
                    # Not using base64 logic here since it might create a name starting with a `-`.
                    container_name = uuid.uuid4().hex
                    print rm, detached, defer
                    A = Job.wrapJobFn(_testDockerCleanFn, data_dir, detached, rm, defer,
                                      container_name)
                    try:
                        Job.Runner.startToil(A, options)
                    except FailedJobsException:
                        # The file created by spooky_container would remain in the directory, and since
                        # it was created inside the container, it would have had uid and gid == 0 (root)
                        # upon creation. If the defer mechanism worked, it should now be non-zero and we
                        # check for that.
                        file_stats = os.stat(test_file)
                        assert file_stats.st_gid != 0
                        assert file_stats.st_uid != 0
                        if (rm and defer != FORGO) or defer == RM:
                            # These containers should not exist
                            assert _containerIsRunning(container_name) is None, \
                                'Container was not removed.'
                        elif defer == STOP:
                            # These containers should exist but be non-running
                            assert _containerIsRunning(container_name) == False, \
                                'Container was not stopped.'
                        else:
                            # These containers will be running
                            assert _containerIsRunning(container_name) == True, \
                                'Container was not running.'
                    finally:
                        # Prepare for the next test.
                        _dockerKill(container_name, RM)
                        os.remove(test_file)

    def testDockerPipeChain(self):
        """
        Test for piping API for dockerCall().  Using this API (activated when list of
        argument lists is given as parameters), commands a piped together into a chain
        ex:  parameters=[ ['printf', 'x\n y\n'], ['wc', '-l'] ] should execute:
        printf 'x\n y\n' | wc -l
        """
        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir, 'jobstore'))
        options.logLevel = 'INFO'
        options.workDir = self.tempDir
        options.clean = 'always'
        A = Job.wrapJobFn(_testDockerPipeChainFn)
        rv = Job.Runner.startToil(A, options)
        assert rv.strip() == '2'

def _testDockerCleanFn(job, workDir, detached=None, rm=None, defer=None, containerName=None):
    """
    Test function for test docker_clean.  Runs a container with given flags and then dies leaving
    behind a zombie container
    :param toil.job.Job job: job
    :param workDir: See `work_dir=` in :func:`dockerCall`
    :param bool rm: See `rm=` in :func:`dockerCall`
    :param bool detached: See `detached=` in :func:`dockerCall`
    :param int defer: See `defer=` in :func:`dockerCall`
    :param str containerName: See `container_name=` in :func:`dockerCall`
    :return:
    """
    dockerParameters = ['--log-driver=none', '-v', os.path.abspath(workDir) + ':/data',
                        '--name', containerName]
    if detached:
        dockerParameters.append('-d')
    if rm:
        dockerParameters.append('--rm')

    def killSelf():
        test_file = os.path.join(workDir, 'test.txt')
        # This will kill the worker once we are sure the docker container started
        while not os.path.exists(test_file):
            _log.debug('Waiting on the file created by spooky_container.')
            time.sleep(1)
        # By the time we reach here, we are sure the container is running.
        os.kill(os.getpid(), signal.SIGKILL)  # signal.SIGINT)
    t = Thread(target=killSelf)
    # Make it a daemon thread so that thread failure doesn't hang tests.
    t.daemon = True
    t.start()
    dockerCall(job, tool='quay.io/ucsc_cgl/spooky_test', workDir=workDir, defer=defer, dockerParameters=dockerParameters)

def _testDockerPipeChainFn(job):
    """
    Return the result of simple pipe chain.  Should be 2
    """
    parameters = [ ['printf', 'x\n y\n'], ['wc', '-l'] ]
    return dockerCheckOutput(job, tool='quay.io/ucsc_cgl/spooky_test', parameters=parameters)
<commit_msg>Add tests for NonCachingFileStore with DockerCall<commit_after>import logging
import signal
import time
import uuid
from threading import Thread

import os
from bd2k.util.files import mkdir_p
from toil.job import Job
from toil.leader import FailedJobsException
from toil.lib.docker import dockerCall, dockerCheckOutput, _containerIsRunning, _dockerKill, STOP, FORGO, RM
from toil.test import ToilTest

_log = logging.getLogger(__name__)

class DockerTest(ToilTest):
    """
    Tests dockerCall and ensures no containers are left around.

    When running tests you may optionally set the TOIL_TEST_TEMP environment variable to the path
    of a directory where you want temporary test files be placed. The directory will be created
    if it doesn't exist. The path may be relative in which case it will be assumed to be relative
    to the project root. If TOIL_TEST_TEMP is not defined, temporary files and directories will
    be created in the system's default location for such files and any temporary files or
    directories left over from tests will be removed automatically removed during tear down.
    Otherwise, left-over files will not be removed.
    """
    def setUp(self):
        self.tempDir = self._createTempDir(purpose='tempDir')

    def testDockerClean(self, caching=True):
        """
        Run the test container that creates a file in the work dir, and sleeps for 5 minutes.  Ensure
        that the calling job gets SIGKILLed after a minute, leaving behind the spooky/ghost/zombie
        container. Ensure that the container is killed on batch system shutdown (through the defer
        mechanism).
        This inherently also tests _docker
        :returns: None
        """
        # We need to test the behaviour of `defer` with `rm` and `detached`. We do not look at the case
        # where `rm` and `detached` are both True.  This is the truth table for the different
        # combinations at the end of the test. R = Running, X = Does not exist, E = Exists but not
        # running.
        #              None     FORGO     STOP    RM
        #    rm        X         R         X      X
        # detached     R         R         E      X
        #  Neither     R         R         E      X
        assert os.getuid() != 0, "Cannot test this if the user is root."
        data_dir = os.path.join(self.tempDir, 'data')
        work_dir = os.path.join(self.tempDir, 'working')
        test_file = os.path.join(data_dir, 'test.txt')
        mkdir_p(data_dir)
        mkdir_p(work_dir)
        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir, 'jobstore'))
        options.logLevel = 'INFO'
        options.workDir = work_dir
        options.clean = 'always'
        if not caching:
            options.disableCaching = True
        for rm in (True, False):
            for detached in (True, False):
                if detached and rm:
                    continue
                for defer in (FORGO, STOP, RM, None):
                    # Not using base64 logic here since it might create a name starting with a `-`.
                    container_name = uuid.uuid4().hex
                    A = Job.wrapJobFn(_testDockerCleanFn, data_dir, detached, rm, defer,
                                      container_name)
                    try:
                        Job.Runner.startToil(A, options)
                    except FailedJobsException:
                        # The file created by spooky_container would remain in the directory, and since
                        # it was created inside the container, it would have had uid and gid == 0 (root)
                        # upon creation. If the defer mechanism worked, it should now be non-zero and we
                        # check for that.
                        file_stats = os.stat(test_file)
                        assert file_stats.st_gid != 0
                        assert file_stats.st_uid != 0
                        if (rm and defer != FORGO) or defer == RM:
                            # These containers should not exist
                            assert _containerIsRunning(container_name) is None, \
                                'Container was not removed.'
                        elif defer == STOP:
                            # These containers should exist but be non-running
                            assert _containerIsRunning(container_name) == False, \
                                'Container was not stopped.'
                        else:
                            # These containers will be running
                            assert _containerIsRunning(container_name) == True, \
                                'Container was not running.'
                    finally:
                        # Prepare for the next test.
                        _dockerKill(container_name, RM)
                        os.remove(test_file)

    def testDockerPipeChain(self, caching=True):
        """
        Test for piping API for dockerCall().  Using this API (activated when list of
        argument lists is given as parameters), commands a piped together into a chain
        ex:  parameters=[ ['printf', 'x\n y\n'], ['wc', '-l'] ] should execute:
        printf 'x\n y\n' | wc -l
        """
        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir, 'jobstore'))
        options.logLevel = 'INFO'
        options.workDir = self.tempDir
        options.clean = 'always'
        if not caching:
            options.disableCaching = True
        A = Job.wrapJobFn(_testDockerPipeChainFn)
        rv = Job.Runner.startToil(A, options)
        assert rv.strip() == '2'

    def testNonCachingDockerChain(self):
        self.testDockerPipeChain(caching=False)

    def testNonCachingDockerClean(self):
        self.testDockerClean(caching=False)

def _testDockerCleanFn(job, workDir, detached=None, rm=None, defer=None, containerName=None):
    """
    Test function for test docker_clean.  Runs a container with given flags and then dies leaving
    behind a zombie container
    :param toil.job.Job job: job
    :param workDir: See `work_dir=` in :func:`dockerCall`
    :param bool rm: See `rm=` in :func:`dockerCall`
    :param bool detached: See `detached=` in :func:`dockerCall`
    :param int defer: See `defer=` in :func:`dockerCall`
    :param str containerName: See `container_name=` in :func:`dockerCall`
    :return:
    """
    dockerParameters = ['--log-driver=none', '-v', os.path.abspath(workDir) + ':/data',
                        '--name', containerName]
    if detached:
        dockerParameters.append('-d')
    if rm:
        dockerParameters.append('--rm')

    def killSelf():
        test_file = os.path.join(workDir, 'test.txt')
        # This will kill the worker once we are sure the docker container started
        while not os.path.exists(test_file):
            _log.debug('Waiting on the file created by spooky_container.')
            time.sleep(1)
        # By the time we reach here, we are sure the container is running.
        os.kill(os.getpid(), signal.SIGKILL)  # signal.SIGINT)
    t = Thread(target=killSelf)
    # Make it a daemon thread so that thread failure doesn't hang tests.
    t.daemon = True
    t.start()
    dockerCall(job, tool='quay.io/ucsc_cgl/spooky_test', workDir=workDir, defer=defer, dockerParameters=dockerParameters)

def _testDockerPipeChainFn(job):
    """
    Return the result of simple pipe chain.  Should be 2
    """
    parameters = [ ['printf', 'x\n y\n'], ['wc', '-l'] ]
    return dockerCheckOutput(job, tool='quay.io/ucsc_cgl/spooky_test', parameters=parameters)
<|endoftext|>


<commit_before>{
    'name': 'Web',
    'category': 'Hidden',
    'version': '7.0.1.0',
    'description':
        """
OpenERP Web core module.
========================

This module provides the core of the OpenERP Web Client.
        """,
    'depends': [],
    'auto_install': True,
    'post_load': 'wsgi_postload',
    'js' : [
        "static/src/fixbind.js",
        "static/lib/datejs/globalization/en-US.js",
        "static/lib/datejs/core.js",
        "static/lib/datejs/parser.js",
        "static/lib/datejs/sugarpak.js",
        "static/lib/datejs/extras.js",
        "static/lib/jquery/jquery-1.8.3.js",
        "static/lib/jquery.MD5/jquery.md5.js",
        "static/lib/jquery.form/jquery.form.js",
        "static/lib/jquery.validate/jquery.validate.js",
        "static/lib/jquery.ba-bbq/jquery.ba-bbq.js",
        "static/lib/spinjs/spin.js",
        "static/lib/jquery.autosize/jquery.autosize.js",
        "static/lib/jquery.blockUI/jquery.blockUI.js",
        "static/lib/jquery.placeholder/jquery.placeholder.js",
        "static/lib/jquery.ui/js/jquery-ui-1.9.1.custom.js",
        "static/lib/jquery.ui.timepicker/js/jquery-ui-timepicker-addon.js",
        "static/lib/jquery.ui.notify/js/jquery.notify.js",
        "static/lib/jquery.deferred-queue/jquery.deferred-queue.js",
        "static/lib/jquery.scrollTo/jquery.scrollTo-min.js",
        "static/lib/jquery.tipsy/jquery.tipsy.js",
        "static/lib/jquery.textext/jquery.textext.js",
        "static/lib/jquery.timeago/jquery.timeago.js",
        "static/lib/qweb/qweb2.js",
        "static/lib/underscore/underscore.js",
        "static/lib/underscore/underscore.string.js",
        "static/lib/backbone/backbone.js",
        "static/lib/cleditor/jquery.cleditor.js",
        "static/lib/py.js/lib/py.js",
        "static/src/js/boot.js",
        "static/src/js/testing.js",
        "static/src/js/pyeval.js",
        "static/src/js/corelib.js",
        "static/src/js/coresetup.js",
        "static/src/js/dates.js",
        "static/src/js/formats.js",
        "static/src/js/chrome.js",
        "static/src/js/views.js",
        "static/src/js/data.js",
        "static/src/js/data_export.js",
        "static/src/js/search.js",
        "static/src/js/view_form.js",
        "static/src/js/view_list.js",
        "static/src/js/view_list_editable.js",
        "static/src/js/view_tree.js",
    ],
    'css' : [
        "static/lib/jquery.ui.bootstrap/css/custom-theme/jquery-ui-1.9.0.custom.css",
        "static/lib/jquery.ui.timepicker/css/jquery-ui-timepicker-addon.css",
        "static/lib/jquery.ui.notify/css/ui.notify.css",
        "static/lib/jquery.tipsy/tipsy.css",
        "static/lib/jquery.textext/jquery.textext.css",
        "static/src/css/base.css",
        "static/src/css/data_export.css",
        "static/lib/cleditor/jquery.cleditor.css",
    ],
    'qweb' : [
        "static/src/xml/*.xml",
    ],
    'test': [
        "static/test/testing.js",
        "static/test/class.js",
        "static/test/registry.js",
        "static/test/form.js",
        "static/test/data.js",
        "static/test/list-utils.js",
        "static/test/formats.js",
        "static/test/rpc.js",
        "static/test/evals.js",
        "static/test/search.js",
        "static/test/Widget.js",
        "static/test/list.js",
        "static/test/list-editable.js",
        "static/test/mutex.js"
    ],
    'bootstrap': True,
}
<commit_msg>[FIX] web: serie agnostic version number<commit_after>{
    'name': 'Web',
    'category': 'Hidden',
    'version': '1.0',
    'description':
        """
OpenERP Web core module.
========================

This module provides the core of the OpenERP Web Client.
        """,
    'depends': [],
    'auto_install': True,
    'post_load': 'wsgi_postload',
    'js' : [
        "static/src/fixbind.js",
        "static/lib/datejs/globalization/en-US.js",
        "static/lib/datejs/core.js",
        "static/lib/datejs/parser.js",
        "static/lib/datejs/sugarpak.js",
        "static/lib/datejs/extras.js",
        "static/lib/jquery/jquery-1.8.3.js",
        "static/lib/jquery.MD5/jquery.md5.js",
        "static/lib/jquery.form/jquery.form.js",
        "static/lib/jquery.validate/jquery.validate.js",
        "static/lib/jquery.ba-bbq/jquery.ba-bbq.js",
        "static/lib/spinjs/spin.js",
        "static/lib/jquery.autosize/jquery.autosize.js",
        "static/lib/jquery.blockUI/jquery.blockUI.js",
        "static/lib/jquery.placeholder/jquery.placeholder.js",
        "static/lib/jquery.ui/js/jquery-ui-1.9.1.custom.js",
        "static/lib/jquery.ui.timepicker/js/jquery-ui-timepicker-addon.js",
        "static/lib/jquery.ui.notify/js/jquery.notify.js",
        "static/lib/jquery.deferred-queue/jquery.deferred-queue.js",
        "static/lib/jquery.scrollTo/jquery.scrollTo-min.js",
        "static/lib/jquery.tipsy/jquery.tipsy.js",
        "static/lib/jquery.textext/jquery.textext.js",
        "static/lib/jquery.timeago/jquery.timeago.js",
        "static/lib/qweb/qweb2.js",
        "static/lib/underscore/underscore.js",
        "static/lib/underscore/underscore.string.js",
        "static/lib/backbone/backbone.js",
        "static/lib/cleditor/jquery.cleditor.js",
        "static/lib/py.js/lib/py.js",
        "static/src/js/boot.js",
        "static/src/js/testing.js",
        "static/src/js/pyeval.js",
        "static/src/js/corelib.js",
        "static/src/js/coresetup.js",
        "static/src/js/dates.js",
        "static/src/js/formats.js",
        "static/src/js/chrome.js",
        "static/src/js/views.js",
        "static/src/js/data.js",
        "static/src/js/data_export.js",
        "static/src/js/search.js",
        "static/src/js/view_form.js",
        "static/src/js/view_list.js",
        "static/src/js/view_list_editable.js",
        "static/src/js/view_tree.js",
    ],
    'css' : [
        "static/lib/jquery.ui.bootstrap/css/custom-theme/jquery-ui-1.9.0.custom.css",
        "static/lib/jquery.ui.timepicker/css/jquery-ui-timepicker-addon.css",
        "static/lib/jquery.ui.notify/css/ui.notify.css",
        "static/lib/jquery.tipsy/tipsy.css",
        "static/lib/jquery.textext/jquery.textext.css",
        "static/src/css/base.css",
        "static/src/css/data_export.css",
        "static/lib/cleditor/jquery.cleditor.css",
    ],
    'qweb' : [
        "static/src/xml/*.xml",
    ],
    'test': [
        "static/test/testing.js",
        "static/test/class.js",
        "static/test/registry.js",
        "static/test/form.js",
        "static/test/data.js",
        "static/test/list-utils.js",
        "static/test/formats.js",
        "static/test/rpc.js",
        "static/test/evals.js",
        "static/test/search.js",
        "static/test/Widget.js",
        "static/test/list.js",
        "static/test/list-editable.js",
        "static/test/mutex.js"
    ],
    'bootstrap': True,
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import time

from caching.tests.utils import assert_invalidates_model_cache
from ted import tasks
from videos.models import Video
from utils.factories import *
from subtitles.pipeline import add_subtitles
from webdriver_testing.webdriver_base import WebdriverTestCase
from webdriver_testing.pages.site_pages.teams_dir_page import TeamsDirPage
from webdriver_testing.pages.site_pages.teams.tasks_tab import TasksTab
from webdriver_testing.pages.site_pages.teams.videos_tab import VideosTab
#from webdriver_testing.data_factories import *
from webdriver_testing.pages.editor_pages import subtitle_editor
from webdriver_testing import data_helpers
from webdriver_testing.pages.site_pages import video_page
from webdriver_testing.pages.site_pages import video_language_page
from webdriver_testing.pages.site_pages import editor_page
from webdriver_testing.pages.site_pages import site_modals
from webdriver_testing.pages.site_pages import edit_video_page

class TestCaseModeratedVideoTitles(WebdriverTestCase):
    NEW_BROWSER_PER_TEST_CASE = False

    @classmethod
    def setUpClass(cls):
        super(TestCaseModeratedVideoTitles, cls).setUpClass()
        cls.data_utils = data_helpers.DataHelpers()
        cls.admin_video_pg = edit_video_page.EditVideoPage(cls)
        cls.tasks_tab = TasksTab(cls)
        cls.videos_tab = VideosTab(cls)
        cls.video_pg = video_page.VideoPage(cls)

        cls.modal = site_modals.SiteModals(cls)
        cls.editor_pg = editor_page.EditorPage(cls)
        cls.video_lang_pg = video_language_page.VideoLanguagePage(cls)

        cls.staff = UserFactory(is_staff=True, is_superuser=True)
        cls.admin = UserFactory()
        cls.manager = UserFactory()
        cls.member = UserFactory()
        cls.team = TeamFactory(admin=cls.admin,
                               manager=cls.manager,
                               member=cls.member,
                               workflow_type='O',
                               workflow_enabled=True
                              )

        cls.workflow = WorkflowFactory(
            team = cls.team,
            autocreate_subtitle = True,
            autocreate_translate = False,
            review_allowed = 10,
            approve_allowed = 20)
        #Create a member of the team
        cls.subs_file = os.path.join(os.getcwd(), 'apps','webdriver_testing',
                                    'subtitle_data', 'basic_subs.dfxp')

    def setUp(self):
        self.tasks_tab.open_team_page(self.team.slug)
        self.tasks_tab.handle_js_alert('accept')

    @classmethod
    def _create_subs(cls, video, lc, user, complete=False):
        subtitles_1 = [
            (0500,2000, 'Hello there'),
        ]
        subtitles_2 = [
            (0500, 2000, 'Hello there'),
            (3000, 5000, 'Hello there'),
        ]
        add_subtitles(video, lc, subtitles_1)
        add_subtitles(video, lc, subtitles_2,
                      author=user,
                      committer=user,
                      action='complete'
                      )

    def perform_task(self, task_type, video, title=None, edit=None):
        self.tasks_tab.perform_task("%s English Subtitles" % task_type,
                                    video.title)
        if edit == 'upload':
            self.editor_pg.upload_subtitles(self.subs_file)
        if edit == 'type':
            self.editor_pg.edit_sub_line('edit', 1)
        if title:
            self.editor_pg.edit_title(title)

    def test_speakername_edit_in_task(self):
        video = TeamVideoFactory(team=self.team,
                                 video__primary_audio_language_code='en'
                                ).video
        self.video_pg.open_video_page(video.video_id)
        self.admin_video_pg.log_in(self.staff.username, 'password')
        self.admin_video_pg.open_edit_video_page(video.id)
        self.admin_video_pg.add_speaker_name('Jerry Garcia')
        self.video_pg.open_video_page(video.video_id)
        self._create_subs(video, 'en', self.member, complete=True)
        self.data_utils.complete_review_task(video.get_team_video(),
                                            20, self.manager)
        self.video_pg.open_video_page(video.video_id)
        self.assertEqual(video.title, self.video_pg.video_title())
        self.data_utils.complete_approve_task(video.get_team_video(),
                                            20, self.manager)

    def test_post_publish_edit(self):
        """Edit title in approve, video title updated after publish """
        video = TeamVideoFactory(team=self.team,
                                 video__primary_audio_language_code='en',
                                ).video
        self._create_subs(video, 'en', self.member, complete=True)
        orig_title = video.title
        self.logger.info(video.title)
        new_title = 'this is a new title'
        self.tasks_tab.log_in(self.manager.username, 'password')
        self.data_utils.complete_review_task(video.get_team_video(), 20, self.manager)
        self.data_utils.complete_approve_task(video.get_team_video(), 20, self.manager)
        self.tasks_tab.log_in(self.admin.username, 'password')
        self.video_lang_pg.open_video_lang_page(video.video_id, 'en')
        self.video_lang_pg.edit_subtitles()
        self.editor_pg.edit_title(new_title)
        with assert_invalidates_model_cache(video):
            self.editor_pg.collab_action('Publish')
            self.assertEqual(new_title, self.video_pg.video_title())

    def test_approve_edit_title(self):
        """Edit title in approve, video title updated after publish """
        video = TeamVideoFactory(team=self.team,
                                 video__primary_audio_language_code='en'
                                ).video
        self._create_subs(video, 'en', self.member, complete=True)
        orig_title = video.title
        new_title = 'this is a new title'
        self.tasks_tab.log_in(self.manager.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.data_utils.complete_review_task(video.get_team_video(), 20, self.manager)
        self.tasks_tab.log_in(self.admin.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Approve Original", video, title=new_title)
        self.editor_pg.collab_action("Approve")
        self.assertEqual(new_title, self.video_pg.video_title())

    def test_review_edit_title(self):
        """Edit title in review, video title updated after publish """
        video = TeamVideoFactory(team=self.team,
                                 video__primary_audio_language_code='en'
                                ).video
        self._create_subs(video, 'en', self.member, complete=True)
        orig_title = video.title
        new_title = 'this is a new title'
        self.tasks_tab.log_in(self.manager.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Review Original", video, title=new_title)
        self.editor_pg.collab_action("Approve")
        self.assertEqual(video.title, self.video_pg.video_title())
        self.tasks_tab.log_in(self.admin.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Approve Original", video, title=None)
        self.editor_pg.collab_action("Approve")
        self.video_pg.open_video_page(video.video_id)
        self.assertEqual(new_title, self.video_pg.video_title())

    def test_review_edit_title_reject(self):
        """Edit title in review, video title updated after publish """
        video = TeamVideoFactory(team=self.team,
                                 video__primary_audio_language_code='en'
                                ).video
        self._create_subs(video, 'en', self.member, complete=True)
        orig_title = video.title
        new_title = 'this is a new title'
        self.tasks_tab.log_in(self.manager.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Review Original", video, title=new_title, edit='type')
        self.editor_pg.collab_action("Send Back")
        self.assertEqual(orig_title, self.video_pg.video_title())
        self.tasks_tab.log_in(self.member.username, 'password')
        self.tasks_tab.open_page('teams/%s/tasks/?assignee=me&/'
                                 % self.team.slug)
        self.perform_task("Transcribe", video, edit='upload', title=None)
        self.editor_pg.collab_action("Complete")
        self.assertEqual(orig_title, self.video_pg.video_title())
        self.data_utils.complete_review_task(video.get_team_video(), 20, self.manager)
        self.data_utils.complete_approve_task(video.get_team_video(), 20, self.admin)
        self.video_pg.open_video_page(video.video_id)
        self.assertEqual(new_title, self.video_pg.video_title())

    def test_transcribe_edit_title_save_draft(self):
        """Edit title in transcribe, and save a draft """
        video = TeamVideoFactory(team=self.team,
                                 video__primary_audio_language_code='en',
                                 video__title = 'test title'
                                ).video
        new_title = "subtitler edited the title"
        self.tasks_tab.log_in(self.member.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Transcribe", video, title=new_title, edit='upload')
        self.editor_pg.save("Exit")
        self.assertEqual(video.title, self.video_pg.video_title())

    def test_youtube_edit_title_save_draft(self):
        """Edit title in transcribe, and save a draft """
        video = YouTubeVideoFactory(title='Youtube video test',
                                    primary_audio_language_code='en')

        TeamVideoFactory(team=self.team, video=video)
        new_title = "subtitler edited the title"
        self.tasks_tab.log_in(self.member.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Transcribe", video, title=new_title, edit='upload')
        self.editor_pg.save("Exit")
        self.assertEqual(video.title, self.video_pg.video_title())

    def test_transcribe_edit_title(self):
        """Edit title in transcribe, video title updated after publish """
        video = VideoFactory(primary_audio_language_code='en',
                             title = 'test title'
                             )
        TeamVideoFactory(video=video, team=self.team)
        new_title = "subtitler edited the title"
        self.tasks_tab.log_in(self.member.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Transcribe", video, title=new_title, edit='upload')
        self.editor_pg.collab_action("Complete")
        self.assertEqual(video.title, self.video_pg.video_title())
        self.data_utils.complete_review_task(video.get_team_video(), 20, self.manager)
        self.tasks_tab.log_in(self.admin.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Approve Original", video, title=new_title)
        self.editor_pg.collab_action("Approve")
        self.assertEqual(new_title, self.video_pg.video_title())

    def test_transcribe_edit_title_reviewer_subs(self):
        """Edit title in transcribe, video title updated after publish """
        video = VideoFactory(primary_audio_language_code='en',
                             title = 'test title'
                             )
        TeamVideoFactory(video=video, team=self.team)
        orig_title = video.title
        new_title = "edited"
        self.tasks_tab.log_in(self.member.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Transcribe", video, title=new_title, edit='upload')
        self.editor_pg.collab_action("Complete")
        self.assertEqual(orig_title, self.video_pg.video_title())
        self.tasks_tab.log_in(self.manager.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Review Original", video, title=None, edit='upload')
        self.editor_pg.collab_action("Approve")
        self.assertEqual(orig_title, self.video_pg.video_title())
        self.tasks_tab.log_in(self.admin.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Approve Original", video, title=None)
        self.editor_pg.collab_action("Approve")
        self.assertEqual(new_title, self.video_pg.video_title())

<commit_msg>update title test<commit_after># -*- coding: utf-8 -*-
import os
import time

from caching.tests.utils import assert_invalidates_model_cache
from ted import tasks
from videos.models import Video
from utils.factories import *
from subtitles.pipeline import add_subtitles
from webdriver_testing.webdriver_base import WebdriverTestCase
from webdriver_testing.pages.site_pages.teams_dir_page import TeamsDirPage
from webdriver_testing.pages.site_pages.teams.tasks_tab import TasksTab
from webdriver_testing.pages.site_pages.teams.videos_tab import VideosTab
#from webdriver_testing.data_factories import *
from webdriver_testing.pages.editor_pages import subtitle_editor
from webdriver_testing import data_helpers
from webdriver_testing.pages.site_pages import video_page
from webdriver_testing.pages.site_pages import video_language_page
from webdriver_testing.pages.site_pages import editor_page
from webdriver_testing.pages.site_pages import site_modals
from webdriver_testing.pages.site_pages import edit_video_page

class TestCaseModeratedVideoTitles(WebdriverTestCase):
    NEW_BROWSER_PER_TEST_CASE = False

    @classmethod
    def setUpClass(cls):
        super(TestCaseModeratedVideoTitles, cls).setUpClass()
        cls.data_utils = data_helpers.DataHelpers()
        cls.admin_video_pg = edit_video_page.EditVideoPage(cls)
        cls.tasks_tab = TasksTab(cls)
        cls.videos_tab = VideosTab(cls)
        cls.video_pg = video_page.VideoPage(cls)

        cls.modal = site_modals.SiteModals(cls)
        cls.editor_pg = editor_page.EditorPage(cls)
        cls.video_lang_pg = video_language_page.VideoLanguagePage(cls)

        cls.staff = UserFactory(is_staff=True, is_superuser=True)
        cls.admin = UserFactory()
        cls.manager = UserFactory()
        cls.member = UserFactory()
        cls.team = TeamFactory(admin=cls.admin,
                               manager=cls.manager,
                               member=cls.member,
                               workflow_type='O',
                               workflow_enabled=True
                              )

        cls.workflow = WorkflowFactory(
            team = cls.team,
            autocreate_subtitle = True,
            autocreate_translate = False,
            review_allowed = 10,
            approve_allowed = 20)
        #Create a member of the team
        cls.subs_file = os.path.join(os.getcwd(), 'apps','webdriver_testing',
                                    'subtitle_data', 'basic_subs.dfxp')

    def setUp(self):
        self.tasks_tab.open_team_page(self.team.slug)
        self.tasks_tab.handle_js_alert('accept')

    @classmethod
    def _create_subs(cls, video, lc, user, complete=False):
        subtitles_1 = [
            (0500,2000, 'Hello there'),
        ]
        subtitles_2 = [
            (0500, 2000, 'Hello there'),
            (3000, 5000, 'Hello there'),
        ]
        add_subtitles(video, lc, subtitles_1)
        add_subtitles(video, lc, subtitles_2,
                      author=user,
                      committer=user,
                      action='complete'
                      )

    def perform_task(self, task_type, video, title=None, edit=None):
        self.tasks_tab.perform_task("%s English Subtitles" % task_type,
                                    video.title)
        if edit == 'upload':
            self.editor_pg.upload_subtitles(self.subs_file)
        if edit == 'type':
            self.editor_pg.edit_sub_line('edit', 1)
        if title:
            self.editor_pg.edit_title(title)

    def test_speakername_edit_in_task(self):
        video = TeamVideoFactory(team=self.team,
                                 video__primary_audio_language_code='en'
                                ).video
        self.video_pg.open_video_page(video.video_id)
        self.admin_video_pg.log_in(self.staff.username, 'password')
        self.admin_video_pg.open_edit_video_page(video.id)
        self.admin_video_pg.add_speaker_name('Jerry Garcia')
        self.video_pg.open_video_page(video.video_id)
        self._create_subs(video, 'en', self.member, complete=True)
        self.data_utils.complete_review_task(video.get_team_video(),
                                            20, self.manager)
        self.video_pg.open_video_page(video.video_id)
        self.assertEqual(video.title, self.video_pg.video_title())
        self.data_utils.complete_approve_task(video.get_team_video(),
                                            20, self.manager)

    def test_post_publish_edit(self):
        """Edit title in approve, video title updated after publish """
        video = TeamVideoFactory(team=self.team,
                                 video__primary_audio_language_code='en',
                                ).video
        self._create_subs(video, 'en', self.member, complete=True)
        orig_title = video.title
        self.logger.info(video.title)
        new_title = 'this is a new title'
        self.tasks_tab.log_in(self.manager.username, 'password')
        self.data_utils.complete_review_task(video.get_team_video(), 20, self.manager)
        self.data_utils.complete_approve_task(video.get_team_video(), 20, self.manager)
        self.tasks_tab.log_in(self.admin.username, 'password')
        self.video_lang_pg.open_video_lang_page(video.video_id, 'en')
        self.video_lang_pg.edit_subtitles()
        self.editor_pg.edit_title(new_title)
        with assert_invalidates_model_cache(video):
            self.editor_pg.collab_action('Publish')
            self.assertEqual(new_title, self.video_pg.video_title())

    def test_approve_edit_title(self):
        """Edit title in approve, video title updated after publish """
        video = TeamVideoFactory(team=self.team,
                                 video__primary_audio_language_code='en'
                                ).video
        self._create_subs(video, 'en', self.member, complete=True)
        orig_title = video.title
        new_title = 'this is a new title'
        self.tasks_tab.log_in(self.manager.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.data_utils.complete_review_task(video.get_team_video(), 20, self.manager)
        self.tasks_tab.log_in(self.admin.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Approve Original", video, title=new_title)
        self.editor_pg.collab_action("Approve")
        self.assertEqual(new_title, self.video_pg.video_title())

    def test_review_edit_title(self):
        """Edit title in review, video title updated after publish """
        video = TeamVideoFactory(team=self.team,
                                 video__primary_audio_language_code='en'
                                ).video
        self._create_subs(video, 'en', self.member, complete=True)
        orig_title = video.title
        new_title = 'this is a new title'
        self.tasks_tab.log_in(self.manager.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Review Original", video, title=new_title)
        self.editor_pg.collab_action("Approve")
        self.assertEqual(video.title, self.video_pg.video_title())
        self.tasks_tab.log_in(self.admin.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Approve Original", video, title=None)
        self.editor_pg.collab_action("Approve")
        self.video_pg.open_video_page(video.video_id)
        self.assertEqual(new_title, self.video_pg.video_title())

    def test_review_edit_title_reject(self):
        """Edit title in review, video title updated after publish """
        video = VideoFactory(title = 'test title',
                             primary_audio_language_code='en'
                            )
        TeamVideoFactory(team=self.team, video=video)
        self._create_subs(video, 'en', self.member, complete=True)
        orig_title = 'test title'
        new_title = 'this is a new title'
        self.tasks_tab.log_in(self.manager.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Review Original", video, title=new_title, edit='type')
        self.editor_pg.collab_action("Send Back")
        self.assertEqual(orig_title, self.video_pg.video_title())
        self.tasks_tab.log_in(self.member.username, 'password')
        self.tasks_tab.open_page('teams/%s/tasks/?assignee=me&/'
                                 % self.team.slug)
        self.perform_task("Transcribe", video, edit='upload', title=None)
        self.editor_pg.collab_action("Complete")
        self.assertEqual(orig_title, self.video_pg.video_title())
        self.data_utils.complete_review_task(video.get_team_video(), 20, self.manager)
        self.data_utils.complete_approve_task(video.get_team_video(), 20, self.admin)
        self.video_pg.open_video_page(video.video_id)
        self.assertEqual(new_title, self.video_pg.video_title())

    def test_transcribe_edit_title_save_draft(self):
        """Edit title in transcribe, and save a draft """
        video = TeamVideoFactory(team=self.team,
                                 video__primary_audio_language_code='en',
                                 video__title = 'test title'
                                ).video
        new_title = "subtitler edited the title"
        self.tasks_tab.log_in(self.member.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Transcribe", video, title=new_title, edit='upload')
        self.editor_pg.save("Exit")
        self.assertEqual(video.title, self.video_pg.video_title())

    def test_youtube_edit_title_save_draft(self):
        """Edit title in transcribe, and save a draft """
        video = YouTubeVideoFactory(title='Youtube video test',
                                    primary_audio_language_code='en')

        TeamVideoFactory(team=self.team, video=video)
        new_title = "subtitler edited the title"
        self.tasks_tab.log_in(self.member.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Transcribe", video, title=new_title, edit='upload')
        self.editor_pg.save("Exit")
        self.assertEqual(video.title, self.video_pg.video_title())

    def test_transcribe_edit_title(self):
        """Edit title in transcribe, video title updated after publish """
        video = VideoFactory(primary_audio_language_code='en',
                             title = 'test title'
                             )
        TeamVideoFactory(video=video, team=self.team)
        new_title = "subtitler edited the title"
        self.tasks_tab.log_in(self.member.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Transcribe", video, title=new_title, edit='upload')
        self.editor_pg.collab_action("Complete")
        self.assertEqual(video.title, self.video_pg.video_title())
        self.data_utils.complete_review_task(video.get_team_video(), 20, self.manager)
        self.tasks_tab.log_in(self.admin.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Approve Original", video, title=new_title)
        self.editor_pg.collab_action("Approve")
        self.assertEqual(new_title, self.video_pg.video_title())

    def test_transcribe_edit_title_reviewer_subs(self):
        """Edit title in transcribe, video title updated after publish """
        video = VideoFactory(primary_audio_language_code='en',
                             title = 'test title'
                             )
        TeamVideoFactory(video=video, team=self.team)
        orig_title = 'test title'
        new_title = "edited"
        self.tasks_tab.log_in(self.member.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Transcribe", video, title=new_title, edit='upload')
        self.editor_pg.collab_action("Complete")
        self.assertEqual(orig_title, self.video_pg.video_title())
        self.tasks_tab.log_in(self.manager.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Review Original", video, title=None, edit='upload')
        self.editor_pg.collab_action("Approve")
        self.assertEqual(orig_title, self.video_pg.video_title())
        self.tasks_tab.log_in(self.admin.username, 'password')
        self.tasks_tab.open_tasks_tab(self.team.slug)
        self.perform_task("Approve Original", video, title=None)
        self.editor_pg.collab_action("Approve")
        self.assertEqual(new_title, self.video_pg.video_title())

<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import cPickle as pkl
import os

from collections import Counter, namedtuple

from htmresearch.frameworks.nlp.runner import Runner
from htmresearch.frameworks.nlp.classify_htm import ClassificationModelHTM
from htmresearch.support.network_text_data_generator import NetworkDataGenerator

try:
  import simplejson as json
except ImportError:
  import json

class HTMRunner(Runner):
  """
  Class to run the HTM NLP experiments with the specified data and evaluation
  metrics.
  """

  def __init__(self,
               dataPath,
               networkConfigPath,
               resultsDir,
               experimentName,
               loadPath,
               modelName,
               retinaScaling=1.0,
               retina="en_associative",
               apiKey=None,
               numClasses=3,
               plots=0,
               orderedSplit=False,
               trainSizes=None,
               verbosity=0,
               generateData=True,
               votingMethod="last",
               classificationFile=""):
    """
    @param networkConfigPath  (str)    Path to JSON specifying network params.
    @param generateData       (bool)   Whether or not we need to generate data.
    @param votingMethod       (str)    Classify with "last" token's score or
                                       "most" frequent of the sequence.
    @param classificationFile (str)    Path to JSON that maps labels to ids.

    See base class constructor for the other parameters.
    """
    super(HTMRunner, self).__init__(dataPath, resultsDir, experimentName,
                                    modelName, retinaScaling, retina, apiKey,
                                    loadPath, numClasses, plots, orderedSplit,
                                    trainSizes, verbosity)

    self.networkConfig = self._getNetworkConfig(networkConfigPath)
    self.model = None
    self.votingMethod = votingMethod
    self.dataFiles = []
    self.actualLabels = None

    if classificationFile == "" and not generateData:
      raise ValueError("Must give classificationFile if not generating data")
    self.classificationFile = classificationFile

    # Setup data now in order to init the network model. If you want to
    # specify data params, just call setupData() again later.
    self.setupNetData(generateData=generateData)

  @staticmethod
  def _getNetworkConfig(networkConfigPath):
    try:
      with open(networkConfigPath, "rb") as fin:
        return json.load(fin)
    except IOError as e:
      print "Could not find network configuration JSON at \'{}\'.".format(
        networkConfigPath)
      raise e

  def initModel(self, trial=0):
    """
    Load or instantiate the classification model. Assumes network data is
    already setup.
    """
    if self.loadPath:
      with open(self.loadPath, "rb") as f:
        self.model = pkl.load(f)
      # TODO: uncomment once we can save TPRegion; do we need this?
      # networkFile = self.model.network
      # self.model.network = Network(networkFile)
      print "Model loaded from \'{0}\'.".format(self.loadPath)
    else:
      self.model = ClassificationModelHTM(self.networkConfig,
                                          self.dataFiles[trial],
                                          retinaScaling=self.retinaScaling,
                                          retina=self.retina,
                                          apiKey=self.apiKey,
                                          verbosity=self.verbosity,
                                          numLabels=self.numClasses,
                                          modelDir=self.modelDir,
                                          prepData=False)

  def setupData(self, _):
    """Passthrough b/c network data generation was done upfront."""
    pass

  def setupNetData(self, preprocess=False, generateData=False, **kwargs):
    """
    Generate the data in network API format if necessary. self.dataFiles is
    populated with the paths of network data files, one for each trial

    Look at runner.py (setupData) and network_text_data_generator.py (split) for
    the parameters.
    """
    if generateData:
      # TODO: use model.prepData()?
      ndg = NetworkDataGenerator()
      ndg.split(self.dataPath, self.numClasses, preprocess, **kwargs)

      filename, ext = os.path.splitext(self.dataPath)
      self.classificationFile = "{}_categories.json".format(filename)

      for i in xrange(len(self.trainSizes)):
        if not self.orderedSplit:
          ndg.randomizeData()
        dataFile = "{}_network_{}{}".format(filename, i, ext)
        ndg.saveData(dataFile, self.classificationFile)
        self.dataFiles.append(dataFile)

      if self.verbosity > 0:
        print "{} file(s) generated at {}".format(len(self.dataFiles),
          self.dataFiles)
        print "Classification JSON is at: {}".format(self.classificationFile)
    else:
      # Use the input file for each trial; maintains the order of samples.
      self.dataFiles = [self.dataPath] * len(self.trainSizes)

    if self.numClasses > 0:
      # Setup labels data objects
      self.actualLabels = [self._getClassifications(size, i)
        for i, size in enumerate(self.trainSizes)]
      self._mapLabelRefs()

  def _getClassifications(self, split, trial):
    """
    Gets the classifications for testing samples for a particular trial
    @param split      (int)       Size of training set
    @param trial      (int)       trial count
    @return           (list)      List of list of ids of classifications for a
                                  sample
    """
    # import pdb; pdb.set_trace()
    dataFile = self.dataFiles[trial]
    classifications = NetworkDataGenerator.getClassifications(dataFile)
    return [[int(c) for c in classes.strip().split(" ")]
             for classes in classifications][split:]

  def _mapLabelRefs(self):
    """Get the mapping from label strings to the corresponding ints."""
    try:
      with open(self.classificationFile, "r") as f:
        labelToId = json.load(f)
      # Convert the dict of strings -> ids to a list of strings ordered by id
      self.labelRefs = zip(*sorted(labelToId.iteritems(), key=lambda x:x[1]))[0]
    except IOError as e:
      print "Must have a valid classification JSON file"
      raise e

  def resetModel(self, trial=0):
    """
    Load or instantiate the classification model; network API doesn't support
    resetting."""
    self.initModel(trial=trial)
    # TODO: change to same as Runner:
    #   self.model.resetModel()
    #   otherwise you're creating a new model instance twice each experiment

  def encodeSamples(self):
    """Passthrough b/c the network encodes the samples."""
    pass

  def _training(self, trial):
    """
    Train the network on all the tokens in the training set for a particular
    trial.
    @param trial      (int)       current trial number
    """
    if self.verbosity > 0:
      i = 0
      indices = []
      for numTokens in self.partitions[trial][0]:
        indices.append(i)
        i += numTokens
      print ("\tRunner selects to train on sequences starting at indices {}.".
            format(indices))

    for numTokens in self.partitions[trial][0]:
      self.model.trainModel(iterations=numTokens)

  def _selectWinners(self, predictions):
    """
    Selects the final classifications for the predictions.  Voting
    method=="last" means the predictions of the last sample are used. Voting
    method=="most" means the most frequent sample is used.
    @param predictions    (list)    List of list of possible classifications
    @return               (list)    List of winning classifications
    """
    if self.votingMethod == "last":
      return predictions[-1]
    elif self.votingMethod == "most":
      counter = Counter()
      for p in predictions:
        counter.update(p)
      return zip(*counter.most_common(self.numClasses))[0]
    else:
      raise ValueError("voting method must be either \'last\' or \'most\'")

  def _testing(self, trial):
    """
    Test the network on the test set for a particular trial and store the
    results
    @param trial      (int)       trial count
    """
    if self.verbosity > 0:
      i = sum(self.partitions[trial][0])
      indices = []
      for numTokens in self.partitions[trial][1]:
        indices.append(i)
        i += numTokens
      print ("\tRunner selects to test on sequences starting at indices "
             "{}".format(indices))

    results = ([], [])
    for i, numTokens in enumerate(self.partitions[trial][1]):
      predictions = []
      for _ in xrange(numTokens):
        predicted = self.model.testModel()
        predictions.append(predicted)
      winningPredictions = self._selectWinners(predictions)

      # TODO: switch to standard (expected, actual) format
      results[0].append(winningPredictions)
      results[1].append(self.actualLabels[trial][i])

    # Prepare data for writeOutClassifications
    trainIdx = range(len(self.partitions[trial][0]))
    testIdx = range(len(self.partitions[trial][0]),
      len(self.partitions[trial][0]) + len(self.partitions[trial][1]))
    self.partitions[trial] = (trainIdx, testIdx)
    self.samples = NetworkDataGenerator.getSamples(self.dataFiles[trial])

    self.results.append(results)

  def partitionIndices(self):
    """
    Sets self.partitions for the number of tokens for each sample in the
    training and test sets (when doing an ordered split).
    """
    for trial, split in enumerate(self.trainSizes):
      dataFile = self.dataFiles[trial]
      numTokens = NetworkDataGenerator.getNumberOfTokens(dataFile)
      self.partitions.append((numTokens[:split], numTokens[split:]))

  # TODO
  # This method is to partition data for which regions are learning, as in the
  # sequence classification experiments.
  def partitionLearning(self):
    """
    Find the number of partitions for the input data based on a specific
    networkConfig.

    @return partitions: (list of namedtuples) Region names and index at which the
      region is to begin learning. The final partition is reserved as a test set.
    """
    Partition = namedtuple("Partition", "partName index")

    # Add regions to partition list in order of learning.
    regionConfigs = ("spRegionConfig", "tmRegionConfig", "tpRegionConfig",
      "classifierRegionConfig")
    partitions = []

    return

  def writeOutClassifications(self):
    # TODO: implement this method after updating HTM network models and runner
    # per nupic.research #277
    pass
<commit_msg>add seed for splitting network api data<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import cPickle as pkl
import os

from collections import Counter, namedtuple

from htmresearch.frameworks.nlp.runner import Runner
from htmresearch.frameworks.nlp.classify_htm import ClassificationModelHTM
from htmresearch.support.network_text_data_generator import NetworkDataGenerator

try:
  import simplejson as json
except ImportError:
  import json

class HTMRunner(Runner):
  """
  Class to run the HTM NLP experiments with the specified data and evaluation
  metrics.
  """

  def __init__(self,
               dataPath,
               networkConfigPath,
               resultsDir,
               experimentName,
               loadPath,
               modelName,
               retinaScaling=1.0,
               retina="en_associative",
               apiKey=None,
               numClasses=3,
               plots=0,
               orderedSplit=False,
               trainSizes=None,
               verbosity=0,
               generateData=True,
               votingMethod="last",
               classificationFile="",
               seed=42):
    """
    @param networkConfigPath  (str)    Path to JSON specifying network params.
    @param generateData       (bool)   Whether or not we need to generate data.
    @param votingMethod       (str)    Classify with "last" token's score or
                                       "most" frequent of the sequence.
    @param classificationFile (str)    Path to JSON that maps labels to ids.

    See base class constructor for the other parameters.
    """
    super(HTMRunner, self).__init__(dataPath, resultsDir, experimentName,
                                    modelName, retinaScaling, retina, apiKey,
                                    loadPath, numClasses, plots, orderedSplit,
                                    trainSizes, verbosity)

    self.networkConfig = self._getNetworkConfig(networkConfigPath)
    self.model = None
    self.votingMethod = votingMethod
    self.dataFiles = []
    self.actualLabels = None

    if classificationFile == "" and not generateData:
      raise ValueError("Must give classificationFile if not generating data")
    self.classificationFile = classificationFile

    # Setup data now in order to init the network model. If you want to
    # specify data params, just call setupData() again later.
    self.setupNetData(generateData=generateData, seed=seed)

  @staticmethod
  def _getNetworkConfig(networkConfigPath):
    try:
      with open(networkConfigPath, "rb") as fin:
        return json.load(fin)
    except IOError as e:
      print "Could not find network configuration JSON at \'{}\'.".format(
        networkConfigPath)
      raise e

  def initModel(self, trial=0):
    """
    Load or instantiate the classification model. Assumes network data is
    already setup.
    """
    if self.loadPath:
      with open(self.loadPath, "rb") as f:
        self.model = pkl.load(f)
      # TODO: uncomment once we can save TPRegion; do we need this?
      # networkFile = self.model.network
      # self.model.network = Network(networkFile)
      print "Model loaded from \'{0}\'.".format(self.loadPath)
    else:
      self.model = ClassificationModelHTM(self.networkConfig,
                                          self.dataFiles[trial],
                                          retinaScaling=self.retinaScaling,
                                          retina=self.retina,
                                          apiKey=self.apiKey,
                                          verbosity=self.verbosity,
                                          numLabels=self.numClasses,
                                          modelDir=self.modelDir,
                                          prepData=False)

  def setupData(self, _):
    """Passthrough b/c network data generation was done upfront."""
    pass

  def setupNetData(self, generateData=False, seed=42, preprocess=False, **kwargs):
    """
    Generate the data in network API format if necessary. self.dataFiles is
    populated with the paths of network data files, one for each trial

    Look at runner.py (setupData) and network_text_data_generator.py (split) for
    the parameters.
    """
    if generateData:
      # TODO: use model.prepData()?
      ndg = NetworkDataGenerator()
      ndg.split(self.dataPath, self.numClasses, preprocess, **kwargs)

      filename, ext = os.path.splitext(self.dataPath)
      self.classificationFile = "{}_categories.json".format(filename)

      for i in xrange(len(self.trainSizes)):
        if not self.orderedSplit:
          ndg.randomizeData(seed)
        dataFile = "{}_network_{}{}".format(filename, i, ext)
        ndg.saveData(dataFile, self.classificationFile)
        self.dataFiles.append(dataFile)

      if self.verbosity > 0:
        print "{} file(s) generated at {}".format(len(self.dataFiles),
          self.dataFiles)
        print "Classification JSON is at: {}".format(self.classificationFile)
    else:
      # Use the input file for each trial; maintains the order of samples.
      self.dataFiles = [self.dataPath] * len(self.trainSizes)

    if self.numClasses > 0:
      # Setup labels data objects
      self.actualLabels = [self._getClassifications(size, i)
        for i, size in enumerate(self.trainSizes)]
      self._mapLabelRefs()

  def _getClassifications(self, split, trial):
    """
    Gets the classifications for testing samples for a particular trial
    @param split      (int)       Size of training set
    @param trial      (int)       trial count
    @return           (list)      List of list of ids of classifications for a
                                  sample
    """
    # import pdb; pdb.set_trace()
    dataFile = self.dataFiles[trial]
    classifications = NetworkDataGenerator.getClassifications(dataFile)
    return [[int(c) for c in classes.strip().split(" ")]
             for classes in classifications][split:]

  def _mapLabelRefs(self):
    """Get the mapping from label strings to the corresponding ints."""
    try:
      with open(self.classificationFile, "r") as f:
        labelToId = json.load(f)
      # Convert the dict of strings -> ids to a list of strings ordered by id
      self.labelRefs = zip(*sorted(labelToId.iteritems(), key=lambda x:x[1]))[0]
    except IOError as e:
      print "Must have a valid classification JSON file"
      raise e

  def resetModel(self, trial=0):
    """
    Load or instantiate the classification model; network API doesn't support
    resetting."""
    self.initModel(trial=trial)
    # TODO: change to same as Runner:
    #   self.model.resetModel()
    #   otherwise you're creating a new model instance twice each experiment

  def encodeSamples(self):
    """Passthrough b/c the network encodes the samples."""
    pass

  def _training(self, trial):
    """
    Train the network on all the tokens in the training set for a particular
    trial.
    @param trial      (int)       current trial number
    """
    if self.verbosity > 0:
      i = 0
      indices = []
      for numTokens in self.partitions[trial][0]:
        indices.append(i)
        i += numTokens
      print ("\tRunner selects to train on sequences starting at indices {}.".
            format(indices))

    for numTokens in self.partitions[trial][0]:
      self.model.trainModel(iterations=numTokens)

  def _selectWinners(self, predictions):
    """
    Selects the final classifications for the predictions.  Voting
    method=="last" means the predictions of the last sample are used. Voting
    method=="most" means the most frequent sample is used.
    @param predictions    (list)    List of list of possible classifications
    @return               (list)    List of winning classifications
    """
    if self.votingMethod == "last":
      return predictions[-1]
    elif self.votingMethod == "most":
      counter = Counter()
      for p in predictions:
        counter.update(p)
      return zip(*counter.most_common(self.numClasses))[0]
    else:
      raise ValueError("voting method must be either \'last\' or \'most\'")

  def _testing(self, trial):
    """
    Test the network on the test set for a particular trial and store the
    results
    @param trial      (int)       trial count
    """
    if self.verbosity > 0:
      i = sum(self.partitions[trial][0])
      indices = []
      for numTokens in self.partitions[trial][1]:
        indices.append(i)
        i += numTokens
      print ("\tRunner selects to test on sequences starting at indices "
             "{}".format(indices))

    results = ([], [])
    for i, numTokens in enumerate(self.partitions[trial][1]):
      predictions = []
      for _ in xrange(numTokens):
        predicted = self.model.testModel()
        predictions.append(predicted)
      winningPredictions = self._selectWinners(predictions)

      # TODO: switch to standard (expected, actual) format
      results[0].append(winningPredictions)
      results[1].append(self.actualLabels[trial][i])

    # Prepare data for writeOutClassifications
    trainIdx = range(len(self.partitions[trial][0]))
    testIdx = range(len(self.partitions[trial][0]),
      len(self.partitions[trial][0]) + len(self.partitions[trial][1]))
    self.partitions[trial] = (trainIdx, testIdx)
    self.samples = NetworkDataGenerator.getSamples(self.dataFiles[trial])

    self.results.append(results)

  def partitionIndices(self, _):
    """
    Sets self.partitions for the number of tokens for each sample in the
    training and test sets.

    The order of sequences is already specified by the network data files; if
    generated by the experiment, these are in order or randomized as specified
    by the orderedSPlit arg.
    """
    for trial, split in enumerate(self.trainSizes):
      import pdb; pdb.set_trace()
      dataFile = self.dataFiles[trial]
      numTokens = NetworkDataGenerator.getNumberOfTokens(dataFile)
      self.partitions.append((numTokens[:split], numTokens[split:]))

  # TODO
  # This method is to partition data for which regions are learning, as in the
  # sequence classification experiments.
  def partitionLearning(self):
    """
    Find the number of partitions for the input data based on a specific
    networkConfig.

    @return partitions: (list of namedtuples) Region names and index at which the
      region is to begin learning. The final partition is reserved as a test set.
    """
    Partition = namedtuple("Partition", "partName index")

    # Add regions to partition list in order of learning.
    regionConfigs = ("spRegionConfig", "tmRegionConfig", "tpRegionConfig",
      "classifierRegionConfig")
    partitions = []

    return

  def writeOutClassifications(self):
    # TODO: implement this method after updating HTM network models and runner
    # per nupic.research #277
    pass
<|endoftext|>


<commit_before><commit_msg>Fixed the "I am willing to mentor this student" button on the review page.<commit_after><|endoftext|>


<commit_before>
__copyright__ = "Copyright 2017-2018, http://radical.rutgers.edu"
__author__    = "Vivek Balasubramanian <vivek.balasubramaniana@rutgers.edu>"
__license__   = "MIT"

import os

import radical.pilot as rp

from ...                     import exceptions as ree
from ..base.resource_manager import Base_ResourceManager

# ------------------------------------------------------------------------------
#
class ResourceManager(Base_ResourceManager):
    """
    A resource manager takes the responsibility of placing resource requests on
    different, possibly multiple, DCIs. This ResourceManager uses the RADICAL
    Pilot as the underlying runtime system.

    :arguments:
        :resource_desc: dictionary with details of the resource request and
                        access credentials of the user
        :example: resource_desc = {
                                    |  'resource'      : 'xsede.stampede',
                                    |  'walltime'      : 120,
                                    |  'cpus'          : 64,
                                    |  'project'       : 'TG-abcxyz',
                                    |  'queue'         : 'abc',    # optional
                                    |  'access_schema' : 'ssh'  # optional}
    """

    # --------------------------------------------------------------------------
    #
    def __init__(self, resource_desc, sid, rts_config):

        super(ResourceManager, self).__init__(resource_desc=resource_desc,
                                              sid=sid,
                                              rts='radical.pilot',
                                              rts_config=rts_config)
        # RP specific parameters
        self._session             = None
        self._pmgr                = None
        self._pilot               = None
        self._download_rp_profile = False

        if "sandbox_cleanup" not in self._rts_config or \
           "db_cleanup"      not in self._rts_config:

            raise ree.ValueError(obj=self._uid,
                                 attribute='config',
                                 expected_value={"sandbox_cleanup": False,
                                                 "db_cleanup"     : False},
                                 actual_value=self._rts_config)

        self._logger.info('Created resource manager object: %s' % self._uid)
        self._prof.prof('rmgr obj created', uid=self._uid)

    # --------------------------------------------------------------------------
    #
    def __del__(self):
        '''
        RE must release the session to avoid leaking threads
        '''

        try:
            if self._session:
                self._session.close()
                self._session = None
        except:
            pass

    # --------------------------------------------------------------------------
    #
    @property
    def session(self):
        """
        :getter: Return the Radical Pilot session currently being used
        """
        return self._session

    @property
    def pmgr(self):
        """
        :getter: Return the Radical Pilot manager currently being used
        """
        return self._pmgr

    @property
    def pilot(self):
        """
        :getter: Return the submitted Pilot
        """
        return self._pilot

    # --------------------------------------------------------------------------
    #
    def get_resource_allocation_state(self):
        """
        **Purpose**: Get the state of the resource allocation

        """

        if self._pilot:
            return self._pilot.state

    # --------------------------------------------------------------------------
    #
    def get_completed_states(self):
        """
        **Purpose**: return states which signal completed resource allocation

        """

        return rp.FINAL

    # --------------------------------------------------------------------------
    #
    def _submit_resource_request(self):
        """
        **Purpose**: Create and submits a RADICAL Pilot Job as per the user
                     provided resource description
        """

        try:

            self._prof.prof('creating rreq', uid=self._uid)

            # ------------------------------------------------------------------
            def _pilot_state_cb(pilot, state):

                self._logger.info('Pilot %s state: %s' % (pilot.uid, state))

                if state == rp.FAILED:
                    self._logger.error('Pilot has failed')

                elif state == rp.DONE:
                    self._logger.error('Pilot has completed')
            # ------------------------------------------------------------------

            self._session = rp.Session(uid=self._sid)
            self._pmgr    = rp.PilotManager(session=self._session)

            self._pmgr.register_callback(_pilot_state_cb)

            cleanup = self._rts_config.get('sandbox_cleanup')
            pd_init = {'resource'      : self._resource,
                       'runtime'       : self._walltime,
                       'cores'         : self._cpus,
                       'project'       : self._project,
                       'gpus'          : self._gpus,
                       'access_schema' : self._access_schema,
                       'queue'         : self._queue,
                       'cleanup'       : cleanup,
                       'input_staging' : self._shared_data,
                       'output_staging': self._outputs
                       }

            # Create Compute Pilot with validated resource description
            pdesc = rp.ComputePilotDescription(pd_init)

            self._prof.prof('rreq created', uid=self._uid)

            # Launch the pilot
            self._pilot = self._pmgr.submit_pilots(pdesc)

            self._prof.prof('rreq submitted', uid=self._uid)

            self._logger.info('Resource request submission successful, waiting'
                              'for pilot to become Active')

            # Wait for pilot to go active
            self._pilot.wait([rp.PMGR_ACTIVE, rp.DONE, rp.FAILED, rp.CANCELED])

            self._prof.prof('resource active', uid=self._uid)
            self._logger.info('Pilot is now active [%s]', self._pilot.state)

        except KeyboardInterrupt:

            if self._session:
                self._session.close()
                self._session = None

            self._logger.exception('Execution interrupted (probably by Ctrl+C) '
                                   'exit callback thread gracefully...')
            raise KeyboardInterrupt

        except Exception:
            self._logger.exception('Resource request submission failed')
            raise

    # --------------------------------------------------------------------------
    #
    def _terminate_resource_request(self):
        """
        **Purpose**: Cancel the RADICAL Pilot Job
        """

        try:

            if self._pilot:

                self._prof.prof('rreq_cancel', uid=self._uid)
                self._logger.debug('Terminating pilot')
                self._pilot.cancel()
                self._logger.debug('Pilot terminated')

                # once the workflow is completed, fetch output data
                if self._outputs:
                    self._pilot.stage_out()

                get_profiles = os.environ.get('RADICAL_PILOT_PROFILE', False)
                cleanup      = self._rts_config.get('db_cleanup', False)

                if self._session:
                    self._session.close(cleanup=cleanup, download=get_profiles)
                    self._session = None

                self._prof.prof('rreq_canceled', uid=self._uid)

        except KeyboardInterrupt:

            self._logger.exception('Execution interrupted (probably by Ctrl+C) '
                                   'exit callback thread gracefully...')
            raise KeyboardInterrupt

        except Exception:
            self._logger.exception('Could not cancel resource request')
            raise

# ------------------------------------------------------------------------------

<commit_msg>attempt to fix #413<commit_after>
__copyright__ = "Copyright 2017-2018, http://radical.rutgers.edu"
__author__    = "Vivek Balasubramanian <vivek.balasubramaniana@rutgers.edu>"
__license__   = "MIT"

import os

import radical.pilot as rp

from ...                     import exceptions as ree
from ..base.resource_manager import Base_ResourceManager

# ------------------------------------------------------------------------------
#
class ResourceManager(Base_ResourceManager):
    """
    A resource manager takes the responsibility of placing resource requests on
    different, possibly multiple, DCIs. This ResourceManager uses the RADICAL
    Pilot as the underlying runtime system.

    :arguments:
        :resource_desc: dictionary with details of the resource request and
                        access credentials of the user
        :example: resource_desc = {
                                    |  'resource'      : 'xsede.stampede',
                                    |  'walltime'      : 120,
                                    |  'cpus'          : 64,
                                    |  'project'       : 'TG-abcxyz',
                                    |  'queue'         : 'abc',    # optional
                                    |  'access_schema' : 'ssh'  # optional}
    """

    # --------------------------------------------------------------------------
    #
    def __init__(self, resource_desc, sid, rts_config):

        super(ResourceManager, self).__init__(resource_desc=resource_desc,
                                              sid=sid,
                                              rts='radical.pilot',
                                              rts_config=rts_config)
        # RP specific parameters
        self._session             = None
        self._pmgr                = None
        self._pilot               = None
        self._download_rp_profile = False

        if "sandbox_cleanup" not in self._rts_config or \
           "db_cleanup"      not in self._rts_config:

            raise ree.ValueError(obj=self._uid,
                                 attribute='config',
                                 expected_value={"sandbox_cleanup": False,
                                                 "db_cleanup"     : False},
                                 actual_value=self._rts_config)

        self._logger.info('Created resource manager object: %s' % self._uid)
        self._prof.prof('rmgr obj created', uid=self._uid)

    # --------------------------------------------------------------------------
    #
    def __del__(self):
        '''
        RE must release the session to avoid leaking threads
        '''

        try:
            if self._session:
                self._session.close()
                self._session = None
        except:
            pass

    # --------------------------------------------------------------------------
    #
    @property
    def session(self):
        """
        :getter: Return the Radical Pilot session currently being used
        """
        return self._session

    @property
    def pmgr(self):
        """
        :getter: Return the Radical Pilot manager currently being used
        """
        return self._pmgr

    @property
    def pilot(self):
        """
        :getter: Return the submitted Pilot
        """
        return self._pilot

    # --------------------------------------------------------------------------
    #
    def get_resource_allocation_state(self):
        """
        **Purpose**: Get the state of the resource allocation

        """

        if self._pilot:
            return self._pilot.state

    # --------------------------------------------------------------------------
    #
    def get_completed_states(self):
        """
        **Purpose**: return states which signal completed resource allocation

        """

        return rp.FINAL

    # --------------------------------------------------------------------------
    #
    def _submit_resource_request(self):
        """
        **Purpose**: Create and submits a RADICAL Pilot Job as per the user
                     provided resource description
        """

        try:

            self._prof.prof('creating rreq', uid=self._uid)

            # ------------------------------------------------------------------
            def _pilot_state_cb(pilot, state):

                self._logger.info('Pilot %s state: %s' % (pilot.uid, state))

                if state == rp.FAILED:
                    self._logger.error('Pilot has failed')

                elif state == rp.DONE:
                    self._logger.error('Pilot has completed')
            # ------------------------------------------------------------------

            self._session = rp.Session(uid=self._sid)
            self._pmgr    = rp.PilotManager(session=self._session)

            self._pmgr.register_callback(_pilot_state_cb)

            cleanup = self._rts_config.get('sandbox_cleanup')
            pd_init = {'resource'      : self._resource,
                       'runtime'       : self._walltime,
                       'cores'         : self._cpus,
                       'project'       : self._project,
                       'gpus'          : self._gpus,
                       'access_schema' : self._access_schema,
                       'queue'         : self._queue,
                       'cleanup'       : cleanup,
                       'input_staging' : self._shared_data,
                       'output_staging': self._outputs
                       }

            # Create Compute Pilot with validated resource description
            pdesc = rp.ComputePilotDescription(pd_init)

            self._prof.prof('rreq created', uid=self._uid)

            # Launch the pilot
            self._pilot = self._pmgr.submit_pilots(pdesc)

            self._prof.prof('rreq submitted', uid=self._uid)

            self._logger.info('Resource request submission successful, waiting'
                              'for pilot to become Active')

            # Wait for pilot to go active
            self._pilot.wait([rp.PMGR_ACTIVE, rp.DONE, rp.FAILED, rp.CANCELED])

            self._prof.prof('resource active', uid=self._uid)
            self._logger.info('Pilot is now active [%s]', self._pilot.state)

        except KeyboardInterrupt:

            if self._session:
                self._session.close()
                self._session = None

            self._logger.exception('Execution interrupted (probably by Ctrl+C) '
                                   'exit callback thread gracefully...')
            raise KeyboardInterrupt

        except Exception:
            self._logger.exception('Resource request submission failed')
            raise

    # --------------------------------------------------------------------------
    #
    def _terminate_resource_request(self):
        """
        **Purpose**: Cancel the RADICAL Pilot Job
        """

        try:

            if self._pilot:

                self._prof.prof('rreq_cancel', uid=self._uid)
                self._logger.debug('Terminating pilot')
                self._pilot.cancel()
                self._logger.debug('Pilot terminated')

                # once the workflow is completed, fetch output data
                if self._outputs:
                    self._pilot.stage_out()

                # make this a config option?
                if 'RADICAL_PILOT_PROFILE' in os.environ or \
                   'RADICAL_ENTK_PROFILE'  in os.environ or \
                   'RADICAL_PROFILE'       in os.environ :
                    get_profiles = True
                else:
                    get_profiles = False

                cleanup = self._rts_config.get('db_cleanup', False)

                if self._session:
                    self._session.close(cleanup=cleanup, download=get_profiles)
                    self._session = None

                self._prof.prof('rreq_canceled', uid=self._uid)

        except KeyboardInterrupt:

            self._logger.exception('Execution interrupted (probably by Ctrl+C) '
                                   'exit callback thread gracefully...')
            raise KeyboardInterrupt

        except Exception:
            self._logger.exception('Could not cancel resource request')
            raise

# ------------------------------------------------------------------------------

<|endoftext|>


<commit_before>import pytest
from django.apps.registry import apps

from modernrpc.apps import check_required_settings_defined
from modernrpc.core import get_all_method_names

def test_init_import_warning(settings):

    settings.MODERNRPC_METHODS_MODULES = [
        'invalid_module'
    ]
    app = apps.get_app_config("modernrpc")

    with pytest.warns(Warning):
        app.ready()

def test_settings_check(settings):

    assert len(check_required_settings_defined(apps.get_app_config("modernrpc"))) == 0

    settings.MODERNRPC_METHODS_MODULES = []

    result = check_required_settings_defined(apps.get_app_config("modernrpc"))
    assert len(result) == 1
    assert result[0].id == 'modernrpc.E001'

def test_init_registry_cleaning(settings):

    orig_modules_list = settings.MODERNRPC_METHODS_MODULES

    app = apps.get_app_config("modernrpc")
    app.ready()

    # Retrieve all methods defined in custom modules (not system.* methods)
    custom_rpc_methods = [m for m in get_all_method_names() if not m.startswith('system.')]
    # Ensure a normal init registered some custom methods
    assert len(custom_rpc_methods) > 0

    # Re-init library, with only invalid module as "methods module"
    settings.MODERNRPC_METHODS_MODULES = ['nope']
    app.ready()

    # Now, we should have removed all custom methods from the registry
    custom_rpc_methods = [m for m in get_all_method_names() if not m.startswith('system.')]
    assert len(custom_rpc_methods) == 0

    # Restore original state for registry
    settings.MODERNRPC_METHODS_MODULES = orig_modules_list
    app.ready()
<commit_msg>Add tiny test for coverage<commit_after>import pytest
from django.apps.registry import apps

from modernrpc.apps import check_required_settings_defined
from modernrpc.core import get_all_method_names

def test_init_import_warning(settings):

    settings.MODERNRPC_METHODS_MODULES = [
        'invalid_module'
    ]
    app = apps.get_app_config("modernrpc")

    with pytest.warns(Warning):
        app.ready()

def test_settings_check(settings):

    assert len(check_required_settings_defined(apps.get_app_config("modernrpc"))) == 0

    settings.MODERNRPC_METHODS_MODULES = []

    result = check_required_settings_defined(apps.get_app_config("modernrpc"))
    assert len(result) == 1
    assert result[0].id == 'modernrpc.E001'

def test_init_registry_cleaning(settings):

    orig_modules_list = settings.MODERNRPC_METHODS_MODULES

    app = apps.get_app_config("modernrpc")
    app.ready()

    # Retrieve all methods defined in custom modules (not system.* methods)
    custom_rpc_methods = [m for m in get_all_method_names() if not m.startswith('system.')]
    # Ensure a normal init registered some custom methods
    assert len(custom_rpc_methods) > 0

    # Re-init library, with only invalid module as "methods module"
    settings.MODERNRPC_METHODS_MODULES = ['nope']
    app.ready()

    # Now, we should have removed all custom methods from the registry
    custom_rpc_methods = [m for m in get_all_method_names() if not m.startswith('system.')]
    assert len(custom_rpc_methods) == 0

    # Re-init library, with None modules list
    settings.MODERNRPC_METHODS_MODULES = None
    app.ready()
    custom_rpc_methods = [m for m in get_all_method_names() if not m.startswith('system.')]
    assert len(custom_rpc_methods) == 0

    # Restore original state for registry
    settings.MODERNRPC_METHODS_MODULES = orig_modules_list
    app.ready()
<|endoftext|>