<commit_before>from __future__ import absolute_import
import logging
import signal
import time
import os
import uuid
import docker
from threading import Thread
from docker.errors import ContainerError

from bd2k.util.files import mkdir_p
from toil.job import Job
from toil.leader import FailedJobsException
from toil.test import ToilTest
from toil.lib import FORGO, STOP, RM
from toil.lib.docker import dockerCall, dockerCheckOutput, \
                            _containerIsRunning, _dockerKill

_logger = logging.getLogger(__name__)

class DockerTest(ToilTest):
    """
    Tests dockerCall and ensures no containers are left around.

    When running tests you may optionally set the TOIL_TEST_TEMP environment
    variable to the path of a directory where you want temporary test files be
    placed. The directory will be created if it doesn't exist. The path may be
    relative in which case it will be assumed to be relative to the project
    root. If TOIL_TEST_TEMP is not defined, temporary files and directories will
    be created in the system's default location for such files and any temporary
    files or directories left over from tests will be removed automatically
    removed during tear down.
    Otherwise, left-over files will not be removed.
    """
    def setUp(self):
        self.tempDir = self._createTempDir(purpose='tempDir')
        self.dockerTestLogLevel = 'INFO'

    def testDockerClean(self,
                        disableCaching=True,
                        detached=True,
                        rm=True,
                        deferParam=None):
        """
        Run the test container that creates a file in the work dir, and sleeps
        for 5 minutes.

        Ensure that the calling job gets SIGKILLed after a minute, leaving
        behind the spooky/ghost/zombie container. Ensure that the container is
        killed on batch system shutdown (through the deferParam mechanism).
        """

        # We need to test the behaviour of `deferParam` with `rm` and
        # `detached`. We do not look at the case where `rm` and `detached` are
        # both True.  This is the truth table for the different combinations at
        # the end of the test. R = Running, X = Does not exist, E = Exists but
        # not running.
        #              None     FORGO     STOP    RM
        #    rm        X         R         X      X
        # detached     R         R         E      X
        #  Neither     R         R         E      X

        data_dir = os.path.join(self.tempDir, 'data')
        work_dir = os.path.join(self.tempDir, 'working')
        test_file = os.path.join(work_dir, 'test.txt')

        mkdir_p(data_dir)
        mkdir_p(work_dir)

        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir,
                                                            'jobstore'))
        options.logLevel = self.dockerTestLogLevel
        options.workDir = work_dir
        options.clean = 'always'
        options.disableCaching = disableCaching

        # No base64 logic since it might create a name starting with a `-`.
        container_name = uuid.uuid4().hex
        A = Job.wrapJobFn(_testDockerCleanFn,
                          work_dir,
                          detached,
                          rm,
                          deferParam,
                          container_name)
        try:
            Job.Runner.startToil(A, options)
        except FailedJobsException:
            # The file created by spooky_container would remain in the directory
            # and since it was created inside the container, it would have had
            # uid and gid == 0 (root) which may cause problems when docker
            # attempts to clean up the jobstore.
            file_stats = os.stat(test_file)
            assert file_stats.st_gid != 0
            assert file_stats.st_uid != 0

            if (rm and (deferParam != FORGO)) or deferParam == RM:
                # These containers should not exist
                assert _containerIsRunning(container_name) is None, \
                    'Container was not removed.'

            elif deferParam == STOP:
                # These containers should exist but be non-running
                assert _containerIsRunning(container_name) == False, \
                    'Container was not stopped.'

            else:
                # These containers will be running
                assert _containerIsRunning(container_name) == True, \
                    'Container was not running.'
        client = docker.from_env(version='auto')
        _dockerKill(container_name, client)
        try:
            os.remove(test_file)
        except:
            pass

    def testDockerClean_CRx_FORGO(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=True,
                             deferParam=FORGO)

    def testDockerClean_CRx_STOP(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=True,
                             deferParam=STOP)

    def testDockerClean_CRx_RM(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=True,
                             deferParam=RM)

    def testDockerClean_CRx_None(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=True,
                             deferParam=None)

    def testDockerClean_CxD_FORGO(self):
        self.testDockerClean(disableCaching=True, detached=True, rm=False,
                             deferParam=FORGO)

    def testDockerClean_CxD_STOP(self):
        self.testDockerClean(disableCaching=True, detached=True, rm=False,
                             deferParam=STOP)

    def testDockerClean_CxD_RM(self):
        self.testDockerClean(disableCaching=True, detached=True, rm=False,
                             deferParam=RM)

    def testDockerClean_CxD_None(self):
        self.testDockerClean(disableCaching=True, detached=True, rm=False,
                             deferParam=None)

    def testDockerClean_Cxx_FORGO(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=False,
                             deferParam=FORGO)

    def testDockerClean_Cxx_STOP(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=False,
                             deferParam=STOP)

    def testDockerClean_Cxx_RM(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=False,
                             deferParam=RM)

    def testDockerClean_Cxx_None(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=False,
                             deferParam=None)

    def testDockerClean_xRx_FORGO(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=True,
                             deferParam=FORGO)

    def testDockerClean_xRx_STOP(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=True,
                             deferParam=STOP)

    def testDockerClean_xRx_RM(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=True,
                             deferParam=RM)

    def testDockerClean_xRx_None(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=True,
                             deferParam=None)

    def testDockerClean_xxD_FORGO(self):
        self.testDockerClean(disableCaching=False, detached=True, rm=False,
                             deferParam=FORGO)

    def testDockerClean_xxD_STOP(self):
        self.testDockerClean(disableCaching=False, detached=True, rm=False,
                             deferParam=STOP)

    def testDockerClean_xxD_RM(self):
        self.testDockerClean(disableCaching=False, detached=True, rm=False,
                             deferParam=RM)

    def testDockerClean_xxD_None(self):
        self.testDockerClean(disableCaching=False, detached=True, rm=False,
                             deferParam=None)

    def testDockerClean_xxx_FORGO(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=False,
                             deferParam=FORGO)

    def testDockerClean_xxx_STOP(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=False,
                             deferParam=STOP)

    def testDockerClean_xxx_RM(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=False,
                             deferParam=RM)

    def testDockerClean_xxx_None(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=False,
                             deferParam=None)

    def testDockerPipeChain(self, disableCaching=True):
        """
        Test for piping API for dockerCall().  Using this API (activated when
        list of argument lists is given as parameters), commands a piped
        together into a chain.
        ex:  parameters=[ ['printf', 'x\n y\n'], ['wc', '-l'] ] should execute:
        printf 'x\n y\n' | wc -l
        """
        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir,
                                                            'jobstore'))
        options.logLevel = self.dockerTestLogLevel
        options.workDir = self.tempDir
        options.clean = 'always'
        options.caching = disableCaching
        A = Job.wrapJobFn(_testDockerPipeChainFn)
        rv = Job.Runner.startToil(A, options)
        assert rv.strip() == '2'

    def testDockerPipeChainErrorDetection(self, disableCaching=True):
        """
        By default, executing cmd1 | cmd2 | ... | cmdN, will only return an
        error if cmdN fails.  This can lead to all manor of errors being
        silently missed.  This tests to make sure that the piping API for
        dockerCall() throws an exception if non-last commands in the chain fail.
        """
        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir,
                                                            'jobstore'))
        options.logLevel = self.dockerTestLogLevel
        options.workDir = self.tempDir
        options.clean = 'always'
        options.caching = disableCaching
        A = Job.wrapJobFn(_testDockerPipeChainErrorFn)
        rv = Job.Runner.startToil(A, options)
        assert rv == True

    def testNonCachingDockerChain(self):
        self.testDockerPipeChain(disableCaching=False)

    def testNonCachingDockerChainErrorDetection(self):
        self.testDockerPipeChainErrorDetection(disableCaching=False)

def _testDockerCleanFn(job,
                       workDir,
                       detached=None,
                       rm=None,
                       deferParam=None,
                       containerName=None):
    """
    Test function for test docker_clean.  Runs a container with given flags and
    then dies leaving behind a zombie container.
    :param toil.job.Job job: job
    :param workDir: See `work_dir=` in :func:`dockerCall`
    :param bool rm: See `rm=` in :func:`dockerCall`
    :param bool detached: See `detached=` in :func:`dockerCall`
    :param int deferParam: See `deferParam=` in :func:`dockerCall`
    :param str containerName: See `container_name=` in :func:`dockerCall`
    """
    def killSelf():
        test_file = os.path.join(workDir, 'test.txt')
        # Kill the worker once we are sure the docker container is started
        while not os.path.exists(test_file):
            _logger.debug('Waiting on the file created by spooky_container.')
            time.sleep(1)
        # By the time we reach here, we are sure the container is running.
        time.sleep(1)
        os.kill(os.getpid(), signal.SIGKILL)

    t = Thread(target=killSelf)
    # Make it a daemon thread so that thread failure doesn't hang tests.
    t.daemon = True
    t.start()
    dockerCall(job,
               tool='quay.io/ucsc_cgl/spooky_test',
               workDir=workDir,
               deferParam=deferParam,
               containerName=containerName,
               detach=detached,
               removeOnExit=rm)

def _testDockerPipeChainFn(job):
    """Return the result of a simple pipe chain.  Should be 2."""
    parameters = [ ['printf', 'x\n y\n'], ['wc', '-l'] ]
    return dockerCheckOutput(job,
                             tool='quay.io/ucsc_cgl/spooky_test',
                             parameters=parameters)

def _testDockerPipeChainErrorFn(job):
    """Return True if the command exit 1 | wc -l raises a ContainerError."""
    parameters = [ ['exit', '1'], ['wc', '-l'] ]
    try:
        dockerCheckOutput(job,
                          tool='quay.io/ucsc_cgl/spooky_test',
                          parameters=parameters)
    except ContainerError:
        return True
    return False
<commit_msg>Marked slow tests.<commit_after>from __future__ import absolute_import
import logging
import signal
import time
import os
import uuid
import docker
from threading import Thread
from docker.errors import ContainerError

from bd2k.util.files import mkdir_p
from toil.job import Job
from toil.leader import FailedJobsException
from toil.test import ToilTest, slow
from toil.lib import FORGO, STOP, RM
from toil.lib.docker import dockerCall, dockerCheckOutput, \
                            _containerIsRunning, _dockerKill

_logger = logging.getLogger(__name__)

class DockerTest(ToilTest):
    """
    Tests dockerCall and ensures no containers are left around.

    When running tests you may optionally set the TOIL_TEST_TEMP environment
    variable to the path of a directory where you want temporary test files be
    placed. The directory will be created if it doesn't exist. The path may be
    relative in which case it will be assumed to be relative to the project
    root. If TOIL_TEST_TEMP is not defined, temporary files and directories will
    be created in the system's default location for such files and any temporary
    files or directories left over from tests will be removed automatically
    removed during tear down.
    Otherwise, left-over files will not be removed.
    """
    def setUp(self):
        self.tempDir = self._createTempDir(purpose='tempDir')
        self.dockerTestLogLevel = 'INFO'

    def testDockerClean(self,
                        disableCaching=True,
                        detached=True,
                        rm=True,
                        deferParam=None):
        """
        Run the test container that creates a file in the work dir, and sleeps
        for 5 minutes.

        Ensure that the calling job gets SIGKILLed after a minute, leaving
        behind the spooky/ghost/zombie container. Ensure that the container is
        killed on batch system shutdown (through the deferParam mechanism).
        """

        # We need to test the behaviour of `deferParam` with `rm` and
        # `detached`. We do not look at the case where `rm` and `detached` are
        # both True.  This is the truth table for the different combinations at
        # the end of the test. R = Running, X = Does not exist, E = Exists but
        # not running.
        #              None     FORGO     STOP    RM
        #    rm        X         R         X      X
        # detached     R         R         E      X
        #  Neither     R         R         E      X

        data_dir = os.path.join(self.tempDir, 'data')
        work_dir = os.path.join(self.tempDir, 'working')
        test_file = os.path.join(work_dir, 'test.txt')

        mkdir_p(data_dir)
        mkdir_p(work_dir)

        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir,
                                                            'jobstore'))
        options.logLevel = self.dockerTestLogLevel
        options.workDir = work_dir
        options.clean = 'always'
        options.disableCaching = disableCaching

        # No base64 logic since it might create a name starting with a `-`.
        container_name = uuid.uuid4().hex
        A = Job.wrapJobFn(_testDockerCleanFn,
                          work_dir,
                          detached,
                          rm,
                          deferParam,
                          container_name)
        try:
            Job.Runner.startToil(A, options)
        except FailedJobsException:
            # The file created by spooky_container would remain in the directory
            # and since it was created inside the container, it would have had
            # uid and gid == 0 (root) which may cause problems when docker
            # attempts to clean up the jobstore.
            file_stats = os.stat(test_file)
            assert file_stats.st_gid != 0
            assert file_stats.st_uid != 0

            if (rm and (deferParam != FORGO)) or deferParam == RM:
                # These containers should not exist
                assert _containerIsRunning(container_name) is None, \
                    'Container was not removed.'

            elif deferParam == STOP:
                # These containers should exist but be non-running
                assert _containerIsRunning(container_name) == False, \
                    'Container was not stopped.'

            else:
                # These containers will be running
                assert _containerIsRunning(container_name) == True, \
                    'Container was not running.'
        client = docker.from_env(version='auto')
        _dockerKill(container_name, client)
        try:
            os.remove(test_file)
        except:
            pass

    def testDockerClean_CRx_FORGO(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=True,
                             deferParam=FORGO)

    def testDockerClean_CRx_STOP(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=True,
                             deferParam=STOP)

    def testDockerClean_CRx_RM(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=True,
                             deferParam=RM)

    def testDockerClean_CRx_None(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=True,
                             deferParam=None)

    @slow
    def testDockerClean_CxD_FORGO(self):
        self.testDockerClean(disableCaching=True, detached=True, rm=False,
                             deferParam=FORGO)

    @slow
    def testDockerClean_CxD_STOP(self):
        self.testDockerClean(disableCaching=True, detached=True, rm=False,
                             deferParam=STOP)

    @slow
    def testDockerClean_CxD_RM(self):
        self.testDockerClean(disableCaching=True, detached=True, rm=False,
                             deferParam=RM)

    @slow
    def testDockerClean_CxD_None(self):
        self.testDockerClean(disableCaching=True, detached=True, rm=False,
                             deferParam=None)

    @slow
    def testDockerClean_Cxx_FORGO(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=False,
                             deferParam=FORGO)

    @slow
    def testDockerClean_Cxx_STOP(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=False,
                             deferParam=STOP)

    @slow
    def testDockerClean_Cxx_RM(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=False,
                             deferParam=RM)

    @slow
    def testDockerClean_Cxx_None(self):
        self.testDockerClean(disableCaching=True, detached=False, rm=False,
                             deferParam=None)

    @slow
    def testDockerClean_xRx_FORGO(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=True,
                             deferParam=FORGO)

    @slow
    def testDockerClean_xRx_STOP(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=True,
                             deferParam=STOP)

    @slow
    def testDockerClean_xRx_RM(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=True,
                             deferParam=RM)

    @slow
    def testDockerClean_xRx_None(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=True,
                             deferParam=None)

    @slow
    def testDockerClean_xxD_FORGO(self):
        self.testDockerClean(disableCaching=False, detached=True, rm=False,
                             deferParam=FORGO)

    @slow
    def testDockerClean_xxD_STOP(self):
        self.testDockerClean(disableCaching=False, detached=True, rm=False,
                             deferParam=STOP)

    @slow
    def testDockerClean_xxD_RM(self):
        self.testDockerClean(disableCaching=False, detached=True, rm=False,
                             deferParam=RM)

    @slow
    def testDockerClean_xxD_None(self):
        self.testDockerClean(disableCaching=False, detached=True, rm=False,
                             deferParam=None)

    @slow
    def testDockerClean_xxx_FORGO(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=False,
                             deferParam=FORGO)

    @slow
    def testDockerClean_xxx_STOP(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=False,
                             deferParam=STOP)

    @slow
    def testDockerClean_xxx_RM(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=False,
                             deferParam=RM)

    @slow
    def testDockerClean_xxx_None(self):
        self.testDockerClean(disableCaching=False, detached=False, rm=False,
                             deferParam=None)

    def testDockerPipeChain(self, disableCaching=True):
        """
        Test for piping API for dockerCall().  Using this API (activated when
        list of argument lists is given as parameters), commands a piped
        together into a chain.
        ex:  parameters=[ ['printf', 'x\n y\n'], ['wc', '-l'] ] should execute:
        printf 'x\n y\n' | wc -l
        """
        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir,
                                                            'jobstore'))
        options.logLevel = self.dockerTestLogLevel
        options.workDir = self.tempDir
        options.clean = 'always'
        options.caching = disableCaching
        A = Job.wrapJobFn(_testDockerPipeChainFn)
        rv = Job.Runner.startToil(A, options)
        assert rv.strip() == '2'

    def testDockerPipeChainErrorDetection(self, disableCaching=True):
        """
        By default, executing cmd1 | cmd2 | ... | cmdN, will only return an
        error if cmdN fails.  This can lead to all manor of errors being
        silently missed.  This tests to make sure that the piping API for
        dockerCall() throws an exception if non-last commands in the chain fail.
        """
        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir,
                                                            'jobstore'))
        options.logLevel = self.dockerTestLogLevel
        options.workDir = self.tempDir
        options.clean = 'always'
        options.caching = disableCaching
        A = Job.wrapJobFn(_testDockerPipeChainErrorFn)
        rv = Job.Runner.startToil(A, options)
        assert rv == True

    @slow
    def testNonCachingDockerChain(self):
        self.testDockerPipeChain(disableCaching=False)

    @slow
    def testNonCachingDockerChainErrorDetection(self):
        self.testDockerPipeChainErrorDetection(disableCaching=False)

def _testDockerCleanFn(job,
                       workDir,
                       detached=None,
                       rm=None,
                       deferParam=None,
                       containerName=None):
    """
    Test function for test docker_clean.  Runs a container with given flags and
    then dies leaving behind a zombie container.
    :param toil.job.Job job: job
    :param workDir: See `work_dir=` in :func:`dockerCall`
    :param bool rm: See `rm=` in :func:`dockerCall`
    :param bool detached: See `detached=` in :func:`dockerCall`
    :param int deferParam: See `deferParam=` in :func:`dockerCall`
    :param str containerName: See `container_name=` in :func:`dockerCall`
    """
    def killSelf():
        test_file = os.path.join(workDir, 'test.txt')
        # Kill the worker once we are sure the docker container is started
        while not os.path.exists(test_file):
            _logger.debug('Waiting on the file created by spooky_container.')
            time.sleep(1)
        # By the time we reach here, we are sure the container is running.
        time.sleep(1)
        os.kill(os.getpid(), signal.SIGKILL)

    t = Thread(target=killSelf)
    # Make it a daemon thread so that thread failure doesn't hang tests.
    t.daemon = True
    t.start()
    dockerCall(job,
               tool='quay.io/ucsc_cgl/spooky_test',
               workDir=workDir,
               deferParam=deferParam,
               containerName=containerName,
               detach=detached,
               removeOnExit=rm,
               privileged=True)

def _testDockerPipeChainFn(job):
    """Return the result of a simple pipe chain.  Should be 2."""
    parameters = [ ['printf', 'x\n y\n'], ['wc', '-l'] ]
    return dockerCheckOutput(job,
                             tool='quay.io/ucsc_cgl/spooky_test',
                             parameters=parameters,
                             privileged=True)

def _testDockerPipeChainErrorFn(job):
    """Return True if the command exit 1 | wc -l raises a ContainerError."""
    parameters = [ ['exit', '1'], ['wc', '-l'] ]
    try:
        dockerCheckOutput(job,
                          tool='quay.io/ucsc_cgl/spooky_test',
                          parameters=parameters)
    except ContainerError:
        return True
    return False
<|endoftext|>


<commit_before>"""
RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
from scrapy.http import Request, Response
from scrapy.exceptions import NotConfigured
from scrapy import signals
from scrapy.utils.python import to_native_str
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.url import strip_url

LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)

POLICY_NO_REFERRER = "no-referrer"
POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade"
POLICY_SAME_ORIGIN = "same-origin"
POLICY_ORIGIN = "origin"
POLICY_STRICT_ORIGIN = "strict-origin"
POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin"
POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin"
POLICY_UNSAFE_URL = "unsafe-url"
POLICY_SCRAPY_DEFAULT = "scrapy-default"

class ReferrerPolicy(object):

    NOREFERRER_SCHEMES = LOCAL_SCHEMES

    def referrer(self, response, request):
        raise NotImplementedError()

    def stripped_referrer(self, req_or_resp):
        return self.strip_url(req_or_resp)

    def origin_referrer(self, req_or_resp):
        return self.strip_url(req_or_resp, origin_only=True)

    def strip_url(self, req_or_resp, origin_only=False):
        """
        https://www.w3.org/TR/referrer-policy/#strip-url

        If url is null, return no referrer.
        If url's scheme is a local scheme, then return no referrer.
        Set url's username to the empty string.
        Set url's password to null.
        Set url's fragment to null.
        If the origin-only flag is true, then:
            Set url's path to null.
            Set url's query to null.
        Return url.
        """
        if req_or_resp.url is None or not req_or_resp.url:
            return None
        parsed_url = urlparse_cached(req_or_resp)
        if parsed_url.scheme not in self.NOREFERRER_SCHEMES:
            return strip_url(parsed_url,
                             strip_credentials=True,
                             strip_fragment=True,
                             strip_default_port=True,
                             origin_only=origin_only)

    def origin(self, req_or_resp):
        """Return serialized origin (scheme, host, path) for a request or response URL."""
        return self.strip_url(req_or_resp, origin_only=True)

class NoReferrerPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer

    The simplest policy is "no-referrer", which specifies that no referrer information
    is to be sent along with requests made from a particular request client to any origin.
    The header will be omitted entirely.
    """
    name = POLICY_NO_REFERRER

    def referrer(self, response, request):
        return None

class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade

    The "no-referrer-when-downgrade" policy sends a full URL
    along with requests from a TLS-protected environment settings object
    to a a priori authenticated URL,
    and requests from request clients which are not TLS-protected to any origin.

    Requests from TLS-protected request clients to non-a priori authenticated URLs,
    on the other hand, will contain no referrer information.
    A Referer HTTP header will not be sent.

    This is a user agent's default behavior, if no policy is otherwise specified.
    """
    name = POLICY_NO_REFERRER_WHEN_DOWNGRADE

    def referrer(self, response, request):
        # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer:
        #
        # If environment is TLS-protected
        # and the origin of request's current URL is not an a priori authenticated URL,
        # then return no referrer.
        if urlparse_cached(response).scheme in ('https', 'ftps') and \
            urlparse_cached(request).scheme in ('http',):
                return None
        return self.stripped_referrer(response)

class SameOriginPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin

    The "same-origin" policy specifies that a full URL, stripped for use as a referrer,
    is sent as referrer information when making same-origin requests from a particular request client.

    Cross-origin requests, on the other hand, will contain no referrer information.
    A Referer HTTP header will not be sent.
    """
    name = POLICY_SAME_ORIGIN

    def referrer(self, response, request):
        if self.origin(response) == self.origin(request):
            return self.stripped_referrer(response)

class OriginPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-origin

    The "origin" policy specifies that only the ASCII serialization
    of the origin of the request client is sent as referrer information
    when making both same-origin requests and cross-origin requests
    from a particular request client.
    """
    name = POLICY_ORIGIN

    def referrer(self, response, request):
        return self.origin_referrer(response)

class StrictOriginPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin

    The "strict-origin" policy sends the ASCII serialization
    of the origin of the request client when making requests:
    - from a TLS-protected environment settings object to a potentially trustworthy URL, and
    - from non-TLS-protected environment settings objects to any origin.

    Requests from TLS-protected request clients to non- potentially trustworthy URLs,
    on the other hand, will contain no referrer information.
    A Referer HTTP header will not be sent.
    """
    name = POLICY_STRICT_ORIGIN

    def referrer(self, response, request):
        if urlparse_cached(response).scheme == urlparse_cached(request).scheme:
            return self.origin_referrer(response)

class OriginWhenCrossOriginPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin

    The "origin-when-cross-origin" policy specifies that a full URL,
    stripped for use as a referrer, is sent as referrer information
    when making same-origin requests from a particular request client,
    and only the ASCII serialization of the origin of the request client
    is sent as referrer information when making cross-origin requests
    from a particular request client.
    """
    name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN

    def referrer(self, response, request):
        origin = self.origin(response)
        if origin == self.origin(request):
            return self.stripped_referrer(response)
        else:
            return origin

class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin

    The "strict-origin-when-cross-origin" policy specifies that a full URL,
    stripped for use as a referrer, is sent as referrer information
    when making same-origin requests from a particular request client,
    and only the ASCII serialization of the origin of the request client
    when making cross-origin requests:

    - from a TLS-protected environment settings object to a potentially trustworthy URL, and
    - from non-TLS-protected environment settings objects to any origin.

    Requests from TLS-protected clients to non- potentially trustworthy URLs,
    on the other hand, will contain no referrer information.
    A Referer HTTP header will not be sent.
    """
    name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN

    def referrer(self, response, request):
        origin = self.origin(response)
        if origin == self.origin(request):
            return self.stripped_referrer(response)
        else:
            return origin

class UnsafeUrlPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url

    The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer,
    is sent along with both cross-origin requests
    and same-origin requests made from a particular request client.

    Note: The policy's name doesn't lie; it is unsafe.
    This policy will leak origins and paths from TLS-protected resources
    to insecure origins.
    Carefully consider the impact of setting such a policy for potentially sensitive documents.
    """
    name = POLICY_UNSAFE_URL

    def referrer(self, response, request):
        return self.stripped_referrer(response)

class LegacyPolicy(ReferrerPolicy):
    def referrer(self, response, request):
        return response.url

class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):

    NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
    name = POLICY_SCRAPY_DEFAULT

_policy_classes = {p.name: p for p in (
    NoReferrerPolicy,
    NoReferrerWhenDowngradePolicy,
    SameOriginPolicy,
    OriginPolicy,
    OriginWhenCrossOriginPolicy,
    UnsafeUrlPolicy,
    DefaultReferrerPolicy,
)}

class RefererMiddleware(object):

    def __init__(self, settings=None):
        self.default_policy = DefaultReferrerPolicy
        if settings is not None:
            policy = settings.get('REFERER_POLICY')
            if policy is not None:
                # expect a string for the path to the policy class
                try:
                    self.default_policy = load_object(policy)
                except ValueError:
                    # otherwise try to interpret the string as standard
                    # https://www.w3.org/TR/referrer-policy/#referrer-policies
                    try:
                        self.default_policy = _policy_classes[policy.lower()]
                    except:
                        raise NotConfigured("Unknown referrer policy name %r" % policy)

    @classmethod
    def from_crawler(cls, crawler):
        if not crawler.settings.getbool('REFERER_ENABLED'):
            raise NotConfigured
        mw = cls(crawler.settings)
        crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled)
        return mw

    def policy(self, response, request):
        # policy set in request's meta dict takes precedence over default policy
        policy_name = request.meta.get('referrer_policy')
        if policy_name is None:
            policy_name = to_native_str(
                response.headers.get('Referrer-Policy', '').decode('latin1'))

        cls = _policy_classes.get(policy_name.lower(), self.default_policy)
        return cls()

    def process_spider_output(self, response, result, spider):
        def _set_referer(r):
            if isinstance(r, Request):
                referrer = self.policy(response, r).referrer(response, r)
                if referrer is not None:
                    r.headers.setdefault('Referer', referrer)
            return r
        return (_set_referer(r) for r in result or ())

    def request_scheduled(self, request, spider):
        # check redirected request to patch "Referer" header if necessary
        redirected_urls = request.meta.get('redirect_urls', [])
        if redirected_urls:
            request_referrer = request.headers.get('Referer')
            # we don't patch the referrer value if there is none
            if request_referrer is not None:
                faked_response = Response(redirected_urls[0])
                policy_referrer = self.policy(faked_response,
                    request).referrer(faked_response, request)
                if policy_referrer != request_referrer:
                    if policy_referrer is None:
                        request.headers.pop('Referer')
                    else:
                        request.headers['Referer'] = policy_referrer
<commit_msg>Fix implementation<commit_after>"""
RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
from scrapy.http import Request, Response
from scrapy.exceptions import NotConfigured
from scrapy import signals
from scrapy.utils.python import to_native_str
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.url import strip_url

LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)

POLICY_NO_REFERRER = "no-referrer"
POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade"
POLICY_SAME_ORIGIN = "same-origin"
POLICY_ORIGIN = "origin"
POLICY_STRICT_ORIGIN = "strict-origin"
POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin"
POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin"
POLICY_UNSAFE_URL = "unsafe-url"
POLICY_SCRAPY_DEFAULT = "scrapy-default"

class ReferrerPolicy(object):

    NOREFERRER_SCHEMES = LOCAL_SCHEMES

    def referrer(self, response, request):
        raise NotImplementedError()

    def stripped_referrer(self, r):
        return self.strip_url(r)

    def origin_referrer(self, r):
        return self.strip_url(r, origin_only=True)

    def strip_url(self, r, origin_only=False):
        """
        https://www.w3.org/TR/referrer-policy/#strip-url

        If url is null, return no referrer.
        If url's scheme is a local scheme, then return no referrer.
        Set url's username to the empty string.
        Set url's password to null.
        Set url's fragment to null.
        If the origin-only flag is true, then:
            Set url's path to null.
            Set url's query to null.
        Return url.
        """
        if r is None or not r.url:
            return None
        parsed_url = urlparse_cached(r)
        if parsed_url.scheme not in self.NOREFERRER_SCHEMES:
            return strip_url(parsed_url,
                             strip_credentials=True,
                             strip_fragment=True,
                             strip_default_port=True,
                             origin_only=origin_only)

    def origin(self, r):
        """Return serialized origin (scheme, host, path) for a request or response URL."""
        return self.strip_url(r, origin_only=True)

    def potentially_trustworthy(self, r):
        # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy
        parsed_url = urlparse_cached(r)
        if parsed_url.scheme in ('data',):
            return False
        return self.tls_protected(r)

    def tls_protected(self, r):
        return urlparse_cached(r).scheme in ('https', 'ftps')

class NoReferrerPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer

    The simplest policy is "no-referrer", which specifies that no referrer information
    is to be sent along with requests made from a particular request client to any origin.
    The header will be omitted entirely.
    """
    name = POLICY_NO_REFERRER

    def referrer(self, response, request):
        return None

class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade

    The "no-referrer-when-downgrade" policy sends a full URL
    along with requests from a TLS-protected environment settings object
    to a a priori authenticated URL,
    and requests from request clients which are not TLS-protected to any origin.

    Requests from TLS-protected request clients to non-a priori authenticated URLs,
    on the other hand, will contain no referrer information.
    A Referer HTTP header will not be sent.

    This is a user agent's default behavior, if no policy is otherwise specified.
    """
    name = POLICY_NO_REFERRER_WHEN_DOWNGRADE

    def referrer(self, response, request):
        # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer:
        #
        # If environment is TLS-protected
        # and the origin of request's current URL is not an a priori authenticated URL,
        # then return no referrer.
        if urlparse_cached(response).scheme in ('https', 'ftps') and \
            urlparse_cached(request).scheme in ('http',):
                return None
        return self.stripped_referrer(response)

class SameOriginPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin

    The "same-origin" policy specifies that a full URL, stripped for use as a referrer,
    is sent as referrer information when making same-origin requests from a particular request client.

    Cross-origin requests, on the other hand, will contain no referrer information.
    A Referer HTTP header will not be sent.
    """
    name = POLICY_SAME_ORIGIN

    def referrer(self, response, request):
        if self.origin(response) == self.origin(request):
            return self.stripped_referrer(response)

class OriginPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-origin

    The "origin" policy specifies that only the ASCII serialization
    of the origin of the request client is sent as referrer information
    when making both same-origin requests and cross-origin requests
    from a particular request client.
    """
    name = POLICY_ORIGIN

    def referrer(self, response, request):
        return self.origin_referrer(response)

class StrictOriginPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin

    The "strict-origin" policy sends the ASCII serialization
    of the origin of the request client when making requests:
    - from a TLS-protected environment settings object to a potentially trustworthy URL, and
    - from non-TLS-protected environment settings objects to any origin.

    Requests from TLS-protected request clients to non- potentially trustworthy URLs,
    on the other hand, will contain no referrer information.
    A Referer HTTP header will not be sent.
    """
    name = POLICY_STRICT_ORIGIN

    def referrer(self, response, request):
        if ((urlparse_cached(response).scheme == 'https' and
             self.potentially_trustworthy(request))
             or urlparse_cached(response).scheme == 'http'):
            return self.origin_referrer(response)

class OriginWhenCrossOriginPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin

    The "origin-when-cross-origin" policy specifies that a full URL,
    stripped for use as a referrer, is sent as referrer information
    when making same-origin requests from a particular request client,
    and only the ASCII serialization of the origin of the request client
    is sent as referrer information when making cross-origin requests
    from a particular request client.
    """
    name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN

    def referrer(self, response, request):
        origin = self.origin(response)
        if origin == self.origin(request):
            return self.stripped_referrer(response)
        else:
            return origin

class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin

    The "strict-origin-when-cross-origin" policy specifies that a full URL,
    stripped for use as a referrer, is sent as referrer information
    when making same-origin requests from a particular request client,
    and only the ASCII serialization of the origin of the request client
    when making cross-origin requests:

    - from a TLS-protected environment settings object to a potentially trustworthy URL, and
    - from non-TLS-protected environment settings objects to any origin.

    Requests from TLS-protected clients to non- potentially trustworthy URLs,
    on the other hand, will contain no referrer information.
    A Referer HTTP header will not be sent.
    """
    name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN

    def referrer(self, response, request):
        origin = self.origin(response)
        if origin == self.origin(request):
            return self.stripped_referrer(response)
        elif ((urlparse_cached(response).scheme in ('https', 'ftps') and
               self.potentially_trustworthy(request))
              or urlparse_cached(response).scheme == 'http'):
            return self.origin_referrer(response)

class UnsafeUrlPolicy(ReferrerPolicy):
    """
    https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url

    The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer,
    is sent along with both cross-origin requests
    and same-origin requests made from a particular request client.

    Note: The policy's name doesn't lie; it is unsafe.
    This policy will leak origins and paths from TLS-protected resources
    to insecure origins.
    Carefully consider the impact of setting such a policy for potentially sensitive documents.
    """
    name = POLICY_UNSAFE_URL

    def referrer(self, response, request):
        return self.stripped_referrer(response)

class LegacyPolicy(ReferrerPolicy):
    def referrer(self, response, request):
        return response.url

class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):

    NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
    name = POLICY_SCRAPY_DEFAULT

_policy_classes = {p.name: p for p in (
    NoReferrerPolicy,
    NoReferrerWhenDowngradePolicy,
    SameOriginPolicy,
    OriginPolicy,
    StrictOriginPolicy,
    OriginWhenCrossOriginPolicy,
    StrictOriginWhenCrossOriginPolicy,
    UnsafeUrlPolicy,
    DefaultReferrerPolicy,
)}

class RefererMiddleware(object):

    def __init__(self, settings=None):
        self.default_policy = DefaultReferrerPolicy
        if settings is not None:
            policy = settings.get('REFERER_POLICY')
            if policy is not None:
                # expect a string for the path to the policy class
                try:
                    self.default_policy = load_object(policy)
                except ValueError:
                    # otherwise try to interpret the string as standard
                    # https://www.w3.org/TR/referrer-policy/#referrer-policies
                    try:
                        self.default_policy = _policy_classes[policy.lower()]
                    except:
                        raise NotConfigured("Unknown referrer policy name %r" % policy)

    @classmethod
    def from_crawler(cls, crawler):
        if not crawler.settings.getbool('REFERER_ENABLED'):
            raise NotConfigured
        mw = cls(crawler.settings)
        crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled)
        return mw

    def policy(self, response, request):
        # policy set in request's meta dict takes precedence over default policy
        policy_name = request.meta.get('referrer_policy')
        if policy_name is None:
            policy_name = to_native_str(
                response.headers.get('Referrer-Policy', '').decode('latin1'))

        cls = _policy_classes.get(policy_name.lower(), self.default_policy)
        return cls()

    def process_spider_output(self, response, result, spider):
        def _set_referer(r):
            if isinstance(r, Request):
                referrer = self.policy(response, r).referrer(response, r)
                if referrer is not None:
                    r.headers.setdefault('Referer', referrer)
            return r
        return (_set_referer(r) for r in result or ())

    def request_scheduled(self, request, spider):
        # check redirected request to patch "Referer" header if necessary
        redirected_urls = request.meta.get('redirect_urls', [])
        if redirected_urls:
            request_referrer = request.headers.get('Referer')
            # we don't patch the referrer value if there is none
            if request_referrer is not None:
                faked_response = Response(redirected_urls[0])
                policy_referrer = self.policy(faked_response,
                    request).referrer(faked_response, request)
                if policy_referrer != request_referrer:
                    if policy_referrer is None:
                        request.headers.pop('Referer')
                    else:
                        request.headers['Referer'] = policy_referrer
<|endoftext|>


<commit_before># coding: utf8
from __future__ import unicode_literals

from ...compat import json_dumps, path2str
from ...util import prints

def conllu2json(input_path, output_path, n_sents=10, use_morphology=False):
    """
    Convert conllu files into JSON format for use with train cli.
    use_morphology parameter enables appending morphology to tags, which is
    useful for languages such as Spanish, where UD tags are not so rich.
    """
    # by @dvsrepo, via #11 explosion/spacy-dev-resources

    docs = []
    sentences = []
    conll_tuples = read_conllx(input_path, use_morphology=use_morphology)

    for i, (raw_text, tokens) in enumerate(conll_tuples):
        sentence, brackets = tokens[0]
        sentences.append(generate_sentence(sentence))
        # Real-sized documents could be extracted using the comments on the
        # conluu document
        if(len(sentences) % n_sents == 0):
            doc = create_doc(sentences, i)
            docs.append(doc)
            sentences = []

    output_filename = input_path.parts[-1].replace(".conllu", ".json")
    output_file = output_path / output_filename
    with output_file.open('w', encoding='utf-8') as f:
        f.write(json_dumps(docs))
    prints("Created %d documents" % len(docs),
           title="Generated output file %s" % path2str(output_file))

def read_conllx(input_path, use_morphology=False, n=0):
    text = input_path.open('r', encoding='utf-8').read()
    i = 0
    for sent in text.strip().split('\n\n'):
        lines = sent.strip().split('\n')
        if lines:
            while lines[0].startswith('#'):
                lines.pop(0)
            tokens = []
            for line in lines:

                id_, word, lemma, pos, tag, morph, head, dep, _1, \
                _2 = line.split('\t')
                if '-' in id_ or '.' in id_:
                    continue
                try:
                    id_ = int(id_) - 1
                    head = (int(head) - 1) if head != '0' else id_
                    dep = 'ROOT' if dep == 'root' else dep
                    tag = pos+'__'+morph  if use_morphology else pos
                    tokens.append((id_, word, tag, head, dep, 'O'))
                except:
                    print(line)
                    raise
            tuples = [list(t) for t in zip(*tokens)]
            yield (None, [[tuples, []]])
            i += 1
            if n >= 1 and i >= n:
                break

def generate_sentence(sent):
    (id_, word, tag, head, dep, _) = sent
    sentence = {}
    tokens = []
    for i, id in enumerate(id_):
        token = {}
        token["orth"] = word[id]
        token["tag"] = tag[id]
        token["head"] = head[id] - i
        token["dep"] = dep[id]
        tokens.append(token)
    sentence["tokens"] = tokens
    return sentence

def create_doc(sentences,id):
    doc = {}
    paragraph = {}
    doc["id"] = id
    doc["paragraphs"] = []
    paragraph["sentences"] = sentences
    doc["paragraphs"].append(paragraph)
    return doc
<commit_msg>Use tag in CoNLL converter, not POS<commit_after># coding: utf8
from __future__ import unicode_literals

from ...compat import json_dumps, path2str
from ...util import prints

def conllu2json(input_path, output_path, n_sents=10, use_morphology=False):
    """
    Convert conllu files into JSON format for use with train cli.
    use_morphology parameter enables appending morphology to tags, which is
    useful for languages such as Spanish, where UD tags are not so rich.
    """
    # by @dvsrepo, via #11 explosion/spacy-dev-resources

    docs = []
    sentences = []
    conll_tuples = read_conllx(input_path, use_morphology=use_morphology)

    for i, (raw_text, tokens) in enumerate(conll_tuples):
        sentence, brackets = tokens[0]
        sentences.append(generate_sentence(sentence))
        # Real-sized documents could be extracted using the comments on the
        # conluu document
        if(len(sentences) % n_sents == 0):
            doc = create_doc(sentences, i)
            docs.append(doc)
            sentences = []

    output_filename = input_path.parts[-1].replace(".conllu", ".json")
    output_file = output_path / output_filename
    with output_file.open('w', encoding='utf-8') as f:
        f.write(json_dumps(docs))
    prints("Created %d documents" % len(docs),
           title="Generated output file %s" % path2str(output_file))

def read_conllx(input_path, use_morphology=False, n=0):
    text = input_path.open('r', encoding='utf-8').read()
    i = 0
    for sent in text.strip().split('\n\n'):
        lines = sent.strip().split('\n')
        if lines:
            while lines[0].startswith('#'):
                lines.pop(0)
            tokens = []
            for line in lines:

                id_, word, lemma, pos, tag, morph, head, dep, _1, \
                _2 = line.split('\t')
                if '-' in id_ or '.' in id_:
                    continue
                try:
                    id_ = int(id_) - 1
                    head = (int(head) - 1) if head != '0' else id_
                    dep = 'ROOT' if dep == 'root' else dep
                    tag = tag+'__'+morph  if use_morphology else tag
                    tokens.append((id_, word, tag, head, dep, 'O'))
                except:
                    print(line)
                    raise
            tuples = [list(t) for t in zip(*tokens)]
            yield (None, [[tuples, []]])
            i += 1
            if n >= 1 and i >= n:
                break

def generate_sentence(sent):
    (id_, word, tag, head, dep, _) = sent
    sentence = {}
    tokens = []
    for i, id in enumerate(id_):
        token = {}
        token["orth"] = word[id]
        token["tag"] = tag[id]
        token["head"] = head[id] - i
        token["dep"] = dep[id]
        tokens.append(token)
    sentence["tokens"] = tokens
    return sentence

def create_doc(sentences,id):
    doc = {}
    paragraph = {}
    doc["id"] = id
    doc["paragraphs"] = []
    paragraph["sentences"] = sentences
    doc["paragraphs"].append(paragraph)
    return doc
<|endoftext|>


<commit_before>"""
Copyright (c) 2020 The Orbit Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
"""

import logging

from pywinauto.keyboard import send_keys

from core.common_controls import DataViewPanel
from core.orbit_e2e import E2ETestCase, wait_for_condition, find_control

from test_cases.capture_window import Capture
from test_cases.live_tab import VerifyFunctionCallCount

def _show_symbols_and_functions_tabs(top_window):
    logging.info("Showing symbols tab")
    find_control(top_window, "TabItem", "Symbols").click_input()

class LoadSymbols(E2ETestCase):
    """
    Load specified modules, wait until the UI marks them as loaded, and verifies the functions list is
    non-empty.

    Selection is done be filtering the module list and loading the first remaining row.
    """
    def _execute(self, module_search_string: str):
        _show_symbols_and_functions_tabs(self.suite.top_window())

        logging.info('Start loading symbols for module %s', module_search_string)
        modules_dataview = DataViewPanel(self.find_control("Group", "ModulesDataView"))

        logging.info('Waiting for module list to be populated...')
        wait_for_condition(lambda: modules_dataview.get_row_count() > 0, 100)

        logging.info('Filtering and loading')
        modules_dataview.filter.set_focus()
        modules_dataview.filter.set_edit_text('')
        send_keys(module_search_string)
        wait_for_condition(lambda: modules_dataview.get_row_count() == 1)
        modules_dataview.get_item_at(0, 0).click_input('right')

        self.find_context_menu_item('Load Symbols').click_input()

        logging.info('Waiting for * to indicate loaded modules')

        wait_for_condition(lambda: modules_dataview.get_item_at(0, 4).texts()[0] == "*")

        functions_dataview = DataViewPanel(self.find_control("Group", "FunctionsDataView"))
        wait_for_condition(lambda: functions_dataview.get_row_count() > 0)

class FilterAndHookFunction(E2ETestCase):
    """
    Hook a function based on a search string, and verify it is indicated as Hooked in the UI.
    """
    def _execute(self, function_search_string):
        _show_symbols_and_functions_tabs(self.suite.top_window())

        logging.info('Hooking function based on search "%s"', function_search_string)
        functions_dataview = DataViewPanel(self.find_control("Group", "FunctionsDataView"))

        logging.info('Waiting for function list to be populated...')
        wait_for_condition(lambda: functions_dataview.get_row_count() > 0, 100)

        logging.info('Filtering and hooking')
        functions_dataview.filter.set_focus()
        functions_dataview.filter.set_edit_text('')
        send_keys(function_search_string)
        wait_for_condition(lambda: functions_dataview.get_row_count() == 1)
        functions_dataview.get_item_at(0, 0).click_input('right')

        self.find_context_menu_item('Hook').click_input()
        wait_for_condition(lambda: '✓' in functions_dataview.get_item_at(0, 0).texts()[0])

class FilterAndHookMultipleFunctions(E2ETestCase):
    """
    Hook multiple functions based on a search string, and verify it is indicated as hooked in the UI.
    """
    def _execute(self, function_search_string):
        _show_symbols_and_functions_tabs(self.suite.top_window())

        logging.info('Hooking functions based on search "%s"', function_search_string)
        functions_dataview = DataViewPanel(self.find_control("Group", "FunctionsDataView"))

        logging.info('Waiting for function list to be populated...')
        wait_for_condition(lambda: functions_dataview.get_row_count() > 0, 100)

        logging.info('Filtering and hooking')
        functions_dataview.filter.set_focus()
        functions_dataview.filter.set_edit_text('')
        send_keys(function_search_string)
        wait_for_condition(lambda: functions_dataview.get_row_count() > 0)

        for i in range(functions_dataview.get_row_count()):
            functions_dataview.get_item_at(i, 0).click_input('right')
            self.find_context_menu_item('Hook').click_input()
            wait_for_condition(lambda: '✓' in functions_dataview.get_item_at(i, 0).texts()[0])

class LoadAndVerifyHelloGgpPreset(E2ETestCase):
    """
    Load the predefined E2E test preset and verify if has been applied correctly.
    TODO: This may need to be updated
    """
    def _execute(self):
        _show_symbols_and_functions_tabs(self.suite.top_window())

        self._load_presets()
        wait_for_condition(lambda: self._try_verify_functions_are_hooked)

        Capture().execute(self.suite)

        logging.info('Verifying function call counts')
        VerifyFunctionCallCount(function_name='DrawFrame', min_calls=30, max_calls=3000).execute(self.suite)
        VerifyFunctionCallCount(function_name='GgpIssueFrameToken', min_calls=30, max_calls=3000).execute(self.suite)

    def _load_presets(self):
        presets_panel = DataViewPanel(self.find_control('Group', 'PresetsDataView'))

        draw_frame_preset_row = presets_panel.find_first_item_row('draw_frame_in_hello_ggp_1_52', 1, True)
        issue_frame_token_preset_row = presets_panel.find_first_item_row(
            'ggp_issue_frame_token_in_hello_ggp_1_52', 1, True)

        self.expect_true(draw_frame_preset_row is not None, 'Found draw_frame preset')
        self.expect_true(issue_frame_token_preset_row is not None, 'Found ggp_issue_frame_token preset')

        presets_panel.get_item_at(draw_frame_preset_row, 0).click_input(button='right')
        self.find_context_menu_item('Load Preset').click_input()
        logging.info('Loaded Preset DrawFrame')

        presets_panel.get_item_at(issue_frame_token_preset_row, 0).click_input(button='right')
        self.find_context_menu_item('Load Preset').click_input()
        logging.info('Loaded Preset GgpIssueFrameToken')

    def _try_verify_functions_are_hooked(self):
        logging.info('Finding rows in the function list')
        functions_panel = DataViewPanel(self.find_control('Group', 'FunctionsDataViewD'))
        draw_frame_row = functions_panel.find_first_item_row('DrawFrame', 1)
        issue_frame_token_row = functions_panel.find_first_item_row('GgpIssueFrameToken', 1)

        if draw_frame_row is None:
            return False
        if issue_frame_token_row is None:
            return False

        logging.info('Verifying hook status of functions')
        self.expect_true('✓' in functions_panel.get_item_at(draw_frame_row, 0).texts()[0],
                         'DrawFrame is marked as hooked')
        self.expect_true('✓' in functions_panel.get_item_at(issue_frame_token_row, 0).texts()[0],
                         'GgpIssueFrameToken is marked as hooked')

        return True
<commit_msg>Increase timeout for symbol loading in E2E tests.<commit_after>"""
Copyright (c) 2020 The Orbit Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
"""

import logging

from pywinauto.keyboard import send_keys

from core.common_controls import DataViewPanel
from core.orbit_e2e import E2ETestCase, wait_for_condition, find_control

from test_cases.capture_window import Capture
from test_cases.live_tab import VerifyFunctionCallCount

def _show_symbols_and_functions_tabs(top_window):
    logging.info("Showing symbols tab")
    find_control(top_window, "TabItem", "Symbols").click_input()

class LoadSymbols(E2ETestCase):
    """
    Load specified modules, wait until the UI marks them as loaded, and verifies the functions list is
    non-empty.

    Selection is done be filtering the module list and loading the first remaining row.
    """
    def _execute(self, module_search_string: str):
        _show_symbols_and_functions_tabs(self.suite.top_window())

        logging.info('Start loading symbols for module %s', module_search_string)
        modules_dataview = DataViewPanel(self.find_control("Group", "ModulesDataView"))

        logging.info('Waiting for module list to be populated...')
        wait_for_condition(lambda: modules_dataview.get_row_count() > 0, 100)

        logging.info('Filtering and loading')
        modules_dataview.filter.set_focus()
        modules_dataview.filter.set_edit_text('')
        send_keys(module_search_string)
        wait_for_condition(lambda: modules_dataview.get_row_count() == 1)
        modules_dataview.get_item_at(0, 0).click_input('right')

        self.find_context_menu_item('Load Symbols').click_input()

        logging.info('Waiting for * to indicate loaded modules')

        wait_for_condition(lambda: modules_dataview.get_item_at(0, 4).texts()[0] == "*", 100)

        functions_dataview = DataViewPanel(self.find_control("Group", "FunctionsDataView"))
        wait_for_condition(lambda: functions_dataview.get_row_count() > 0)

class FilterAndHookFunction(E2ETestCase):
    """
    Hook a function based on a search string, and verify it is indicated as Hooked in the UI.
    """
    def _execute(self, function_search_string):
        _show_symbols_and_functions_tabs(self.suite.top_window())

        logging.info('Hooking function based on search "%s"', function_search_string)
        functions_dataview = DataViewPanel(self.find_control("Group", "FunctionsDataView"))

        logging.info('Waiting for function list to be populated...')
        wait_for_condition(lambda: functions_dataview.get_row_count() > 0, 100)

        logging.info('Filtering and hooking')
        functions_dataview.filter.set_focus()
        functions_dataview.filter.set_edit_text('')
        send_keys(function_search_string)
        wait_for_condition(lambda: functions_dataview.get_row_count() == 1)
        functions_dataview.get_item_at(0, 0).click_input('right')

        self.find_context_menu_item('Hook').click_input()
        wait_for_condition(lambda: '✓' in functions_dataview.get_item_at(0, 0).texts()[0])

class FilterAndHookMultipleFunctions(E2ETestCase):
    """
    Hook multiple functions based on a search string, and verify it is indicated as hooked in the UI.
    """
    def _execute(self, function_search_string):
        _show_symbols_and_functions_tabs(self.suite.top_window())

        logging.info('Hooking functions based on search "%s"', function_search_string)
        functions_dataview = DataViewPanel(self.find_control("Group", "FunctionsDataView"))

        logging.info('Waiting for function list to be populated...')
        wait_for_condition(lambda: functions_dataview.get_row_count() > 0, 100)

        logging.info('Filtering and hooking')
        functions_dataview.filter.set_focus()
        functions_dataview.filter.set_edit_text('')
        send_keys(function_search_string)
        wait_for_condition(lambda: functions_dataview.get_row_count() > 0)

        for i in range(functions_dataview.get_row_count()):
            functions_dataview.get_item_at(i, 0).click_input('right')
            self.find_context_menu_item('Hook').click_input()
            wait_for_condition(lambda: '✓' in functions_dataview.get_item_at(i, 0).texts()[0])

class LoadAndVerifyHelloGgpPreset(E2ETestCase):
    """
    Load the predefined E2E test preset and verify if has been applied correctly.
    TODO: This may need to be updated
    """
    def _execute(self):
        _show_symbols_and_functions_tabs(self.suite.top_window())

        self._load_presets()
        wait_for_condition(lambda: self._try_verify_functions_are_hooked)

        Capture().execute(self.suite)

        logging.info('Verifying function call counts')
        VerifyFunctionCallCount(function_name='DrawFrame', min_calls=30, max_calls=3000).execute(self.suite)
        VerifyFunctionCallCount(function_name='GgpIssueFrameToken', min_calls=30, max_calls=3000).execute(self.suite)

    def _load_presets(self):
        presets_panel = DataViewPanel(self.find_control('Group', 'PresetsDataView'))

        draw_frame_preset_row = presets_panel.find_first_item_row('draw_frame_in_hello_ggp_1_52', 1, True)
        issue_frame_token_preset_row = presets_panel.find_first_item_row(
            'ggp_issue_frame_token_in_hello_ggp_1_52', 1, True)

        self.expect_true(draw_frame_preset_row is not None, 'Found draw_frame preset')
        self.expect_true(issue_frame_token_preset_row is not None, 'Found ggp_issue_frame_token preset')

        presets_panel.get_item_at(draw_frame_preset_row, 0).click_input(button='right')
        self.find_context_menu_item('Load Preset').click_input()
        logging.info('Loaded Preset DrawFrame')

        presets_panel.get_item_at(issue_frame_token_preset_row, 0).click_input(button='right')
        self.find_context_menu_item('Load Preset').click_input()
        logging.info('Loaded Preset GgpIssueFrameToken')

    def _try_verify_functions_are_hooked(self):
        logging.info('Finding rows in the function list')
        functions_panel = DataViewPanel(self.find_control('Group', 'FunctionsDataViewD'))
        draw_frame_row = functions_panel.find_first_item_row('DrawFrame', 1)
        issue_frame_token_row = functions_panel.find_first_item_row('GgpIssueFrameToken', 1)

        if draw_frame_row is None:
            return False
        if issue_frame_token_row is None:
            return False

        logging.info('Verifying hook status of functions')
        self.expect_true('✓' in functions_panel.get_item_at(draw_frame_row, 0).texts()[0],
                         'DrawFrame is marked as hooked')
        self.expect_true('✓' in functions_panel.get_item_at(issue_frame_token_row, 0).texts()[0],
                         'GgpIssueFrameToken is marked as hooked')

        return True
<|endoftext|>


<commit_before>from sqlalchemy.dialects.postgresql.base import ischema_names, PGTypeCompiler, ARRAY
from sqlalchemy import types as sqltypes
from sqlalchemy.sql import expression

class LTREE(sqltypes.Concatenable, sqltypes.TypeEngine):
    """Postgresql LTREE type.

    The LTREE datatype can be used for representing labels of data stored in
    hierarchial tree-like structure. For more detailed information please refer
    to http://www.postgresql.org/docs/9.1/static/ltree.html.

    .. note::

        Using :class:`LTREE`, :class:`LQUERY` and :class:`LTXTQUERY` types may
        require installation of Postgresql ltree extension on the server side.
        Please visit http://www.postgres.org for details.
    """

    class Comparator(sqltypes.Concatenable.Comparator):

        def ancestor_of(self, other):
            if isinstance(other, list):
                return self.op('@>')(expression.cast(other, ARRAY(LTREE)))
            else:
                return self.op('@>')(other)

        def descendant_of(self, other):
            if isinstance(other, list):
                return self.op('<@')(expression.cast(other, ARRAY(LTREE)))
            else:
                return self.op('<@')(other)

        def lquery(self, other):
            if isinstance(other, list):
                return self.op('?')(expression.cast(other, ARRAY(LQUERY)))
            else:
                return self.op('~')(other)

        def ltxtquery(self, other):
            return self.op('@')(other)

    comparator_factory = Comparator

    __visit_name__ = 'LTREE'

class LQUERY(sqltypes.TypeEngine):
    """Postresql LQUERY type.

    See :class:`LTREE` for details.
    """
    __visit_name__ = 'LQUERY'

class LTXTQUERY(sqltypes.TypeEngine):
    """Postresql LTXTQUERY type.

    See :class:`LTREE` for details.
    """
    __visit_name__ = 'LTXTQUERY'

ischema_names['ltree'] = LTREE
ischema_names['lquery'] = LQUERY
ischema_names['ltxtquery'] = LTXTQUERY

PGTypeCompiler.visit_LTREE = lambda type_: 'LTREE'
PGTypeCompiler.visit_LQUERY = lambda type_: 'LQUERY'
PGTypeCompiler.visit_LTXTQUERY = lambda type_: 'LTXTQUERY'<commit_msg>Update ltree again<commit_after>from sqlalchemy.dialects.postgresql.base import ischema_names, PGTypeCompiler, ARRAY
from sqlalchemy import types as sqltypes
from sqlalchemy.sql import expression

class LTREE(sqltypes.Concatenable, sqltypes.TypeEngine):
    """Postgresql LTREE type.

    The LTREE datatype can be used for representing labels of data stored in
    hierarchial tree-like structure. For more detailed information please refer
    to http://www.postgresql.org/docs/9.1/static/ltree.html.

    .. note::

        Using :class:`LTREE`, :class:`LQUERY` and :class:`LTXTQUERY` types may
        require installation of Postgresql ltree extension on the server side.
        Please visit http://www.postgres.org for details.
    """

    class Comparator(sqltypes.Concatenable.Comparator):

        def ancestor_of(self, other):
            if isinstance(other, list):
                return self.op('@>')(expression.cast(other, ARRAY(LTREE)))
            else:
                return self.op('@>')(other)

        def descendant_of(self, other):
            if isinstance(other, list):
                return self.op('<@')(expression.cast(other, ARRAY(LTREE)))
            else:
                return self.op('<@')(other)

        def lquery(self, other):
            if isinstance(other, list):
                return self.op('?')(expression.cast(other, ARRAY(LQUERY)))
            else:
                return self.op('~')(other)

        def ltxtquery(self, other):
            return self.op('@')(other)

    comparator_factory = Comparator

    __visit_name__ = 'LTREE'

class LQUERY(sqltypes.TypeEngine):
    """Postresql LQUERY type.

    See :class:`LTREE` for details.
    """
    __visit_name__ = 'LQUERY'

class LTXTQUERY(sqltypes.TypeEngine):
    """Postresql LTXTQUERY type.

    See :class:`LTREE` for details.
    """
    __visit_name__ = 'LTXTQUERY'

ischema_names['ltree'] = LTREE
ischema_names['lquery'] = LQUERY
ischema_names['ltxtquery'] = LTXTQUERY

PGTypeCompiler.visit_LTREE = lambda self, type_: 'LTREE'
PGTypeCompiler.visit_LQUERY = lambda self, type_: 'LQUERY'
PGTypeCompiler.visit_LTXTQUERY = lambda self, type_: 'LTXTQUERY'<|endoftext|>


<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals
import re

from flask import render_template
from flask_pluginengine import render_plugin_template

from indico.util.decorators import classproperty
from indico.modules.vc.forms import VCPluginSettingsFormBase
from indico.modules.vc.models.vc_rooms import VCRoomLinkType
from indico.util.string import remove_accents
from indico.util.user import retrieve_principals, retrieve_principal
from indico.web.flask.templating import get_overridable_template_name
from indico.web.forms.base import FormDefaults

PREFIX_RE = re.compile('^vc_')

class VCPluginMixin(object):
    settings_form = VCPluginSettingsFormBase
    #: the :class:`IndicoForm` to use for the video conference room form
    vc_room_form = None
    #: the :class:`IndicoForm` to use for the video conference room attach form
    vc_room_attach_form = None
    #: the readable name of the VC plugin
    friendly_name = None

    def init(self):
        super(VCPluginMixin, self).init()
        if not self.name.startswith('vc_'):
            raise Exception('Video conference plugins must be named vc_*')

    def get_vc_room_form_defaults(self, event):
        return {
            'name': re.sub(r'[^\w_-]', '_', remove_accents(event.getTitle(), reencode=False)),
            'show': True,
            'linking': 'event',
            'contribution': '',
            'block': ''
        }

    def get_vc_room_attach_form_defaults(self, event):
        return {
            'room': None,
            'contribution': None,
            'block': None,
            'linking': 'event',
            'show': True
        }

    def get_notification_cc_list(self, action, vc_room, event):
        owner = retrieve_principal(vc_room.data['owner'])
        return {owner.getEmail()} if owner else {}

    def get_notification_bcc_list(self, action, vc_room, event):
        if self.settings.get('notify_managers'):
            return {u.getEmail() for u in retrieve_principals(self.settings.get('managers'))}
        else:
            return {}

    @property
    def service_name(self):
        return PREFIX_RE.sub('', self.name)

    @classproperty
    @classmethod
    def category(self):
        from indico.core.plugins import PluginCategory
        return PluginCategory.videoconference

    def render_form(self, **kwargs):
        """Renders the video conference room form
        :param kwargs: arguments passed to the template
        """
        return render_template('vc/manage_event_create_room.html', **kwargs)

    def render_custom_create_button(self, **kwargs):
        tpl = get_overridable_template_name('create_button.html', self, 'vc/')
        return render_template(tpl, plugin=self, **kwargs)

    def render_info_box(self, vc_room, event_vc_room, event, **kwargs):
        """Renders the information shown in the expandable box of a VC room row
        :param vc_room: the VC room object
        :param event_vc_room: the association of an event and a VC room
        :param event: the event with the current VC room attached to it
        :param kwargs: arguments passed to the template
        """
        return render_plugin_template('{}:info_box.html'.format(self.name), plugin=self, event_vc_room=event_vc_room,
                                      event=event, vc_room=vc_room, retrieve_principal=retrieve_principal,
                                      settings=self.settings, **kwargs)

    def render_manage_event_info_box(self, vc_room, event_vc_room, event, **kwargs):
        """Renders the information shown in the expandable box on a VC room in the management area
        :param vc_room: the VC room object
        :param event_vc_room: the association of an event and a VC room
        :param event: the event with the current VC room attached to it
        :param kwargs: arguments passed to the template
        """
        return render_plugin_template('{}:manage_event_info_box.html'.format(self.name), plugin=self,
                                      event_vc_room=event_vc_room, event=event, vc_room=vc_room,
                                      retrieve_principal=retrieve_principal, settings=self.settings, **kwargs)

    def render_buttons(self, vc_room, event_vc_room, **kwargs):
        """Renders a list of plugin specific buttons (eg: Join URL, etc) in the management area
        :param vc_room: the VC room object
        :param event_vc_room: the association of an event and a VC room
        :param kwargs: arguments passed to the template
        """
        return render_plugin_template('{}:buttons.html'.format(self.name), plugin=self, vc_room=vc_room,
                                      event_vc_room=event_vc_room, **kwargs)

    def render_event_buttons(self, vc_room, event_vc_room, **kwargs):
            """Renders a list of plugin specific buttons (eg: Join URL, etc) in the event page
            :param vc_room: the VC room object
            :param event_vc_room: the association of an event and a VC room
            :param kwargs: arguments passed to the template
            """
            return render_plugin_template('{}:event_buttons.html'.format(self.name), plugin=self, vc_room=vc_room,
                                          event_vc_room=event_vc_room, **kwargs)

    def create_form(self, event, existing_vc_room=None, existing_event_vc_room=None):
        """Creates the video conference room form

        :param event: the event the video conference room is for
        :param existing_vc_room: a vc_room from which to retrieve data for the form
        :param \*\*kwargs: extra data to pass to the form if an existing vc room is passed
        :return: an instance of an :class:`IndicoForm` subclass
        """
        if existing_vc_room and existing_event_vc_room:
            kwargs = {
                'name': existing_vc_room.name,
                'linking': existing_event_vc_room.link_type.name,
                'show': existing_event_vc_room.show
            }

            if existing_event_vc_room.link_type != VCRoomLinkType.event:
                kwargs[existing_event_vc_room.link_type.name] = existing_event_vc_room.link_id

            data = existing_vc_room.data
            data.update(existing_event_vc_room.data)

            defaults = FormDefaults(data, **kwargs)
        else:
            defaults = FormDefaults(self.get_vc_room_form_defaults(event))
        with self.plugin_context():
            return self.vc_room_form(prefix='vc-', obj=defaults, event=event, vc_room=existing_vc_room)

    def handle_form_data_association(self, event, vc_room, event_vc_room, data):
        contribution_id = data.pop('contribution')
        block_id = data.pop('block')
        link_type = VCRoomLinkType[data.pop('linking')]

        if link_type == VCRoomLinkType.event:
            link_id = None
        else:
            link_id = contribution_id if link_type == VCRoomLinkType.contribution else block_id

        event_vc_room.event_id = event.id
        event_vc_room.vc_room = vc_room
        event_vc_room.link_type = link_type
        event_vc_room.link_id = link_id
        event_vc_room.show = data.pop('show')

        if event_vc_room.data is None:
            event_vc_room.data = {}

    def handle_form_data_vc_room(self, vc_room, data):
        vc_room.name = data.pop('name')

        if vc_room.data is None:
            vc_room.data = {}

    def create_room(self, vc_room, event):
        raise NotImplementedError('Plugin must implement create_room()')

    def can_manage_vc_rooms(self, user, event):
        """Checks if a user can manage vc rooms on an event"""
        acl = self.settings.get('acl')
        if not acl:
            return True

        principals = retrieve_principals(acl)
        return any(principal.containsUser(user) for principal in principals)

    def can_manage_vc_room(self, user, room):
        """Checks if a user can manage a vc room"""
        return (user.isAdmin() or
                user in retrieve_principals(room.plugin.settings.get('managers')) or
                any(evt_assoc.event.canUserModify(user) for evt_assoc in room.events))

    def can_manage_vc(self, user):
        """Checks if a user has management rights on this VC system"""
        return user.isAdmin() or user in retrieve_principals(self.settings.get('managers'))
<commit_msg>Fix improper {} usage (should have been set())<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals
import re

from flask import render_template
from flask_pluginengine import render_plugin_template

from indico.util.decorators import classproperty
from indico.modules.vc.forms import VCPluginSettingsFormBase
from indico.modules.vc.models.vc_rooms import VCRoomLinkType
from indico.util.string import remove_accents
from indico.util.user import retrieve_principals, retrieve_principal
from indico.web.flask.templating import get_overridable_template_name
from indico.web.forms.base import FormDefaults

PREFIX_RE = re.compile('^vc_')

class VCPluginMixin(object):
    settings_form = VCPluginSettingsFormBase
    #: the :class:`IndicoForm` to use for the video conference room form
    vc_room_form = None
    #: the :class:`IndicoForm` to use for the video conference room attach form
    vc_room_attach_form = None
    #: the readable name of the VC plugin
    friendly_name = None

    def init(self):
        super(VCPluginMixin, self).init()
        if not self.name.startswith('vc_'):
            raise Exception('Video conference plugins must be named vc_*')

    def get_vc_room_form_defaults(self, event):
        return {
            'name': re.sub(r'[^\w_-]', '_', remove_accents(event.getTitle(), reencode=False)),
            'show': True,
            'linking': 'event',
            'contribution': '',
            'block': ''
        }

    def get_vc_room_attach_form_defaults(self, event):
        return {
            'room': None,
            'contribution': None,
            'block': None,
            'linking': 'event',
            'show': True
        }

    def get_notification_cc_list(self, action, vc_room, event):
        owner = retrieve_principal(vc_room.data['owner'])
        return {owner.getEmail()} if owner else set()

    def get_notification_bcc_list(self, action, vc_room, event):
        if self.settings.get('notify_managers'):
            return {u.getEmail() for u in retrieve_principals(self.settings.get('managers'))}
        else:
            return set()

    @property
    def service_name(self):
        return PREFIX_RE.sub('', self.name)

    @classproperty
    @classmethod
    def category(self):
        from indico.core.plugins import PluginCategory
        return PluginCategory.videoconference

    def render_form(self, **kwargs):
        """Renders the video conference room form
        :param kwargs: arguments passed to the template
        """
        return render_template('vc/manage_event_create_room.html', **kwargs)

    def render_custom_create_button(self, **kwargs):
        tpl = get_overridable_template_name('create_button.html', self, 'vc/')
        return render_template(tpl, plugin=self, **kwargs)

    def render_info_box(self, vc_room, event_vc_room, event, **kwargs):
        """Renders the information shown in the expandable box of a VC room row
        :param vc_room: the VC room object
        :param event_vc_room: the association of an event and a VC room
        :param event: the event with the current VC room attached to it
        :param kwargs: arguments passed to the template
        """
        return render_plugin_template('{}:info_box.html'.format(self.name), plugin=self, event_vc_room=event_vc_room,
                                      event=event, vc_room=vc_room, retrieve_principal=retrieve_principal,
                                      settings=self.settings, **kwargs)

    def render_manage_event_info_box(self, vc_room, event_vc_room, event, **kwargs):
        """Renders the information shown in the expandable box on a VC room in the management area
        :param vc_room: the VC room object
        :param event_vc_room: the association of an event and a VC room
        :param event: the event with the current VC room attached to it
        :param kwargs: arguments passed to the template
        """
        return render_plugin_template('{}:manage_event_info_box.html'.format(self.name), plugin=self,
                                      event_vc_room=event_vc_room, event=event, vc_room=vc_room,
                                      retrieve_principal=retrieve_principal, settings=self.settings, **kwargs)

    def render_buttons(self, vc_room, event_vc_room, **kwargs):
        """Renders a list of plugin specific buttons (eg: Join URL, etc) in the management area
        :param vc_room: the VC room object
        :param event_vc_room: the association of an event and a VC room
        :param kwargs: arguments passed to the template
        """
        return render_plugin_template('{}:buttons.html'.format(self.name), plugin=self, vc_room=vc_room,
                                      event_vc_room=event_vc_room, **kwargs)

    def render_event_buttons(self, vc_room, event_vc_room, **kwargs):
            """Renders a list of plugin specific buttons (eg: Join URL, etc) in the event page
            :param vc_room: the VC room object
            :param event_vc_room: the association of an event and a VC room
            :param kwargs: arguments passed to the template
            """
            return render_plugin_template('{}:event_buttons.html'.format(self.name), plugin=self, vc_room=vc_room,
                                          event_vc_room=event_vc_room, **kwargs)

    def create_form(self, event, existing_vc_room=None, existing_event_vc_room=None):
        """Creates the video conference room form

        :param event: the event the video conference room is for
        :param existing_vc_room: a vc_room from which to retrieve data for the form
        :param \*\*kwargs: extra data to pass to the form if an existing vc room is passed
        :return: an instance of an :class:`IndicoForm` subclass
        """
        if existing_vc_room and existing_event_vc_room:
            kwargs = {
                'name': existing_vc_room.name,
                'linking': existing_event_vc_room.link_type.name,
                'show': existing_event_vc_room.show
            }

            if existing_event_vc_room.link_type != VCRoomLinkType.event:
                kwargs[existing_event_vc_room.link_type.name] = existing_event_vc_room.link_id

            data = existing_vc_room.data
            data.update(existing_event_vc_room.data)

            defaults = FormDefaults(data, **kwargs)
        else:
            defaults = FormDefaults(self.get_vc_room_form_defaults(event))
        with self.plugin_context():
            return self.vc_room_form(prefix='vc-', obj=defaults, event=event, vc_room=existing_vc_room)

    def handle_form_data_association(self, event, vc_room, event_vc_room, data):
        contribution_id = data.pop('contribution')
        block_id = data.pop('block')
        link_type = VCRoomLinkType[data.pop('linking')]

        if link_type == VCRoomLinkType.event:
            link_id = None
        else:
            link_id = contribution_id if link_type == VCRoomLinkType.contribution else block_id

        event_vc_room.event_id = event.id
        event_vc_room.vc_room = vc_room
        event_vc_room.link_type = link_type
        event_vc_room.link_id = link_id
        event_vc_room.show = data.pop('show')

        if event_vc_room.data is None:
            event_vc_room.data = {}

    def handle_form_data_vc_room(self, vc_room, data):
        vc_room.name = data.pop('name')

        if vc_room.data is None:
            vc_room.data = {}

    def create_room(self, vc_room, event):
        raise NotImplementedError('Plugin must implement create_room()')

    def can_manage_vc_rooms(self, user, event):
        """Checks if a user can manage vc rooms on an event"""
        acl = self.settings.get('acl')
        if not acl:
            return True

        principals = retrieve_principals(acl)
        return any(principal.containsUser(user) for principal in principals)

    def can_manage_vc_room(self, user, room):
        """Checks if a user can manage a vc room"""
        return (user.isAdmin() or
                user in retrieve_principals(room.plugin.settings.get('managers')) or
                any(evt_assoc.event.canUserModify(user) for evt_assoc in room.events))

    def can_manage_vc(self, user):
        """Checks if a user has management rights on this VC system"""
        return user.isAdmin() or user in retrieve_principals(self.settings.get('managers'))
<|endoftext|>


<commit_before>import requests
from .processor import SifProcessor

base_url = ('https://git-r3lab.uni.lu/covid/models/-/raw/master/'
            'Executable%20Modules/SBML_qual_build/sif/')
model_ids_to_filenames = {
    786: 'Virus_replication_cycle_stable_raw.sif',
    787: 'PAMP_signalling_stable_raw.sif',
    788: 'Interferon1_stable_raw.sif',
    789: 'Orf3a_stable_raw.sif',
    790: 'TGFB_pathway_stable_raw.sif',
    791: 'IFN-lambda_stable_raw.sif',
    792: 'Kynurenine_pathway_stable_raw.sif',
    793: 'HMOX1_Pathway_stable_raw.sif',
    794: 'Orf10_Cul2_pathway_stable_raw.sif',
    795: 'E_protein_stable_raw.sif',
    796: 'RTC-and-transcription_stable_raw.sif',
    797: 'JNK_pathway_stable_raw.sif',
    798: 'ER_Stress_stable_raw.sif',
    799: 'Apoptosis_stable_raw.sif',
    801: 'Coagulation-pathway_stable_raw.sif',
    802: 'Nsp4_Nsp6_stable_raw.sif',
    803: 'Pyrimidine_deprivation_stable_raw.sif',
    804: 'ETC_stable_raw.sif',
    805: 'Renin_angiotensin_stable_raw.sif',
    806: 'Nsp9_protein_stable_raw.sif',
    807: 'NLRP3_Activation_stable_raw.sif'
    }

def process_file(filename, model_id):
    return process_files({model_id: filename})

def process_files(ids_to_filenames):
    model_id_to_sif_strs = {}
    for model_id, filename in ids_to_filenames.items():
        with open(filename, 'r') as f:
            sif_strs = f.readlines()
        model_id_to_sif_strs[model_id] = sif_strs
    return process_sif_strs(model_id_to_sif_strs)

def process_from_web(model_ids='all'):
    if model_ids == 'all':
        model_ids = list(model_ids_to_filenames.keys())
    model_id_to_sif_strs = {}
    for model_id in model_ids:
        fname = model_ids_to_filenames[model_id]
        url = base_url + fname
        res = requests.get(url)
        if res.status_code == 200:
            sif_strs = res.text.split('\n')
            model_id_to_sif_strs[model_id] = sif_strs
    return process_sif_strs(model_id_to_sif_strs)

def process_sif_strs(model_id_to_sif_strs):
    sp = SifProcessor(model_id_to_sif_strs)
    sp.extract_statements()
    return sp
<commit_msg>Document api<commit_after>import requests
import logging
from .processor import SifProcessor

logger = logging.getLogger()
base_url = ('https://git-r3lab.uni.lu/covid/models/-/raw/master/'
            'Executable%20Modules/SBML_qual_build/sif/')
model_ids_to_filenames = {
    786: 'Virus_replication_cycle_stable_raw.sif',
    787: 'PAMP_signalling_stable_raw.sif',
    788: 'Interferon1_stable_raw.sif',
    789: 'Orf3a_stable_raw.sif',
    790: 'TGFB_pathway_stable_raw.sif',
    791: 'IFN-lambda_stable_raw.sif',
    792: 'Kynurenine_pathway_stable_raw.sif',
    793: 'HMOX1_Pathway_stable_raw.sif',
    794: 'Orf10_Cul2_pathway_stable_raw.sif',
    795: 'E_protein_stable_raw.sif',
    796: 'RTC-and-transcription_stable_raw.sif',
    797: 'JNK_pathway_stable_raw.sif',
    798: 'ER_Stress_stable_raw.sif',
    799: 'Apoptosis_stable_raw.sif',
    801: 'Coagulation-pathway_stable_raw.sif',
    802: 'Nsp4_Nsp6_stable_raw.sif',
    803: 'Pyrimidine_deprivation_stable_raw.sif',
    804: 'ETC_stable_raw.sif',
    805: 'Renin_angiotensin_stable_raw.sif',
    806: 'Nsp9_protein_stable_raw.sif',
    807: 'NLRP3_Activation_stable_raw.sif'
    }

def process_file(filename, model_id):
    """Get statements by processing a single local SIF file.

    Parameters
    ----------
    filename : str
        A name (or path) of a local SIF file to process.
    model_id : int
        ID of a model corresponding to file content. Model ID is needed to
        find relevant references.

    Returns
    -------
    sp : indra.source.minerva.SifProcessor
        An instance of a SifProcessor with extracted INDRA statements.
    """
    return process_files({model_id: filename})

def process_files(ids_to_filenames):
    """Get statements by processing one or more local SIF files.

    Parameters
    ----------
    ids_to_file_names : dict
        A dictionary mapping model IDs to files containing model content as
        SIF. Model IDs are needed to find relevant references.

    Returns
    -------
    sp : indra.source.minerva.SifProcessor
        An instance of a SifProcessor with extracted INDRA statements.
    """
    model_id_to_sif_strs = {}
    for model_id, filename in ids_to_filenames.items():
        with open(filename, 'r') as f:
            sif_strs = f.readlines()
        model_id_to_sif_strs[model_id] = sif_strs
    return process_sif_strs(model_id_to_sif_strs)

def process_from_web(model_ids='all'):
    """Get statements by processing remote SIF files.

    Parameters
    ----------
    model_ids : list or str('all')
        Model IDs for models that need to be processed. If set to 'all'
        (default), then all available models will be processed.

    Returns
    -------
    sp : indra.source.minerva.SifProcessor
        An instance of a SifProcessor with extracted INDRA statements.
    """
    if model_ids == 'all':
        model_ids = list(model_ids_to_filenames.keys())
    model_id_to_sif_strs = {}
    for model_id in model_ids:
        fname = model_ids_to_filenames[model_id]
        url = base_url + fname
        res = requests.get(url)
        if res.status_code == 200:
            sif_strs = res.text.split('\n')
            model_id_to_sif_strs[model_id] = sif_strs
        else:
            logger.warning('Could not get content from file %s, skipping '
                           'model %d' % (fname, model_id))
    return process_sif_strs(model_id_to_sif_strs)

def process_sif_strs(model_id_to_sif_strs):
    sp = SifProcessor(model_id_to_sif_strs)
    sp.extract_statements()
    return sp
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import logging
import xml.etree.ElementTree as ET
from indra.util import UnicodeXMLTreeBuilder as UTB
from indra.sparser.processor import SparserProcessor

logger = logging.getLogger('sparser')

def process_xml(xml_str):
    try:
        tree = ET.XML(xml_str, parser=UTB())
    except ET.ParseError:
        logger.error('Could not parse XML string')
        return None
    sp = _process_elementtree(tree)
    return sp

def _process_elementtree(tree):
    sp = SparserProcessor(tree)
    sp.get_modifications()
    sp.get_activations()
    return sp
<commit_msg>Print XML parse errors in Sparser API<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import logging
import xml.etree.ElementTree as ET
from indra.util import UnicodeXMLTreeBuilder as UTB
from indra.sparser.processor import SparserProcessor

logger = logging.getLogger('sparser')

def process_xml(xml_str):
    try:
        tree = ET.XML(xml_str, parser=UTB())
    except ET.ParseError as e:
        logger.error('Could not parse XML string')
        logger.error(e)
        return None
    sp = _process_elementtree(tree)
    return sp

def _process_elementtree(tree):
    sp = SparserProcessor(tree)
    sp.get_modifications()
    sp.get_activations()
    return sp
<|endoftext|>