<commit_before>#!/usr/bin/env python3
# -*- coding: ascii -*-

# An init script for running Instant and a number of bots.

import os, re, time
import errno, signal
import socket
import shlex
import subprocess
import argparse

import coroutines

try: # Py3K
    from configparser import ConfigParser as _ConfigParser
except ImportError: # Py2K
    from ConfigParser import SafeConfigParser as _ConfigParser

DEFAULT_CONFFILE = 'config/instant.ini'
DEFAULT_PIDFILE_TEMPLATE = 'run/%s.pid'

REDIRECTION_RE = re.compile(r'^[<>|&]+')
PID_LINE_RE = re.compile(r'^[0-9]+\s*$')

class RunnerError(Exception): pass

class ConfigurationError(RunnerError): pass

def open_mkdirs(path, mode, do_mkdirs=True):
    try:
        return open(path, mode)
    except IOError as e:
        if not (e.errno == errno.ENOENT and ('a' in mode or 'w' in mode) and
                do_mkdirs):
            raise
    try:
        os.makedirs(os.path.dirname(path))
    except IOError as e:
        if e.errno != errno.EEXIST:
            raise
    return open(path, mode)

class VerboseExit(coroutines.Exit):
    def __init__(self, result, verbose=False, context=None):
        coroutines.Exit.__init__(self, result)
        self.verbose = verbose
        self.context = context

    def log(self):
        if not self.verbose: return
        context_str = '' if self.context is None else '%s: ' % (self.context,)
        result_str = 'OK' if self.result is None else str(self.result)
        print (context_str + result_str)

    def apply(self, wake, executor, routine):
        self.log()
        coroutines.Exit.apply(self, wake, executor, routine)

class Redirection:
    @classmethod
    def parse(cls, text):
        text = text.strip()
        try:
            constant = {'': None,
                        'stdout': subprocess.STDOUT,
                        'devnull': subprocess.DEVNULL}[text]
        except KeyError:
            pass
        else:
            return cls(constant)
        m = REDIRECTION_RE.match(text)
        if not m:
            raise ValueError('Invalid redirection string: %r' % (text,))
        prefix, filename = m.group(0), text[m.end():].strip()
        try:
            mode = {'<': 'rb', '>': 'wb', '>>': 'ab', '<>': 'r+b'}[prefix]
        except KeyError:
            raise ValueError('Invalid redirection operator: %s' % prefix)
        return cls(filename, mode)

    def __init__(self, target, mode=None):
        self.target = target
        self.mode = mode
        self.mkdirs = True

    def open(self, _retry=True):
        if isinstance(self.target, str):
            return open_mkdirs(self.target, self.mode, self.mkdirs)
        else:
            return self.target

    def close(self, fp):
        if hasattr(fp, 'close'):
            try:
                fp.close()
            except Exception:
                pass

class PIDFile:
    def __init__(self, path):
        self.path = path
        self._pid = Ellipsis

    def _read_file(self):
        f = None
        try:
            f = open(self.path)
            data = f.read()
            if not PID_LINE_RE.match(data):
                raise ValueError('Invalid PID file contents')
            ret = int(data)
            if ret <= 0:
                raise ValueError('Invalid PID in PID file')
            return ret
        except IOError as e:
            if e.errno == errno.ENOENT: return None
            raise
        finally:
            if f: f.close()

    def _write_file(self, pid):
        if pid is None:
            try:
                os.unlink(self.path)
            except OSError as e:
                if e.errno == errno.ENOENT: return
                raise
        else:
            with open_mkdirs(self.path, 'w') as f:
                f.write('%s\n' % pid)

    def get_pid(self, force=False):
        if self._pid is Ellipsis or force:
            self._pid = self._read_file()
        return self._pid

    def set_pid(self, pid):
        self._pid = pid
        self._write_file(pid)

class Remote:
    class RemoteServer:
        def __init__(self, parent, path):
            self.parent = parent
            self.path = path
            self.sock = None

        def listen(self):
            self.sock = socket.socket(socket.AF_UNIX)
            try:
                self.sock.bind(self.path)
            except socket.error as e:
                if e.errno != errno.EADDRINUSE: raise
                os.unlink(self.path)
                self.sock.bind(self.path)
            self.sock.listen(5)

        def accept(self):
            sock, addr = self.sock.accept()
            return self.parent.RemoteConnection(self.parent, self.path,
                                                self.sock)

        def close(self):
            try:
                os.unlink(self.path)
            except IOError:
                pass
            try:
                self.sock.close()
            except IOError:
                pass

    class RemoteConnection:
        def __init__(self, parent, path, sock=None):
            self.parent = parent
            self.path = path
            self.sock = sock
            self.rfile = None
            self.wfile = None
            if self.sock is not None: self._make_files()

        def _make_files(self):
            self.rfile = self.sock.makefile('rb', 0)
            self.wfile = self.sock.makefile('wb', 0)

        def connect(self):
            self.sock = socket.socket(socket.AF_UNIX)
            self.sock.connect(self.path)
            self._make_files()

        def close(self):
            for item in (self.rfile, self.wfile, self.sock):
                if isinstance(item, socket.socket):
                    try:
                        item.shutdown(socket.SHUT_RDWR)
                    except IOError:
                        pass
                try:
                    item.close()
                except IOError:
                    pass
            self.rfile = self.wfile = self.sock = None

    def __init__(self, path):
        self.path = path

    def listen(self):
        res = self.RemoteServer(self, self.path)
        res.listen()
        return res

    def connect(self):
        res = self.RemoteConnection(self, self.path)
        res.connect()
        return res

class Process:
    def __init__(self, name, command, config=None):
        if config is None: config = {}
        self.name = name
        self.command = command
        self.pidfile = PIDFile(config.get('pidfile',
                                          DEFAULT_PIDFILE_TEMPLATE % name))
        self.workdir = config.get('workdir')
        self.stdin = Redirection.parse(config.get('stdin', ''))
        self.stdout = Redirection.parse(config.get('stdout', ''))
        self.min_stop = float(config.get('min-stop', 0))
        self.stderr = Redirection.parse(config.get('stderr', ''))
        self._child = None

    def _redirectors(self):
        return (self.stdin, self.stdout, self.stderr)

    def _make_exit(self, operation, verbose):
        def callback(value):
            return VerboseExit(value, verbose, context)
        if verbose and operation:
            context = '%s (%s)' % (self.name, operation)
        elif verbose:
            context = self.name
        else:
            context = None
        return callback

    def start(self, wait=True, verbose=False):
        exit = self._make_exit('start', verbose)
        cur_status = yield coroutines.Call(self.status(verbose=False))
        if cur_status == 'RUNNING':
            yield exit('ALREADY_RUNNING')
        files = []
        try:
            # NOTE: The finally clause relies on every file being added to
            #       files as soon as it is created -- [r.open() for r in
            #       self._redirectors()] may leak file descriptors.
            for r in self._redirectors():
                files.append(r.open())
            self._child = subprocess.Popen(self.command, cwd=self.workdir,
                close_fds=False, stdin=files[0], stdout=files[1],
                stderr=files[2])
        finally:
            for r, f in zip(self._redirectors(), files):
                r.close(f)
        self.pidfile.set_pid(self._child.pid)
        yield exit('OK')

    def stop(self, wait=True, verbose=False):
        exit = self._make_exit('stop', verbose)
        prev_child = self._child
        if self._child is not None:
            self._child.terminate()
            self._child = None
        else:
            # We could theoretically wait for the PID below, but that would be
            # even more fragile than what we already do in status().
            pid = self.pidfile.get_pid()
            if pid is None:
                yield exit('NOT_RUNNING')
            else:
                os.kill(pid, signal.SIGTERM)
        self.pidfile.set_pid(None)
        if wait:
            if prev_child:
                raw_status = yield coroutines.WaitProcess(prev_child)
                status = 'OK %s' % (raw_status,)
            else:
                status = None
            delay = self.min_stop
            if delay > 0: yield coroutines.Sleep(delay)
        else:
            status = 'OK'
        yield exit(status)

    def status(self, verbose=True):
        exit = self._make_exit(None, verbose)
        pid = self.pidfile.get_pid()
        if pid is None:
            yield exit('NOT_RUNNING')
        try:
            os.kill(pid, 0)
            yield exit('RUNNING')
        except OSError as e:
            if e.errno == errno.ESRCH: yield exit('STALEFILE')
            raise

class ProcessGroup:
    def __init__(self):
        self.processes = []

    def add(self, proc):
        self.processes.append(proc)

    def _for_each(self, handler):
        calls = [coroutines.Call(handler(p)) for p in self.processes]
        result = yield coroutines.All(*calls)
        yield coroutines.Exit(result)

    def start(self, wait=True, verbose=False):
        return self._for_each(lambda p: p.start(wait, verbose))

    def stop(self, wait=True, verbose=False):
        return self._for_each(lambda p: p.stop(wait, verbose))

    def status(self, verbose=True):
        return self._for_each(lambda p: p.status(verbose))

class InstantManager(ProcessGroup):
    def __init__(self, conffile=None):
        ProcessGroup.__init__(self)
        if conffile is None: conffile = DEFAULT_CONFFILE
        self.conffile = conffile
        self.config = None

    def init(self):
        self.config = _ConfigParser()
        self.config.read(self.conffile)
        sections = [s for s in self.config.sections()
                    if s == 'instant' or s.startswith('scribe-')]
        sections.sort()
        for s in sections:
            values = dict(self.config.items(s))
            try:
                cmdline = values['cmdline']
            except KeyError:
                raise ConfigurationError('Missing required key "cmdline" in '
                    'section "%s"' % s)
            command = tuple(shlex.split(cmdline))
            self.add(Process(s, command, values))

    def dispatch(self, cmd, arguments=None):
        try:
            func = {'start': self.do_start, 'stop': self.do_stop,
                    'restart': self.do_restart, 'status': self.do_status}[cmd]
        except KeyError:
            raise RunnerError('Unknown command: ' + cmd)
        kwds = {}
        if cmd in ('start', 'stop'):
            kwds['wait'] = getattr(arguments, 'wait')
        func(**kwds)

    def _run_routine(self, routine):
        coroutines.run([routine], sigpipe=True)

    def do_start(self, wait=True):
        self._run_routine(self.start(verbose=True))

    def do_stop(self, wait=True):
        self._run_routine(self.stop(verbose=True))

    def do_restart(self):
        self.do_stop()
        self.do_start()

    def do_status(self):
        self._run_routine(self.status(verbose=True))

def main():
    p = argparse.ArgumentParser(
        description='Manage an Instant backend and a group of bots')
    p.add_argument('--config', '-c', default=DEFAULT_CONFFILE,
                   help='Configuration file location (default %(default)s)')
    sp = p.add_subparsers(dest='cmd', description='The action to perform')
    sp.required = True
    p_start = sp.add_parser('start', help='Start the backend and bots')
    p_stop = sp.add_parser('stop', help='Stop the backend and bots')
    p_restart = sp.add_parser('restart',
                              help='Perform "stop" and then "start"')
    p_status = sp.add_parser('status',
                             help='Check whether backend or bots are running')
    p_start.add_argument('--no-wait', action='store_false', dest='wait',
                         help='Exit immediately after commencing the start')
    p_stop.add_argument('--no-wait', action='store_false', dest='wait',
                        help='Exit immediately after commencing the stop')
    arguments = p.parse_args()
    mgr = InstantManager(arguments.config)
    try:
        mgr.init()
    except ConfigurationError as exc:
        raise SystemExit('Configuration error: ' + str(exc))
    mgr.dispatch(arguments.cmd, arguments)

if __name__ == '__main__': main()
<commit_msg>[Scripts] Remove redundant words from class names; fix bugs<commit_after>#!/usr/bin/env python3
# -*- coding: ascii -*-

# An init script for running Instant and a number of bots.

import os, re, time
import errno, signal
import socket
import shlex
import subprocess
import argparse

import coroutines

try: # Py3K
    from configparser import ConfigParser as _ConfigParser
except ImportError: # Py2K
    from ConfigParser import SafeConfigParser as _ConfigParser

DEFAULT_CONFFILE = 'config/instant.ini'
DEFAULT_PIDFILE_TEMPLATE = 'run/%s.pid'

REDIRECTION_RE = re.compile(r'^[<>|&]+')
PID_LINE_RE = re.compile(r'^[0-9]+\s*$')

class RunnerError(Exception): pass

class ConfigurationError(RunnerError): pass

def open_mkdirs(path, mode, do_mkdirs=True):
    try:
        return open(path, mode)
    except IOError as e:
        if not (e.errno == errno.ENOENT and ('a' in mode or 'w' in mode) and
                do_mkdirs):
            raise
    try:
        os.makedirs(os.path.dirname(path))
    except IOError as e:
        if e.errno != errno.EEXIST:
            raise
    return open(path, mode)

class VerboseExit(coroutines.Exit):
    def __init__(self, result, verbose=False, context=None):
        coroutines.Exit.__init__(self, result)
        self.verbose = verbose
        self.context = context

    def log(self):
        if not self.verbose: return
        context_str = '' if self.context is None else '%s: ' % (self.context,)
        result_str = 'OK' if self.result is None else str(self.result)
        print (context_str + result_str)

    def apply(self, wake, executor, routine):
        self.log()
        coroutines.Exit.apply(self, wake, executor, routine)

class Redirection:
    @classmethod
    def parse(cls, text):
        text = text.strip()
        try:
            constant = {'': None,
                        'stdout': subprocess.STDOUT,
                        'devnull': subprocess.DEVNULL}[text]
        except KeyError:
            pass
        else:
            return cls(constant)
        m = REDIRECTION_RE.match(text)
        if not m:
            raise ValueError('Invalid redirection string: %r' % (text,))
        prefix, filename = m.group(0), text[m.end():].strip()
        try:
            mode = {'<': 'rb', '>': 'wb', '>>': 'ab', '<>': 'r+b'}[prefix]
        except KeyError:
            raise ValueError('Invalid redirection operator: %s' % prefix)
        return cls(filename, mode)

    def __init__(self, target, mode=None):
        self.target = target
        self.mode = mode
        self.mkdirs = True

    def open(self, _retry=True):
        if isinstance(self.target, str):
            return open_mkdirs(self.target, self.mode, self.mkdirs)
        else:
            return self.target

    def close(self, fp):
        if hasattr(fp, 'close'):
            try:
                fp.close()
            except Exception:
                pass

class PIDFile:
    def __init__(self, path):
        self.path = path
        self._pid = Ellipsis

    def _read_file(self):
        f = None
        try:
            f = open(self.path)
            data = f.read()
            if not PID_LINE_RE.match(data):
                raise ValueError('Invalid PID file contents')
            ret = int(data)
            if ret <= 0:
                raise ValueError('Invalid PID in PID file')
            return ret
        except IOError as e:
            if e.errno == errno.ENOENT: return None
            raise
        finally:
            if f: f.close()

    def _write_file(self, pid):
        if pid is None:
            try:
                os.unlink(self.path)
            except OSError as e:
                if e.errno == errno.ENOENT: return
                raise
        else:
            with open_mkdirs(self.path, 'w') as f:
                f.write('%s\n' % pid)

    def get_pid(self, force=False):
        if self._pid is Ellipsis or force:
            self._pid = self._read_file()
        return self._pid

    def set_pid(self, pid):
        self._pid = pid
        self._write_file(pid)

class Remote:
    class Server:
        def __init__(self, parent, path):
            self.parent = parent
            self.path = path
            self.sock = None

        def listen(self):
            self.sock = socket.socket(socket.AF_UNIX)
            try:
                self.sock.bind(self.path)
            except socket.error as e:
                if e.errno != errno.EADDRINUSE: raise
                os.unlink(self.path)
                self.sock.bind(self.path)
            self.sock.listen(5)

        def accept(self):
            sock, addr = self.sock.accept()
            return self.parent.Connection(self.parent, self.path, sock)

        def close(self):
            try:
                os.unlink(self.path)
            except IOError:
                pass
            try:
                self.sock.close()
            except IOError:
                pass

    class Connection:
        def __init__(self, parent, path, sock=None):
            self.parent = parent
            self.path = path
            self.sock = sock
            self.rfile = None
            self.wfile = None
            if self.sock is not None: self._make_files()

        def _make_files(self):
            self.rfile = self.sock.makefile('rb', 0)
            self.wfile = self.sock.makefile('wb', 0)

        def connect(self):
            self.sock = socket.socket(socket.AF_UNIX)
            self.sock.connect(self.path)
            self._make_files()

        def close(self):
            try:
                socket.sock.shutdown(socket.SHUT_RDWR)
            except IOError:
                pass
            for item in (self.rfile, self.wfile, self.sock):
                try:
                    item.close()
                except IOError:
                    pass
            self.rfile = self.wfile = self.sock = None

    def __init__(self, path):
        self.path = path

    def listen(self):
        res = self.Server(self, self.path)
        res.listen()
        return res

    def connect(self):
        res = self.Connection(self, self.path)
        res.connect()
        return res

class Process:
    def __init__(self, name, command, config=None):
        if config is None: config = {}
        self.name = name
        self.command = command
        self.pidfile = PIDFile(config.get('pidfile',
                                          DEFAULT_PIDFILE_TEMPLATE % name))
        self.workdir = config.get('workdir')
        self.stdin = Redirection.parse(config.get('stdin', ''))
        self.stdout = Redirection.parse(config.get('stdout', ''))
        self.min_stop = float(config.get('min-stop', 0))
        self.stderr = Redirection.parse(config.get('stderr', ''))
        self._child = None

    def _redirectors(self):
        return (self.stdin, self.stdout, self.stderr)

    def _make_exit(self, operation, verbose):
        def callback(value):
            return VerboseExit(value, verbose, context)
        if verbose and operation:
            context = '%s (%s)' % (self.name, operation)
        elif verbose:
            context = self.name
        else:
            context = None
        return callback

    def start(self, wait=True, verbose=False):
        exit = self._make_exit('start', verbose)
        cur_status = yield coroutines.Call(self.status(verbose=False))
        if cur_status == 'RUNNING':
            yield exit('ALREADY_RUNNING')
        files = []
        try:
            # NOTE: The finally clause relies on every file being added to
            #       files as soon as it is created -- [r.open() for r in
            #       self._redirectors()] may leak file descriptors.
            for r in self._redirectors():
                files.append(r.open())
            self._child = subprocess.Popen(self.command, cwd=self.workdir,
                close_fds=False, stdin=files[0], stdout=files[1],
                stderr=files[2])
        finally:
            for r, f in zip(self._redirectors(), files):
                r.close(f)
        self.pidfile.set_pid(self._child.pid)
        yield exit('OK')

    def stop(self, wait=True, verbose=False):
        exit = self._make_exit('stop', verbose)
        prev_child = self._child
        if self._child is not None:
            self._child.terminate()
            self._child = None
        else:
            # We could theoretically wait for the PID below, but that would be
            # even more fragile than what we already do in status().
            pid = self.pidfile.get_pid()
            if pid is None:
                yield exit('NOT_RUNNING')
            else:
                os.kill(pid, signal.SIGTERM)
        self.pidfile.set_pid(None)
        if wait:
            if prev_child:
                raw_status = yield coroutines.WaitProcess(prev_child)
                status = 'OK %s' % (raw_status,)
            else:
                status = None
            delay = self.min_stop
            if delay > 0: yield coroutines.Sleep(delay)
        else:
            status = 'OK'
        yield exit(status)

    def status(self, verbose=True):
        exit = self._make_exit(None, verbose)
        pid = self.pidfile.get_pid()
        if pid is None:
            yield exit('NOT_RUNNING')
        try:
            os.kill(pid, 0)
            yield exit('RUNNING')
        except OSError as e:
            if e.errno == errno.ESRCH: yield exit('STALEFILE')
            raise

class ProcessGroup:
    def __init__(self):
        self.processes = []

    def add(self, proc):
        self.processes.append(proc)

    def _for_each(self, handler):
        calls = [coroutines.Call(handler(p)) for p in self.processes]
        result = yield coroutines.All(*calls)
        yield coroutines.Exit(result)

    def start(self, wait=True, verbose=False):
        return self._for_each(lambda p: p.start(wait, verbose))

    def stop(self, wait=True, verbose=False):
        return self._for_each(lambda p: p.stop(wait, verbose))

    def status(self, verbose=True):
        return self._for_each(lambda p: p.status(verbose))

class InstantManager(ProcessGroup):
    def __init__(self, conffile=None):
        ProcessGroup.__init__(self)
        if conffile is None: conffile = DEFAULT_CONFFILE
        self.conffile = conffile
        self.config = None

    def init(self):
        self.config = _ConfigParser()
        self.config.read(self.conffile)
        sections = [s for s in self.config.sections()
                    if s == 'instant' or s.startswith('scribe-')]
        sections.sort()
        for s in sections:
            values = dict(self.config.items(s))
            try:
                cmdline = values['cmdline']
            except KeyError:
                raise ConfigurationError('Missing required key "cmdline" in '
                    'section "%s"' % s)
            command = tuple(shlex.split(cmdline))
            self.add(Process(s, command, values))

    def dispatch(self, cmd, arguments=None):
        try:
            func = {'start': self.do_start, 'stop': self.do_stop,
                    'restart': self.do_restart, 'status': self.do_status}[cmd]
        except KeyError:
            raise RunnerError('Unknown command: ' + cmd)
        kwds = {}
        if cmd in ('start', 'stop'):
            kwds['wait'] = getattr(arguments, 'wait')
        func(**kwds)

    def _run_routine(self, routine):
        coroutines.run([routine], sigpipe=True)

    def do_start(self, wait=True):
        self._run_routine(self.start(verbose=True))

    def do_stop(self, wait=True):
        self._run_routine(self.stop(verbose=True))

    def do_restart(self):
        self.do_stop()
        self.do_start()

    def do_status(self):
        self._run_routine(self.status(verbose=True))

def main():
    p = argparse.ArgumentParser(
        description='Manage an Instant backend and a group of bots')
    p.add_argument('--config', '-c', default=DEFAULT_CONFFILE,
                   help='Configuration file location (default %(default)s)')
    sp = p.add_subparsers(dest='cmd', description='The action to perform')
    sp.required = True
    p_start = sp.add_parser('start', help='Start the backend and bots')
    p_stop = sp.add_parser('stop', help='Stop the backend and bots')
    p_restart = sp.add_parser('restart',
                              help='Perform "stop" and then "start"')
    p_status = sp.add_parser('status',
                             help='Check whether backend or bots are running')
    p_start.add_argument('--no-wait', action='store_false', dest='wait',
                         help='Exit immediately after commencing the start')
    p_stop.add_argument('--no-wait', action='store_false', dest='wait',
                        help='Exit immediately after commencing the stop')
    arguments = p.parse_args()
    mgr = InstantManager(arguments.config)
    try:
        mgr.init()
    except ConfigurationError as exc:
        raise SystemExit('Configuration error: ' + str(exc))
    mgr.dispatch(arguments.cmd, arguments)

if __name__ == '__main__': main()
<|endoftext|>


<commit_before>import os.path
from django.urls import include, path, register_converter, re_path
from django.contrib import admin
from django.conf import settings
from django.conf.urls.static import static

# This URLs should be always be served, even during maintenance mode
urlpatterns = static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

# Check for maintenace mode
from django.conf import settings

if settings.MAINTENANCE:
    from mygpo.web import utils

    urlpatterns += [re_path("", utils.maintenance)]

# URLs are still registered during maintenace mode because we need to
# build links from them (eg login-link).
urlpatterns += [
    path("", include("mygpo.web.urls")),
    path("", include("mygpo.podcasts.urls")),
    path("", include("mygpo.directory.urls")),
    path("", include("mygpo.api.urls")),
    path("", include("mygpo.userfeeds.urls")),
    path("", include("mygpo.share.urls")),
    path("", include("mygpo.history.urls")),
    path("", include("mygpo.subscriptions.urls")),
    path("", include("mygpo.users.urls")),
    path("", include("mygpo.podcastlists.urls")),
    path("suggestions/", include("mygpo.suggestions.urls")),
    path("publisher/", include("mygpo.publisher.urls")),
    path("administration/", include("mygpo.administration.urls")),
    path("pubsub/", include("mygpo.pubsub.urls")),
    path("admin/", admin.site.urls),
]
<commit_msg>Include debug_toolbars urls when running in DEBUG mode<commit_after>import os.path
from django.urls import include, path, register_converter, re_path
from django.contrib import admin
from django.conf import settings
from django.conf.urls.static import static

# This URLs should be always be served, even during maintenance mode
urlpatterns = static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

# Check for maintenace mode
from django.conf import settings

if settings.MAINTENANCE:
    from mygpo.web import utils

    urlpatterns += [re_path("", utils.maintenance)]

# Add debug urlpattern for debug_toolbar
if settings.DEBUG:
	import debug_toolbar
	urlpatterns += [
		path('__debug__/', include(debug_toolbar.urls)),
	]

# URLs are still registered during maintenace mode because we need to
# build links from them (eg login-link).
urlpatterns += [
    path("", include("mygpo.web.urls")),
    path("", include("mygpo.podcasts.urls")),
    path("", include("mygpo.directory.urls")),
    path("", include("mygpo.api.urls")),
    path("", include("mygpo.userfeeds.urls")),
    path("", include("mygpo.share.urls")),
    path("", include("mygpo.history.urls")),
    path("", include("mygpo.subscriptions.urls")),
    path("", include("mygpo.users.urls")),
    path("", include("mygpo.podcastlists.urls")),
    path("suggestions/", include("mygpo.suggestions.urls")),
    path("publisher/", include("mygpo.publisher.urls")),
    path("administration/", include("mygpo.administration.urls")),
    path("pubsub/", include("mygpo.pubsub.urls")),
    path("admin/", admin.site.urls),
]
<|endoftext|>


<commit_before>import time
import threading
import json
import logging

from gantryd.componentstate import ComponentState, STOPPED_STATUS, KILLED_STATUS, READY_STATUS, PULL_FAIL
from util import report, fail, getDockerClient, ReportLevels

CHECK_SLEEP_TIME = 30 # 30 seconds
CHECK_SHORT_SLEEP_TIME = 10 # 10 seconds
MONITOR_SLEEP_TIME = 30 # 30 seconds

class ComponentWatcher(object):
  """ Helper class which watches a specific component's status in etcd and
      manages the update/stop/kill process (if necessary). Also watches the
      component itself once started, and ensures that it remains running (restarting
      it if it failed).
  """
  def __init__(self, component, project_name, machine_id, etcd_client):
    self.component = component
    self.project_name = project_name
    self.machine_id = machine_id
    self.is_running = False

    # Logging.
    self.logger = logging.getLogger(__name__)

    # Setup the state helper for the component.
    self.state = ComponentState(project_name, component, etcd_client)

    # Setup the watcher thread.
    self.watcher_thread = threading.Thread(target=self.waitForCommand, args=[])
    self.watcher_thread.daemon = True

    # Setup the monitor thread.
    self.monitor_thread = threading.Thread(target=self.monitorComponent, args=[])
    self.monitor_thread.daemon = True

    # Setup an event to ping the monitor thread when it should restart checking in
    # on the component.
    self.monitor_event = threading.Event()

    # Setup a lock to prevent multiple threads from trying to (re)start a container.
    self.update_lock = threading.Lock()

  def start(self):
    """ Starts the watcher. """
    self.watcher_thread.start()
    self.monitor_thread.start()

  def monitorComponent(self):
    """ Monitors a component by pinging it every MONITOR_SLEEP_TIME seconds or so. If a component
        fails, then the system will try to restart it. If that fails, the component is marked
        as dead.
    """
    while True:
      # Wait for the component to be running.
      self.monitor_event.wait()

      # Sleep MONITOR_SLEEP_TIME seconds.
      time.sleep(MONITOR_SLEEP_TIME)

      # Check the component.
      report('Checking in on component', project=self.project_name, component=self.component,
             level=ReportLevels.BACKGROUND)

      if not self.component.isHealthy():
        self.logger.debug('Component %s is not healty', self.component.getName())
        with self.update_lock:
          # Just to be sure...
          if not self.is_running:
            continue

          # Ensure that the component is still ready.
          state = self.state.getState()
          current_status = ComponentState.getStatusOf(state)
          if current_status == READY_STATUS:
            report('Component ' + self.component.getName() + ' is not healthy. Restarting...',
                   project=self.project_name, component=self.component)

            if not self.component.update():
              report('Could not restart component ' + self.component.getName(),
                     project=self.project_name, component=self.component,
                     level=ReportLevels.IMPORTANT)
              self.monitor_event.clear()
              continue

  def waitForCommand(self):
    """ Waits for an command notification on the component in etcd. If one is received,
        processes it by attempting to update the component.
    """
    is_initial_loop = True
    sleep_time = 0
    while True:
      # Sleep and then check again.
      time.sleep(sleep_time)
      sleep_time = CHECK_SLEEP_TIME

      # Check the component's status.
      self.logger.debug('Checking state for component %s', self.component.getName())
      state = self.state.getState()
      self.logger.debug('Found state %s for component %s', state, self.component.getName())

      # Determine whether we should give initial status messages.
      was_initial_loop = is_initial_loop
      is_initial_loop = False

      # Take actions based on the status requested.
      current_status = ComponentState.getStatusOf(state)
      sleep_time = self.handleStatus(current_status, state, was_initial_loop)

  def handleStatus(self, current_status, state, was_initial_check):
    """ Handles the various status states for the component, returning the
        amount of time after which to retry lookup up the state or -1 for
        terminated.
    """
    if current_status == STOPPED_STATUS:
      return self.handleStopped(was_initial_check)
    elif current_status == KILLED_STATUS:
      return self.handleKilled(was_initial_check)
    elif current_status == READY_STATUS or current_status = PULL_FAIL:
      with self.update_lock:
        return self.handleReady(state, was_initial_check)

    return CHECK_SLEEP_TIME

  def handleStopped(self, was_initial_check):
    """ Handles when the component has been marked to be stopped. """
    self.monitor_event.clear()

    if was_initial_check:
      report('Component %s is marked as stopped' % self.component.getName(),
             project=self.project_name, component=self.component)

    self.is_running = False
    self.component.stop(kill=False)
    return CHECK_SLEEP_TIME

  def handleKilled(self, was_initial_check):
    """ Handles when the component has been marked to be killed. """
    self.monitor_event.clear()

    if was_initial_check:
      report('Component %s is marked as killed' % self.component.getName(),
             project=self.project_name, component=self.component)

    self.is_running = False
    self.component.stop(kill=True)
    return CHECK_SLEEP_TIME

  def handleReady(self, state, was_initial_check):
    """ Handles when the component has been marked as ready. """

    # If the status is ready, we update the component if:
    #   - The ID of the component's image does not match that found in the status.
    #   - The process is not running.
    imageid = ComponentState.getImageIdOf(state)
    imageid_different = imageid != self.component.getImageId()
    should_update = not self.is_running or imageid_different

    if should_update:
      self.is_running = False
      self.monitor_event.clear()

      # We need to update this machine's copy. First, do a test and set to ensure that
      # we are the only machine allowed to update. If the test and set fails, we'll
      # try again in 10s.
      if imageid_different:
        report('Detected pushed update for component ' + self.component.getName(),
               project=self.project_name, component=self.component)
      else:
        report('Component %s is not running; starting' % self.component.getName(),
               project=self.project_name, component=self.component)

      result = self.state.setUpdatingStatus('updating', self.machine_id, state)
      if not result:
        # The exchange failed. Sleep CHECK_SHORT_SLEEP_TIME seconds and try again.
        report('Could not grab update lock. Will try again in %s seconds' % CHECK_SHORT_SLEEP_TIME,
               project=self.project_name, component=self.component)
        return CHECK_SHORT_SLEEP_TIME

      # Start the update by pulling the repo for the component.
      if imageid_different:
        report('Pulling the image for component ' + self.component.getName())
        if not self.component.pullRepo():
          # The pull failed.
          report('Pull failed of image %s for component %s' % (imageid[0:12],
                                                               self.component.getName()),
                 project=self.project_name, component=self.component, level=ReportLevels.IMPORTANT)
          self.state.setUpdatingStatus('pullfail', self.machine_id, result)
          return CHECK_SLEEP_TIME

      # Run the update on the component and wait for it to finish.
      if imageid_different:
        report('Starting update for component ' + self.component.getName(),
               project=self.project_name, component=self.component)

      if not self.component.update():
        # The update failed.
        self.state.setUpdatingStatus('updatefail', self.machine_id, result)
        return CHECK_SLEEP_TIME

      # Otherwise, the update has succeeded. Mark the component as ready, so another
      # gantryd can start its update.
      if imageid_different:
        report('Update completed for component ' + self.component.getName(),
               project=self.project_name, component=self.component)
      else:
        report('Component ' + self.component.getName() + ' is now running',
               project=self.project_name, component=self.component)

      self.state.setReadyStatus(self.component.getImageId())
      self.is_running = True
      self.monitor_event.set()

    return CHECK_SLEEP_TIME
<commit_msg>Fix pull fail condition<commit_after>import time
import threading
import json
import logging

from gantryd.componentstate import ComponentState, STOPPED_STATUS, KILLED_STATUS, READY_STATUS, PULL_FAIL
from util import report, fail, getDockerClient, ReportLevels

CHECK_SLEEP_TIME = 30 # 30 seconds
CHECK_SHORT_SLEEP_TIME = 10 # 10 seconds
MONITOR_SLEEP_TIME = 30 # 30 seconds

class ComponentWatcher(object):
  """ Helper class which watches a specific component's status in etcd and
      manages the update/stop/kill process (if necessary). Also watches the
      component itself once started, and ensures that it remains running (restarting
      it if it failed).
  """
  def __init__(self, component, project_name, machine_id, etcd_client):
    self.component = component
    self.project_name = project_name
    self.machine_id = machine_id
    self.is_running = False

    # Logging.
    self.logger = logging.getLogger(__name__)

    # Setup the state helper for the component.
    self.state = ComponentState(project_name, component, etcd_client)

    # Setup the watcher thread.
    self.watcher_thread = threading.Thread(target=self.waitForCommand, args=[])
    self.watcher_thread.daemon = True

    # Setup the monitor thread.
    self.monitor_thread = threading.Thread(target=self.monitorComponent, args=[])
    self.monitor_thread.daemon = True

    # Setup an event to ping the monitor thread when it should restart checking in
    # on the component.
    self.monitor_event = threading.Event()

    # Setup a lock to prevent multiple threads from trying to (re)start a container.
    self.update_lock = threading.Lock()

  def start(self):
    """ Starts the watcher. """
    self.watcher_thread.start()
    self.monitor_thread.start()

  def monitorComponent(self):
    """ Monitors a component by pinging it every MONITOR_SLEEP_TIME seconds or so. If a component
        fails, then the system will try to restart it. If that fails, the component is marked
        as dead.
    """
    while True:
      # Wait for the component to be running.
      self.monitor_event.wait()

      # Sleep MONITOR_SLEEP_TIME seconds.
      time.sleep(MONITOR_SLEEP_TIME)

      # Check the component.
      report('Checking in on component', project=self.project_name, component=self.component,
             level=ReportLevels.BACKGROUND)

      if not self.component.isHealthy():
        self.logger.debug('Component %s is not healty', self.component.getName())
        with self.update_lock:
          # Just to be sure...
          if not self.is_running:
            continue

          # Ensure that the component is still ready.
          state = self.state.getState()
          current_status = ComponentState.getStatusOf(state)
          if current_status == READY_STATUS:
            report('Component ' + self.component.getName() + ' is not healthy. Restarting...',
                   project=self.project_name, component=self.component)

            if not self.component.update():
              report('Could not restart component ' + self.component.getName(),
                     project=self.project_name, component=self.component,
                     level=ReportLevels.IMPORTANT)
              self.monitor_event.clear()
              continue

  def waitForCommand(self):
    """ Waits for an command notification on the component in etcd. If one is received,
        processes it by attempting to update the component.
    """
    is_initial_loop = True
    sleep_time = 0
    while True:
      # Sleep and then check again.
      time.sleep(sleep_time)
      sleep_time = CHECK_SLEEP_TIME

      # Check the component's status.
      self.logger.debug('Checking state for component %s', self.component.getName())
      state = self.state.getState()
      self.logger.debug('Found state %s for component %s', state, self.component.getName())

      # Determine whether we should give initial status messages.
      was_initial_loop = is_initial_loop
      is_initial_loop = False

      # Take actions based on the status requested.
      current_status = ComponentState.getStatusOf(state)
      sleep_time = self.handleStatus(current_status, state, was_initial_loop)

  def handleStatus(self, current_status, state, was_initial_check):
    """ Handles the various status states for the component, returning the
        amount of time after which to retry lookup up the state or -1 for
        terminated.
    """
    if current_status == STOPPED_STATUS:
      return self.handleStopped(was_initial_check)
    elif current_status == KILLED_STATUS:
      return self.handleKilled(was_initial_check)
    elif current_status == READY_STATUS or current_status == PULL_FAIL:
      with self.update_lock:
        return self.handleReady(state, was_initial_check)

    return CHECK_SLEEP_TIME

  def handleStopped(self, was_initial_check):
    """ Handles when the component has been marked to be stopped. """
    self.monitor_event.clear()

    if was_initial_check:
      report('Component %s is marked as stopped' % self.component.getName(),
             project=self.project_name, component=self.component)

    self.is_running = False
    self.component.stop(kill=False)
    return CHECK_SLEEP_TIME

  def handleKilled(self, was_initial_check):
    """ Handles when the component has been marked to be killed. """
    self.monitor_event.clear()

    if was_initial_check:
      report('Component %s is marked as killed' % self.component.getName(),
             project=self.project_name, component=self.component)

    self.is_running = False
    self.component.stop(kill=True)
    return CHECK_SLEEP_TIME

  def handleReady(self, state, was_initial_check):
    """ Handles when the component has been marked as ready. """

    # If the status is ready, we update the component if:
    #   - The ID of the component's image does not match that found in the status.
    #   - The process is not running.
    imageid = ComponentState.getImageIdOf(state)
    imageid_different = imageid != self.component.getImageId()
    should_update = not self.is_running or imageid_different

    if should_update:
      self.is_running = False
      self.monitor_event.clear()

      # We need to update this machine's copy. First, do a test and set to ensure that
      # we are the only machine allowed to update. If the test and set fails, we'll
      # try again in 10s.
      if imageid_different:
        report('Detected pushed update for component ' + self.component.getName(),
               project=self.project_name, component=self.component)
      else:
        report('Component %s is not running; starting' % self.component.getName(),
               project=self.project_name, component=self.component)

      result = self.state.setUpdatingStatus('updating', self.machine_id, state)
      if not result:
        # The exchange failed. Sleep CHECK_SHORT_SLEEP_TIME seconds and try again.
        report('Could not grab update lock. Will try again in %s seconds' % CHECK_SHORT_SLEEP_TIME,
               project=self.project_name, component=self.component)
        return CHECK_SHORT_SLEEP_TIME

      # Start the update by pulling the repo for the component.
      if imageid_different:
        report('Pulling the image for component ' + self.component.getName())
        if not self.component.pullRepo():
          # The pull failed.
          report('Pull failed of image %s for component %s' % (imageid[0:12],
                                                               self.component.getName()),
                 project=self.project_name, component=self.component, level=ReportLevels.IMPORTANT)
          self.state.setUpdatingStatus('pullfail', self.machine_id, result)
          return CHECK_SLEEP_TIME

      # Run the update on the component and wait for it to finish.
      if imageid_different:
        report('Starting update for component ' + self.component.getName(),
               project=self.project_name, component=self.component)

      if not self.component.update():
        # The update failed.
        self.state.setUpdatingStatus('updatefail', self.machine_id, result)
        return CHECK_SLEEP_TIME

      # Otherwise, the update has succeeded. Mark the component as ready, so another
      # gantryd can start its update.
      if imageid_different:
        report('Update completed for component ' + self.component.getName(),
               project=self.project_name, component=self.component)
      else:
        report('Component ' + self.component.getName() + ' is now running',
               project=self.project_name, component=self.component)

      self.state.setReadyStatus(self.component.getImageId())
      self.is_running = True
      self.monitor_event.set()

    return CHECK_SLEEP_TIME
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging
from optparse import make_option

from django.core.management.base import BaseCommand
from ..ipgeobase import IpGeobase

class Command(BaseCommand):

    help = 'Updates django-geoip data stored in db'
    option_list = BaseCommand.option_list + (
        make_option('--clear',
            action='store_true',
            default=False,
            help="Clear tables prior import"
        ),
    )

    def get_logger(self, verbosity):
        logger = logging.getLogger('import')
        logger.addHandler(logging.StreamHandler())
        VERBOSITY_MAPPING = {
            0: logging.CRITICAL, # no
            1: logging.INFO, # means normal output (default)
            2: logging.DEBUG, # means verbose output
            3: logging.DEBUG, # means very verbose output
        }
        logger.setLevel(VERBOSITY_MAPPING[int(verbosity)])
        return logger

    def handle(self, *args, **options):
        logger = self.get_logger(options['verbosity'])
        backend = IpGeobase(logger=logger)

        if options.get('clear'):
            backend.clear_database()

        backend.download_files()
        backend.sync_database()<commit_msg>fix geoip_update command for django 1.10<commit_after># -*- coding: utf-8 -*-
import logging
from optparse import make_option

from django.core.management.base import BaseCommand
from ..ipgeobase import IpGeobase

class Command(BaseCommand):

    help = 'Updates django-geoip data stored in db'

    def add_arguments(self, parser):
        parser.add_argument('--clear',
            action='store_true',
            default=False,
            help="Clear tables prior import"
        )

    def get_logger(self, verbosity):
        logger = logging.getLogger('import')
        logger.addHandler(logging.StreamHandler())
        VERBOSITY_MAPPING = {
            0: logging.CRITICAL, # no
            1: logging.INFO, # means normal output (default)
            2: logging.DEBUG, # means verbose output
            3: logging.DEBUG, # means very verbose output
        }
        logger.setLevel(VERBOSITY_MAPPING[int(verbosity)])
        return logger

    def handle(self, *args, **options):
        logger = self.get_logger(options['verbosity'])
        backend = IpGeobase(logger=logger)

        if options.get('clear'):
            backend.clear_database()

        backend.download_files()
        backend.sync_database()
<|endoftext|>


<commit_before>"""Jinja2's i18n functionality is not exactly the same as Django's.
In particular, the tags names and their syntax are different:

  1. The Django ``trans`` tag is replaced by a _() global.
  2. The Django ``blocktrans`` tag is called ``trans``.

(1) isn't an issue, since the whole ``makemessages`` process is based on
converting the template tags to ``_()`` calls. However, (2) means that
those Jinja2 ``trans`` tags will not be picked up my Django's
``makemessage`` command.

There aren't any nice solutions here. While Jinja2's i18n extension does
come with extraction capabilities built in, the code behind ``makemessages``
unfortunately isn't extensible, so we can:

  * Duplicate the command + code behind it.
  * Offer a separate command for Jinja2 extraction.
  * Try to get Django to offer hooks into makemessages().
  * Monkey-patch.

We are currently doing that last thing. It turns out there we are lucky
for once: It's simply a matter of extending two regular expressions.
Credit for the approach goes to:
http://stackoverflow.com/questions/2090717/getting-translation-strings-for-jinja2-templates-integrated-with-django-1-x
"""

import re
from django.core.management.commands import makemessages
from django.utils.translation import trans_real
from django.template.base import BLOCK_TAG_START, BLOCK_TAG_END

strip_whitespace_right = re.compile(r"(%s-?\s*(trans|pluralize).*?-%s)\s+" % (BLOCK_TAG_START, BLOCK_TAG_END), re.U)
strip_whitespace_left = re.compile(r"\s+(%s-\s*(endtrans|pluralize).*?-?%s)" % (BLOCK_TAG_START, BLOCK_TAG_END), re.U)

def strip_whitespaces(src):
    src = strip_whitespace_left.sub(r'\1', src)
    src = strip_whitespace_right.sub(r'\1', src)
    return src

class Command(makemessages.Command):

    def handle(self, *args, **options):
        old_endblock_re = trans_real.endblock_re
        old_block_re = trans_real.block_re
        old_constant_re = trans_real.constant_re

        old_templatize = trans_real.templatize
        # Extend the regular expressions that are used to detect
        # translation blocks with an "OR jinja-syntax" clause.
        trans_real.endblock_re = re.compile(
            trans_real.endblock_re.pattern + '|' + r"""^-?\s*endtrans\s*-?$""")
        trans_real.block_re = re.compile(
            trans_real.block_re.pattern + '|' + r"""^-?\s*trans(?:\s+(?!'|")(?=.*?=.*?)|\s*-?$)""")
        trans_real.plural_re = re.compile(
            trans_real.plural_re.pattern + '|' + r"""^-?\s*pluralize(?:\s+.+|-?$)""")
        trans_real.constant_re = re.compile(r"""_\(((?:".*?")|(?:'.*?')).*\)""")

        def my_templatize(src, origin=None):
            new_src = strip_whitespaces(src)
            return old_templatize(new_src, origin)

        trans_real.templatize = my_templatize

        try:
            super(Command, self).handle(*args, **options)
        finally:
            trans_real.endblock_re = old_endblock_re
            trans_real.block_re = old_block_re
            trans_real.templatize = old_templatize
            trans_real.constant_re = old_constant_re
<commit_msg>Fix compatibility with django 1.11.<commit_after>"""Jinja2's i18n functionality is not exactly the same as Django's.
In particular, the tags names and their syntax are different:

  1. The Django ``trans`` tag is replaced by a _() global.
  2. The Django ``blocktrans`` tag is called ``trans``.

(1) isn't an issue, since the whole ``makemessages`` process is based on
converting the template tags to ``_()`` calls. However, (2) means that
those Jinja2 ``trans`` tags will not be picked up my Django's
``makemessage`` command.

There aren't any nice solutions here. While Jinja2's i18n extension does
come with extraction capabilities built in, the code behind ``makemessages``
unfortunately isn't extensible, so we can:

  * Duplicate the command + code behind it.
  * Offer a separate command for Jinja2 extraction.
  * Try to get Django to offer hooks into makemessages().
  * Monkey-patch.

We are currently doing that last thing. It turns out there we are lucky
for once: It's simply a matter of extending two regular expressions.
Credit for the approach goes to:
http://stackoverflow.com/questions/2090717/getting-translation-strings-for-jinja2-templates-integrated-with-django-1-x
"""

import re
from django.core.management.commands import makemessages
from django.template.base import BLOCK_TAG_START, BLOCK_TAG_END

try:
    from django.utils.translation import trans_real
except ImportError:
    from django.utils.translation import template as trans_real

strip_whitespace_right = re.compile(r"(%s-?\s*(trans|pluralize).*?-%s)\s+" % (BLOCK_TAG_START, BLOCK_TAG_END), re.U)
strip_whitespace_left = re.compile(r"\s+(%s-\s*(endtrans|pluralize).*?-?%s)" % (BLOCK_TAG_START, BLOCK_TAG_END), re.U)

def strip_whitespaces(src):
    src = strip_whitespace_left.sub(r'\1', src)
    src = strip_whitespace_right.sub(r'\1', src)
    return src

class Command(makemessages.Command):

    def handle(self, *args, **options):
        old_endblock_re = trans_real.endblock_re
        old_block_re = trans_real.block_re
        old_constant_re = trans_real.constant_re

        old_templatize = trans_real.templatize
        # Extend the regular expressions that are used to detect
        # translation blocks with an "OR jinja-syntax" clause.
        trans_real.endblock_re = re.compile(
            trans_real.endblock_re.pattern + '|' + r"""^-?\s*endtrans\s*-?$""")
        trans_real.block_re = re.compile(
            trans_real.block_re.pattern + '|' + r"""^-?\s*trans(?:\s+(?!'|")(?=.*?=.*?)|\s*-?$)""")
        trans_real.plural_re = re.compile(
            trans_real.plural_re.pattern + '|' + r"""^-?\s*pluralize(?:\s+.+|-?$)""")
        trans_real.constant_re = re.compile(r"""_\(((?:".*?")|(?:'.*?')).*\)""")

        def my_templatize(src, origin=None):
            new_src = strip_whitespaces(src)
            return old_templatize(new_src, origin)

        trans_real.templatize = my_templatize

        try:
            super(Command, self).handle(*args, **options)
        finally:
            trans_real.endblock_re = old_endblock_re
            trans_real.block_re = old_block_re
            trans_real.templatize = old_templatize
            trans_real.constant_re = old_constant_re
<|endoftext|>


<commit_before>from django.db import models
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.paginator import EmptyPage, PageNotAnInteger
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q

from paginator.paginator import CustomPaginator

from unidecode import unidecode

class Blob(models.Model):
    url = models.CharField(max_length=255)
    title = models.CharField(max_length=255)
    blob = models.TextField()
    is_public = models.BooleanField(default=False)

    record_type = models.ForeignKey(
        ContentType,
        related_name='search_blob',
        # on_delete=models.DELETE
    )
    record_id = models.PositiveIntegerField()
    record_object = GenericForeignKey('record_type', 'record_id')

    @classmethod
    def search(cls, query, is_public=False):
        if not query:
            return []

        return cls.objects.filter(is_public=is_public).filter(
            Q(blob__icontains=query) | Q(blob__icontains=unidecode(query))
        )

    @classmethod
    def search_paginator(cls, query, page, is_public=False):
        paginator = CustomPaginator(cls.search(query, is_public), 12)
        try:
            results = paginator.page(page)
        except PageNotAnInteger:
            results = paginator.page(1)
        except EmptyPage:
            results = paginator.page(1)
        return results

class SearchModel:
    def get_search_title(self):
        raise NotImplementedError

    def get_search_url(self):
        raise NotImplementedError

    def get_search_blob(self):
        raise NotImplementedError

    def get_public_search_blob(self):
        return None

    def get_search_public_url(self):
        return self.get_search_url()

    def delete_blob(self):
        Blob.objects.filter(
            record_type=ContentType.objects.get_for_model(self),
            record_id=self.id,
        ).delete()

    def update_search_blob(self):
        if hasattr(self, 'active') and not self.active:
            return

        # Add version without diacritics:
        search_blob = self.get_search_blob()
        blob_all = search_blob + unidecode(search_blob)

        Blob.objects.update_or_create(
            record_type=ContentType.objects.get_for_model(self),
            record_id=self.id,
            is_public=False,
            defaults={
                "title": self.get_search_title(),
                "url": self.get_search_url(),
                "blob": blob_all
            }
        )

        blob_public = self.get_public_search_blob()
        if blob_public:
            blob_all = blob_public + unidecode(blob_public)
            Blob.objects.update_or_create(
                record_type=ContentType.objects.get_for_model(self),
                record_id=self.id,
                is_public=True,
                defaults={
                    "title": self.get_search_title(),
                    "url": self.get_search_public_url(),
                    "blob": blob_all
                }
            )

def update_search(instance, **kwargs):
    instance.update_search_blob()
# post_save.connect(update_search, sender=<SearchModel instance>)
<commit_msg>delete search blobs without any url...<commit_after>from django.db import models
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.paginator import EmptyPage, PageNotAnInteger
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q

from paginator.paginator import CustomPaginator

from unidecode import unidecode

class Blob(models.Model):
    url = models.CharField(max_length=255)
    title = models.CharField(max_length=255)
    blob = models.TextField()
    is_public = models.BooleanField(default=False)

    record_type = models.ForeignKey(
        ContentType,
        related_name='search_blob',
        # on_delete=models.DELETE
    )
    record_id = models.PositiveIntegerField()
    record_object = GenericForeignKey('record_type', 'record_id')

    @classmethod
    def search(cls, query, is_public=False):
        if not query:
            return []

        return cls.objects.filter(is_public=is_public).filter(
            Q(blob__icontains=query) | Q(blob__icontains=unidecode(query))
        )

    @classmethod
    def search_paginator(cls, query, page, is_public=False):
        paginator = CustomPaginator(cls.search(query, is_public), 12)
        try:
            results = paginator.page(page)
        except PageNotAnInteger:
            results = paginator.page(1)
        except EmptyPage:
            results = paginator.page(1)
        return results

class SearchModel:
    def get_search_title(self):
        raise NotImplementedError

    def get_search_url(self):
        raise NotImplementedError

    def get_search_blob(self):
        raise NotImplementedError

    def get_public_search_blob(self):
        return None

    def get_search_public_url(self):
        return self.get_search_url()

    def delete_blob(self):
        Blob.objects.filter(
            record_type=ContentType.objects.get_for_model(self),
            record_id=self.id,
        ).delete()

    def update_search_blob(self):
        if hasattr(self, 'active') and not self.active:
            return

        # Add version without diacritics:
        search_blob = self.get_search_blob()
        blob_all = search_blob + unidecode(search_blob)

        # if url is empty then we will delete this blob.
        url = self.get_search_url()
        if not url:
            Blob.objects.filter(
                record_type=ContentType.objects.get_for_model(self),
                record_id=self.id,
            ).delete()
            return

        Blob.objects.update_or_create(
            record_type=ContentType.objects.get_for_model(self),
            record_id=self.id,
            is_public=False,
            defaults={
                "title": self.get_search_title(),
                "url": self.get_search_url(),
                "blob": blob_all
            }
        )

        blob_public = self.get_public_search_blob()
        if blob_public:
            blob_all = blob_public + unidecode(blob_public)
            Blob.objects.update_or_create(
                record_type=ContentType.objects.get_for_model(self),
                record_id=self.id,
                is_public=True,
                defaults={
                    "title": self.get_search_title(),
                    "url": self.get_search_public_url(),
                    "blob": blob_all
                }
            )

def update_search(instance, **kwargs):
    instance.update_search_blob()
# post_save.connect(update_search, sender=<SearchModel instance>)
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function

import numpy as np
import scipy.sparse as ss
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.preprocessing import normalize

from graphs import Graph
from ..mini_six import range
from .neighbors import neighbor_graph

__all__ = ['saffron']

def saffron(X, q=32, k=4, tangent_dim=1, curv_thresh=0.95, decay_rate=0.9,
            max_iter=15, verbose=False):
  '''
  SAFFRON graph construction method.

    X : (n,d)-array of coordinates
    q : int, median number of candidate friends per vertex
    k : int, number of friends to select per vertex, k < q
    tangent_dim : int, dimensionality of manifold tangent space
    curv_thresh : float, tolerance to curvature, lambda in the paper
    decay_rate : float, controls step size per iteration, between 0 and 1
    max_iter : int, cap on number of iterations
    verbose : bool, print goodness measure per iteration when True

  From "Tangent Space Guided Intelligent Neighbor Finding",
    by Gashler & Martinez, 2011.
  See http://axon.cs.byu.edu/papers/gashler2011ijcnn1.pdf
  '''
  n = len(X)
  dist = pairwise_distances(X)
  idx = np.argpartition(dist, q)[:, q]
  # radius for finding candidate friends: median distance to qth neighbor
  r = np.median(dist[np.arange(n), idx])

  # make candidate graph + weights
  W = neighbor_graph(dist, precomputed=True, epsilon=r).matrix('csr')
  # NOTE: this differs from the paper, where W.data[:] = 1 initially
  W.data[:] = 1 / W.data
  # row normalize
  normalize(W, norm='l1', axis=1, copy=False)
  # XXX: hacky densify
  W = W.toarray()

  # iterate to learn optimal weights
  prev_goodness = 1e-12
  for it in range(max_iter):
    goodness = 0
    S = _estimate_tangent_spaces(X, W, tangent_dim)
    # find aligned candidates
    for i, row in enumerate(W):
      nbrs = row.nonzero()[-1]

      # compute alignment scores
      edges = X[nbrs] - X[i]
      edge_norms = (edges**2).sum(axis=1)
      a1 = (edges.dot(S[i])**2).sum(axis=1) / edge_norms
      a2 = (np.einsum('ij,ijk->ik', edges, S[nbrs])**2).sum(axis=1) / edge_norms
      a3 = _principal_angle(S[i], S[nbrs]) ** 2
      x = (np.minimum(curv_thresh, a1) *
           np.minimum(curv_thresh, a2) *
           np.minimum(curv_thresh, a3))

      # decay weight of least-aligned candidates
      excess = x.shape[0] - k
      if excess > 0:
        bad_idx = np.argpartition(x, excess-1)[:excess]
        W[i, nbrs[bad_idx]] *= decay_rate
        W[i] /= W[i].sum()

      # update goodness measure (weighted alignment)
      goodness += x.dot(W[i,nbrs])

    if verbose:  # pragma: no cover
      goodness /= n
      print(it, goodness, goodness / prev_goodness)
    if goodness / prev_goodness <= 1.0001:
      break
    prev_goodness = goodness
  else:
    print('Failed to converge after %d iterations' % max_iter)

  # use the largest k weights for each row of W, weighted by original distance
  indptr, indices, data = [0], [], []
  for i, row in enumerate(W):
    nbrs = row.nonzero()[-1]
    if len(nbrs) > k:
      nbrs = nbrs[np.argpartition(row[nbrs], len(nbrs)-k)[-k:]]
    indices.extend(nbrs)
    indptr.append(len(nbrs))
    data.extend(dist[i, nbrs])
  indptr = np.cumsum(indptr)
  data = np.array(data)
  indices = np.array(indices)
  W = ss.csr_matrix((data, indices, indptr), shape=W.shape)
  return Graph.from_adj_matrix(W)

def _estimate_tangent_spaces(X, W, dim):
  # compute many PCAs in batch
  covs = np.empty(X.shape + (X.shape[1],))
  for i, row in enumerate(W):
    nbrs = row.nonzero()[-1]
    xx = X[nbrs] * row[nbrs,None]  # weight samples by W
    xx -= xx.mean(axis=0)
    covs[i] = xx.T.dot(xx)
  # compute all the PCs at once
  _, vecs = np.linalg.eigh(covs)
  return vecs[:,:,-dim:]

def _principal_angle(a, B):
  '''a is (d,t), B is (k,d,t)'''
  # TODO: check case for t = d-1
  if a.shape[1] == 1:
    return a.T.dot(B)[0,:,0]

  # find normals that maximize distance when projected
  x1 = np.einsum('abc,adc->abd', B, B).dot(a) - a   # b.dot(b.T).dot(a) - a
  x2 = np.einsum('ab,cad->cbd', a.dot(a.T), B) - B  # a.dot(a.T).dot(b) - b
  xx = np.vstack((x1, x2))

  # batch PCA (1st comp. only)
  xx -= xx.mean(axis=1)[:,None]
  c = np.einsum('abc,abd->acd', xx, xx)
  _, vecs = np.linalg.eigh(c)
  fpc = vecs[:,:,-1]
  fpc1 = fpc[:len(x1)]
  fpc2 = fpc[len(x1):]

  # a.dot(fpc1).dot(b.dot(fpc2))
  lhs = a.dot(fpc1.T).T
  rhs = np.einsum('abc,ac->ab', B, fpc2)
  return np.einsum('ij,ij->i', lhs, rhs)
<commit_msg>Use a real warning<commit_after>from __future__ import absolute_import, print_function

import numpy as np
import scipy.sparse as ss
import warnings
from sklearn.metrics.pairwise import pairwise_distances
from sklearn.preprocessing import normalize

from graphs import Graph
from ..mini_six import range
from .neighbors import neighbor_graph

__all__ = ['saffron']

def saffron(X, q=32, k=4, tangent_dim=1, curv_thresh=0.95, decay_rate=0.9,
            max_iter=15, verbose=False):
  '''
  SAFFRON graph construction method.

    X : (n,d)-array of coordinates
    q : int, median number of candidate friends per vertex
    k : int, number of friends to select per vertex, k < q
    tangent_dim : int, dimensionality of manifold tangent space
    curv_thresh : float, tolerance to curvature, lambda in the paper
    decay_rate : float, controls step size per iteration, between 0 and 1
    max_iter : int, cap on number of iterations
    verbose : bool, print goodness measure per iteration when True

  From "Tangent Space Guided Intelligent Neighbor Finding",
    by Gashler & Martinez, 2011.
  See http://axon.cs.byu.edu/papers/gashler2011ijcnn1.pdf
  '''
  n = len(X)
  dist = pairwise_distances(X)
  idx = np.argpartition(dist, q)[:, q]
  # radius for finding candidate friends: median distance to qth neighbor
  r = np.median(dist[np.arange(n), idx])

  # make candidate graph + weights
  W = neighbor_graph(dist, precomputed=True, epsilon=r).matrix('csr')
  # NOTE: this differs from the paper, where W.data[:] = 1 initially
  W.data[:] = 1 / W.data
  # row normalize
  normalize(W, norm='l1', axis=1, copy=False)
  # XXX: hacky densify
  W = W.toarray()

  # iterate to learn optimal weights
  prev_goodness = 1e-12
  for it in range(max_iter):
    goodness = 0
    S = _estimate_tangent_spaces(X, W, tangent_dim)
    # find aligned candidates
    for i, row in enumerate(W):
      nbrs = row.nonzero()[-1]

      # compute alignment scores
      edges = X[nbrs] - X[i]
      edge_norms = (edges**2).sum(axis=1)
      a1 = (edges.dot(S[i])**2).sum(axis=1) / edge_norms
      a2 = (np.einsum('ij,ijk->ik', edges, S[nbrs])**2).sum(axis=1) / edge_norms
      a3 = _principal_angle(S[i], S[nbrs]) ** 2
      x = (np.minimum(curv_thresh, a1) *
           np.minimum(curv_thresh, a2) *
           np.minimum(curv_thresh, a3))

      # decay weight of least-aligned candidates
      excess = x.shape[0] - k
      if excess > 0:
        bad_idx = np.argpartition(x, excess-1)[:excess]
        W[i, nbrs[bad_idx]] *= decay_rate
        W[i] /= W[i].sum()

      # update goodness measure (weighted alignment)
      goodness += x.dot(W[i,nbrs])

    if verbose:  # pragma: no cover
      goodness /= n
      print(it, goodness, goodness / prev_goodness)
    if goodness / prev_goodness <= 1.0001:
      break
    prev_goodness = goodness
  else:
    warnings.warn('Failed to converge after %d iterations.' % max_iter)

  # use the largest k weights for each row of W, weighted by original distance
  indptr, indices, data = [0], [], []
  for i, row in enumerate(W):
    nbrs = row.nonzero()[-1]
    if len(nbrs) > k:
      nbrs = nbrs[np.argpartition(row[nbrs], len(nbrs)-k)[-k:]]
    indices.extend(nbrs)
    indptr.append(len(nbrs))
    data.extend(dist[i, nbrs])
  indptr = np.cumsum(indptr)
  data = np.array(data)
  indices = np.array(indices)
  W = ss.csr_matrix((data, indices, indptr), shape=W.shape)
  return Graph.from_adj_matrix(W)

def _estimate_tangent_spaces(X, W, dim):
  # compute many PCAs in batch
  covs = np.empty(X.shape + (X.shape[1],))
  for i, row in enumerate(W):
    nbrs = row.nonzero()[-1]
    xx = X[nbrs] * row[nbrs,None]  # weight samples by W
    xx -= xx.mean(axis=0)
    covs[i] = xx.T.dot(xx)
  # compute all the PCs at once
  _, vecs = np.linalg.eigh(covs)
  return vecs[:,:,-dim:]

def _principal_angle(a, B):
  '''a is (d,t), B is (k,d,t)'''
  # TODO: check case for t = d-1
  if a.shape[1] == 1:
    return a.T.dot(B)[0,:,0]

  # find normals that maximize distance when projected
  x1 = np.einsum('abc,adc->abd', B, B).dot(a) - a   # b.dot(b.T).dot(a) - a
  x2 = np.einsum('ab,cad->cbd', a.dot(a.T), B) - B  # a.dot(a.T).dot(b) - b
  xx = np.vstack((x1, x2))

  # batch PCA (1st comp. only)
  xx -= xx.mean(axis=1)[:,None]
  c = np.einsum('abc,abd->acd', xx, xx)
  _, vecs = np.linalg.eigh(c)
  fpc = vecs[:,:,-1]
  fpc1 = fpc[:len(x1)]
  fpc2 = fpc[len(x1):]

  # a.dot(fpc1).dot(b.dot(fpc2))
  lhs = a.dot(fpc1.T).T
  rhs = np.einsum('abc,ac->ab', B, fpc2)
  return np.einsum('ij,ij->i', lhs, rhs)
<|endoftext|>


<commit_before>"""
Test end to end django views.
"""
from django.test import TestCase
from django.test.client import Client
from django.core.urlresolvers import reverse
from django.contrib.auth import get_user_model

from sga.backend.constants import Roles
from sga.models import Assignment, Course, Submission, Student, Grader

class TestViews(TestCase):  # pylint: disable=too-many-public-methods
    """
    Test that the views work as expected.
    """

    ###
    # Setup
    ###
    def setUp(self):
        """
        Common test setup
        """
        super(TestViews, self).setUp()
        self.client = Client()
        self.user_model = get_user_model()

    ###
    # Helper functions
    ###
    def setup_lti_params(self):
        """
        Initializes LTI parameters in session
        """
        lti_params = {
            "context_id": "test_course",
            "resource_link_id": "test_assignment",
            "user_id": "test_user_id",
        }
        session = self.client.session
        session["LTI_LAUNCH"] = lti_params
        session.save()

    def log_in_as(self, role):
        """
        Logs in as the role provided
        """
        if role == Roles.student:
            self.log_in_as_student()
        elif role == Roles.grader:
            self.log_in_as_grader()
        elif role == Roles.admin:
            self.log_in_as_admin()
        else:
            # Assume anonymous, but logged in, user
            self.log_in_as_non_role_user()

    def log_in_as_admin(self):
        """
        Logs in as an admin in the test course
        """
        user, _ = self.user_model.objects.get_or_create(username="test_admin_id")
        course = self.get_test_course()
        course.administrators.add(user)
        self.client.force_login(user)
        self.setup_lti_params()

    def log_in_as_grader(self):
        """
        Logs in as a grader in the test course
        """
        grader = self.get_test_grader()
        self.client.force_login(grader.user)
        self.setup_lti_params()

    def log_in_as_student(self):
        """
        Logs in as a grader in the test course
        """
        student_user = self.get_test_student_user()
        self.client.force_login(student_user)
        self.setup_lti_params()

    def log_in_as_non_role_user(self):
        """
        Logs in as user with no role
        """
        user = self.get_test_user()
        self.client.force_login(user)
        self.setup_lti_params()

    def get_test_course(self):  # pylint: disable=no-self-use
        """
        Creates or retrieves a course object for testing. Returns the Course object.
        """
        return Course.objects.get_or_create(edx_id="test_course")[0]

    def get_test_assignment(self):
        """
        Creates or retrieves an assignment object for testing. Returns the Assignment object.
        Returns the Assignment object.
        """
        return Assignment.objects.get_or_create(
            edx_id="test_assignment",
            course=self.get_test_course()
        )[0]

    def get_test_user(self):  # pylint: disable=no-self-use
        """
        Creates or retrieves test user (with no role). Returns the User object.
        """
        return self.user_model.objects.get_or_create(username="test_user_id")[0]

    def get_test_student(self):
        """
        Creates or retrieves a user object for testing and attaches it to a course as a student.
        Returns the Student object.
        """
        student_user, _ = self.user_model.objects.get_or_create(username="test_student_id")
        course = self.get_test_course()
        student, _ = Student.objects.get_or_create(course=course, user=student_user)
        return student

    def get_test_student_user(self):
        """
        Returns the User object attached to the Student from get_test_student()
        """
        student = self.get_test_student()
        return student.user

    def get_test_grader(self):
        """
        Creates or retrieves a user object for testing and attaches it to a course as a grader.
        Returns the Grader object
        """
        grader_user, _ = self.user_model.objects.get_or_create(username="test_grader_id")
        course = self.get_test_course()
        grader, _ = Grader.objects.get_or_create(course=course, user=grader_user)
        return grader

    def get_test_grader_user(self):
        """
        Returns the User object attached to the Grader from get_test_grader()
        """
        grader = self.get_test_grader()
        return grader.user

    def get_test_submission(self):
        """
        Creates or retrieves a submission object for testing. Returns the Submission object.
        """
        return Submission.objects.get_or_create(
            student=self.get_test_student_user(),
            assignment=self.get_test_assignment()
        )[0]

    def do_test_forbidden_view(self, url_path, role):
        """
        Runs general tests for view functions to ensure the view is forbidden for the role provided

        @param url_path: (str) url path for self.client.get()
        @param role: (st) role to log in as; must be in of [Roles.student, Roles.grader, Roles.admin]
        """
        self.log_in_as(role)
        response = self.client.get(url_path, follow=True)
        self.assertEqual(response.status_code, 403)
        return response

    def do_test_successful_view(self, url_path, role, template=None, contains=None, context_keys=None):
        # pylint: disable-msg=too-many-arguments
        """
        Runs general tests for view functions to ensure 200 status code, template used, context variables

        @param url_path: (str) url path for self.client.get()
        @param role: (str) role to log in as; must be in of [Roles.student, Roles.grader, Roles.admin]
        @param template: (str) template path
        @param context_keys: (list) keys expected to be in context
        @param contains: (str) str expected to occur in html of view
        """
        self.log_in_as(role)
        response = self.client.get(url_path, follow=True)
        self.assertEqual(response.status_code, 200)
        if template:
            self.assertTemplateUsed(response, template)
        if context_keys:
            for key in context_keys:
                self.assertTrue(key in response.context, msg="{key} not in context".format(key=key))
        if contains:
            self.assertContains(response, contains)
        return response

    ###
    # Tests
    ###
    def test_index_view(self):
        """ Verify the index view is as expected """
        self.do_test_successful_view(
            reverse('sga-index'),
            None,
            template="sga/index.html",
            contains="Logged In As",
            context_keys=[
                "course",
                "assignments",
                "users",
                "students",
                "graders",
                "admins"
            ]
        )

    def test_unsubmit_submission(self):
        """
        Verify that unset_submission returns 200 and updates the submission object
        """
        self.log_in_as_admin()
        assignment = self.get_test_assignment()
        student_user = self.get_test_student_user()
        submission = self.get_test_submission()
        submission.update(submitted=True, graded=True)
        kwargs = {"student_user_id": student_user.id, "assignment_id": assignment.id}
        response = self.client.post(reverse("unsubmit_submission", kwargs=kwargs), follow=True)
        self.assertEqual(response.status_code, 200)
        submission = self.get_test_submission()
        self.assertFalse(submission.graded)
        self.assertFalse(submission.submitted)

    def test_unsubmit_submission_admin_only(self):
        """
        Verify that unset_submission is not allowed for graders or students
        """
        assignment = self.get_test_assignment()
        student_user = self.get_test_student_user()
        submission = self.get_test_submission()
        submission.update(submitted=True, graded=True)
        kwargs = {"student_user_id": student_user.id, "assignment_id": assignment.id}
        url = reverse("unsubmit_submission", kwargs=kwargs)
        for role in [Roles.grader, Roles.student]:
            self.do_test_forbidden_view(url, role)
            submission = self.get_test_submission()
            self.assertTrue(submission.graded)
            self.assertTrue(submission.submitted)

    def test_view_submission_as_student(self):
        """
        Verify view submission page is as expected
        """
        assignment = self.get_test_assignment()
        url = reverse("view_submission_as_student", kwargs={"assignment_id": assignment.id})
        self.do_test_successful_view(
            url,
            Roles.student,
            template="sga/view_submission_as_student.html",
            context_keys=["submission_form", "submission", "assignment"]
        )

    def test_view_submission_as_staff(self):
        """
        Verify view submission page is as expected
        """
        student_user = self.get_test_student_user()
        assignment = self.get_test_assignment()
        kwargs = {"assignment_id": assignment.id, "student_user_id": student_user.id}
        url = reverse("view_submission_as_staff", kwargs=kwargs)
        for role in [Roles.grader, Roles.admin]:
            self.do_test_successful_view(
                url,
                role,
                template="sga/view_submission_as_staff.html",
                context_keys=[
                    "submission_form",
                    "submission",
                    "assignment",
                    "student_user",
                    "next_not_graded_submission_url"
                ]
            )

    def test_view_student_list(self):
        """
        Verify view student list page is as expected
        """
        course = self.get_test_course()
        url = reverse("view_student_list", kwargs={"course_id": course.id})
        for role in [Roles.grader, Roles.admin]:
            self.do_test_successful_view(
                url,
                role,
                template="sga/view_student_list.html",
                context_keys=["course", "students", "grader_user"]
            )

    def test_view_assignment_list(self):
        """
        Verify view assignment list page is as expected
        """
        course = self.get_test_course()
        url = reverse("view_assignment_list", kwargs={"course_id": course.id})
        for role in [Roles.grader, Roles.admin]:
            self.do_test_successful_view(
                url,
                role,
                template="sga/view_assignment_list.html",
                context_keys=["course", "assignments", "grader_user"]
            )

    def test_view_student(self):
        """
        Verify view student page is as expected
        """
        student_user = self.get_test_student_user()
        course = self.get_test_course()
        url = reverse("view_student", kwargs={"course_id": course.id, "student_user_id": student_user.id})
        self.do_test_successful_view(
            url,
            Roles.student,
            template="sga/view_student.html",
            context_keys=[
                "course",
                "student",
                "assignments",
                "STUDENT_TO_GRADER_CONFIRM",
                "UNASSIGN_GRADER_CONFIRM",
                "assign_grader_form"
            ]
        )

    def test_view_grader(self):
        """
        Verify view grader page is as expected
        """
        grader_user = self.get_test_grader_user()
        course = self.get_test_course()
        url = reverse("view_grader", kwargs={"course_id": course.id, "grader_user_id": grader_user.id})
        for role in [Roles.grader, Roles.admin]:
            self.do_test_successful_view(
                url,
                role,
                template="sga/view_grader.html",
                context_keys=[
                    "course",
                    "grader",
                    "graded_submissions",
                    "max_students_form",
                    "assign_student_form",
                    "students",
                    "GRADER_TO_STUDENT_CONFIRM",
                    "UNASSIGN_STUDENT_CONFIRM"
                ]
            )
<commit_msg>More tests<commit_after>"""
Test end to end django views.
"""
from django.test import TestCase
from django.test.client import Client
from django.core.urlresolvers import reverse
from django.contrib.auth import get_user_model

from sga.backend.constants import Roles
from sga.models import Assignment, Course, Submission, Student, Grader

class TestViews(TestCase):  # pylint: disable=too-many-public-methods
    """
    Test that the views work as expected.
    """

    ###
    # Setup
    ###
    def setUp(self):
        """
        Common test setup
        """
        super(TestViews, self).setUp()
        self.client = Client()
        self.user_model = get_user_model()

    ###
    # Helper functions
    ###
    def setup_lti_params(self):
        """
        Initializes LTI parameters in session
        """
        lti_params = {
            "context_id": "test_course",
            "resource_link_id": "test_assignment",
            "user_id": "test_user_id",
        }
        session = self.client.session
        session["LTI_LAUNCH"] = lti_params
        session.save()

    def log_in_as(self, role):
        """
        Logs in as the role provided
        """
        if role == Roles.student:
            self.log_in_as_student()
        elif role == Roles.grader:
            self.log_in_as_grader()
        elif role == Roles.admin:
            self.log_in_as_admin()
        else:
            # Assume anonymous, but logged in, user
            self.log_in_as_non_role_user()

    def log_in_as_admin(self):
        """
        Logs in as an admin in the test course
        """
        user, _ = self.user_model.objects.get_or_create(username="test_admin_id")
        course = self.get_test_course()
        course.administrators.add(user)
        self.client.force_login(user)
        self.setup_lti_params()

    def log_in_as_grader(self):
        """
        Logs in as a grader in the test course
        """
        grader = self.get_test_grader()
        self.client.force_login(grader.user)
        self.setup_lti_params()

    def log_in_as_student(self):
        """
        Logs in as a grader in the test course
        """
        student_user = self.get_test_student_user()
        self.client.force_login(student_user)
        self.setup_lti_params()

    def log_in_as_non_role_user(self):
        """
        Logs in as user with no role
        """
        user = self.get_test_user()
        self.client.force_login(user)
        self.setup_lti_params()

    def get_test_course(self):  # pylint: disable=no-self-use
        """
        Creates or retrieves a course object for testing. Returns the Course object.
        """
        return Course.objects.get_or_create(edx_id="test_course")[0]

    def get_test_assignment(self):
        """
        Creates or retrieves an assignment object for testing. Returns the Assignment object.
        Returns the Assignment object.
        """
        return Assignment.objects.get_or_create(
            edx_id="test_assignment",
            course=self.get_test_course()
        )[0]

    def get_test_user(self):  # pylint: disable=no-self-use
        """
        Creates or retrieves test user (with no role). Returns the User object.
        """
        return self.user_model.objects.get_or_create(username="test_user_id")[0]

    def get_test_student(self):
        """
        Creates or retrieves a user object for testing and attaches it to a course as a student.
        Returns the Student object.
        """
        student_user, _ = self.user_model.objects.get_or_create(username="test_student_id")
        course = self.get_test_course()
        student, _ = Student.objects.get_or_create(course=course, user=student_user)
        return student

    def get_test_student_user(self):
        """
        Returns the User object attached to the Student from get_test_student()
        """
        student = self.get_test_student()
        return student.user

    def get_test_grader(self, username="test_grader_id"):
        """
        Creates or retrieves a user object for testing and attaches it to a course as a grader.
        Returns the Grader object
        """
        grader_user, _ = self.user_model.objects.get_or_create(username=username)
        course = self.get_test_course()
        grader, _ = Grader.objects.get_or_create(course=course, user=grader_user)
        return grader

    def get_test_grader_user(self):
        """
        Returns the User object attached to the Grader from get_test_grader()
        """
        grader = self.get_test_grader()
        return grader.user

    def get_test_submission(self):
        """
        Creates or retrieves a submission object for testing. Returns the Submission object.
        """
        return Submission.objects.get_or_create(
            student=self.get_test_student_user(),
            assignment=self.get_test_assignment()
        )[0]

    def do_test_forbidden_view(self, url_path, role):
        """
        Runs general tests for view functions to ensure the view is forbidden for the role provided

        @param url_path: (str) url path for self.client.get()
        @param role: (st) role to log in as; must be in of [Roles.student, Roles.grader, Roles.admin]
        """
        self.log_in_as(role)
        response = self.client.get(url_path, follow=True)
        self.assertEqual(response.status_code, 403)
        return response

    def do_test_successful_view(self, url_path, role, template=None, contains=None, context_keys=None):
        # pylint: disable-msg=too-many-arguments
        """
        Runs general tests for view functions to ensure 200 status code, template used, context variables

        @param url_path: (str) url path for self.client.get()
        @param role: (str) role to log in as; must be in of [Roles.student, Roles.grader, Roles.admin]
        @param template: (str) template path
        @param context_keys: (list) keys expected to be in context
        @param contains: (str) str expected to occur in html of view
        """
        self.log_in_as(role)
        response = self.client.get(url_path, follow=True)
        self.assertEqual(response.status_code, 200)
        if template:
            self.assertTemplateUsed(response, template)
        if context_keys:
            for key in context_keys:
                self.assertTrue(key in response.context, msg="{key} not in context".format(key=key))
        if contains:
            self.assertContains(response, contains)
        return response

    ###
    # Tests
    ###
    def test_index_view(self):
        """ Verify the index view is as expected """
        self.do_test_successful_view(
            reverse('sga-index'),
            None,
            template="sga/index.html",
            contains="Logged In As",
            context_keys=[
                "course",
                "assignments",
                "users",
                "students",
                "graders",
                "admins"
            ]
        )

    def test_unsubmit_submission(self):
        """
        Verify that unset_submission returns 200 and updates the submission object
        """
        self.log_in_as_admin()
        assignment = self.get_test_assignment()
        student_user = self.get_test_student_user()
        submission = self.get_test_submission()
        submission.update(submitted=True, graded=True)
        kwargs = {"student_user_id": student_user.id, "assignment_id": assignment.id}
        response = self.client.post(reverse("unsubmit_submission", kwargs=kwargs), follow=True)
        self.assertEqual(response.status_code, 200)
        submission = self.get_test_submission()
        self.assertFalse(submission.graded)
        self.assertFalse(submission.submitted)

    def test_unsubmit_submission_admin_only(self):
        """
        Verify that unset_submission is not allowed for graders or students
        """
        assignment = self.get_test_assignment()
        student_user = self.get_test_student_user()
        submission = self.get_test_submission()
        submission.update(submitted=True, graded=True)
        kwargs = {"student_user_id": student_user.id, "assignment_id": assignment.id}
        url = reverse("unsubmit_submission", kwargs=kwargs)
        for role in [Roles.grader, Roles.student]:
            self.do_test_forbidden_view(url, role)
            submission = self.get_test_submission()
            self.assertTrue(submission.graded)
            self.assertTrue(submission.submitted)

    def test_view_submission_as_student(self):
        """
        Verify view submission page is as expected
        """
        assignment = self.get_test_assignment()
        url = reverse("view_submission_as_student", kwargs={"assignment_id": assignment.id})
        self.do_test_successful_view(
            url,
            Roles.student,
            template="sga/view_submission_as_student.html",
            context_keys=["submission_form", "submission", "assignment"]
        )

    def test_view_submission_as_staff(self):
        """
        Verify view submission page is as expected
        """
        student_user = self.get_test_student_user()
        assignment = self.get_test_assignment()
        kwargs = {"assignment_id": assignment.id, "student_user_id": student_user.id}
        url = reverse("view_submission_as_staff", kwargs=kwargs)
        for role in [Roles.grader, Roles.admin]:
            self.do_test_successful_view(
                url,
                role,
                template="sga/view_submission_as_staff.html",
                context_keys=[
                    "submission_form",
                    "submission",
                    "assignment",
                    "student_user",
                    "next_not_graded_submission_url"
                ]
            )

    def test_view_assignment(self):
        """
        Verify view assignment is as expected
        """
        assignment = self.get_test_assignment()
        url = reverse("view_assignment", kwargs={"assignment_id": assignment.id})
        for role in [Roles.grader, Roles.admin]:
            self.do_test_successful_view(
                url,
                role,
                template="sga/view_assignment.html",
                context_keys=["student_users", "course", "assignment"]
            )

    def test_view_assignment_staff_only(self):
        """
        Verify view assignment page is only accessible for staff
        """
        assignment = self.get_test_assignment()
        url = reverse("view_assignment", kwargs={"assignment_id": assignment.id})
        self.do_test_forbidden_view(url, Roles.student)

    def test_view_student_list(self):
        """
        Verify view student list page is as expected
        """
        course = self.get_test_course()
        url = reverse("view_student_list", kwargs={"course_id": course.id})
        for role in [Roles.grader, Roles.admin]:
            self.do_test_successful_view(
                url,
                role,
                template="sga/view_student_list.html",
                context_keys=["course", "students", "grader_user"]
            )

    def test_view_student_list_staff_only(self):
        """
        Verify view student list page is only accessible for staff
        """
        course = self.get_test_course()
        url = reverse("view_student_list", kwargs={"course_id": course.id})
        self.do_test_forbidden_view(url, Roles.student)

    def test_view_assignment_list(self):
        """
        Verify view assignment list page is as expected
        """
        course = self.get_test_course()
        url = reverse("view_assignment_list", kwargs={"course_id": course.id})
        for role in [Roles.grader, Roles.admin]:
            self.do_test_successful_view(
                url,
                role,
                template="sga/view_assignment_list.html",
                context_keys=["course", "assignments", "grader_user"]
            )

    def test_view_assignment_list_staff_only(self):
        """
        Verify view assignment list page is only accessible for staff
        """
        course = self.get_test_course()
        url = reverse("view_assignment_list", kwargs={"course_id": course.id})
        self.do_test_forbidden_view(url, Roles.student)

    def test_view_grader_list(self):
        """
        Verify view grader list page is as expected
        """
        course = self.get_test_course()
        url = reverse("view_grader_list", kwargs={"course_id": course.id})
        self.do_test_successful_view(
            url,
            Roles.admin,
            template="sga/view_grader_list.html",
            context_keys=["course", "graders"]
        )

    def test_view_grader_list_admin_only(self):
        """
        Verify view grader list page is only accessible for admins
        """
        course = self.get_test_course()
        url = reverse("view_grader_list", kwargs={"course_id": course.id})
        for role in [Roles.grader, Roles.student]:
            self.do_test_forbidden_view(url, role)

    def test_view_student(self):
        """
        Verify view student page is as expected
        """
        student_user = self.get_test_student_user()
        course = self.get_test_course()
        url = reverse("view_student", kwargs={"course_id": course.id, "student_user_id": student_user.id})
        self.do_test_successful_view(
            url,
            Roles.student,
            template="sga/view_student.html",
            context_keys=[
                "course",
                "student",
                "assignments",
                "STUDENT_TO_GRADER_CONFIRM",
                "UNASSIGN_GRADER_CONFIRM",
                "assign_grader_form"
            ]
        )

    def test_view_grader(self):
        """
        Verify view grader page is as expected
        """
        grader_user = self.get_test_grader_user()
        course = self.get_test_course()
        url = reverse("view_grader", kwargs={"course_id": course.id, "grader_user_id": grader_user.id})
        for role in [Roles.grader, Roles.admin]:
            self.do_test_successful_view(
                url,
                role,
                template="sga/view_grader.html",
                context_keys=[
                    "course",
                    "grader",
                    "graded_submissions",
                    "max_students_form",
                    "assign_student_form",
                    "students",
                    "GRADER_TO_STUDENT_CONFIRM",
                    "UNASSIGN_STUDENT_CONFIRM"
                ]
            )

    def test_view_grader_admin_or_self_grader_only(self):
        """
        Verify view grader list page is only accessible for admins and the logged in grader if it's the
        logged in grader's own page.
        """
        # We want to try to view the grader that isn't the default grader that's logged in
        grader_2 = self.get_test_grader(username="test_grader_2_id")
        course = self.get_test_course()
        url = reverse("view_grader", kwargs={"course_id": course.id, "grader_user_id": grader_2.user.id})
        for role in [Roles.grader, Roles.student]:
            self.do_test_forbidden_view(url, role)
<|endoftext|>