<commit_before>#!/usr/bin/env python

#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# INFO:
#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

"""
Author: Evan Hubinger
Date Created: 2014
Description: The Coconut Compiler.
"""

#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# IMPORTS:
#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

from __future__ import with_statement, print_function, absolute_import, unicode_literals, division

from .util import *
from . import parser
import argparse
import os
import os.path

#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# UTILITIES:
#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

def fixpath(path):
    """Properly Formats A Weapon."""
    return os.path.normpath(os.path.realpath(path))

class executor(object):
    """Compiled Python Executor."""
    def __init__(self, extras=None):
        """Creates The Executor."""
        self.variables = {}
        if extras is not None:
            for k,v in extras.items():
                self.variables[k] = v
    def run(__self, __code, __error=False):
        """Executes Python Code."""
        __globals = globals().copy()
        __locals = locals().copy()
        for __k, __v in __self.variables.items():
            globals()[__k] = __v
        try:
            exec(__code)
        except Exception:
            if __error:
                raise
            else:
                print_error()
        __overrides = {}
        for __k, __v in list(globals().items()):
            if __k in __self.variables:
                __self.variables[__k] = __v
                del globals()[__k]
            elif __k not in __globals:
                __overrides[__k] = __v
                del globals()[__k]
        for __k, __v in locals().items():
            if __k in __self.variables:
                __self.variables[__k] = __v
            elif __k not in __locals:
                __self.variables[__k] = __v
        for __k, __v in __overrides.items():
            __self.variables[__k] = __v
        for __k, __v in __globals.items():
            globals()[__k] = __v

#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# MAIN:
#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

class cli(object):
    """The Coconut Command-Line Interface."""
    code_ext = ".coc"
    comp_ext = ".py"
    max_debug = 2
    commandline = argparse.ArgumentParser(description="The Coconut Programming Language.")
    commandline.add_argument("source", metavar="source", type=str, nargs="?", default=None, help="path to the coconut file/folder to compile")
    commandline.add_argument("dest", metavar="dest", type=str, nargs="?", default=None, help="destination directory for compiled files (defaults to the source directory)")
    commandline.add_argument("-v", "--version", action="store_const", const=True, default=False, help="print coconut and python version information")
    commandline.add_argument("-s", "--strict", action="store_const", const=True, default=False, help="enforce code cleanliness standards")
    commandline.add_argument("-p", "--print", action="store_const", const=True, default=False, help="print the compiled source")
    commandline.add_argument("-r", "--run", action="store_const", const=True, default=False, help="run the compiled source")
    commandline.add_argument("-n", "--nowrite", action="store_const", const=True, default=False, help="disable writing the compiled source")
    commandline.add_argument("-i", "--interact", action="store_const", const=True, default=False, help="force the interpreter to start (otherwise starts if no other command is given)")
    commandline.add_argument("-q", "--quiet", action="store_const", const=True, default=False, help="suppress all info and debug output")
    commandline.add_argument("-d", "--debug", metavar="level", type=int, nargs="?", default=0, const=1, help="enable debug output (0 is off, no arg defaults to 1, max is "+str(max_debug)+")")
    commandline.add_argument("-c", "--code", metavar="code", type=str, nargs=1, default=None, help="run a line of coconut passed in as a string")
    commandline.add_argument("--autopep8", type=str, nargs=argparse.REMAINDER, default=None, help="use autopep8 to format compiled code (remaining args passed to autopep8)")
    processor = None
    debug = False
    running = False
    runner = None

    def __init__(self, main_color=None, debug_color=None, prompt=">>> ", moreprompt="    ", main_sig="Coconut: ", debug_sig=None):
        """Creates The CLI."""
        self.console = terminal(main_color, debug_color, main_sig, debug_sig)
        self.prompt = self.console.addcolor(prompt, main_color)
        self.moreprompt = self.console.addcolor(moreprompt, main_color)

    def start(self):
        """Gets Command-Line Arguments."""
        args = self.commandline.parse_args()
        self.cmd(args)

    def setup(self, strict=False):
        """Creates The Processor."""
        self.processor = parser.processor(strict)
        self.processor.TRACER.show = self.console.debug

    def debug_set(self, level=0):
        """Sets The Debug Level."""
        self.debug = level >= 1
        self.processor.debug(level >= 2)

    def debug_level(self, level=0):
        """Processes A Debug Level."""
        if level < 0 or self.max_debug < level:
            raise parser.CoconutException("debug level must be in range(0, "+str(self.max_debug+1)+")")
        else:
            self.debug_set(level)

    def quiet(self, state=None):
        """Quiets Output."""
        if state is None:
            state = self.console.on
        self.console.on = not state

    def version(self):
        """Gets Version Info."""
        return "Version "+repr(VERSION)+" running on Python "+sys.version

    def cmd(self, args):
        """Parses Command-Line Arguments."""
        self.setup(args.strict)
        self.debug_level(args.debug)
        if args.quiet:
            self.quiet(True)
        if args.version:
            self.console.print(self.version())
        if args.autopep8 is not None:
            self.processor.autopep8(args.autopep8)
        if args.code is not None:
            self.execute(self.processor.parse_single(args.code[0]), True)
        if args.source is not None:
            if args.run and os.path.isdir(args.source):
                raise parser.CoconutException("source path can't point to file when --run is enabled")
            if args.dest is None:
                if args.nowrite:
                    self.compile_path(args.source, None, show=args.print, run=args.run)
                else:
                    self.compile_path(args.source, show=args.print, run=args.run)
            elif args.nowrite:
                raise parser.CoconutException("destination path can't be given when --nowrite is enabled")
            elif os.path.isfile(args.dest):
                raise parser.CoconutException("destination path can't point to file")
            else:
                self.compile_path(args.source, args.dest, show=args.print, run=args.run)
        elif args.run or args.print:
            raise parser.CoconutException("a source file must be specified when --run or --print is enabled")
        if args.interact or not (args.source or args.code or args.version):
            self.start_prompt()

    def compile_path(self, path, write=True, show=False, run=False):
        """Compiles A Path."""
        if os.path.isfile(path):
            if write is None:
                module = None
            else:
                module = False
            self.compile_file(path, write, module, show, run)
        elif os.path.isdir(path):
            self.compile_module(path, write, show, run)
        else:
            raise parser.CoconutException("could not find source path "+repr(path))

    def compile_module(self, directory, write=True, show=False, run=False):
        """Compiles A Module."""
        for dirpath, dirnames, filenames in os.walk(directory):
            writedir = write
            module = True
            if writedir is None:
                tocreate = None
                module = None
            elif writedir is True:
                tocreate = dirpath
            else:
                writedir = os.path.join(writedir, os.path.relpath(dirpath, directory))
                tocreate = writedir
            wrote = False
            for filename in filenames:
                if os.path.splitext(filename)[1] == self.code_ext:
                    self.compile_file(os.path.join(dirpath, filename), writedir, module, show, run)
                    wrote = True
            if wrote and tocreate is not None:
                self.create_module(tocreate)

    def compile_file(self, filepath, write=True, module=False, show=False, run=False):
        """Compiles A File."""
        if write is None:
            destpath = None
        elif write is True:
            destpath = os.path.splitext(filepath)[0]+self.comp_ext
        else:
            destpath = os.path.join(write, os.path.splitext(os.path.basename(filepath))[0]+self.comp_ext)
        self.compile(filepath, destpath, module, show, run)

    def compile(self, codepath, destpath=None, module=False, show=False, run=False):
        """Compiles A Source Coconut File To A Destination Python File."""
        codepath = fixpath(codepath)
        self.console.print("Compiling "+repr(codepath)+"...")
        with openfile(codepath, "r") as opened:
            code = readfile(opened)
        if module is True:
            compiled = self.processor.parse_module(code)
        elif module is None:
            compiled = self.processor.parse_block(code)
        elif module is False:
            compiled = self.processor.parse_file(code)
        else:
            raise parser.CoconutException("invalid value for module boolean of "+repr(module))
        if show:
            print(compiled)
        if run:
            self.execute(compiled, True)
        if destpath is not None:
            destpath = fixpath(destpath)
            destdir = os.path.dirname(destpath)
            if not os.path.exists(destdir):
                os.makedirs(destdir)
            with openfile(destpath, "w") as opened:
                writefile(opened, compiled)
            self.console.print("Compiled "+repr(destpath)+".")

    def create_module(self, dirpath):
        """Sets Up A Module Directory."""
        with openfile(os.path.join(dirpath, "__coconut__.py"), "w") as opened:
            writefile(opened, parser.headers["package"])

    def start_prompt(self):
        """Starts The Interpreter."""
        self.console.print("[Interpreter:]")
        self.running = True
        while self.running:
            self.execute(self.handle(input(self.prompt)))

    def exit(self):
        """Exits The Interpreter."""
        self.running = False

    def handle(self, code):
        """Compiles Coconut Interpreter Input."""
        try:
            compiled = self.processor.parse_single(code)
        except (parser.ParseFatalException, parser.ParseException):
            while True:
                line = input(self.moreprompt)
                if line:
                    code += "\n"+line
                else:
                    break
            try:
                compiled = self.processor.parse_single(code)
            except (parser.ParseFatalException, parser.ParseException):
                return print_error()
        return compiled

    def execute(self, compiled=None, error=False):
        """Executes Compiled Code."""
        if self.runner is None:
            self.start_runner()
        if compiled is not None:
            if self.debug:
                self.console.debug("Executing "+repr(compiled)+"...")
            self.runner.run(compiled, error)

    def start_runner(self):
        """Starts The Runner."""
        self.runner = executor({
            "exit" : self.exit
            })
        self.runner.run(parser.headers["code"])
<commit_msg>Allows compilation to non-.py<commit_after>#!/usr/bin/env python

#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# INFO:
#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

"""
Author: Evan Hubinger
Date Created: 2014
Description: The Coconut Compiler.
"""

#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# IMPORTS:
#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

from __future__ import with_statement, print_function, absolute_import, unicode_literals, division

from .util import *
from . import parser
import argparse
import os
import os.path

#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# UTILITIES:
#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

def fixpath(path):
    """Properly Formats A Weapon."""
    return os.path.normpath(os.path.realpath(path))

class executor(object):
    """Compiled Python Executor."""
    def __init__(self, extras=None):
        """Creates The Executor."""
        self.variables = {}
        if extras is not None:
            for k,v in extras.items():
                self.variables[k] = v
    def run(__self, __code, __error=False):
        """Executes Python Code."""
        __globals = globals().copy()
        __locals = locals().copy()
        for __k, __v in __self.variables.items():
            globals()[__k] = __v
        try:
            exec(__code)
        except Exception:
            if __error:
                raise
            else:
                print_error()
        __overrides = {}
        for __k, __v in list(globals().items()):
            if __k in __self.variables:
                __self.variables[__k] = __v
                del globals()[__k]
            elif __k not in __globals:
                __overrides[__k] = __v
                del globals()[__k]
        for __k, __v in locals().items():
            if __k in __self.variables:
                __self.variables[__k] = __v
            elif __k not in __locals:
                __self.variables[__k] = __v
        for __k, __v in __overrides.items():
            __self.variables[__k] = __v
        for __k, __v in __globals.items():
            globals()[__k] = __v

#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
# MAIN:
#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

class cli(object):
    """The Coconut Command-Line Interface."""
    code_ext = ".coc"
    comp_ext = ".py"
    max_debug = 2
    commandline = argparse.ArgumentParser(description="The Coconut Programming Language.")
    commandline.add_argument("source", metavar="source", type=str, nargs="?", default=None, help="path to the coconut file/folder to compile")
    commandline.add_argument("dest", metavar="dest", type=str, nargs="?", default=None, help="destination directory for compiled files (defaults to the source directory)")
    commandline.add_argument("-v", "--version", action="store_const", const=True, default=False, help="print coconut and python version information")
    commandline.add_argument("-s", "--strict", action="store_const", const=True, default=False, help="enforce code cleanliness standards")
    commandline.add_argument("-p", "--print", action="store_const", const=True, default=False, help="print the compiled source")
    commandline.add_argument("-r", "--run", action="store_const", const=True, default=False, help="run the compiled source")
    commandline.add_argument("-n", "--nowrite", action="store_const", const=True, default=False, help="disable writing the compiled source")
    commandline.add_argument("-i", "--interact", action="store_const", const=True, default=False, help="force the interpreter to start (otherwise starts if no other command is given)")
    commandline.add_argument("-q", "--quiet", action="store_const", const=True, default=False, help="suppress all info and debug output")
    commandline.add_argument("-d", "--debug", metavar="level", type=int, nargs="?", default=0, const=1, help="enable debug output (0 is off, no arg defaults to 1, max is "+str(max_debug)+")")
    commandline.add_argument("-c", "--code", metavar="code", type=str, nargs=1, default=None, help="run a line of coconut passed in as a string")
    commandline.add_argument("--autopep8", type=str, nargs=argparse.REMAINDER, default=None, help="use autopep8 to format compiled code (remaining args passed to autopep8)")
    processor = None
    debug = False
    running = False
    runner = None

    def __init__(self, main_color=None, debug_color=None, prompt=">>> ", moreprompt="    ", main_sig="Coconut: ", debug_sig=None):
        """Creates The CLI."""
        self.console = terminal(main_color, debug_color, main_sig, debug_sig)
        self.prompt = self.console.addcolor(prompt, main_color)
        self.moreprompt = self.console.addcolor(moreprompt, main_color)

    def start(self):
        """Gets Command-Line Arguments."""
        args = self.commandline.parse_args()
        self.cmd(args)

    def setup(self, strict=False):
        """Creates The Processor."""
        self.processor = parser.processor(strict)
        self.processor.TRACER.show = self.console.debug

    def debug_set(self, level=0):
        """Sets The Debug Level."""
        self.debug = level >= 1
        self.processor.debug(level >= 2)

    def debug_level(self, level=0):
        """Processes A Debug Level."""
        if level < 0 or self.max_debug < level:
            raise parser.CoconutException("debug level must be in range(0, "+str(self.max_debug+1)+")")
        else:
            self.debug_set(level)

    def quiet(self, state=None):
        """Quiets Output."""
        if state is None:
            state = self.console.on
        self.console.on = not state

    def version(self):
        """Gets Version Info."""
        return "Version "+repr(VERSION)+" running on Python "+sys.version

    def cmd(self, args):
        """Parses Command-Line Arguments."""
        self.setup(args.strict)
        self.debug_level(args.debug)
        if args.quiet:
            self.quiet(True)
        if args.version:
            self.console.print(self.version())
        if args.autopep8 is not None:
            self.processor.autopep8(args.autopep8)
        if args.code is not None:
            self.execute(self.processor.parse_single(args.code[0]), True)
        if args.source is not None:
            if args.run and os.path.isdir(args.source):
                raise parser.CoconutException("source path can't point to file when --run is enabled")
            if args.dest is None:
                if args.nowrite:
                    self.compile_path(args.source, None, show=args.print, run=args.run)
                else:
                    self.compile_path(args.source, show=args.print, run=args.run)
            elif args.nowrite:
                raise parser.CoconutException("destination path can't be given when --nowrite is enabled")
            elif os.path.isfile(args.dest):
                raise parser.CoconutException("destination path can't point to file")
            else:
                self.compile_path(args.source, args.dest, show=args.print, run=args.run)
        elif args.run or args.print:
            raise parser.CoconutException("a source file must be specified when --run or --print is enabled")
        if args.interact or not (args.source or args.code or args.version):
            self.start_prompt()

    def compile_path(self, path, write=True, show=False, run=False):
        """Compiles A Path."""
        if os.path.isfile(path):
            if write is None:
                module = None
            else:
                module = False
            self.compile_file(path, write, module, show, run)
        elif os.path.isdir(path):
            self.compile_module(path, write, show, run)
        else:
            raise parser.CoconutException("could not find source path "+repr(path))

    def compile_module(self, directory, write=True, show=False, run=False):
        """Compiles A Module."""
        for dirpath, dirnames, filenames in os.walk(directory):
            writedir = write
            module = True
            if writedir is None:
                tocreate = None
                module = None
            elif writedir is True:
                tocreate = dirpath
            else:
                writedir = os.path.join(writedir, os.path.relpath(dirpath, directory))
                tocreate = writedir
            wrote = False
            for filename in filenames:
                if os.path.splitext(filename)[1] == self.code_ext:
                    self.compile_file(os.path.join(dirpath, filename), writedir, module, show, run)
                    wrote = True
            if wrote and tocreate is not None:
                self.create_module(tocreate)

    def compile_file(self, filepath, write=True, module=False, show=False, run=False):
        """Compiles A File."""
        if write is None:
            destpath = None
        elif write is True:
            destpath = filepath
        else:
            destpath = os.path.join(write, os.path.basename(filepath))
        if destpath is not None:
            base, ext = os.path.splitext(os.path.splitext(destpath)[0])
            if not ext:
                ext = self.comp_ext
            destpath = base + ext
        self.compile(filepath, destpath, module, show, run)

    def compile(self, codepath, destpath=None, module=False, show=False, run=False):
        """Compiles A Source Coconut File To A Destination Python File."""
        codepath = fixpath(codepath)
        self.console.print("Compiling "+repr(codepath)+"...")
        with openfile(codepath, "r") as opened:
            code = readfile(opened)
        if module is True:
            compiled = self.processor.parse_module(code)
        elif module is None:
            compiled = self.processor.parse_block(code)
        elif module is False:
            compiled = self.processor.parse_file(code)
        else:
            raise parser.CoconutException("invalid value for module boolean of "+repr(module))
        if show:
            print(compiled)
        if run:
            self.execute(compiled, True)
        if destpath is not None:
            destpath = fixpath(destpath)
            destdir = os.path.dirname(destpath)
            if not os.path.exists(destdir):
                os.makedirs(destdir)
            with openfile(destpath, "w") as opened:
                writefile(opened, compiled)
            self.console.print("Compiled "+repr(destpath)+".")

    def create_module(self, dirpath):
        """Sets Up A Module Directory."""
        with openfile(os.path.join(dirpath, "__coconut__.py"), "w") as opened:
            writefile(opened, parser.headers["package"])

    def start_prompt(self):
        """Starts The Interpreter."""
        self.console.print("[Interpreter:]")
        self.running = True
        while self.running:
            self.execute(self.handle(input(self.prompt)))

    def exit(self):
        """Exits The Interpreter."""
        self.running = False

    def handle(self, code):
        """Compiles Coconut Interpreter Input."""
        try:
            compiled = self.processor.parse_single(code)
        except (parser.ParseFatalException, parser.ParseException):
            while True:
                line = input(self.moreprompt)
                if line:
                    code += "\n"+line
                else:
                    break
            try:
                compiled = self.processor.parse_single(code)
            except (parser.ParseFatalException, parser.ParseException):
                return print_error()
        return compiled

    def execute(self, compiled=None, error=False):
        """Executes Compiled Code."""
        if self.runner is None:
            self.start_runner()
        if compiled is not None:
            if self.debug:
                self.console.debug("Executing "+repr(compiled)+"...")
            self.runner.run(compiled, error)

    def start_runner(self):
        """Starts The Runner."""
        self.runner = executor({
            "exit" : self.exit
            })
        self.runner.run(parser.headers["code"])
<|endoftext|>


<commit_before>#! /usr/bin/env python

import rospy
import actionlib
import cpp_controller_msgs.msg

def waypoint_client():
    pass

if __name__ == '__main__':
    try:
        rospy.init_node("waypoint_client_py")
        result = waypoint_client()
        print "Result:", result
    except rospy.ROSInterruptException as e:
        print "Client interrupted before completion!", str(e)
<commit_msg>[py_controller_client] Add a simple Python action client<commit_after>#! /usr/bin/env python

import rospy
import actionlib

from cpp_controller_msgs.msg import *
from geometry_msgs.msg import Pose2D

def waypoint_client(waypoints = []):

    # Create the client, passing the type of the action to the constructor.
    client = actionlib.SimpleActionClient("waypoint_following",
                                          WaypointFollowingAction)
    # Wait until the action server has started up.
    client.wait_for_server()

    # Create a goal to be sent to the action server.
    action_goal = WaypointFollowingGoal()

    # Fill out the request part of the message.
    action_goal.waypoints = waypoints

    # Send the goal to the   action server.
    client.send_goal(action_goal)

    # Wait for the server to finish performing the action.
    client.wait_for_result()

    return client.get_state()

if __name__ == '__main__':

    waypoints = list()
    waypoints.append(Pose2D(x = 9.0, y = 1.0))
    waypoints.append(Pose2D(x = 9.0, y = 9.0))
    waypoints.append(Pose2D(x = 1.0, y = 9.0))
    waypoints.append(Pose2D(x = 1.0, y = 1.0))

    for wp in waypoints:
        print wp

    try:
        rospy.init_node("waypoint_client_py")
        result = waypoint_client(waypoints)
        print "Result:", result, ("SUCCEEDED = 3")
    except rospy.ROSInterruptException as e:
        print "Client interrupted before completion!", str(e)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from datetime import datetime, timedelta

from django.test import TestCase
from django.utils.six.moves import reload_module
from jose import jwt
from rest_framework.exceptions import AuthenticationFailed
from rest_framework.test import APIRequestFactory
from rest_framework_simplejwt import authentication
from rest_framework_simplejwt.settings import api_settings

from .utils import override_api_settings

class TestJWTAuthentication(TestCase):
    def setUp(self):
        self.factory = APIRequestFactory()
        self.backend = authentication.JWTAuthentication()

        self.fake_token = b'TokenMcTokenface'
        self.fake_header = b'Bearer ' + self.fake_token

    def test_get_header(self):
        # Should pull correct header off request
        request = self.factory.get('/test-url/', HTTP_AUTHORIZATION=self.fake_header)
        self.assertEqual(self.backend.get_header(request), self.fake_header)

        # Should work for unicode headers
        request = self.factory.get('/test-url/', HTTP_AUTHORIZATION=self.fake_header.decode('utf-8'))
        self.assertEqual(self.backend.get_header(request), self.fake_header)

    def test_get_token(self):
        # Should return None if header lacks correct type keyword
        with override_api_settings(AUTH_HEADER_TYPE='JWT'):
            reload_module(authentication)
            self.assertIsNone(self.backend.get_token(self.fake_header))
        reload_module(authentication)

        # Should raise error if header is malformed
        with self.assertRaises(AuthenticationFailed):
            self.backend.get_token(b'Bearer one two')

        with self.assertRaises(AuthenticationFailed):
            self.backend.get_token(b'Bearer')

        # Otherwise, should return token in header
        self.assertEqual(self.backend.get_token(self.fake_header), self.fake_token)

    def test_get_payload(self):
        # No expiry tokens should cause exception
        payload = {'foo': 'bar'}
        no_exp_token = jwt.encode(payload, api_settings.SECRET_KEY, algorithm='HS256')
        with self.assertRaises(AuthenticationFailed):
            self.backend.get_payload(no_exp_token)

        # Expired tokens should cause exception
        payload['exp'] = datetime.utcnow() - timedelta(seconds=1)
        expired_token = jwt.encode(payload, api_settings.SECRET_KEY, algorithm='HS256')
        with self.assertRaises(AuthenticationFailed):
            self.backend.get_payload(expired_token)

        # Token with invalid signature should cause exception
        payload['exp'] = datetime.utcnow() + timedelta(days=1)
        token = jwt.encode(payload, api_settings.SECRET_KEY, algorithm='HS256')

        payload['foo'] = 'baz'
        other_token = jwt.encode(payload, api_settings.SECRET_KEY, algorithm='HS256')

        incorrect_payload = other_token.rsplit('.', 1)[0]
        correct_sig = token.rsplit('.', 1)[-1]
        invalid_token = incorrect_payload + '.' + correct_sig

        with self.assertRaises(AuthenticationFailed):
            self.backend.get_payload(invalid_token)
<commit_msg>Update TestJWTAuthentication.test_get_header<commit_after>from __future__ import unicode_literals

from datetime import datetime, timedelta

from django.test import TestCase
from django.utils.six.moves import reload_module
from jose import jwt
from rest_framework.exceptions import AuthenticationFailed
from rest_framework.test import APIRequestFactory
from rest_framework_simplejwt import authentication
from rest_framework_simplejwt.settings import api_settings

from .utils import override_api_settings

class TestJWTAuthentication(TestCase):
    def setUp(self):
        self.factory = APIRequestFactory()
        self.backend = authentication.JWTAuthentication()

        self.fake_token = b'TokenMcTokenface'
        self.fake_header = b'Bearer ' + self.fake_token

    def test_get_header(self):
        # Should return None if no authorization header
        request = self.factory.get('/test-url/')
        self.assertIsNone(self.backend.get_header(request))

        # Should pull correct header off request
        request = self.factory.get('/test-url/', HTTP_AUTHORIZATION=self.fake_header)
        self.assertEqual(self.backend.get_header(request), self.fake_header)

        # Should work for unicode headers
        request = self.factory.get('/test-url/', HTTP_AUTHORIZATION=self.fake_header.decode('utf-8'))
        self.assertEqual(self.backend.get_header(request), self.fake_header)

    def test_get_token(self):
        # Should return None if header lacks correct type keyword
        with override_api_settings(AUTH_HEADER_TYPE='JWT'):
            reload_module(authentication)
            self.assertIsNone(self.backend.get_token(self.fake_header))
        reload_module(authentication)

        # Should raise error if header is malformed
        with self.assertRaises(AuthenticationFailed):
            self.backend.get_token(b'Bearer one two')

        with self.assertRaises(AuthenticationFailed):
            self.backend.get_token(b'Bearer')

        # Otherwise, should return token in header
        self.assertEqual(self.backend.get_token(self.fake_header), self.fake_token)

    def test_get_payload(self):
        # No expiry tokens should cause exception
        payload = {'foo': 'bar'}
        no_exp_token = jwt.encode(payload, api_settings.SECRET_KEY, algorithm='HS256')
        with self.assertRaises(AuthenticationFailed):
            self.backend.get_payload(no_exp_token)

        # Expired tokens should cause exception
        payload['exp'] = datetime.utcnow() - timedelta(seconds=1)
        expired_token = jwt.encode(payload, api_settings.SECRET_KEY, algorithm='HS256')
        with self.assertRaises(AuthenticationFailed):
            self.backend.get_payload(expired_token)

        # Token with invalid signature should cause exception
        payload['exp'] = datetime.utcnow() + timedelta(days=1)
        token = jwt.encode(payload, api_settings.SECRET_KEY, algorithm='HS256')

        payload['foo'] = 'baz'
        other_token = jwt.encode(payload, api_settings.SECRET_KEY, algorithm='HS256')

        incorrect_payload = other_token.rsplit('.', 1)[0]
        correct_sig = token.rsplit('.', 1)[-1]
        invalid_token = incorrect_payload + '.' + correct_sig

        with self.assertRaises(AuthenticationFailed):
            self.backend.get_payload(invalid_token)
<|endoftext|>


<commit_before>__author__ = 'feurerm'

import unittest

import sklearn.datasets
import sklearn.decomposition
import sklearn.ensemble
import sklearn.svm

from HPOlibConfigSpace.configuration_space import ConfigurationSpace
from HPOlibConfigSpace.hyperparameters import CategoricalHyperparameter

from ParamSklearn.classification import ParamSklearnClassifier
from ParamSklearn.components.classification_base import ParamSklearnClassificationAlgorithm
from ParamSklearn.components.preprocessor_base import ParamSklearnPreprocessingAlgorithm
import ParamSklearn.components.classification as classification_components
import ParamSklearn.components.preprocessing as preprocessing_components
from ParamSklearn.util import get_dataset, DENSE, SPARSE, PREDICTIONS

class TestParamSklearnClassifier(unittest.TestCase):
    # TODO: test for both possible ways to initialize ParamSklearn
    # parameters and other...

    def test_io_dict(self):
        classifiers = classification_components._classifiers
        for c in classifiers:
            self.assertIn('input', classifiers[c].get_properties())
            self.assertIn('output', classifiers[c].get_properties())
            inp = classifiers[c].get_properties()['input']
            output = classifiers[c].get_properties()['output']

            self.assertIsInstance(inp, tuple)
            self.assertIsInstance(output, str)
            for i in inp:
                self.assertIn(i, (SPARSE, DENSE))
            self.assertEqual(output, PREDICTIONS)

    def test_find_classifiers(self):
        classifiers = classification_components._classifiers
        self.assertGreaterEqual(len(classifiers), 1)
        for key in classifiers:
            self.assertIn(ParamSklearnClassificationAlgorithm,
                            classifiers[key].__bases__)

    def test_find_preprocessors(self):
        preprocessors = preprocessing_components._preprocessors
        self.assertGreaterEqual(len(preprocessors),  1)
        for key in preprocessors:
            self.assertIn(ParamSklearnPreprocessingAlgorithm,
                            preprocessors[key].__bases__)

    def test_default_configuration(self):
        for i in range(2):
            cs = ParamSklearnClassifier.get_hyperparameter_search_space()
            default = cs.get_default_configuration()
            X_train, Y_train, X_test, Y_test = get_dataset(dataset='iris')
            auto = ParamSklearnClassifier(default)
            auto = auto.fit(X_train, Y_train)
            predictions = auto.predict(X_test)
            self.assertAlmostEqual(0.95999999999999996,
                sklearn.metrics.accuracy_score(predictions, Y_test))
            scores = auto.predict_proba(X_test)

    def test_get_hyperparameter_search_space(self):
        cs = ParamSklearnClassifier.get_hyperparameter_search_space()
        self.assertIsInstance(cs, ConfigurationSpace)
        conditions = cs.get_conditions()
        hyperparameters = cs.get_hyperparameters()
        self.assertEqual(86, len(hyperparameters))
        # The four parameters which are always active are classifier,
        # preprocessor, imputation strategy and scaling strategy
        self.assertEqual(len(hyperparameters) - 4, len(conditions))

    def test_get_hyperparameter_search_space_include_exclude_models(self):
        cs = ParamSklearnClassifier.get_hyperparameter_search_space(
            include_estimators=['libsvm_svc'])
        self.assertEqual(cs.get_hyperparameter('classifier'),
            CategoricalHyperparameter('classifier', ['libsvm_svc']))

        cs = ParamSklearnClassifier.get_hyperparameter_search_space(
            exclude_estimators=['libsvm_svc'])
        self.assertNotIn('libsvm_svc', str(cs))

        cs = ParamSklearnClassifier.get_hyperparameter_search_space(
            include_preprocessors=['pca'])
        self.assertEqual(cs.get_hyperparameter('preprocessor'),
            CategoricalHyperparameter('preprocessor', ["None", 'pca']))

        cs = ParamSklearnClassifier.get_hyperparameter_search_space(
            exclude_preprocessors=['pca'])
        self.assertNotIn('pca', str(cs))

    def test_get_hyperparameter_search_space_dataset_properties(self):
        cs_mc = ParamSklearnClassifier.get_hyperparameter_search_space(
            dataset_properties={'multiclass': True})
        self.assertNotIn('bernoulli_nb', str(cs_mc))

        cs_ml = ParamSklearnClassifier.get_hyperparameter_search_space(
            dataset_properties={'multilabel': True})
        self.assertNotIn('k_nearest_neighbors', str(cs_ml))
        self.assertNotIn('liblinear', str(cs_ml))
        self.assertNotIn('libsvm_svc', str(cs_ml))
        self.assertNotIn('sgd', str(cs_ml))

        cs_sp = ParamSklearnClassifier.get_hyperparameter_search_space(
            dataset_properties={'sparse': True})
        self.assertNotIn('extra_trees', str(cs_sp))
        self.assertNotIn('gradient_boosting', str(cs_sp))
        self.assertNotIn('random_forest', str(cs_sp))

        cs_mc_ml = ParamSklearnClassifier.get_hyperparameter_search_space(
            dataset_properties={'multilabel': True, 'multiclass': True})
        self.assertEqual(cs_ml, cs_mc_ml)

        self.assertRaisesRegexp(ValueError,
                                "No classifier to build a configuration space "
                                "for...", ParamSklearnClassifier.
                                get_hyperparameter_search_space,
                                dataset_properties={'multilabel': True,
                                                    'multiclass': True,
                                                    'sparse': True})

    @unittest.skip("test_check_random_state Not yet Implemented")
    def test_check_random_state(self):
        raise NotImplementedError()

    @unittest.skip("test_validate_input_X Not yet Implemented")
    def test_validate_input_X(self):
        raise NotImplementedError()

    @unittest.skip("test_validate_input_Y Not yet Implemented")
    def test_validate_input_Y(self):
        raise NotImplementedError()

    def test_set_params(self):
        pass

    def test_get_params(self):
        pass
<commit_msg>add checks for propertiy keys and remove no longer valid checks<commit_after>__author__ = 'feurerm'

import unittest

import sklearn.datasets
import sklearn.decomposition
import sklearn.ensemble
import sklearn.svm

from HPOlibConfigSpace.configuration_space import ConfigurationSpace
from HPOlibConfigSpace.hyperparameters import CategoricalHyperparameter

from ParamSklearn.classification import ParamSklearnClassifier
from ParamSklearn.components.classification_base import ParamSklearnClassificationAlgorithm
from ParamSklearn.components.preprocessor_base import ParamSklearnPreprocessingAlgorithm
import ParamSklearn.components.classification as classification_components
import ParamSklearn.components.preprocessing as preprocessing_components
from ParamSklearn.util import get_dataset, DENSE, SPARSE, PREDICTIONS

class TestParamSklearnClassifier(unittest.TestCase):
    # TODO: test for both possible ways to initialize ParamSklearn
    # parameters and other...

    def test_io_dict(self):
        classifiers = classification_components._classifiers
        for c in classifiers:
            props = classifiers[c].get_properties()
            self.assertIn('input', props)
            self.assertIn('output', props)
            inp = props['input']
            output = props['output']

            self.assertIsInstance(inp, tuple)
            self.assertIsInstance(output, str)
            for i in inp:
                self.assertIn(i, (SPARSE, DENSE))
            self.assertEqual(output, PREDICTIONS)
            self.assertIn('handles_regression', props)
            self.assertFalse(props['handles_regression'])
            self.assertIn('handles_classification', props)
            self.assertIn('handles_multiclass', props)
            self.assertIn('handles_multilabel', props)

    def test_find_classifiers(self):
        classifiers = classification_components._classifiers
        self.assertGreaterEqual(len(classifiers), 1)
        for key in classifiers:
            self.assertIn(ParamSklearnClassificationAlgorithm,
                            classifiers[key].__bases__)

    def test_find_preprocessors(self):
        preprocessors = preprocessing_components._preprocessors
        self.assertGreaterEqual(len(preprocessors),  1)
        for key in preprocessors:
            self.assertIn(ParamSklearnPreprocessingAlgorithm,
                            preprocessors[key].__bases__)

    def test_default_configuration(self):
        for i in range(2):
            cs = ParamSklearnClassifier.get_hyperparameter_search_space()
            default = cs.get_default_configuration()
            X_train, Y_train, X_test, Y_test = get_dataset(dataset='iris')
            auto = ParamSklearnClassifier(default)
            auto = auto.fit(X_train, Y_train)
            predictions = auto.predict(X_test)
            self.assertAlmostEqual(0.95999999999999996,
                sklearn.metrics.accuracy_score(predictions, Y_test))
            scores = auto.predict_proba(X_test)

    def test_get_hyperparameter_search_space(self):
        cs = ParamSklearnClassifier.get_hyperparameter_search_space()
        self.assertIsInstance(cs, ConfigurationSpace)
        conditions = cs.get_conditions()
        hyperparameters = cs.get_hyperparameters()
        self.assertEqual(86, len(hyperparameters))
        # The four parameters which are always active are classifier,
        # preprocessor, imputation strategy and scaling strategy
        self.assertEqual(len(hyperparameters) - 4, len(conditions))

    def test_get_hyperparameter_search_space_include_exclude_models(self):
        cs = ParamSklearnClassifier.get_hyperparameter_search_space(
            include_estimators=['libsvm_svc'])
        self.assertEqual(cs.get_hyperparameter('classifier'),
            CategoricalHyperparameter('classifier', ['libsvm_svc']))

        cs = ParamSklearnClassifier.get_hyperparameter_search_space(
            exclude_estimators=['libsvm_svc'])
        self.assertNotIn('libsvm_svc', str(cs))

        cs = ParamSklearnClassifier.get_hyperparameter_search_space(
            include_preprocessors=['pca'])
        self.assertEqual(cs.get_hyperparameter('preprocessor'),
            CategoricalHyperparameter('preprocessor', ["None", 'pca']))

        cs = ParamSklearnClassifier.get_hyperparameter_search_space(
            exclude_preprocessors=['pca'])
        self.assertNotIn('pca', str(cs))

    def test_get_hyperparameter_search_space_dataset_properties(self):
        cs_mc = ParamSklearnClassifier.get_hyperparameter_search_space(
            dataset_properties={'multiclass': True})
        self.assertNotIn('bernoulli_nb', str(cs_mc))

        cs_ml = ParamSklearnClassifier.get_hyperparameter_search_space(
            dataset_properties={'multilabel': True})
        self.assertNotIn('k_nearest_neighbors', str(cs_ml))
        self.assertNotIn('liblinear', str(cs_ml))
        self.assertNotIn('libsvm_svc', str(cs_ml))
        self.assertNotIn('sgd', str(cs_ml))

        cs_sp = ParamSklearnClassifier.get_hyperparameter_search_space(
            dataset_properties={'sparse': True})
        self.assertIn('extra_trees', str(cs_sp))
        self.assertIn('gradient_boosting', str(cs_sp))
        self.assertIn('random_forest', str(cs_sp))

        cs_mc_ml = ParamSklearnClassifier.get_hyperparameter_search_space(
            dataset_properties={'multilabel': True, 'multiclass': True})
        self.assertEqual(cs_ml, cs_mc_ml)

        # We now have a preprocessing method that handles this case
        #self.assertRaisesRegexp(ValueError,
        #                        "No classifier to build a configuration space "
        #                        "for...", ParamSklearnClassifier.
        #                        get_hyperparameter_search_space,
        #                        dataset_properties={'multilabel': True,
        #                                            'multiclass': True,
        #                                            'sparse': True})

    @unittest.skip("test_check_random_state Not yet Implemented")
    def test_check_random_state(self):
        raise NotImplementedError()

    @unittest.skip("test_validate_input_X Not yet Implemented")
    def test_validate_input_X(self):
        raise NotImplementedError()

    @unittest.skip("test_validate_input_Y Not yet Implemented")
    def test_validate_input_Y(self):
        raise NotImplementedError()

    def test_set_params(self):
        pass

    def test_get_params(self):
        pass
<|endoftext|>


<commit_before>""" Report on reapplications for issue https://github.com/DOAJ/doaj/issues/1376 - this is copy of reporting.py, but
will be discarded in time as reapplications were a one-off. """

from portality import models
from portality.clcsv import UnicodeWriter
from portality.lib import dates

from portality.background import BackgroundApi, BackgroundTask
from portality.tasks.redis_huey import main_queue
from portality.decorators import write_required

import codecs, os, shutil, re

def reapplication_reports(outdir):
    pipeline = [
        RejectedReapplicationReporter(),
        NoReapplicationReporter(),
        ContinuationNoteReporter(),
        TickReporter()
    ]

    for j in models.Journal.all_in_doaj(page_size=1000):
        for filt in pipeline:
            filt.test(j)

    outfiles = []
    for p in pipeline:
        table = p.tabulate()
        outfile = os.path.join(outdir, p.filename())
        outfiles.append(outfile)
        with codecs.open(outfile, "wb", "utf-8") as f:
            writer = UnicodeWriter(f)
            for row in table:
                writer.writerow(row)

    return outfiles

def _tabulate_journal(report, entity_key):
    categories = report.values()[0].keys()
    categories.sort()
    table = [[entity_key] + categories]

    for k, v in report.iteritems():
        row = [k]
        for c in categories:
            row.append(v[c])
        table.append(row)
    return table

class JournalReporter(object):

    def __init__(self):
        self.report = {}

    def include(self, j):
        self.report[j.id] = {
            'Title': j.bibjson().title,
            'EISSN': j.bibjson().get_one_identifier('eissn') or '',
            'PISSN': j.bibjson().get_one_identifier('pissn') or ''
        }

    def tabulate(self):
        return _tabulate_journal(self.report, "Journal ID")

    def test(self, j):
        raise NotImplementedError()

    def filename(self):
        raise NotImplementedError()

class RejectedReapplicationReporter(JournalReporter):

    def test(self, j):
        if j.current_application is not None:
            cur_app = models.Suggestion.pull(j.current_application)
            if cur_app is not None:
                if cur_app.current_journal and cur_app.application_status == 'rejected':        # rejected reapplication
                    self.include(j)

    def filename(self):
        return '1_journals_with_rejected_reapplication_on_' + dates.today() + '.csv'

class NoReapplicationReporter(JournalReporter):

    def test(self, j):
        if j.current_application is None and not j.is_ticked():
            self.include(j)

    def filename(self):
        return '2_journals_without_reapplications_on_' + dates.today() + '.csv'

class ContinuationNoteReporter(JournalReporter):

    notes_regex = re.compile('^Continuation automatically extracted from journal .* during migration$', re.IGNORECASE)

    def test(self, j):
        for n in j.notes:
            if self.notes_regex.match(n['note']):
                self.include(j)

    def filename(self):
        return '3_journals_with_continued_note_on_' + dates.today() + '.csv'

class TickReporter(JournalReporter):

    def test(self, j):
        if not j.is_ticked():
            self.include(j)

    def filename(self):
        return '4_journals_without_tick_on_' + dates.today() + '.csv'

def email(data_dir, archv_name):
    """
    Compress and email the reports to the specified email address.
    :param data_dir: Directory containing the reports
    :param archv_name: Filename for the archive and resulting email attachment
    """
    import shutil
    from portality import app_email
    from portality.core import app

    email_to = app.config.get('REPORTS_EMAIL_TO', ['feedback@doaj.org'])
    email_from = app.config.get('SYSTEM_EMAIL_FROM', 'feedback@doaj.org')
    email_sub = app.config.get('SERVICE_NAME', '') + ' - generated {0}'.format(archv_name)
    msg = "Attached: {0}.zip\n".format(archv_name)

    # Create an archive of the reports
    archv = shutil.make_archive(archv_name, "zip", root_dir=data_dir)

    # Read the archive to create an attachment, send it with the app email
    with open(archv) as f:
        dat = f.read()
        attach = [app_email.make_attachment(filename=archv_name, content_type='application/zip', data=dat)]
        app_email.send_mail(to=email_to, fro=email_from, subject=email_sub, msg_body=msg, files=attach)

    # Clean up the archive
    os.remove(archv)

#########################################################
# Background task implementation

class ReportingBackgroundTask(BackgroundTask):

    __action__ = "reapplication_reporting"

    def run(self):
        """
        Execute the task as specified by the background_job
        :return:
        """
        job = self.background_job
        params = job.params

        outdir = self.get_param(params, "outdir", "reapplication_report_" + dates.today())

        job.add_audit_message("Saving reports to " + outdir)
        if not os.path.exists(outdir):
            os.makedirs(outdir)

        reapp_outfiles = reapplication_reports(outdir)
        refs = {}
        self.set_reference(refs, "reapplication_outfiles", reapp_outfiles)
        job.reference = refs

        msg = u"Generated reapplication reports as of {0}".format(dates.now())
        job.add_audit_message(msg)

        send_email = self.get_param(params, "email", False)
        if send_email:
            archive_name = "reapplication_reports_" + dates.today()
            email(outdir, archive_name)
            job.add_audit_message("email alert sent")
        else:
            job.add_audit_message("no email alert sent")

    def cleanup(self):
        """
        Cleanup after a successful OR failed run of the task
        :return:
        """
        failed = self.background_job.is_failed()
        if not failed:
            return

        params = self.background_job.params
        outdir = self.get_param(params, "outdir")

        if outdir is not None and os.path.exists(outdir):
            shutil.rmtree(outdir)

        self.background_job.add_audit_message(u"Deleted directory {x} due to job failure".format(x=outdir))

    @classmethod
    def prepare(cls, username, **kwargs):
        """
        Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob,
        or fail with a suitable exception

        :param username: user to run this job as
        :param kwargs: arbitrary keyword arguments pertaining to this task type
        :return: a BackgroundJob instance representing this task
        """

        job = models.BackgroundJob()
        job.user = username
        job.action = cls.__action__

        params = {}
        cls.set_param(params, "outdir", kwargs.get("outdir", "reapplication_report_" + dates.today()))
        cls.set_param(params, "email", kwargs.get("email", False))
        job.params = params

        return job

    @classmethod
    def submit(cls, background_job):
        """
        Submit the specified BackgroundJob to the background queue

        :param background_job: the BackgroundJob instance
        :return:
        """
        background_job.save()
        run_reports.schedule(args=(background_job.id,), delay=10)

@main_queue.task()
@write_required(script=True)
def run_reports(job_id):
    job = models.BackgroundJob.pull(job_id)
    task = ReportingBackgroundTask(job)
    BackgroundApi.execute(task)
<commit_msg>Added error catching and reporting for one of the reapplication reports.<commit_after>""" Report on reapplications for issue https://github.com/DOAJ/doaj/issues/1376 - this is copy of reporting.py, but
will be discarded in time as reapplications were a one-off. """

from portality import models
from portality.clcsv import UnicodeWriter
from portality.lib import dates
from portality.lib.dataobj import DataStructureException

from portality.background import BackgroundApi, BackgroundTask
from portality.tasks.redis_huey import main_queue
from portality.decorators import write_required

import codecs, os, shutil, re

def reapplication_reports(outdir):
    pipeline = [
        RejectedReapplicationReporter(),
        NoReapplicationReporter(),
        ContinuationNoteReporter(),
        TickReporter()
    ]

    for j in models.Journal.all_in_doaj(page_size=1000):
        for filt in pipeline:
            filt.test(j)

    outfiles = []
    all_errors = {}
    for p in pipeline:
        table = p.tabulate()
        outfile = os.path.join(outdir, p.filename())
        outfiles.append(outfile)
        merge_errors(all_errors, p.errors)
        with codecs.open(outfile, "wb", "utf-8") as f:
            writer = UnicodeWriter(f)
            for row in table:
                writer.writerow(row)

    return outfiles, all_errors

def _tabulate_journal(report, entity_key):
    categories = report.values()[0].keys()
    categories.sort()
    table = [[entity_key] + categories]

    for k, v in report.iteritems():
        row = [k]
        for c in categories:
            row.append(v[c])
        table.append(row)
    return table

class JournalReporter(object):

    def __init__(self):
        self.report = {}
        self.errors = {}

    def include(self, j):
        self.report[j.id] = {
            'Title': j.bibjson().title,
            'EISSN': j.bibjson().get_one_identifier('eissn') or '',
            'PISSN': j.bibjson().get_one_identifier('pissn') or ''
        }

    def tabulate(self):
        return _tabulate_journal(self.report, "Journal ID")

    def test(self, j):
        raise NotImplementedError()

    def filename(self):
        raise NotImplementedError()

class RejectedReapplicationReporter(JournalReporter):

    def test(self, j):
        if j.current_application is not None:
            try:
                cur_app = models.Suggestion.pull(j.current_application)
                if cur_app is not None:
                    if cur_app.current_journal and cur_app.application_status == 'rejected':    # rejected reapplication
                        self.include(j)
            except DataStructureException as e:
                self.errors[j.id] = '1_journals_with_rejected_reapplication: Reapp {0} - DataStructureException: {1}'.format(j.current_application, e)

    def filename(self):
        return '1_journals_with_rejected_reapplication_on_' + dates.today() + '.csv'

class NoReapplicationReporter(JournalReporter):

    def test(self, j):
        if j.current_application is None and not j.is_ticked():
            self.include(j)

    def filename(self):
        return '2_journals_without_reapplications_on_' + dates.today() + '.csv'

class ContinuationNoteReporter(JournalReporter):

    notes_regex = re.compile('^Continuation automatically extracted from journal .* during migration$', re.IGNORECASE)

    def test(self, j):
        for n in j.notes:
            if self.notes_regex.match(n['note']):
                self.include(j)

    def filename(self):
        return '3_journals_with_continued_note_on_' + dates.today() + '.csv'

class TickReporter(JournalReporter):

    def test(self, j):
        if not j.is_ticked():
            self.include(j)

    def filename(self):
        return '4_journals_without_tick_on_' + dates.today() + '.csv'

def email(data_dir, archv_name):
    """
    Compress and email the reports to the specified email address.
    :param data_dir: Directory containing the reports
    :param archv_name: Filename for the archive and resulting email attachment
    """
    import shutil
    from portality import app_email
    from portality.core import app

    email_to = app.config.get('REPORTS_EMAIL_TO', ['feedback@doaj.org'])
    email_from = app.config.get('SYSTEM_EMAIL_FROM', 'feedback@doaj.org')
    email_sub = app.config.get('SERVICE_NAME', '') + ' - generated {0}'.format(archv_name)
    msg = "Attached: {0}.zip\n".format(archv_name)

    # Create an archive of the reports
    archv = shutil.make_archive(archv_name, "zip", root_dir=data_dir)

    # Read the archive to create an attachment, send it with the app email
    with open(archv) as f:
        dat = f.read()
        attach = [app_email.make_attachment(filename=archv_name, content_type='application/zip', data=dat)]
        app_email.send_mail(to=email_to, fro=email_from, subject=email_sub, msg_body=msg, files=attach)

    # Clean up the archive
    os.remove(archv)

def merge_errors(base, update):
    """ Copy the errors from update into base. """
    for k, v in update.iteritems():
        try:
            base[k].append(v)
        except KeyError:
            base[k] = [v]

#########################################################
# Background task implementation

class ReportingBackgroundTask(BackgroundTask):

    __action__ = "reapplication_reporting"

    def run(self):
        """
        Execute the task as specified by the background_job
        :return:
        """
        job = self.background_job
        params = job.params

        outdir = self.get_param(params, "outdir", "reapplication_report_" + dates.today())

        job.add_audit_message("Saving reports to " + outdir)
        if not os.path.exists(outdir):
            os.makedirs(outdir)

        reapp_outfiles, errors = reapplication_reports(outdir)
        refs = {}
        self.set_reference(refs, "reapplication_outfiles", reapp_outfiles)
        job.reference = refs
        [job.add_audit_message('Journal {0} errors: {1}'.format(k, ', '.join(v))) for k, v in errors.iteritems()]

        msg = u"Generated reapplication reports as of {0}".format(dates.now())
        job.add_audit_message(msg)

        send_email = self.get_param(params, "email", False)
        if send_email:
            archive_name = "reapplication_reports_" + dates.today()
            email(outdir, archive_name)
            job.add_audit_message("email alert sent")
        else:
            job.add_audit_message("no email alert sent")

    def cleanup(self):
        """
        Cleanup after a successful OR failed run of the task
        :return:
        """
        failed = self.background_job.is_failed()
        if not failed:
            return

        params = self.background_job.params
        outdir = self.get_param(params, "outdir")

        if outdir is not None and os.path.exists(outdir):
            shutil.rmtree(outdir)

        self.background_job.add_audit_message(u"Deleted directory {x} due to job failure".format(x=outdir))

    @classmethod
    def prepare(cls, username, **kwargs):
        """
        Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob,
        or fail with a suitable exception

        :param username: user to run this job as
        :param kwargs: arbitrary keyword arguments pertaining to this task type
        :return: a BackgroundJob instance representing this task
        """

        job = models.BackgroundJob()
        job.user = username
        job.action = cls.__action__

        params = {}
        cls.set_param(params, "outdir", kwargs.get("outdir", "reapplication_report_" + dates.today()))
        cls.set_param(params, "email", kwargs.get("email", False))
        job.params = params

        return job

    @classmethod
    def submit(cls, background_job):
        """
        Submit the specified BackgroundJob to the background queue

        :param background_job: the BackgroundJob instance
        :return:
        """
        background_job.save()
        run_reports.schedule(args=(background_job.id,), delay=10)

@main_queue.task()
@write_required(script=True)
def run_reports(job_id):
    job = models.BackgroundJob.pull(job_id)
    task = ReportingBackgroundTask(job)
    BackgroundApi.execute(task)
<|endoftext|>


<commit_before># Do the necessary imports
import argparse
import shutil
import base64
from datetime import datetime
import os
import cv2
import numpy as np
import socketio
import eventlet
import eventlet.wsgi
from PIL import Image
from flask import Flask
from io import BytesIO, StringIO
import json
import pickle
import matplotlib.image as mpimg
import time

# Import functions for perception and decision making
from perception import perception_step
from decision import decision_step
from supporting_functions import update_rover, create_output_images
# Initialize socketio server and Flask application
# (learn more at: https://python-socketio.readthedocs.io/en/latest/)
sio = socketio.Server()
app = Flask(__name__)

# Read in ground truth map and create 3-channel green version for overplotting
# NOTE: images are read in by default with the origin (0, 0) in the upper left
# and y-axis increasing downward.
ground_truth = mpimg.imread('../calibration_images/map_bw.png')
# This next line creates arrays of zeros in the red and blue channels
# and puts the map into the green channel.  This is why the underlying
# map output looks green in the display image
ground_truth_3d = np.dstack((ground_truth*0, ground_truth*255, ground_truth*0)).astype(np.float)

# Define RoverState() class to retain rover state parameters
class RoverState():
    def __init__(self):
        self.start_time = None # To record the start time of navigation
        self.total_time = None # To record total duration of naviagation
        self.img = None # Current camera image
        self.pos = None # Current position (x, y)
        self.yaw = None # Current yaw angle
        self.pitch = None # Current pitch angle
        self.roll = None # Current roll angle
        self.vel = None # Current velocity
        self.steer = 0 # Current steering angle
        self.throttle = 0 # Current throttle value
        self.brake = 0 # Current brake value
        self.nav_angles = None # Angles of navigable terrain pixels
        self.nav_dists = None # Distances of navigable terrain pixels
        self.ground_truth = ground_truth_3d # Ground truth worldmap
        self.mode = 'forward' # Current mode (can be forward or stop)
        self.throttle_set = 0.2 # Throttle setting when accelerating
        self.brake_set = 10 # Brake setting when braking
        # The stop_forward and go_forward fields below represent total count
        # of navigable terrain pixels.  This is a very crude form of knowing
        # when you can keep going and when you should stop.  Feel free to
        # get creative in adding new fields or modifying these!
        self.stop_forward = 50 # Threshold to initiate stopping
        self.go_forward = 500 # Threshold to go forward again
        self.max_vel = 2 # Maximum velocity (meters/second)
        # Image output from perception step
        # Update this image to display your intermediate analysis steps
        # on screen in autonomous mode
        self.vision_image = np.zeros((160, 320, 3), dtype=np.float)
        # Worldmap
        # Update this image with the positions of navigable terrain
        # obstacles and rock samples
        self.worldmap = np.zeros((200, 200, 3), dtype=np.float)
        self.samples_pos = None # To store the actual sample positions
        self.samples_found = None # To count the number of samples found
# Initialize our rover
Rover = RoverState()

# Define telemetry function for what to do with incoming data
@sio.on('telemetry')
def telemetry(sid, data):
    if data:
        global Rover

        # Initialize / update Rover with current telemetry
        Rover = update_rover(Rover, data)

        if np.isfinite(Rover.vel):

            # Execute the perception and decision steps to update the Rover's state
            Rover = perception_step(Rover)
            Rover = decision_step(Rover)

            # Create output images to send to server
            out_image_string1, out_image_string2 = create_output_images(Rover)

            # The action step!  Send commands to the rover!

            commands = (Rover.throttle, Rover.brake, Rover.steer)
            #print(commands)
            send_control(commands, out_image_string1, out_image_string2)

        else:

            # Send zeros for throttle, brake and steer and empty images
            # if we got a bad velocity value in telemetry
            send_control((0, 0, 0), '', '')

        # If you want to save camera images from autonomous driving specify a path
        # Example: $ python drive_rover.py image_folder_path
        # Conditioal to save image frame if folder was specified
        if args.image_folder != '':
            timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3]
            image_filename = os.path.join(args.image_folder, timestamp)
            image.save('{}.jpg'.format(image_filename))
    else:
        sio.emit('manual', data={}, skip_sid=True)

@sio.on('connect')
def connect(sid, environ):
    print("connect ", sid)
    send_control((0, 0, 0), '', '')
    sample_data = {}
    sio.emit(
        "get_samples",
        sample_data,
        skip_sid=True)

def send_control(commands, image_string1, image_string2):
    # Define commands to be sent to the rover
    data={
        'throttle': commands[0].__str__(),
        'brake': commands[1].__str__(),
        'steering_angle': commands[2].__str__(),
        'inset_image1': image_string1,
        'inset_image2': image_string2,
        }
    # Send commands via socketIO server
    sio.emit(
        "data",
        data,
        skip_sid=True)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Remote Driving')
    parser.add_argument(
        'image_folder',
        type=str,
        nargs='?',
        default='',
        help='Path to image folder. This is where the images from the run will be saved.'
    )
    args = parser.parse_args()

    os.system('rm -rf IMG_stream/*')
    if args.image_folder != '':
        print("Creating image folder at {}".format(args.image_folder))
        if not os.path.exists(args.image_folder):
            os.makedirs(args.image_folder)
        else:
            shutil.rmtree(args.image_folder)
            os.makedirs(args.image_folder)
        print("Recording this run ...")
    else:
        print("NOT recording this run ...")

    # wrap Flask application with socketio's middleware
    app = socketio.Middleware(sio, app)

    # deploy as an eventlet WSGI server
    eventlet.wsgi.server(eventlet.listen(('', 4567)), app)
<commit_msg>Update telemetry() func in drive_rover.py<commit_after># Do the necessary imports
import argparse
import shutil
import base64
from datetime import datetime
import os
import cv2
import numpy as np
import socketio
import eventlet
import eventlet.wsgi
from PIL import Image
from flask import Flask
from io import BytesIO, StringIO
import json
import pickle
import matplotlib.image as mpimg
import time

# Import functions for perception and decision making
from perception import perception_step
from decision import decision_step
from supporting_functions import update_rover, create_output_images
# Initialize socketio server and Flask application
# (learn more at: https://python-socketio.readthedocs.io/en/latest/)
sio = socketio.Server()
app = Flask(__name__)

# Read in ground truth map and create 3-channel green version for overplotting
# NOTE: images are read in by default with the origin (0, 0) in the upper left
# and y-axis increasing downward.
ground_truth = mpimg.imread('../calibration_images/map_bw.png')
# This next line creates arrays of zeros in the red and blue channels
# and puts the map into the green channel.  This is why the underlying
# map output looks green in the display image
ground_truth_3d = np.dstack((ground_truth*0, ground_truth*255, ground_truth*0)).astype(np.float)

# Define RoverState() class to retain rover state parameters
class RoverState():
    def __init__(self):
        self.start_time = None # To record the start time of navigation
        self.total_time = None # To record total duration of naviagation
        self.img = None # Current camera image
        self.pos = None # Current position (x, y)
        self.yaw = None # Current yaw angle
        self.pitch = None # Current pitch angle
        self.roll = None # Current roll angle
        self.vel = None # Current velocity
        self.steer = 0 # Current steering angle
        self.throttle = 0 # Current throttle value
        self.brake = 0 # Current brake value
        self.nav_angles = None # Angles of navigable terrain pixels
        self.nav_dists = None # Distances of navigable terrain pixels
        self.ground_truth = ground_truth_3d # Ground truth worldmap
        self.mode = 'forward' # Current mode (can be forward or stop)
        self.throttle_set = 0.2 # Throttle setting when accelerating
        self.brake_set = 10 # Brake setting when braking
        # The stop_forward and go_forward fields below represent total count
        # of navigable terrain pixels.  This is a very crude form of knowing
        # when you can keep going and when you should stop.  Feel free to
        # get creative in adding new fields or modifying these!
        self.stop_forward = 50 # Threshold to initiate stopping
        self.go_forward = 500 # Threshold to go forward again
        self.max_vel = 2 # Maximum velocity (meters/second)
        # Image output from perception step
        # Update this image to display your intermediate analysis steps
        # on screen in autonomous mode
        self.vision_image = np.zeros((160, 320, 3), dtype=np.float)
        # Worldmap
        # Update this image with the positions of navigable terrain
        # obstacles and rock samples
        self.worldmap = np.zeros((200, 200, 3), dtype=np.float)
        self.samples_pos = None # To store the actual sample positions
        self.samples_found = None # To count the number of samples found
# Initialize our rover
Rover = RoverState()

# Define telemetry function for what to do with incoming data
@sio.on('telemetry')
def telemetry(sid, data):
    if data:
        global Rover

        # Initialize / update Rover with current telemetry
        Rover = update_rover(Rover, data)

        if np.isfinite(Rover.vel):

            # Execute the perception and decision steps to update the Rover's state
            Rover = perception_step(Rover)
            Rover = decision_step(Rover)

            # Create output images to send to server
            out_image_string1, out_image_string2 = create_output_images(Rover)

            # The action step!  Send commands to the rover!
            commands = (Rover.throttle, Rover.brake, Rover.steer)
            send_control(commands, out_image_string1, out_image_string2)

        # In case of invalid telemetry, send null commands
        else:

            # Send zeros for throttle, brake and steer and empty images
            send_control((0, 0, 0), '', '')

        # If you want to save camera images from autonomous driving specify a path
        # Example: $ python drive_rover.py image_folder_path
        # Conditional to save image frame if folder was specified
        if args.image_folder != '':
            timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3]
            image_filename = os.path.join(args.image_folder, timestamp)
            image.save('{}.jpg'.format(image_filename))

    else:
        sio.emit('manual', data={}, skip_sid=True)

@sio.on('connect')
def connect(sid, environ):
    print("connect ", sid)
    send_control((0, 0, 0), '', '')
    sample_data = {}
    sio.emit(
        "get_samples",
        sample_data,
        skip_sid=True)

def send_control(commands, image_string1, image_string2):
    # Define commands to be sent to the rover
    data={
        'throttle': commands[0].__str__(),
        'brake': commands[1].__str__(),
        'steering_angle': commands[2].__str__(),
        'inset_image1': image_string1,
        'inset_image2': image_string2,
        }
    # Send commands via socketIO server
    sio.emit(
        "data",
        data,
        skip_sid=True)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Remote Driving')
    parser.add_argument(
        'image_folder',
        type=str,
        nargs='?',
        default='',
        help='Path to image folder. This is where the images from the run will be saved.'
    )
    args = parser.parse_args()

    os.system('rm -rf IMG_stream/*')
    if args.image_folder != '':
        print("Creating image folder at {}".format(args.image_folder))
        if not os.path.exists(args.image_folder):
            os.makedirs(args.image_folder)
        else:
            shutil.rmtree(args.image_folder)
            os.makedirs(args.image_folder)
        print("Recording this run ...")
    else:
        print("NOT recording this run ...")

    # wrap Flask application with socketio's middleware
    app = socketio.Middleware(sio, app)

    # deploy as an eventlet WSGI server
    eventlet.wsgi.server(eventlet.listen(('', 4567)), app)
<|endoftext|>


<commit_before>import unittest

import numpy
import six

from chainer import cuda
from chainer import gradient_check
from chainer.testing import attr
from chainer.testing import condition

if cuda.available:
    cuda.init()

def _uniform(*shape):
    return numpy.random.uniform(-1, 1, shape).astype(numpy.float32)

def _full_like(x, val):
    if isinstance(x, numpy.ndarray):
        return numpy.full_like(x, val)
    else:
        return cuda.full_like(x, val)

def _zeros_like(x):
    if isinstance(x, numpy.ndarray):
        return numpy.zeros_like(x)
    else:
        return cuda.zeros_like(x)

class NumeraicalGradientTest(unittest.TestCase):

    def f(self, xs):
        return (xs[0] ** 2,)

    def df(self, xs):
        return ((2 * xs[0],),)

    def setUp(self):
        self.xs = (_uniform(2, 1),)
        self.gys = (_uniform(2, 1),)

    def check_numerical_grad(self, f, df, xs, gys):
        dfxs = df(xs)

        gys = tuple(0 if gy is None else gy for gy in gys)
        # matrix-vector multiplication of dfxs and dys
        dx_expect = map(lambda dfx:
                        sum(map(lambda (a, b): a*b, zip(dfx, gys))), dfxs)

        func = lambda: f(xs)
        dx_actual = gradient_check.numerical_grad(func, xs, gys)

        self.assertEqual(len(dx_expect), len(dx_actual))
        for e, a in zip(dx_expect, dx_actual):
            gradient_check.assert_allclose(e, a)

    @condition.retry(3)
    def test_numerical_grad_cpu(self):
        self.check_numerical_grad(self.f, self.df, self.xs, self.gys)

    @condition.retry(3)
    @attr.gpu
    def test_numerical_grad_gpu(self):
        gys = tuple(None if gy is None else cuda.to_gpu(gy)
                    for gy in self.gys)

        self.check_numerical_grad(self.f, self.df,
                                  map(cuda.to_gpu, self.xs), gys)

class NumericalGradientTest2(NumeraicalGradientTest):

    f = lambda self, xs: (1,)
    df = lambda self, xs: ((0,),)

def _exp(x):
    if isinstance(x, numpy.ndarray):
        return numpy.exp(x)
    else:
        return cuda.cumath.exp(x)

class NumericalGradientTest3(NumeraicalGradientTest):

    def f(self, xs):
        return (_exp(xs[0]),)

    def df(self, xs):
        return ((_exp(xs[0]),),)

    def setUp(self):
        self.xs = (_uniform(2, 1),)
        self.gys = (_uniform(2, 1),)

class NumericalGradientTest4(NumeraicalGradientTest):

    def f(self, xs):
        assert len(xs) == 2
        return (2 * xs[0] + 3 * xs[1],
                4 * xs[0] + 5 * xs[1],
                6 * xs[0] + 7 * xs[1])

    def df(self, xs):
        assert len(xs) == 2
        return (
            (_full_like(xs[0], 2), _full_like(xs[0], 4), _full_like(xs[0], 6)),
            (_full_like(xs[1], 3), _full_like(xs[1], 5), _full_like(xs[1], 7)))

    def setUp(self):
        self.xs = tuple(_uniform(2, 1) for _ in six.moves.range(2))
        self.gys = tuple(_uniform(2, 1) for _ in six.moves.range(3))

class NumericalGradientTest5(NumericalGradientTest4):

    def f(self, xs):
        assert len(xs) == 2
        return (2 * xs[0] + 3 * xs[1],
                4 * xs[0] + 5 * xs[1],
                6 * xs[0] + 7 * xs[1])

    def df(self, xs):
        assert len(xs) == 2
        return (
            (_full_like(xs[0], 2), _zeros_like(xs[0]), _full_like(xs[0], 6)),
            (_full_like(xs[1], 3), _zeros_like(xs[1]), _full_like(xs[1], 7)))

    def setUp(self):
        super(NumericalGradientTest5, self).setUp()
        self.gys = (_uniform(2, 1), None, _uniform(2, 1))

class NumericalGradientTest6(NumeraicalGradientTest):

    def setUp(self):
        self.xs = (_uniform(2, 1),)
        self.gys = (None,)

class AssertAllCloseTest(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)
        self.y = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)

    def check_identical(self, x):
        gradient_check.assert_allclose(x, x, atol=0, rtol=0)

    @condition.repeat(5)
    def test_identical_cpu(self):
        self.check_identical(self.x)

    @condition.repeat(5)
    @attr.gpu
    def test_identical_gpu(self):
        self.check_identical(cuda.to_gpu(self.x))

    def check_atol(self, x, y):
        x_cpu = cuda.to_cpu(x)
        y_cpu = cuda.to_cpu(y)
        max_abs_diff = numpy.max(numpy.abs(x_cpu - y_cpu))
        with self.assertRaises(AssertionError):
            gradient_check.assert_allclose(x, y, atol=max_abs_diff - 1, rtol=0)
        gradient_check.assert_allclose(x, y, atol=max_abs_diff + 1, rtol=0)

    @condition.repeat(5)
    def test_atol_cpu(self):
        self.check_atol(self.x, self.y)

    @condition.repeat(5)
    @attr.gpu
    def test_atol_gpu(self):
        self.check_atol(cuda.to_gpu(self.x), cuda.to_gpu(self.y))

class AssertAllCloseTest2(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)
        self.y = numpy.random.uniform(1, 2, (2, 3)).astype(numpy.float32)

    def check_rtol(self, x, y):
        x_cpu = cuda.to_cpu(x)
        y_cpu = cuda.to_cpu(y)
        max_ratio = numpy.max(numpy.abs(x_cpu - y_cpu) / y_cpu)
        with self.assertRaises(AssertionError):
            gradient_check.assert_allclose(x, y, atol=0, rtol=max_ratio - 1)
        gradient_check.assert_allclose(x, y, atol=0, rtol=max_ratio + 1)

    @condition.repeat(5)
    def test_rtol_cpu(self):
        self.check_rtol(self.x, self.y)

    @condition.repeat(5)
    @attr.gpu
    def test_rtol_gpu(self):
        self.check_rtol(cuda.to_gpu(self.x), cuda.to_gpu(self.y))
<commit_msg>Fix typo<commit_after>import unittest

import numpy
import six

from chainer import cuda
from chainer import gradient_check
from chainer.testing import attr
from chainer.testing import condition

if cuda.available:
    cuda.init()

def _uniform(*shape):
    return numpy.random.uniform(-1, 1, shape).astype(numpy.float32)

def _full_like(x, val):
    if isinstance(x, numpy.ndarray):
        return numpy.full_like(x, val)
    else:
        return cuda.full_like(x, val)

def _zeros_like(x):
    if isinstance(x, numpy.ndarray):
        return numpy.zeros_like(x)
    else:
        return cuda.zeros_like(x)

class NumericalGradientTest(unittest.TestCase):

    def f(self, xs):
        return (xs[0] ** 2,)

    def df(self, xs):
        return ((2 * xs[0],),)

    def setUp(self):
        self.xs = (_uniform(2, 1),)
        self.gys = (_uniform(2, 1),)

    def check_numerical_grad(self, f, df, xs, gys):
        dfxs = df(xs)

        gys = tuple(0 if gy is None else gy for gy in gys)
        # matrix-vector multiplication of dfxs and dys
        dx_expect = map(lambda dfx:
                        sum(map(lambda (a, b): a*b, zip(dfx, gys))), dfxs)

        func = lambda: f(xs)
        dx_actual = gradient_check.numerical_grad(func, xs, gys)

        self.assertEqual(len(dx_expect), len(dx_actual))
        for e, a in zip(dx_expect, dx_actual):
            gradient_check.assert_allclose(e, a)

    @condition.retry(3)
    def test_numerical_grad_cpu(self):
        self.check_numerical_grad(self.f, self.df, self.xs, self.gys)

    @condition.retry(3)
    @attr.gpu
    def test_numerical_grad_gpu(self):
        gys = tuple(None if gy is None else cuda.to_gpu(gy)
                    for gy in self.gys)

        self.check_numerical_grad(self.f, self.df,
                                  map(cuda.to_gpu, self.xs), gys)

class NumericalGradientTest2(NumericalGradientTest):

    f = lambda self, xs: (1,)
    df = lambda self, xs: ((0,),)

def _exp(x):
    if isinstance(x, numpy.ndarray):
        return numpy.exp(x)
    else:
        return cuda.cumath.exp(x)

class NumericalGradientTest3(NumericalGradientTest):

    def f(self, xs):
        return (_exp(xs[0]),)

    def df(self, xs):
        return ((_exp(xs[0]),),)

    def setUp(self):
        self.xs = (_uniform(2, 1),)
        self.gys = (_uniform(2, 1),)

class NumericalGradientTest4(NumericalGradientTest):

    def f(self, xs):
        assert len(xs) == 2
        return (2 * xs[0] + 3 * xs[1],
                4 * xs[0] + 5 * xs[1],
                6 * xs[0] + 7 * xs[1])

    def df(self, xs):
        assert len(xs) == 2
        return (
            (_full_like(xs[0], 2), _full_like(xs[0], 4), _full_like(xs[0], 6)),
            (_full_like(xs[1], 3), _full_like(xs[1], 5), _full_like(xs[1], 7)))

    def setUp(self):
        self.xs = tuple(_uniform(2, 1) for _ in six.moves.range(2))
        self.gys = tuple(_uniform(2, 1) for _ in six.moves.range(3))

class NumericalGradientTest5(NumericalGradientTest4):

    def f(self, xs):
        assert len(xs) == 2
        return (2 * xs[0] + 3 * xs[1],
                4 * xs[0] + 5 * xs[1],
                6 * xs[0] + 7 * xs[1])

    def df(self, xs):
        assert len(xs) == 2
        return (
            (_full_like(xs[0], 2), _zeros_like(xs[0]), _full_like(xs[0], 6)),
            (_full_like(xs[1], 3), _zeros_like(xs[1]), _full_like(xs[1], 7)))

    def setUp(self):
        super(NumericalGradientTest5, self).setUp()
        self.gys = (_uniform(2, 1), None, _uniform(2, 1))

class NumericalGradientTest6(NumericalGradientTest):

    def setUp(self):
        self.xs = (_uniform(2, 1),)
        self.gys = (None,)

class AssertAllCloseTest(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)
        self.y = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)

    def check_identical(self, x):
        gradient_check.assert_allclose(x, x, atol=0, rtol=0)

    @condition.repeat(5)
    def test_identical_cpu(self):
        self.check_identical(self.x)

    @condition.repeat(5)
    @attr.gpu
    def test_identical_gpu(self):
        self.check_identical(cuda.to_gpu(self.x))

    def check_atol(self, x, y):
        x_cpu = cuda.to_cpu(x)
        y_cpu = cuda.to_cpu(y)
        max_abs_diff = numpy.max(numpy.abs(x_cpu - y_cpu))
        with self.assertRaises(AssertionError):
            gradient_check.assert_allclose(x, y, atol=max_abs_diff - 1, rtol=0)
        gradient_check.assert_allclose(x, y, atol=max_abs_diff + 1, rtol=0)

    @condition.repeat(5)
    def test_atol_cpu(self):
        self.check_atol(self.x, self.y)

    @condition.repeat(5)
    @attr.gpu
    def test_atol_gpu(self):
        self.check_atol(cuda.to_gpu(self.x), cuda.to_gpu(self.y))

class AssertAllCloseTest2(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)
        self.y = numpy.random.uniform(1, 2, (2, 3)).astype(numpy.float32)

    def check_rtol(self, x, y):
        x_cpu = cuda.to_cpu(x)
        y_cpu = cuda.to_cpu(y)
        max_ratio = numpy.max(numpy.abs(x_cpu - y_cpu) / y_cpu)
        with self.assertRaises(AssertionError):
            gradient_check.assert_allclose(x, y, atol=0, rtol=max_ratio - 1)
        gradient_check.assert_allclose(x, y, atol=0, rtol=max_ratio + 1)

    @condition.repeat(5)
    def test_rtol_cpu(self):
        self.check_rtol(self.x, self.y)

    @condition.repeat(5)
    @attr.gpu
    def test_rtol_gpu(self):
        self.check_rtol(cuda.to_gpu(self.x), cuda.to_gpu(self.y))
<|endoftext|>