<commit_before>#   Copyright 2012 Marcelo Anton
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""
 nose plugin to facilitate the creation of automated tests that access Mongo
 Engine structures.
"""
import os
import shutil
import socket
import sys
import tempfile
import time
import uuid
import inspect
import subprocess

from subprocess import Popen
from nose.plugins import Plugin
from mongoengine.connection import connect

def scan_path(executable="mongod"):
    """Scan the path for a binary.
    """
    for path in os.environ.get("PATH", "").split(":"):
        path = os.path.abspath(path)
        executable_path = os.path.join(path, executable)
        if os.path.exists(executable_path):
            return executable_path

def get_open_port(host="localhost"):
    """Get an open port on the machine.
    """
    temp_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    temp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    temp_sock.bind((host, 0))
    port = temp_sock.getsockname()[1]
    temp_sock.close()
    del temp_sock
    return port

class MongoEnginePlugin(Plugin):
    """A nose plugin to facilitate the creation of automated tests that access
       Mongo Engine structures.
    """

    def __init__(self):
        super(MongoEnginePlugin, self).__init__()
        self.mongodb_param = {}
        self.process = None
        self._running = False
        self.database_name = None
        self.connection = None
        self.clear_context = {}

    def options(self, parser, env=None):
        parser.add_option(
            "--mongoengine",
            action="store_true",
            default=False,
            help="Enable the mongoengine plugin.")

        parser.add_option(
            "--mongoengine-mongodb-bin",
            dest="mongodb_bin",
            action="store",
            default=None,
            help="Optionally specify the path to the mongod executable.")

        parser.add_option(
            "--mongoengine-clear-after-module",
            dest="mongoengine_clear_after_module",
            action="store_true",
            default=False,
            help="Optionally clear data in db after every module of tests.")

        parser.add_option(
            "--mongoengine-clear-after-class",
            dest="mongoengine_clear_after_class",
            action="store_true",
            default=False,
            help="Optionally clear data in db after every class of tests.")

        parser.add_option(
            "--mongoengine-mongodb-port",
            action="store",
            dest="mongodb_port",
            type="int",
            default=0,
            help="Optionally specify the port to run mongodb on.")
        parser.add_option(
            "--mongoengine-mongodb-scripting",
            action="store_true",
            dest="mongodb_scripting",
            default=False,
            help="Optionally enables mongodb script engine.")
        parser.add_option(
            "--mongoengine-mongodb-logpath",
            action="store",
            dest="mongodb_logpath",
            default="/dev/null",
            help=("Optionally store the mongodb "
                  "log here (default is /dev/null)"))
        parser.add_option(
            "--mongoengine-mongodb-prealloc",
            action="store_true",
            dest="mongodb_prealloc",
            default=False,
            help=("Optionally preallocate db files"))

    def configure(self, options, conf):
        """Parse the command line options and start an instance of mongodb
        """
        # This option has to be specified on the command line, to enable the
        # plugin.
        if not options.mongoengine or options.mongodb_bin:
            return

        if not options.mongodb_bin:
            self.mongodb_param['mongodb_bin'] = scan_path()
            if self.mongodb_param['mongodb_bin'] is None:
                raise AssertionError(
                    "Mongodb plugin enabled, but no mongod on path, "
                    "please specify path to binary\n"
                    "ie. --mongodb=/path/to/mongod")
        else:
            self.mongodb_param['mongodb_bin'] = os.path.abspath(
                os.path.expanduser(os.path.expandvars(options.mongodb_bin)))
            if not os.path.exists(self.mongodb_param['mongodb_bin']):
                raise AssertionError(
                    "Invalid mongodb binary %r" % \
                    self.mongodb_param['mongodb_bin'])

        # Its necessary to enable in nose
        self.enabled = True

        db_log_path = os.path.expandvars(os.path.expanduser(
            options.mongodb_logpath))
        try:
            db_file = open(db_log_path, "w")
            db_file.close()
        except Exception, exc:
            raise AssertionError("Invalid log path %r" % exc)

        if not options.mongodb_port:
            self.mongodb_param['db_port'] = get_open_port()
        else:
            self.mongodb_param['db_port'] = options.mongodb_port

        db_prealloc = options.mongodb_prealloc
        db_scripting = options.mongodb_scripting

        self.clear_context['module'] = options.mongoengine_clear_after_module
        self.clear_context['class'] = options.mongoengine_clear_after_class

        # generate random database name
        self.database_name = str(uuid.uuid1())

        #########################################
        # Start a instance of mongo
        #########################################

        # Stores data here
        self.mongodb_param['db_path'] = tempfile.mkdtemp()
        if not os.path.exists(self.mongodb_param['db_path']):
            os.mkdir(self.mongodb_param['db_path'])

        args = [
            self.mongodb_param['mongodb_bin'],
            "--dbpath",
            self.mongodb_param['db_path'],
            "--port",
            str(self.mongodb_param['db_port']),
            # don't flood stdout, we're not reading it
            "--quiet",
            # save the port
            "--nohttpinterface",
            # disable unused.
            "--nounixsocket",
            # use a smaller default file size
            "--smallfiles",
            # journaling on by default in 2.0 and makes it to slow
            # for tests, can causes failures in jenkins
            "--nojournal",
            # Default is /dev/null
            "--logpath",
            db_log_path,
            "-vvvvv"
            ]

        if not db_prealloc:
            args.append("--noprealloc")

        if not db_scripting:
            args.append("--noscripting")

        self.process = Popen(
            args,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT
            )

        self._running = True
        os.environ["TEST_MONGODB"] = "localhost:%s" % \
                                     self.mongodb_param['db_port']
        os.environ["TEST_MONGODB_DATABASE"] = self.database_name

        # Give a moment for mongodb to finish coming up
        time.sleep(0.3)

        # Connecting using mongoengine
        self.connection = connect(self.database_name, host="localhost",
                port=self.mongodb_param['db_port'])

    def stopContext(self, context):
        """Clear the database if so configured for this
        """

        # Use pymongo directly to drop all collections of created db
        if ((self.clear_context['module'] and inspect.ismodule(context)) or
           (self.clear_context['class'] and inspect.isclass(context))):
            self.connection.drop_database(self.database_name)

    def finalize(self, result):
        """Stop the mongodb instance.
        """

        if not self._running:
            return

        # Clear out the env variable.
        del os.environ["TEST_MONGODB"]
        del os.environ["TEST_MONGODB_DATABASE"]

        # Kill the mongod process
        if sys.platform == 'darwin':
            self.process.kill()
        else:
            self.process.terminate()
        self.process.wait()

        # Clean out the test data.
        shutil.rmtree(self.mongodb_param['db_path'])
        self._running = False
<commit_msg>Correct error message if not found mongodb binary.<commit_after>#   Copyright 2012 Marcelo Anton
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""
 nose plugin to facilitate the creation of automated tests that access Mongo
 Engine structures.
"""
import os
import shutil
import socket
import sys
import tempfile
import time
import uuid
import inspect
import subprocess

from subprocess import Popen
from nose.plugins import Plugin
from mongoengine.connection import connect

def scan_path(executable="mongod"):
    """Scan the path for a binary.
    """
    for path in os.environ.get("PATH", "").split(":"):
        path = os.path.abspath(path)
        executable_path = os.path.join(path, executable)
        if os.path.exists(executable_path):
            return executable_path

def get_open_port(host="localhost"):
    """Get an open port on the machine.
    """
    temp_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    temp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    temp_sock.bind((host, 0))
    port = temp_sock.getsockname()[1]
    temp_sock.close()
    del temp_sock
    return port

class MongoEnginePlugin(Plugin):
    """A nose plugin to facilitate the creation of automated tests that access
       Mongo Engine structures.
    """

    def __init__(self):
        super(MongoEnginePlugin, self).__init__()
        self.mongodb_param = {}
        self.process = None
        self._running = False
        self.database_name = None
        self.connection = None
        self.clear_context = {}

    def options(self, parser, env=None):
        parser.add_option(
            "--mongoengine",
            action="store_true",
            default=False,
            help="Enable the mongoengine plugin.")

        parser.add_option(
            "--mongoengine-mongodb-bin",
            dest="mongodb_bin",
            action="store",
            default=None,
            help="Optionally specify the path to the mongod executable.")

        parser.add_option(
            "--mongoengine-clear-after-module",
            dest="mongoengine_clear_after_module",
            action="store_true",
            default=False,
            help="Optionally clear data in db after every module of tests.")

        parser.add_option(
            "--mongoengine-clear-after-class",
            dest="mongoengine_clear_after_class",
            action="store_true",
            default=False,
            help="Optionally clear data in db after every class of tests.")

        parser.add_option(
            "--mongoengine-mongodb-port",
            action="store",
            dest="mongodb_port",
            type="int",
            default=0,
            help="Optionally specify the port to run mongodb on.")
        parser.add_option(
            "--mongoengine-mongodb-scripting",
            action="store_true",
            dest="mongodb_scripting",
            default=False,
            help="Optionally enables mongodb script engine.")
        parser.add_option(
            "--mongoengine-mongodb-logpath",
            action="store",
            dest="mongodb_logpath",
            default="/dev/null",
            help=("Optionally store the mongodb "
                  "log here (default is /dev/null)"))
        parser.add_option(
            "--mongoengine-mongodb-prealloc",
            action="store_true",
            dest="mongodb_prealloc",
            default=False,
            help=("Optionally preallocate db files"))

    def configure(self, options, conf):
        """Parse the command line options and start an instance of mongodb
        """
        # This option has to be specified on the command line, to enable the
        # plugin.
        if not options.mongoengine or options.mongodb_bin:
            return

        if not options.mongodb_bin:
            self.mongodb_param['mongodb_bin'] = scan_path()
            if self.mongodb_param['mongodb_bin'] is None:
                raise AssertionError(
                    "Mongoengine plugin enabled, but no mongod on path, "
                    "please specify path to binary\n"
                    "ie. --mongoengine-mongodb=/path/to/mongod")
        else:
            self.mongodb_param['mongodb_bin'] = os.path.abspath(
                os.path.expanduser(os.path.expandvars(options.mongodb_bin)))
            if not os.path.exists(self.mongodb_param['mongodb_bin']):
                raise AssertionError(
                    "Invalid mongodb binary %r" % \
                    self.mongodb_param['mongodb_bin'])

        # Its necessary to enable in nose
        self.enabled = True

        db_log_path = os.path.expandvars(os.path.expanduser(
            options.mongodb_logpath))
        try:
            db_file = open(db_log_path, "w")
            db_file.close()
        except Exception, exc:
            raise AssertionError("Invalid log path %r" % exc)

        if not options.mongodb_port:
            self.mongodb_param['db_port'] = get_open_port()
        else:
            self.mongodb_param['db_port'] = options.mongodb_port

        db_prealloc = options.mongodb_prealloc
        db_scripting = options.mongodb_scripting

        self.clear_context['module'] = options.mongoengine_clear_after_module
        self.clear_context['class'] = options.mongoengine_clear_after_class

        # generate random database name
        self.database_name = str(uuid.uuid1())

        #########################################
        # Start a instance of mongo
        #########################################

        # Stores data here
        self.mongodb_param['db_path'] = tempfile.mkdtemp()
        if not os.path.exists(self.mongodb_param['db_path']):
            os.mkdir(self.mongodb_param['db_path'])

        args = [
            self.mongodb_param['mongodb_bin'],
            "--dbpath",
            self.mongodb_param['db_path'],
            "--port",
            str(self.mongodb_param['db_port']),
            # don't flood stdout, we're not reading it
            "--quiet",
            # save the port
            "--nohttpinterface",
            # disable unused.
            "--nounixsocket",
            # use a smaller default file size
            "--smallfiles",
            # journaling on by default in 2.0 and makes it to slow
            # for tests, can causes failures in jenkins
            "--nojournal",
            # Default is /dev/null
            "--logpath",
            db_log_path,
            "-vvvvv"
            ]

        if not db_prealloc:
            args.append("--noprealloc")

        if not db_scripting:
            args.append("--noscripting")

        self.process = Popen(
            args,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT
            )

        self._running = True
        os.environ["TEST_MONGODB"] = "localhost:%s" % \
                                     self.mongodb_param['db_port']
        os.environ["TEST_MONGODB_DATABASE"] = self.database_name

        # Give a moment for mongodb to finish coming up
        time.sleep(0.3)

        # Connecting using mongoengine
        self.connection = connect(self.database_name, host="localhost",
                port=self.mongodb_param['db_port'])

    def stopContext(self, context):
        """Clear the database if so configured for this
        """

        # Use pymongo directly to drop all collections of created db
        if ((self.clear_context['module'] and inspect.ismodule(context)) or
           (self.clear_context['class'] and inspect.isclass(context))):
            self.connection.drop_database(self.database_name)

    def finalize(self, result):
        """Stop the mongodb instance.
        """

        if not self._running:
            return

        # Clear out the env variable.
        del os.environ["TEST_MONGODB"]
        del os.environ["TEST_MONGODB_DATABASE"]

        # Kill the mongod process
        if sys.platform == 'darwin':
            self.process.kill()
        else:
            self.process.terminate()
        self.process.wait()

        # Clean out the test data.
        shutil.rmtree(self.mongodb_param['db_path'])
        self._running = False
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
venvfreezer -- helper script for pip and venv

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import hashlib
import logging
import os
import os.path
import subprocess
import sys
import venv
import itertools
import glob

logger = logging.getLogger(__name__)

class PipHelper:
    """
    Helper Class to run pip commands in the venv
    """

    def __init__(self, python_exe):
        """
        :param python_exe: path to the venv or the python exe in the venv
        """
        if os.path.isdir(python_exe):
            if sys.platform == 'win32':
                binname = 'Scripts'
            else:
                binname = 'bin'
            binpath = os.path.join(os.path.abspath(python_exe), binname)
            if sys.platform == 'darwin' and \
                    '__PYVENV_LAUNCHER__' in os.environ:
                executable = os.environ['__PYVENV_LAUNCHER__']
            else:
                executable = sys.executable
            _, exename = os.path.split(os.path.abspath(executable))
            python_exe = os.path.join(binpath, exename)
        self.pip_cmd = [python_exe, '-Im', 'pip']

    def install(self, *params):
        """
        Calls pip install with the given parameters.

        :param params: List of parameters
        """
        return subprocess.check_call(
            self.pip_cmd + ['install'] +  list(params))

    def freeze(self, freeze_filepath):
        """
        Calls pip freeze and stores the output under the given filepath.

        :param freeze_filepath: Path to store the freeze output
        """
        with open(freeze_filepath, mode='w', newline='\n') as freeze_file:
            # Using check_output for line ending conversion to \n
            output = subprocess.check_output(
                self.pip_cmd + ['freeze'],
                universal_newlines=True)
            freeze_file.write(output)

    def download(self, download_folder, *params):
        """
        Calls pip download and stores all requirements in the given folder.

        :param download_folder: Path to the download folder
        """
        return subprocess.check_call(
            self.pip_cmd + ['download', '-d', download_folder] + list(params))

def checksum_of_file(filepath):
    """
    Creates a checksum of the given filepath.

    :params filepath: Path to the file
    """
    with open(filepath, mode='rb') as f:
        checksum = hashlib.sha256(f.read()).hexdigest()
        logger.debug("Checksum of %s was %s", filepath, checksum)
        return checksum

class VenvFreezerException(Exception):
    """ Base Exception for AuteEnv """

    def __init__(self, message):
        """
        :params message: Message that desribes the error
        """
        self.message = message
        super().__init__(message)

class VenvFreezer(venv.EnvBuilder):
    """
    EnvBuilder that automatically installs all dependencies from
    the requirements.txt
    """

    CHECKSUM_FILENAME = 'requirements.sha256'
    REQUIREMENTS_FILENAMES = ['requirements.txt', 'requirements.*.txt']
    FREEZE_FILENAME = 'requirements.lock'
    DOWNLOAD_FOLDER = 'requirements'

    def __init__(self, use_index=None, update_pip=False,
                 update_setuptools=False, symlinks=False):
        self.use_index = use_index
        self.update_pip = update_pip
        self.update_setuptools = update_setuptools
        super().__init__(
            system_site_packages=False, clear=True,
            symlinks=False, upgrade=False, with_pip=True)

    def _freeze(self, pip):
        filepath = os.path.abspath(self.FREEZE_FILENAME)
        logger.info('Creating freeze file: %s', filepath)
        pip.freeze(filepath)

    def download(self, env_dir):
        """
        Downloads all requirements into the download folder.
        """
        if not self._is_frozen():
            raise VenvFreezerException(
                "VENV is not frozen! Run setup before download.")
        pip = PipHelper(env_dir)
        pip.download(self.DOWNLOAD_FOLDER, '-r', self.FREEZE_FILENAME)

    def create(self, env_dir):
        if not self._check_venv(env_dir):
            logger.info('Recreating venv.')
            super().create(env_dir)

    def _check_venv(self, env_dir):
        if not os.path.isdir(env_dir):
            logger.info('No venv found.')
            return False
        if not self._is_frozen():
            logger.info('Venv not frozen.')
            return False
        logger.info('Checking current venv.')
        checksum_filepath = os.path.join(env_dir, self.CHECKSUM_FILENAME)
        if not os.path.isfile(checksum_filepath):
            logger.info('No checksum file found.')
            return False
        with open(checksum_filepath, mode='r') as f:
            checksum = f.read()
        if checksum != checksum_of_file(self.FREEZE_FILENAME):
            logger.info('Invalid checksum found.')
            return False
        logger.info('Venv is up to date.')
        return True

    def post_setup(self, context):
        """
        Set up any packages which need to be pre-installed into the
        environment being created.
        :param context: The information for the environment creation request
                        being processed.
        """
        os.environ['VIRTUAL_ENV'] = context.env_dir
        pip = PipHelper(context.env_exe)
        if self.update_setuptools:
            logger.info('Updateing setuptools ...')
            pip.install('-U', 'setuptools')
        if self.update_pip:
            logger.info('Updateing pip ...')
            pip.install('-U', 'pip')
        self._install_requirements(pip)
        if not self._is_frozen():
            self._freeze(pip)
        self._create_checksum_file(context)

    def _get_requirements_files_parameters(self):
        if self._is_frozen():
            logger.info('Using freeze file: %s', self.FREEZE_FILENAME)
            return ['-r', self.FREEZE_FILENAME]
        req_files = [
            filename for filename in itertools.chain.from_iterable(
                glob.glob(filepattern) for filepattern
                in self.REQUIREMENTS_FILENAMES)
            if filename]
        logger.info("Using requirements files: %s", req_files)
        return list(itertools.chain.from_iterable(
            ('-r', filename) for filename in req_files))

    def _install_requirements(self, pip):
        logger.info('Running pip install ...')
        cmd = self._get_requirements_files_parameters()
        # Checking requirements folder
        if os.path.isdir('requirements'):
            cmd.extend(['-f', 'requirements'])
        # setting no-index
        if self.use_index is False:
            cmd.append('--no-index')
        pip.install(*cmd)

    def _create_checksum_file(self, context):
        logger.info('Creating checksum of %s ...', self.FREEZE_FILENAME)
        checksum = checksum_of_file(self.FREEZE_FILENAME)
        checksum_filepath = os.path.join(
            context.env_dir, self.CHECKSUM_FILENAME)
        with open(checksum_filepath, mode='w') as f:
            f.write(checksum)

    def _is_frozen(self):
        return os.path.isfile(self.FREEZE_FILENAME)

def main(args=None):
    """ Main function """
    # Setup argparse.
    import argparse
    parser = argparse.ArgumentParser(
        prog=__name__ if __name__ != '__main__' else 'venvfreeze.py',
        description="Creates a virtual Python environment with all"
                    "dependencies from the requirements.txt installed.")
    parser.add_argument(
        '--update-setuptools',
        default=False, action='store_true', dest='update_setuptools',
        help='Updates setuptools in the environment.')
    parser.add_argument(
        '--update-pip',
        default=False, action='store_true', dest='update_pip',
        help='Updates pip in the environment.')
    parser.add_argument(
        '--symlinks',
        default=False, action='store_true', dest='symlinks',
        help='Try to use symlinks rather than copies, when symlinks are not '
             'the default for the platform.')
    venv_path_basedir = '.'
    venv_path = os.path.join(venv_path_basedir, 'venv')
    parser.add_argument('-p', '--path', dest='venv_path', default=venv_path,
                        help="Sets the path to the venv.")
    parser.add_argument("-v", "--verbosity", action="count",
                        help="increase output verbosity")
    parser.add_argument(
        '--no-index', action='store_false', dest='use_index', default=True,
        help='Force disable pip index.')
    parser.add_argument(
        'mode', choices=['setup', 'download'],
        default='setup', nargs='?',
        help="Use 'setup' to create the venv.\n"
             "Use 'download' to download all requirements into a folder "
             "('./requirements').")

    # Parse the arguments
    options = parser.parse_args(args)

    # Setup Logger
    logger.setLevel(logging.DEBUG)
    console_handler = logging.StreamHandler()
    if options.verbosity:
        console_handler.setLevel(logging.DEBUG)
    else:
        console_handler.setLevel(logging.INFO)
    # create formatter
    formatter = logging.Formatter(
        '%(asctime)s %(levelname)s - %(message)s')
    console_handler.setFormatter(formatter)
    # add console_handler to logger
    logger.addHandler(console_handler)

    # Run VenvFreezer
    builder = VenvFreezer(update_pip=options.update_pip,
                          update_setuptools=options.update_setuptools,
                          symlinks=options.symlinks)
    try:
        if options.mode == 'download':
            builder.download(options.venv_path)
        else:
            builder.create(options.venv_path)
    except VenvFreezerException as venv_exception:
        logger.error(venv_exception.message)

if __name__ == '__main__':
    main()
<commit_msg>Removed _freeze method<commit_after>#!/usr/bin/env python
"""
venvfreezer -- helper script for pip and venv

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import hashlib
import logging
import os
import os.path
import subprocess
import sys
import venv
import itertools
import glob

logger = logging.getLogger(__name__)

class PipHelper:
    """
    Helper Class to run pip commands in the venv
    """

    def __init__(self, python_exe):
        """
        :param python_exe: path to the venv or the python exe in the venv
        """
        if os.path.isdir(python_exe):
            if sys.platform == 'win32':
                binname = 'Scripts'
            else:
                binname = 'bin'
            binpath = os.path.join(os.path.abspath(python_exe), binname)
            if sys.platform == 'darwin' and \
                    '__PYVENV_LAUNCHER__' in os.environ:
                executable = os.environ['__PYVENV_LAUNCHER__']
            else:
                executable = sys.executable
            _, exename = os.path.split(os.path.abspath(executable))
            python_exe = os.path.join(binpath, exename)
        self.pip_cmd = [python_exe, '-Im', 'pip']

    def install(self, *params):
        """
        Calls pip install with the given parameters.

        :param params: List of parameters
        """
        return subprocess.check_call(
            self.pip_cmd + ['install'] +  list(params))

    def freeze(self, freeze_filepath):
        """
        Calls pip freeze and stores the output under the given filepath.

        :param freeze_filepath: Path to store the freeze output
        """
        with open(freeze_filepath, mode='w', newline='\n') as freeze_file:
            # Using check_output for line ending conversion to \n
            output = subprocess.check_output(
                self.pip_cmd + ['freeze'],
                universal_newlines=True)
            freeze_file.write(output)

    def download(self, download_folder, *params):
        """
        Calls pip download and stores all requirements in the given folder.

        :param download_folder: Path to the download folder
        """
        return subprocess.check_call(
            self.pip_cmd + ['download', '-d', download_folder] + list(params))

def checksum_of_file(filepath):
    """
    Creates a checksum of the given filepath.

    :params filepath: Path to the file
    """
    with open(filepath, mode='rb') as f:
        checksum = hashlib.sha256(f.read()).hexdigest()
        logger.debug("Checksum of %s was %s", filepath, checksum)
        return checksum

class VenvFreezerException(Exception):
    """ Base Exception for AuteEnv """

    def __init__(self, message):
        """
        :params message: Message that desribes the error
        """
        self.message = message
        super().__init__(message)

class VenvFreezer(venv.EnvBuilder):
    """
    EnvBuilder that automatically installs all dependencies from
    the requirements.txt
    """

    CHECKSUM_FILENAME = 'requirements.sha256'
    REQUIREMENTS_FILENAMES = ['requirements.txt', 'requirements.*.txt']
    FREEZE_FILENAME = 'requirements.lock'
    DOWNLOAD_FOLDER = 'requirements'

    def __init__(self, use_index=None, update_pip=False,
                 update_setuptools=False, symlinks=False):
        self.use_index = use_index
        self.update_pip = update_pip
        self.update_setuptools = update_setuptools
        super().__init__(
            system_site_packages=False, clear=True,
            symlinks=False, upgrade=False, with_pip=True)

    def download(self, env_dir):
        """
        Downloads all requirements into the download folder.
        """
        if not self._is_frozen():
            raise VenvFreezerException(
                "VENV is not frozen! Run setup before download.")
        pip = PipHelper(env_dir)
        pip.download(self.DOWNLOAD_FOLDER, '-r', self.FREEZE_FILENAME)

    def create(self, env_dir):
        if not self._check_venv(env_dir):
            logger.info('Recreating venv.')
            super().create(env_dir)

    def _check_venv(self, env_dir):
        if not os.path.isdir(env_dir):
            logger.info('No venv found.')
            return False
        if not self._is_frozen():
            logger.info('Venv not frozen.')
            return False
        logger.info('Checking current venv.')
        checksum_filepath = os.path.join(env_dir, self.CHECKSUM_FILENAME)
        if not os.path.isfile(checksum_filepath):
            logger.info('No checksum file found.')
            return False
        with open(checksum_filepath, mode='r') as f:
            checksum = f.read()
        if checksum != checksum_of_file(self.FREEZE_FILENAME):
            logger.info('Invalid checksum found.')
            return False
        logger.info('Venv is up to date.')
        return True

    def post_setup(self, context):
        """
        Set up any packages which need to be pre-installed into the
        environment being created.
        :param context: The information for the environment creation request
                        being processed.
        """
        os.environ['VIRTUAL_ENV'] = context.env_dir
        pip = PipHelper(context.env_exe)
        if self.update_setuptools:
            logger.info('Updateing setuptools ...')
            pip.install('-U', 'setuptools')
        if self.update_pip:
            logger.info('Updateing pip ...')
            pip.install('-U', 'pip')
        self._install_requirements(pip)
        if not self._is_frozen():
            filepath = os.path.abspath(self.FREEZE_FILENAME)
            logger.info('Creating freeze file: %s', filepath)
            pip.freeze(filepath)
        self._create_checksum_file(context)

    def _get_requirements_files_parameters(self):
        if self._is_frozen():
            logger.info('Using freeze file: %s', self.FREEZE_FILENAME)
            return ['-r', self.FREEZE_FILENAME]
        req_files = [
            filename for filename in itertools.chain.from_iterable(
                glob.glob(filepattern) for filepattern
                in self.REQUIREMENTS_FILENAMES)
            if filename]
        logger.info("Using requirements files: %s", req_files)
        return list(itertools.chain.from_iterable(
            ('-r', filename) for filename in req_files))

    def _install_requirements(self, pip):
        logger.info('Running pip install ...')
        cmd = self._get_requirements_files_parameters()
        # Checking requirements folder
        if os.path.isdir(self.DOWNLOAD_FOLDER):
            cmd.extend(['-f', self.DOWNLOAD_FOLDER])
        # setting no-index
        if self.use_index is False:
            cmd.append('--no-index')
        pip.install(*cmd)

    def _create_checksum_file(self, context):
        logger.info('Creating checksum of %s ...', self.FREEZE_FILENAME)
        checksum = checksum_of_file(self.FREEZE_FILENAME)
        checksum_filepath = os.path.join(
            context.env_dir, self.CHECKSUM_FILENAME)
        with open(checksum_filepath, mode='w') as f:
            f.write(checksum)

    def _is_frozen(self):
        return os.path.isfile(self.FREEZE_FILENAME)

def main(args=None):
    """ Main function """
    # Setup argparse.
    import argparse
    parser = argparse.ArgumentParser(
        prog=__name__ if __name__ != '__main__' else 'venvfreeze.py',
        description="Creates a virtual Python environment with all"
                    "dependencies from the requirements.txt installed.")
    parser.add_argument(
        '--update-setuptools',
        default=False, action='store_true', dest='update_setuptools',
        help='Updates setuptools in the environment.')
    parser.add_argument(
        '--update-pip',
        default=False, action='store_true', dest='update_pip',
        help='Updates pip in the environment.')
    parser.add_argument(
        '--symlinks',
        default=False, action='store_true', dest='symlinks',
        help='Try to use symlinks rather than copies, when symlinks are not '
             'the default for the platform.')
    venv_path_basedir = '.'
    venv_path = os.path.join(venv_path_basedir, 'venv')
    parser.add_argument('-p', '--path', dest='venv_path', default=venv_path,
                        help="Sets the path to the venv.")
    parser.add_argument("-v", "--verbosity", action="count",
                        help="increase output verbosity")
    parser.add_argument(
        '--no-index', action='store_false', dest='use_index', default=True,
        help='Force disable pip index.')
    parser.add_argument(
        'mode', choices=['setup', 'download'],
        default='setup', nargs='?',
        help="Use 'setup' to create the venv.\n"
             "Use 'download' to download all requirements into a folder "
             "('./requirements').")

    # Parse the arguments
    options = parser.parse_args(args)

    # Setup Logger
    logger.setLevel(logging.DEBUG)
    console_handler = logging.StreamHandler()
    if options.verbosity:
        console_handler.setLevel(logging.DEBUG)
    else:
        console_handler.setLevel(logging.INFO)
    # create formatter
    formatter = logging.Formatter(
        '%(asctime)s %(levelname)s - %(message)s')
    console_handler.setFormatter(formatter)
    # add console_handler to logger
    logger.addHandler(console_handler)

    # Run VenvFreezer
    builder = VenvFreezer(update_pip=options.update_pip,
                          update_setuptools=options.update_setuptools,
                          symlinks=options.symlinks)
    try:
        if options.mode == 'download':
            builder.download(options.venv_path)
        else:
            builder.create(options.venv_path)
    except VenvFreezerException as venv_exception:
        logger.error(venv_exception.message)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import standard_play
import behavior
import skills
import tactics
import main
import robocup
import evaluation
import constants
import math

class Basic122(standard_play.StandardPlay):

    # how far the 2 support robots should stay away from the striker
    SupportAvoidTeammateRadius = 0.5

    OffenseSupportRatio = 0.7
    DefenseSupportRatio = 0.9

    # if an opponent is this close to the striker, we don't instruct the support bots to mark him
    SupportBackoffThresh = 1.5

    # multiplier used to decide when it's worth it to reassign the bot a support is marking
    MarkHysteresisCoeff = 0.9

    def __init__(self):
        super().__init__(continuous=False)  # FIXME: continuous?

        striker = skills.pivot_kick.PivotKick()
        striker.aim_params['error_threshold'] = 0.15
        striker.aim_params['max_steady_ang_vel'] = 7
        striker.aim_params['min_steady_duration'] = 0.1
        striker.aim_params['desperate_timeout'] = 2.5

        self.add_transition(behavior.Behavior.State.start,
                            behavior.Behavior.State.running, lambda: True,
                            "immediately")

        striker.add_transition(behavior.Behavior.State.completed,
                               behavior.Behavior.State.start, lambda: True,
                               "immediately")
        self.add_subbehavior(striker, 'striker', required=False, priority=3)

        support1 = skills.mark.Mark()
        support1.mark_line_thresh = 1.0
        self.add_subbehavior(support1, 'support1', required=False, priority=2)

        support2 = skills.mark.Mark()
        support2.mark_line_thresh = 1.0
        self.add_subbehavior(support2, 'support2', required=False, priority=1)

    @classmethod
    def score(cls):
        return 10 if main.game_state().is_playing() else float("inf")

    def execute_running(self):
        super().execute_running()
        striker = self.subbehavior_with_name('striker')
        support1 = self.subbehavior_with_name('support1')
        support2 = self.subbehavior_with_name('support2')
        supports = [support1, support2]

        # project ball location a bit into the future
        ball_proj = evaluation.ball.predict(main.ball().pos,
                                            main.ball().vel,
                                            t=0.75)

        # find closest opponent to striker
        closest_dist_to_striker, closest_opp_to_striker = float("inf"), None
        if striker.robot != None:
            for opp in main.their_robots():
                d = opp.pos.dist_to(striker.robot.pos)
                if d < closest_dist_to_striker:
                    closest_dist_to_striker, closest_opp_to_striker = d, opp

        striker_engaged = striker.robot != None and closest_dist_to_striker < Basic122.SupportBackoffThresh

        # pick out which opponents our defenders should 'mark'
        # TODO: explain
        nrOppClose = 0
        bestOpp1, bestOpp2 = None, None
        bestDistSq1, bestDistSq2 = float("inf"), float("inf")
        for opp in main.their_robots():
            # use dist from goal rather than just y-coord to handle corners better
            oppDistSq = opp.pos.magsq()
            if not (striker_engaged and opp == closest_opp_to_striker):
                if oppDistSq < bestDistSq1:
                    bestDistSq2, bestOpp2 = bestDistSq1, bestOpp1
                    bestDistSq1, bestOpp1 = oppDistSq, opp
                elif oppDistSq < bestDistSq2:
                    bestDistSq2, bestOpp2 = oppDistSq, opp

                if oppDistSq < constants.Field.Length**2 / 4.0:
                    nrOppClose += 1

        # handle the case of having no good robots to mark
        if bestOpp1 == None and support1.robot != None:
            support1.mark_robot = None
            support1.robot.add_text("No mark target", (255, 255, 255),
                                    "RobotText")
            if striker.robot != None:
                support1.robot.add_text("Static Support", (255, 255, 255),
                                        "RobotText")
                support_goal = striker.robot.pos
                support_goal.x *= -1.0
                if abs(support_goal.x) < 0.2:
                    support_goal.x = -1.0 if support_goal.x < 0 else 1.0

                if ball_proj.y > constants.Field.Length / 2.0 and nrOppClose > 0:
                    support_goal.y = max(support_goal.y *
                                         Basic122.OffenseSupportRatio, 0.3)
                else:
                    support_goal.y = max(support_goal.y *
                                         Basic122.DefenseSupportRatio, 0.3)

                support1.robot.move_to(support_goal)
                support1.robot.face(ball_proj)

        # sit around and do jack shit...
        # TODO: make it do something useful
        if bestOpp2 == None and support2.robot != None:
            support2.mark_robot = None
            support2.robot.add_text("No mark target", (255, 255, 255),
                                    "RobotText")
            if striker.robot != None:
                support2.robot.add_text("Static Support", (255, 255, 255),
                                        "RobotText")
                support_goal = striker.robot.pos
                support_goal.x *= -1.0
                if abs(support_goal.x) < 0.2:
                    support_goal.x = -1.0 if support_goal.x < 0 else 1.0

                if ball_proj.y > constants.Field.Length / 2.0 and nrOppClose > 0:
                    support_goal.y = max(support_goal.y *
                                         Basic122.OffenseSupportRatio, 0.3)
                else:
                    support_goal.y = max(support_goal.y *
                                         Basic122.DefenseSupportRatio, 0.3)

                support2.robot.move_to(support_goal)
                support2.robot.face(ball_proj)

        # reassign support robots' mark targets based on dist sq and hysteresis coeff
        new_dists = [bestDistSq1, bestDistSq2]
        new_bots = [bestOpp1, bestOpp2]
        for i in range(2):
            support = supports[i]
            if new_bots[i] != None:
                cur_dist_sq = (
                    support.mark_robot.pos -
                    ball_proj).magsq() if support.mark_robot else float("inf")
                if new_dists[i] < cur_dist_sq * Basic122.MarkHysteresisCoeff:
                    support.mark_robot = new_bots[i]

        # if the supports are farther from the ball, they can mark further away
        if ball_proj.y > constants.Field.Length / 2.0 and nrOppClose > 0:
            for support in supports:
                support.ratio = Basic122.OffenseSupportRatio
        else:
            for support in supports:
                support.ratio = Basic122.DefenseSupportRatio

            # raise NotImplementedError("Make support robots avoid the shot channel")
            # FROM C++:
            # Polygon shot_obs;
            # shot_obs.vertices.push_back(Geometry2d::Point(Field_GoalWidth / 2, Field_Length));
            # shot_obs.vertices.push_back(Geometry2d::Point(-Field_GoalWidth / 2, Field_Length));
            # shot_obs.vertices.push_back(ballProj);

            # TODO: this would be a good place for a "keep-trying container behavior"
            # make the kicker try again if it already kicked
        if not striker.is_in_state(behavior.Behavior.State.running):
            striker.restart()
<commit_msg>Fixed both robots going to the ball<commit_after>import standard_play
import behavior
import skills
import tactics
import main
import robocup
import evaluation
import constants
import math

class Basic122(standard_play.StandardPlay):

    # how far the 2 support robots should stay away from the striker
    SupportAvoidTeammateRadius = 5#0.5

    OffenseSupportRatio = 0.7
    DefenseSupportRatio = 0.9

    # if an opponent is this close to the striker, we don't instruct the support bots to mark him
    SupportBackoffThresh = 10#1.5

    # multiplier used to decide when it's worth it to reassign the bot a support is marking
    MarkHysteresisCoeff = 0.9

    def __init__(self):
        super().__init__(continuous=False)  # FIXME: continuous?

        striker = skills.pivot_kick.PivotKick()
        striker.aim_params['error_threshold'] = 0.15
        striker.aim_params['max_steady_ang_vel'] = 7
        striker.aim_params['min_steady_duration'] = 0.1
        striker.aim_params['desperate_timeout'] = 2.5

        self.add_transition(behavior.Behavior.State.start,
                            behavior.Behavior.State.running, lambda: True,
                            "immediately")

        striker.add_transition(behavior.Behavior.State.completed,
                               behavior.Behavior.State.start, lambda: True,
                               "immediately")
        self.add_subbehavior(striker, 'striker', required=False, priority=3)

        support1 = skills.mark.Mark()
        support1.mark_line_thresh = 1.0
        self.add_subbehavior(support1, 'support1', required=False, priority=2)

        support2 = skills.mark.Mark()
        support2.mark_line_thresh = 1.0
        self.add_subbehavior(support2, 'support2', required=False, priority=1)

    @classmethod
    def score(cls):
        return 10 if main.game_state().is_playing() else float("inf")

    def execute_running(self):
        super().execute_running()
        striker = self.subbehavior_with_name('striker')
        support1 = self.subbehavior_with_name('support1')
        support2 = self.subbehavior_with_name('support2')
        supports = [support1, support2]

        # project ball location a bit into the future
        ball_proj = evaluation.ball.predict(main.ball().pos,
                                            main.ball().vel,
                                            t=0.75)

        # find closest opponent to striker
        closest_dist_to_striker, closest_opp_to_striker = float("inf"), None
        if striker.robot != None:
            for opp in main.their_robots():
                d = opp.pos.dist_to(striker.robot.pos)
                if d < closest_dist_to_striker:
                    closest_dist_to_striker, closest_opp_to_striker = d, opp

        striker_engaged = striker.robot != None and closest_dist_to_striker < Basic122.SupportBackoffThresh

        # pick out which opponents our defenders should 'mark'
        # TODO: explain
        nrOppClose = 0
        bestOpp1, bestOpp2 = None, None
        bestDistSq1, bestDistSq2 = float("inf"), float("inf")
        for opp in main.their_robots():
            # use dist from goal rather than just y-coord to handle corners better
            oppDistSq = opp.pos.magsq()
            if not (striker_engaged and opp == closest_opp_to_striker):
                if oppDistSq < bestDistSq1:
                    bestDistSq2, bestOpp2 = bestDistSq1, bestOpp1
                    bestDistSq1, bestOpp1 = oppDistSq, opp
                elif oppDistSq < bestDistSq2:
                    bestDistSq2, bestOpp2 = oppDistSq, opp

                if oppDistSq < constants.Field.Length**2 / 4.0:
                    nrOppClose += 1

        # handle the case of having no good robots to mark
        if bestOpp1 == None and support1.robot != None:
            support1.mark_robot = None
            support1.robot.add_text("No mark target", (255, 255, 255),
                                    "RobotText")
            if striker.robot != None:
                support1.robot.add_text("Static Support", (255, 255, 255),
                                        "RobotText")
                support_goal = striker.robot.pos
                support_goal.x *= -1.0
                if abs(support_goal.x) < 0.2:
                    support_goal.x = -1.0 if support_goal.x < 0 else 1.0

                if ball_proj.y > constants.Field.Length / 2.0 and nrOppClose > 0:
                    support_goal.y = max(support_goal.y *
                                         Basic122.OffenseSupportRatio, 0.3)
                else:
                    support_goal.y = max(support_goal.y *
                                         Basic122.DefenseSupportRatio, 0.3)

                support1.robot.move_to(support_goal)
                support1.robot.face(ball_proj)

        # sit around and do jack shit...
        # TODO: make it do something useful
        if bestOpp2 == None and support2.robot != None:
            support2.mark_robot = None
            support2.robot.add_text("No mark target", (255, 255, 255),
                                    "RobotText")
            if striker.robot != None:
                support2.robot.add_text("Static Support", (255, 255, 255),
                                        "RobotText")
                support_goal = striker.robot.pos
                support_goal.x *= -1.0
                if abs(support_goal.x) < 0.2:
                    support_goal.x = -1.0 if support_goal.x < 0 else 1.0

                if ball_proj.y > constants.Field.Length / 2.0 and nrOppClose > 0:
                    support_goal.y = max(support_goal.y *
                                         Basic122.OffenseSupportRatio, 0.3)
                else:
                    support_goal.y = max(support_goal.y *
                                         Basic122.DefenseSupportRatio, 0.3)

                support2.robot.move_to(support_goal)
                support2.robot.face(ball_proj)

        # reassign support robots' mark targets based on dist sq and hysteresis coeff
        new_dists = [bestDistSq1, bestDistSq2]
        new_bots = [bestOpp1, bestOpp2]

        # If one of the robots is marking the robot near the ball
        # Reset them both to another robot
        # Both are done so they won't have the same robot
        clear_all_marks = False
        for i in range(2):
            if (striker_engaged and
                supports[i].mark_robot is not None and
                supports[i].mark_robot == closest_opp_to_striker):
                clear_all_marks = True

        for i in range(2):
            support = supports[i]
            if new_bots[i] != None:
                cur_dist_sq = (
                    support.mark_robot.pos -
                    ball_proj).magsq() if support.mark_robot else float("inf")
                if (new_dists[i] < cur_dist_sq * Basic122.MarkHysteresisCoeff or
                    clear_all_marks):
                    support.mark_robot = new_bots[i]

        # if the supports are farther from the ball, they can mark further away
        if ball_proj.y > constants.Field.Length / 2.0 and nrOppClose > 0:
            for support in supports:
                support.ratio = Basic122.OffenseSupportRatio
        else:
            for support in supports:
                support.ratio = Basic122.DefenseSupportRatio

            # raise NotImplementedError("Make support robots avoid the shot channel")
            # FROM C++:
            # Polygon shot_obs;
            # shot_obs.vertices.push_back(Geometry2d::Point(Field_GoalWidth / 2, Field_Length));
            # shot_obs.vertices.push_back(Geometry2d::Point(-Field_GoalWidth / 2, Field_Length));
            # shot_obs.vertices.push_back(ballProj);

            # TODO: this would be a good place for a "keep-trying container behavior"
            # make the kicker try again if it already kicked
        if not striker.is_in_state(behavior.Behavior.State.running):
            striker.restart()
<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.translation import ugettext_lazy as _
from django.views.generic.base import View
from socialregistration.contrib.openid.client import OpenIDClient
from socialregistration.contrib.openid.models import OpenIDProfile
from socialregistration.mixins import SocialRegistration
from socialregistration.views import SetupCallback

class OpenIDRedirect(SocialRegistration, View):
    client = OpenIDClient

    def post(self, request):
        request.session['next'] = self.get_next(request)

        # We don't want to pass in the whole session object as this might not
        # be pickleable depending on what session backend one is using.
        # See issue #73
        client = self.get_client()(dict(request.session.items()),
            request.POST.get('openid_provider'))

        request.session[self.get_client().get_session_key()] = client

        return HttpResponseRedirect(client.get_redirect_url())

class OpenIDCallback(SocialRegistration, View):
    template_name = 'socialregistration/openid/openid.html'
    profile = OpenIDProfile
    client = OpenIDClient

    def get(self, request):

        client = request.session[self.get_client().get_session_key()]

        client.complete(dict(request.GET.items()), request.get_full_path())

        if not client.is_valid():
            return self.render_to_response(dict(
                error=_("Unfortunately we couldn't validate your identity: %s") % client.result.message))

        # Save the client back to the session or we're not carrying the result
        # to the next view.
        request.session[self.get_client().get_session_key()] = client

        return HttpResponseRedirect(reverse('socialregistration:openid:setup'))

class OpenIDSetup(SetupCallback):
    template_name = 'socialregistration/openid/openid.html'
    profile = OpenIDProfile
    client = OpenIDClient

    def get_lookup_kwargs(self, request, client):
        return {'identity': client.get_identity()}
<commit_msg>handle case when client error does not include message<commit_after>from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.utils.translation import ugettext_lazy as _
from django.views.generic.base import View
from socialregistration.contrib.openid.client import OpenIDClient
from socialregistration.contrib.openid.models import OpenIDProfile
from socialregistration.mixins import SocialRegistration
from socialregistration.views import SetupCallback

class OpenIDRedirect(SocialRegistration, View):
    client = OpenIDClient

    def post(self, request):
        request.session['next'] = self.get_next(request)

        # We don't want to pass in the whole session object as this might not
        # be pickleable depending on what session backend one is using.
        # See issue #73
        client = self.get_client()(dict(request.session.items()),
            request.POST.get('openid_provider'))

        request.session[self.get_client().get_session_key()] = client

        return HttpResponseRedirect(client.get_redirect_url())

class OpenIDCallback(SocialRegistration, View):
    template_name = 'socialregistration/openid/openid.html'
    profile = OpenIDProfile
    client = OpenIDClient

    def get(self, request):

        client = request.session[self.get_client().get_session_key()]

        client.complete(dict(request.GET.items()), request.get_full_path())

        if not client.is_valid():
            if hasattr(client.result, 'message'):
                msg = _("Unfortunately we couldn't validate your identity: %s") % client.result.message
            else:
                msg = _("Unfortunately we couldn't validate your identity")
            return self.render_to_response({'error': msg})

        # Save the client back to the session or we're not carrying the result
        # to the next view.
        request.session[self.get_client().get_session_key()] = client

        return HttpResponseRedirect(reverse('socialregistration:openid:setup'))

class OpenIDSetup(SetupCallback):
    template_name = 'socialregistration/openid/openid.html'
    profile = OpenIDProfile
    client = OpenIDClient

    def get_lookup_kwargs(self, request, client):
        return {'identity': client.get_identity()}
<|endoftext|>


<commit_before>"""
Transform a subset of (tractable) VGDL games into explicit Markov Decision Processes (MDPS).

Given a game, we want to produce a list of unique, non-redundant discrete states S
There is a set of permissible discrete actions A.
Actions and states are identified by their index.

We produce a 3D-array of transition probabilities Ts:
    Ts[action_id][from_state_id][to_state_id] = Prob(nextstate=to_state | from_state,action)

We also produce a reward vector for entering each state
(by default: -1 for losing, +1 for winning, 0 elsewhere)

Finally, we produce a set of features/observations. Two possibilities:
  - fully observable: the observations uniquely determine the underlying state,
     then they are basically a factored representation of the state
  - partially observable: generally from first-person avatar perspective

"""
from scipy import zeros
from pybrain.utilities import flood
from ontology import MovingAvatar, BASEDIRS, RotatingAvatar
from core import VGDLSprite
from tools import listRotate

class MDPconverter(object):
    """ Simplest case: Assume the game has a single avatar,
        with 4 actions, outcomes are deterministic, physics are grid-based,
        and all other sprites are Immovables. """

    def __init__(self, game, verbose=False):
        self.game = game
        self.verbose=verbose
        self.actions = BASEDIRS
        self.sas_tuples = []
        self.rewards = {}
        self.oriented = False

    def convert(self, observations=True):
        if observations:
            obstypes = []
        for _, ss in self.game.sprite_groups.items():
            #find avatar
            if isinstance(ss[0], MovingAvatar):
                self.avatar = ss[0]
            elif observations:
                # retain observable features
                tmp = [self.sprite2state(sprite) for sprite in ss if sprite.is_static]
                obstypes.append(tmp)
        # what type of movement dynamics do we use?
        if isinstance(self.avatar, RotatingAvatar):
            allPos = set([(col, row, dir_) for row in range(self.game.height)
                          for col in range(self.game.width)
                          for dir_ in BASEDIRS])
            self.oriented = True
        else:
            allPos = set([(col, row) for row in range(self.game.height)
                          for col in range(self.game.width)])
        if self.verbose:
            if observations:
                print 'Number of features:', 5*len(obstypes)
            print 'Maximum state space:', len(allPos)
        initSet = [self.sprite2state(self.avatar)]
        self.states = sorted(flood(self.tryMoves, allPos, initSet))
        dim = len(self.states)
        if self.verbose:
            print 'Actual states:', dim
            print 'Non-negative rewards:', self.rewards
            print 'Initial state', initSet[0]
        Ts = [zeros((dim, dim)) for _ in self.actions]
        R = zeros(dim)
        statedic= {}
        actiondic = {}
        for si, pos in enumerate(self.states):
            statedic[pos] = si
        for ai, a in enumerate(self.actions):
            actiondic[a] = ai
        for pos, val in self.rewards.items():
            R[statedic[pos]] += val
        for pos, a, dest in self.sas_tuples:
            ai = actiondic[a]
            si = statedic[pos]
            di = statedic[dest]
            Ts[ai][si, di] += 1
        if self.verbose:
            print 'Built Ts.'
        for T in Ts:
            for row in T:
                row /= sum(row)
        if self.verbose:
            print 'Normalized Ts.'
        if observations:
            # one observation for current position and each of the 4 neighbors.
            fMap = zeros((len(obstypes)*5, dim))
            for si, pos in enumerate(self.states):
                for i, p in enumerate([pos]+self.stateNeighbors(pos)):
                    for j, obs in enumerate(obstypes):
                        if p in obs:
                            fMap[j*5+i, si] = 1
            if self.verbose:
                print 'Built features.'
            return Ts, R, fMap
        else:
            return Ts, R

    def stateNeighbors(self, state):
        if self.oriented:
            pos = (state[0], state[1])
        else:
            pos = state
        ns = [(a[0]+pos[0], a[1]+pos[1]) for a in self.actions]
        if self.oriented:
            # subjective perspective, so we rotate the view according to the current orientation
            ns = listRotate(ns, BASEDIRS.index(state[2]))
            return ns
        else:
            return ns

    def sprite2state(self, s):
        pos = self.rect2pos(s.rect)
        if self.oriented:
            return (pos[0], pos[1], s.orientation)
        else:
            return pos

    def rect2pos(self, r):
        return (r.left / self.game.block_size, r.top / self.game.block_size)

    def setRectPos(self, rect, pos):
        rect.left = pos[0] * self.game.block_size
        rect.top = pos[1] * self.game.block_size

    def setSpriteState(self, s, state):
        if self.oriented:
            s.orientation = state[2]
            self.setRectPos(s.rect, (state[0], state[1]))
        else:
            self.setRectPos(s.rect, state)

    def setState(self, state):
        self.setSpriteState(self.avatar, state)
        self.game.kill_list = []
        VGDLSprite.update(self.avatar, self.game)

    def tryMoves(self, pos):
        res = []
        for a in self.actions:
            # reset game to starting state
            self.setState(pos)
            # take action and compute consequences
            self.avatar._readMultiActions = lambda *x: [a]
            #self.avatar.physics.activeMovement(self.avatar, a)
            self.avatar.update(self.game)
            self.game._updateCollisionDict()
            self.game._eventHandling()
            # remember the outcome of the action
            dest = self.sprite2state(self.avatar)
            res.append(dest)
            self.sas_tuples.append((pos, a, dest))
            # remember reward if the final state ends the game
            for t in self.game.terminations[1:]:
                # Convention: the first criterion is for keyboard-interrupt termination
                ended, win = t.isDone(self.game)
                if ended:
                    if self.verbose:
                        print '    win', win, pos, a, dest
                    if win:
                        self.rewards[dest] = 1
                    else:
                        self.rewards[dest] = -1
        # pass on the list of neighboring states
        return res

def testMaze():
    from core import VGDLParser
    from examples.gridphysics.mazes import * #@UnusedWildImport
    game_str, map_str = polarmaze_game, maze_level_1
    g = VGDLParser().parseGame(game_str)
    g.buildLevel(map_str)
    C = MDPconverter(g, verbose=True)
    Ts, R, fMap = C.convert()
    print C.states
    print R
    for T in Ts:
        print T
    print fMap

if __name__ == '__main__':
    testMaze()
<commit_msg>minor bugfix<commit_after>"""
Transform a subset of (tractable) VGDL games into explicit Markov Decision Processes (MDPS).

Given a game, we want to produce a list of unique, non-redundant discrete states S
There is a set of permissible discrete actions A.
Actions and states are identified by their index.

We produce a 3D-array of transition probabilities Ts:
    Ts[action_id][from_state_id][to_state_id] = Prob(nextstate=to_state | from_state,action)

We also produce a reward vector for entering each state
(by default: -1 for losing, +1 for winning, 0 elsewhere)

Finally, we produce a set of features/observations. Two possibilities:
  - fully observable: the observations uniquely determine the underlying state,
     then they are basically a factored representation of the state
  - partially observable: generally from first-person avatar perspective

"""
from scipy import zeros
from pybrain.utilities import flood
from ontology import MovingAvatar, BASEDIRS, RotatingAvatar
from core import VGDLSprite
from tools import listRotate

class MDPconverter(object):
    """ Simplest case: Assume the game has a single avatar,
        with 4 actions, outcomes are deterministic, physics are grid-based,
        and all other sprites are Immovables. """

    def __init__(self, game, verbose=False):
        self.game = game
        self.verbose=verbose
        self.actions = BASEDIRS
        self.sas_tuples = []
        self.rewards = {}
        self.oriented = False

    def convert(self, observations=True):
        if observations:
            obstypes = []
        for _, ss in self.game.sprite_groups.items():
            #find avatar
            if isinstance(ss[0], MovingAvatar):
                self.avatar = ss[0]
            elif observations:
                # retain observable features
                tmp = [self.sprite2state(sprite) for sprite in ss if sprite.is_static]
                obstypes.append(tmp)
        # what type of movement dynamics do we use?
        if isinstance(self.avatar, RotatingAvatar):
            allPos = set([(col, row, dir_) for row in range(self.game.height)
                          for col in range(self.game.width)
                          for dir_ in BASEDIRS])
            self.oriented = True
        else:
            allPos = set([(col, row) for row in range(self.game.height)
                          for col in range(self.game.width)])
        if self.verbose:
            if observations:
                print 'Number of features:', 5*len(obstypes)
            print 'Maximum state space:', len(allPos)
        initSet = [self.sprite2state(self.avatar)]
        self.states = sorted(flood(self.tryMoves, allPos, initSet))
        dim = len(self.states)
        if self.verbose:
            print 'Actual states:', dim
            print 'Non-negative rewards:', self.rewards
            print 'Initial state', initSet[0]
        Ts = [zeros((dim, dim)) for _ in self.actions]
        R = zeros(dim)
        statedic= {}
        actiondic = {}
        for si, pos in enumerate(self.states):
            statedic[pos] = si
        for ai, a in enumerate(self.actions):
            actiondic[a] = ai
        for pos, val in self.rewards.items():
            R[statedic[pos]] += val
        for pos, a, dest in self.sas_tuples:
            ai = actiondic[a]
            si = statedic[pos]
            di = statedic[dest]
            Ts[ai][si, di] += 1
        if self.verbose:
            print 'Built Ts.'
        for T in Ts:
            for row in T:
                row /= sum(row)
        if self.verbose:
            print 'Normalized Ts.'
        if observations:
            # one observation for current position and each of the 4 neighbors.
            fMap = zeros((len(obstypes)*5, dim))
            for si, state in enumerate(self.states):
                if self.oriented:
                    pos = (state[0], state[1])
                else:
                    pos = state
                for i, p in enumerate([pos]+self.stateNeighbors(state)):
                    for j, obs in enumerate(obstypes):
                        if p in obs:
                            fMap[j*5+i, si] = 1
            if self.verbose:
                print 'Built features.'
            return Ts, R, fMap
        else:
            return Ts, R

    def stateNeighbors(self, state):
        if self.oriented:
            pos = (state[0], state[1])
        else:
            pos = state
        ns = [(a[0]+pos[0], a[1]+pos[1]) for a in BASEDIRS]
        if self.oriented:
            # subjective perspective, so we rotate the view according to the current orientation
            ns = listRotate(ns, BASEDIRS.index(state[2]))
            return ns
        else:
            return ns

    def sprite2state(self, s):
        pos = self.rect2pos(s.rect)
        if self.oriented:
            return (pos[0], pos[1], s.orientation)
        else:
            return pos

    def rect2pos(self, r):
        return (r.left / self.game.block_size, r.top / self.game.block_size)

    def setRectPos(self, rect, pos):
        rect.left = pos[0] * self.game.block_size
        rect.top = pos[1] * self.game.block_size

    def setSpriteState(self, s, state):
        if self.oriented:
            s.orientation = state[2]
            self.setRectPos(s.rect, (state[0], state[1]))
        else:
            self.setRectPos(s.rect, state)

    def setState(self, state):
        self.setSpriteState(self.avatar, state)
        self.game.kill_list = []
        VGDLSprite.update(self.avatar, self.game)

    def tryMoves(self, pos):
        res = []
        for a in self.actions:
            # reset game to starting state
            self.setState(pos)
            # take action and compute consequences
            self.avatar._readMultiActions = lambda *x: [a]
            #self.avatar.physics.activeMovement(self.avatar, a)
            self.avatar.update(self.game)
            self.game._updateCollisionDict()
            self.game._eventHandling()
            # remember the outcome of the action
            dest = self.sprite2state(self.avatar)
            res.append(dest)
            self.sas_tuples.append((pos, a, dest))
            # remember reward if the final state ends the game
            for t in self.game.terminations[1:]:
                # Convention: the first criterion is for keyboard-interrupt termination
                ended, win = t.isDone(self.game)
                if ended:
                    if self.verbose:
                        print '    win', win, pos, a, dest
                    if win:
                        self.rewards[dest] = 1
                    else:
                        self.rewards[dest] = -1
        # pass on the list of neighboring states
        return res

def testMaze():
    from core import VGDLParser
    from examples.gridphysics.mazes import * #@UnusedWildImport
    game_str, map_str = polarmaze_game, maze_level_1
    g = VGDLParser().parseGame(game_str)
    g.buildLevel(map_str)
    C = MDPconverter(g, verbose=True)
    Ts, R, fMap = C.convert()
    print C.states
    print R
    for T in Ts:
        print T
    print fMap

if __name__ == '__main__':
    testMaze()
<|endoftext|>


<commit_before>import collections
import json

vi_user_setting = collections.namedtuple('vi_editor_setting', 'scope values default parser action noable')

WINDOW_SETTINGS = [
    'last_buffer_search',
    '_cmdline_cd',
]

SCOPE_WINDOW = 1
SCOPE_VIEW = 2
SCOPE_VI_VIEW = 3
SCOPE_VI_WINDOW = 4

def set_generic_view_setting(view, name, value, opt):
    if opt.scope == SCOPE_VI_VIEW:
        name = 'vintageous_' + name
    if not opt.parser:
        view.settings().set(name, value)
        return
    else:
        view.settings().set(name, opt.parser(value))
        return
    raise ValueError("Vintageous: bad option value")

def set_minimap(view, name, value, opt):
    # TODO: Ensure the minimap gets hidden when so desired.
    view.window().run_command('toggle_minimap')

def opt_bool_parser(value):
    if value.lower() in ('false', 'true', '0', '1', 'yes', 'no'):
        if value.lower() in ('true', '1', 'yes'):
            return True
        return False

def opt_rulers_parser(value):
    try:
        converted = json.loads(value)
        if isinstance(converted, list):
            return converted
        else:
            raise ValueError
    except ValueError:
        raise
    except TypeError:
        raise ValueError

VI_OPTIONS = {
    # TODO: BUG - unrelated to this code: D,p,u,redo doesn't do what we want.
    'hlsearch': vi_user_setting(scope=SCOPE_VI_VIEW, values=(True, False, '0', '1'), default=True, parser=opt_bool_parser, action=set_generic_view_setting, noable=False),
    'incsearch': vi_user_setting(scope=SCOPE_VI_VIEW, values=(True, False, '0', '1'), default=True, parser=None, action=set_generic_view_setting, noable=False),
    'autoindent': vi_user_setting(scope=SCOPE_VI_VIEW, values=(True, False, '0', '1'), default=True, parser=None, action=set_generic_view_setting, noable=False),
    'showminimap': vi_user_setting(scope=SCOPE_WINDOW, values=(True, False, '0', '1'), default=True, parser=None, action=set_minimap, noable=True),
    'rulers': vi_user_setting(scope=SCOPE_VIEW, values=None, default=[], parser=opt_rulers_parser, action=set_generic_view_setting, noable=False),
}

# For completions.
def iter_settings(prefix=''):
    for k in sorted(VI_OPTIONS.keys()):
        if (prefix == '') or k.startswith(prefix):
            yield k

def set_local(view, name, value):
    try:
        opt = VI_OPTIONS[name]
        opt.action(view, name, value, opt)
    except KeyError as e:
        if name.startswith('no'):
            try:
                opt = VI_OPTIONS[name[2:]]
                if opt.noable:
                    opt.action(view, name, value, opt)
                return
            except KeyError:
                pass
        raise

def get_option(view, name):
    # TODO: Should probably return global, local values.
    option_data = VI_OPTIONS[name]
    if option_data.scope == SCOPE_WINDOW:
        value = view.window().settings().get('vintageous_' + name)
    else:
        value = view.settings().get('vintageous_' + name)

    return value if (value in option_data.values) else option_data.default

class SublimeSettings(object):
    """ Helper class for accessing settings values from views """

    def __init__(self, view=None):
        self.view = view

    def __get__(self, instance, owner):
        if instance is not None:
            return SublimeSettings(instance.v)
        return SublimeSettings()

    def __getitem__(self, key):
        return self.view.settings().get(key)

    def __setitem__(self, key, value):
        self.view.settings().set(key, value)

class VintageSettings(object):
    """ Helper class for accessing settings related to Vintage. """

    def __init__(self, view=None):
        self.view = view

        if view is not None and not isinstance(self.view.settings().get('vintage'), dict):
            self.view.settings().set('vintage', dict())

        if view is not None and view.window() is not None and not isinstance(self.view.window().settings().get('vintage'), dict):
            self.view.window().settings().set('vintage', dict())

    def __get__(self, instance, owner):
        if instance is not None:
            return VintageSettings(instance.v)
        return VintageSettings()

    def __getitem__(self, key):

        # Vi editor options.
        if key in VI_OPTIONS:
            return get_option(self.view, key)

        # Vintageous settings.
        try:
            if key not in WINDOW_SETTINGS:
                value = self.view.settings().get('vintage').get(key)
            else:
                value = self.view.window().settings().get('vintage').get(key)
        except (KeyError, AttributeError):
            value = None
        return value

    def __setitem__(self, key, value):
        if key not in WINDOW_SETTINGS:
            setts, target = self.view.settings().get('vintage'), self.view
        else:
            setts, target = self.view.window().settings().get('vintage'), self.view.window()

        setts[key] = value
        target.settings().set('vintage', setts)

# TODO: Make this a descriptor; avoid instantiation.
class SettingsManager(object):
    view = SublimeSettings()
    vi = VintageSettings()

    def __init__(self, view):
        self.v = view
<commit_msg>improve :setlocal<commit_after>import collections
import json

vi_user_setting = collections.namedtuple('vi_editor_setting', 'scope values default parser action noable')

WINDOW_SETTINGS = [
    'last_buffer_search',
    '_cmdline_cd',
]

SCOPE_WINDOW = 1
SCOPE_VIEW = 2
SCOPE_VI_VIEW = 3
SCOPE_VI_WINDOW = 4

def set_generic_view_setting(view, name, value, opt):
    if opt.scope == SCOPE_VI_VIEW:
        name = 'vintageous_' + name
    if not opt.parser:
        view.settings().set(name, value)
        return
    else:
        view.settings().set(name, opt.parser(value))
        return
    raise ValueError("Vintageous: bad option value")

def set_minimap(view, name, value, opt):
    # TODO: Ensure the minimap gets hidden when so desired.
    view.window().run_command('toggle_minimap')

def opt_bool_parser(value):
    if value.lower() in ('false', 'true', '0', '1', 'yes', 'no'):
        if value.lower() in ('true', '1', 'yes'):
            return True
        return False

def opt_rulers_parser(value):
    try:
        converted = json.loads(value)
        if isinstance(converted, list):
            return converted
        else:
            raise ValueError
    except ValueError:
        raise
    except TypeError:
        raise ValueError

VI_OPTIONS = {
    # TODO: BUG - unrelated to this code: D,p,u,redo doesn't do what we want.
    'hlsearch': vi_user_setting(scope=SCOPE_VI_VIEW, values=(True, False, '0', '1'), default=True, parser=opt_bool_parser, action=set_generic_view_setting, noable=True),
    'incsearch': vi_user_setting(scope=SCOPE_VI_VIEW, values=(True, False, '0', '1'), default=True, parser=opt_bool_parser, action=set_generic_view_setting, noable=True),
    'autoindent': vi_user_setting(scope=SCOPE_VI_VIEW, values=(True, False, '0', '1'), default=True, parser=None, action=set_generic_view_setting, noable=False),
    'showminimap': vi_user_setting(scope=SCOPE_WINDOW, values=(True, False, '0', '1'), default=True, parser=None, action=set_minimap, noable=True),
    'rulers': vi_user_setting(scope=SCOPE_VIEW, values=None, default=[], parser=opt_rulers_parser, action=set_generic_view_setting, noable=False),
}

# For completions.
def iter_settings(prefix=''):
    if prefix.startswith('no'):
        for item in (x for (x, y) in VI_OPTIONS.items() if y.noable):
            if ('no' + item).startswith(prefix):
                yield 'no' + item
    else:
        for k in sorted(VI_OPTIONS.keys()):
            if (prefix == '') or k.startswith(prefix):
                yield k

def set_local(view, name, value):
    try:
        opt = VI_OPTIONS[name]
        if not value and opt.noable:
            opt.action(view, name, '1', opt)
            return
        opt.action(view, name, value, opt)
    except KeyError as e:
        if name.startswith('no'):
            try:
                opt = VI_OPTIONS[name[2:]]
                if opt.noable:
                    opt.action(view, name[2:], '0', opt)
                return
            except KeyError:
                pass
        raise

def get_option(view, name):
    # TODO: Should probably return global, local values.
    option_data = VI_OPTIONS[name]
    if option_data.scope == SCOPE_WINDOW:
        value = view.window().settings().get('vintageous_' + name)
    else:
        value = view.settings().get('vintageous_' + name)

    return value if (value in option_data.values) else option_data.default

class SublimeSettings(object):
    """ Helper class for accessing settings values from views """

    def __init__(self, view=None):
        self.view = view

    def __get__(self, instance, owner):
        if instance is not None:
            return SublimeSettings(instance.v)
        return SublimeSettings()

    def __getitem__(self, key):
        return self.view.settings().get(key)

    def __setitem__(self, key, value):
        self.view.settings().set(key, value)

class VintageSettings(object):
    """ Helper class for accessing settings related to Vintage. """

    def __init__(self, view=None):
        self.view = view

        if view is not None and not isinstance(self.view.settings().get('vintage'), dict):
            self.view.settings().set('vintage', dict())

        if view is not None and view.window() is not None and not isinstance(self.view.window().settings().get('vintage'), dict):
            self.view.window().settings().set('vintage', dict())

    def __get__(self, instance, owner):
        if instance is not None:
            return VintageSettings(instance.v)
        return VintageSettings()

    def __getitem__(self, key):

        # Vi editor options.
        if key in VI_OPTIONS:
            return get_option(self.view, key)

        # Vintageous settings.
        try:
            if key not in WINDOW_SETTINGS:
                value = self.view.settings().get('vintage').get(key)
            else:
                value = self.view.window().settings().get('vintage').get(key)
        except (KeyError, AttributeError):
            value = None
        return value

    def __setitem__(self, key, value):
        if key not in WINDOW_SETTINGS:
            setts, target = self.view.settings().get('vintage'), self.view
        else:
            setts, target = self.view.window().settings().get('vintage'), self.view.window()

        setts[key] = value
        target.settings().set('vintage', setts)

# TODO: Make this a descriptor; avoid instantiation.
class SettingsManager(object):
    view = SublimeSettings()
    vi = VintageSettings()

    def __init__(self, view):
        self.v = view
<|endoftext|>