<commit_before>#!/usr/bin/env python
"""Execute tasks on a remote EMR cluster."""

import argparse
import json
import os
import pipes
from subprocess import Popen, PIPE
import sys
import uuid

STATIC_FILES_PATH = os.path.join(sys.prefix, 'share', 'edx.analytics.tasks')
EC2_INVENTORY_PATH = os.path.join(STATIC_FILES_PATH, 'ec2.py')

REMOTE_DATA_DIR = '/var/lib/analytics-tasks'
REMOTE_LOG_DIR = '/var/log/analytics-tasks'

def main():
    """Parse arguments and run the remote task."""
    parser = argparse.ArgumentParser()
    parser.add_argument('--job-flow-id', help='EMR job flow to run the task', default=None)
    parser.add_argument('--job-flow-name', help='EMR job flow to run the task', default=None)
    parser.add_argument('--host', help='host:port to run the task on', default=None)
    parser.add_argument('--vagrant-path', help='path to the root directory containing a running vagrant container', default=None)
    parser.add_argument('--branch', help='git branch to checkout before running the task', default='release')
    parser.add_argument('--repo', help='git repository to clone')
    parser.add_argument('--remote-name', help='an identifier for this remote task')
    parser.add_argument('--wait', action='store_true', help='wait for the task to complete')
    parser.add_argument('--verbose', action='store_true', help='display very verbose output')
    parser.add_argument('--log-path', help='download luigi output streams after completing the task', default=None)
    parser.add_argument('--user', help='remote user name to connect as', default=None)
    parser.add_argument('--private-key', help='a private key file to use to connect to the host', default=None)
    parser.add_argument('--override-config', help='config file to use to run the job', default=None)
    parser.add_argument('--secure-config', help='config file in secure config repo to use to run the job', default=None)
    parser.add_argument('--secure-config-branch', help='git branch to checkout to find the secure config file', default=None)
    parser.add_argument('--secure-config-repo', help='git repository to clone to find the secure config file', default=os.getenv('ANALYTICS_SECURE_REPO'))
    parser.add_argument('--shell', help='execute a shell command on the cluster and exit', default=None)
    parser.add_argument('--sudo-user', help='execute the shell command as this user on the cluster', default='hadoop')
    parser.add_argument('--workflow-profiler', choices=['pyinstrument'], help='profiler to run on the launch-task process', default=None)
    parser.add_argument('--wheel-url', help='url of the wheelhouse', default=None)
    parser.add_argument('--skip-setup', action='store_true', help='assumes the environment has already been configured and you can simply run the task')
    arguments, extra_args = parser.parse_known_args()
    arguments.launch_task_arguments = extra_args

    log('Parsed arguments = {0}'.format(arguments))
    log('Running commands from path = {0}'.format(STATIC_FILES_PATH))
    uid = arguments.remote_name or str(uuid.uuid4())
    log('Remote name = {0}'.format(uid))

    if arguments.vagrant_path:
        parse_vagrant_ssh_config(arguments)

    if arguments.host:
        inventory = {}
    else:
        inventory = get_ansible_inventory()

    if arguments.shell:
        return_code = run_remote_shell(inventory, arguments, arguments.shell)
    else:
        return_code = run_task_playbook(inventory, arguments, uid)

    log('Exiting with status = {0}'.format(return_code))
    sys.exit(return_code)

def run_task_playbook(inventory, arguments, uid):
    """
    Execute the ansible playbook that triggers and monitors the remote task execution.

    Args:
        arguments (argparse.Namespace): The arguments that were passed in on the command line.
        uid (str): A unique identifier for this task execution.
    """
    if not arguments.skip_setup:
        extra_vars = convert_args_to_extra_vars(arguments, uid)
        args = ['task.yml', '-e', extra_vars]
        prep_result = run_ansible(tuple(args), arguments, executable='ansible-playbook')
        if prep_result != 0:
            return prep_result

    data_dir = os.path.join(REMOTE_DATA_DIR, uid)
    log_dir = os.path.join(REMOTE_LOG_DIR, uid)
    sudo_user = arguments.sudo_user

    env_vars = {}
    if arguments.workflow_profiler:
        env_vars['WORKFLOW_PROFILER'] = arguments.workflow_profiler
        env_vars['WORKFLOW_PROFILER_PATH'] = log_dir

    env_var_string = ' '.join('{0}={1}'.format(k, v) for k, v in env_vars.iteritems())

    command = 'cd {data_dir}/repo && . $HOME/.bashrc && {env_vars}{bg}{data_dir}/venv/bin/launch-task {task_arguments}{end_bg}'.format(
        env_vars=env_var_string + ' ' if env_var_string else '',
        data_dir=data_dir,
        task_arguments=' '.join(arguments.launch_task_arguments),
        log_dir=log_dir,
        bg='nohup ' if not arguments.wait else '',
        end_bg=' &' if not arguments.wait else '',
        sudo_user=sudo_user,
    )

    result = run_remote_shell(inventory, arguments, command)
    if arguments.wait and arguments.log_path:
        host_group = get_ansible_inventory_host(arguments)
        fetch_arguments = [host_group, '-m', 'fetch']
        if arguments.user:
            fetch_arguments.extend(['-u', arguments.user])
        for filename in ('edx_analytics.log', 'launch-task.trace'):
            module_arguments = 'src={src} dest={dest} flat=yes'.format(
                src=os.path.join(log_dir, filename),
                dest=os.path.join(arguments.log_path, filename)
            )
            run_ansible(fetch_arguments + ['-a', module_arguments], arguments)

    return result

def get_ansible_inventory_host(arguments):
    return 'mr_{0}_master'.format(arguments.job_flow_id or arguments.job_flow_name)

def convert_args_to_extra_vars(arguments, uid):
    """
    Generate the set of variables that need to be passed in to ansible since they are expected to be set by the
    playbook.

    Args:
        arguments (argparse.Namespace): The arguments that were passed in on the command line.
        uid (str): A unique identifier for this task execution.
    """
    if arguments.host:
        name = 'all'
    else:
        name = get_ansible_inventory_host(arguments)
    extra_vars = {
        'name': name,
        'branch': arguments.branch,
        'uuid': uid,
        'root_data_dir': REMOTE_DATA_DIR,
        'root_log_dir': REMOTE_LOG_DIR,
    }
    if arguments.repo:
        extra_vars['repo'] = arguments.repo
    if arguments.override_config:
        extra_vars['override_config'] = arguments.override_config
    if arguments.secure_config_repo:
        extra_vars['secure_config_repo'] = arguments.secure_config_repo
    if arguments.secure_config_branch:
        extra_vars['secure_config_branch'] = arguments.secure_config_branch
    if arguments.secure_config:
        extra_vars['secure_config'] = arguments.secure_config
    if arguments.wheel_url:
        extra_vars['install_env'] = {
            'WHEEL_URL': arguments.wheel_url,
            'WHEEL_PYVER': '2.7'
        }
    if arguments.vagrant_path:
        extra_vars['write_luigi_config'] = False
    return json.dumps(extra_vars)

def parse_vagrant_ssh_config(arguments):
    log('Connecting to vagrant container in {0}'.format(arguments.vagrant_path))
    command = 'vagrant ssh-config'
    log('Running command = {0}'.format(command))
    with open('/dev/null', 'r+') as devnull:
        proc = Popen(
            command,
            stdin=devnull,
            stdout=PIPE,
            cwd=arguments.vagrant_path,
            shell=True
        )
        stdout = proc.communicate()[0]

    if proc.returncode != 0:
        raise RuntimeError('Unable to determine vagrant connectivity parameters.')

    hostname = '127.0.0.1'
    port = '2222'
    for line in stdout.split('\n'):
        split_line = line.strip().split(' ')
        if len(split_line) != 2:
            continue

        key, value = split_line
        if key == "HostName":
            hostname = value
        elif key == "User":
            arguments.user = value
        elif key == "Port":
            port = value
        elif key == "IdentityFile":
            arguments.private_key = value

    arguments.host = hostname + ':' + port

def get_ansible_inventory():
    """
    Ensure the EC2 inventory cache is cleared before running ansible.

    Otherwise new resources will not be present in the inventory which will cause ansible to fail to connect to them.

    """
    command = [EC2_INVENTORY_PATH, '--refresh-cache']
    log('Running command = {0}'.format(command))
    with open('/dev/null', 'r+') as devnull:
        proc = Popen(
            command,
            stdin=devnull,
            stdout=PIPE,
            cwd=STATIC_FILES_PATH
        )
        stdout = proc.communicate()[0]

    if proc.returncode != 0:
        raise RuntimeError('Unable to refresh ansible inventory cache.')

    return json.loads(stdout)

def run_ansible(args, arguments, executable='ansible'):
    """
    Execute ansible passing in the provided arguments.

    Args:
        args (iterable): A collection of arguments to pass to ansible on the command line.
        verbose (bool): Tell ansible to produce verbose output detailing exactly what commands it is executing.
        executable (str): The executable script to invoke on the command line.  Defaults to "ansible".

    """
    if arguments.host:
        inventory_file_path = arguments.host + ','
    else:
        inventory_file_path = EC2_INVENTORY_PATH
    executable_path = os.path.join(sys.prefix, 'bin', executable)
    command = [executable_path, '-i', inventory_file_path] + list(args)
    if arguments.user:
        command.extend(['-u', arguments.user])
    if arguments.private_key:
        command.extend(['--private-key', arguments.private_key])
    if arguments.verbose:
        command.append('-vvvv')

    env = dict(os.environ)
    env.update({
        # Ansible may be pulling down private git repos on the remote machine.  Forward the local agent so that the
        # remote machine can access any repos this one can. These machines are dynamically created, so we don't know
        # their host key.  In an ideal world we would store the host key at provisioning time, however, that doesn't
        # happen, so just trust we have the right machine.
        'ANSIBLE_SSH_ARGS': '-o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
    })
    log('Running command = {0}'.format(command))
    with open('/dev/null', 'r+') as devnull:
        proc = Popen(
            command,
            stdin=devnull,
            env=env,
            cwd=STATIC_FILES_PATH
        )
        proc.wait()

    return proc.returncode

def run_remote_shell(inventory, arguments, shell_command):
    """Run a shell command on a hadoop cluster."""
    port = None
    if not arguments.host:
        ansible_group_name = get_ansible_inventory_host(arguments)
        hostname = inventory[ansible_group_name][0]
    else:
        split_host = arguments.host.split(':')
        hostname = split_host[0]
        if len(split_host) > 1:
            port = split_host[1]
    sudo_user = arguments.sudo_user
    if sudo_user:
        shell_command = 'sudo -u {0} /bin/bash -c {1}'.format(sudo_user, pipes.quote(shell_command))
    command = [
        'ssh',
        '-tt',
        '-o', 'ForwardAgent=yes',
        '-o', 'StrictHostKeyChecking=no',
        '-o', 'UserKnownHostsFile=/dev/null',
        '-o', 'KbdInteractiveAuthentication=no',
        '-o', 'PasswordAuthentication=no',
        '-o', 'User=' + arguments.user,
        '-o', 'ConnectTimeout=10',
    ]
    if port:
        command.extend(['-p', port])
    if arguments.private_key:
        command.extend(['-i', arguments.private_key])
    command.extend([hostname, shell_command])
    log('Running command = {0}'.format(command))
    proc = Popen(
        command,
        cwd=STATIC_FILES_PATH
    )
    proc.wait()
    return proc.returncode

def log(message):
    """Writes debugging information to stderr."""
    sys.stderr.write(message)
    sys.stderr.write('\n')

if __name__ == '__main__':
    main()
<commit_msg>Ensure the $HOME variable is expanded properly in the sudo command<commit_after>#!/usr/bin/env python
"""Execute tasks on a remote EMR cluster."""

import argparse
import json
import os
import pipes
from subprocess import Popen, PIPE
import sys
import uuid

STATIC_FILES_PATH = os.path.join(sys.prefix, 'share', 'edx.analytics.tasks')
EC2_INVENTORY_PATH = os.path.join(STATIC_FILES_PATH, 'ec2.py')

REMOTE_DATA_DIR = '/var/lib/analytics-tasks'
REMOTE_LOG_DIR = '/var/log/analytics-tasks'

def main():
    """Parse arguments and run the remote task."""
    parser = argparse.ArgumentParser()
    parser.add_argument('--job-flow-id', help='EMR job flow to run the task', default=None)
    parser.add_argument('--job-flow-name', help='EMR job flow to run the task', default=None)
    parser.add_argument('--host', help='host:port to run the task on', default=None)
    parser.add_argument('--vagrant-path', help='path to the root directory containing a running vagrant container', default=None)
    parser.add_argument('--branch', help='git branch to checkout before running the task', default='release')
    parser.add_argument('--repo', help='git repository to clone')
    parser.add_argument('--remote-name', help='an identifier for this remote task')
    parser.add_argument('--wait', action='store_true', help='wait for the task to complete')
    parser.add_argument('--verbose', action='store_true', help='display very verbose output')
    parser.add_argument('--log-path', help='download luigi output streams after completing the task', default=None)
    parser.add_argument('--user', help='remote user name to connect as', default=None)
    parser.add_argument('--private-key', help='a private key file to use to connect to the host', default=None)
    parser.add_argument('--override-config', help='config file to use to run the job', default=None)
    parser.add_argument('--secure-config', help='config file in secure config repo to use to run the job', default=None)
    parser.add_argument('--secure-config-branch', help='git branch to checkout to find the secure config file', default=None)
    parser.add_argument('--secure-config-repo', help='git repository to clone to find the secure config file', default=os.getenv('ANALYTICS_SECURE_REPO'))
    parser.add_argument('--shell', help='execute a shell command on the cluster and exit', default=None)
    parser.add_argument('--sudo-user', help='execute the shell command as this user on the cluster', default='hadoop')
    parser.add_argument('--workflow-profiler', choices=['pyinstrument'], help='profiler to run on the launch-task process', default=None)
    parser.add_argument('--wheel-url', help='url of the wheelhouse', default=None)
    parser.add_argument('--skip-setup', action='store_true', help='assumes the environment has already been configured and you can simply run the task')
    arguments, extra_args = parser.parse_known_args()
    arguments.launch_task_arguments = extra_args

    log('Parsed arguments = {0}'.format(arguments))
    log('Running commands from path = {0}'.format(STATIC_FILES_PATH))
    uid = arguments.remote_name or str(uuid.uuid4())
    log('Remote name = {0}'.format(uid))

    if arguments.vagrant_path:
        parse_vagrant_ssh_config(arguments)

    if arguments.host:
        inventory = {}
    else:
        inventory = get_ansible_inventory()

    if arguments.shell:
        return_code = run_remote_shell(inventory, arguments, arguments.shell)
    else:
        return_code = run_task_playbook(inventory, arguments, uid)

    log('Exiting with status = {0}'.format(return_code))
    sys.exit(return_code)

def run_task_playbook(inventory, arguments, uid):
    """
    Execute the ansible playbook that triggers and monitors the remote task execution.

    Args:
        arguments (argparse.Namespace): The arguments that were passed in on the command line.
        uid (str): A unique identifier for this task execution.
    """
    if not arguments.skip_setup:
        extra_vars = convert_args_to_extra_vars(arguments, uid)
        args = ['task.yml', '-e', extra_vars]
        prep_result = run_ansible(tuple(args), arguments, executable='ansible-playbook')
        if prep_result != 0:
            return prep_result

    data_dir = os.path.join(REMOTE_DATA_DIR, uid)
    log_dir = os.path.join(REMOTE_LOG_DIR, uid)
    sudo_user = arguments.sudo_user

    env_vars = {}
    if arguments.workflow_profiler:
        env_vars['WORKFLOW_PROFILER'] = arguments.workflow_profiler
        env_vars['WORKFLOW_PROFILER_PATH'] = log_dir

    env_var_string = ' '.join('{0}={1}'.format(k, v) for k, v in env_vars.iteritems())

    command = 'cd {data_dir}/repo && . $HOME/.bashrc && {env_vars}{bg}{data_dir}/venv/bin/launch-task {task_arguments}{end_bg}'.format(
        env_vars=env_var_string + ' ' if env_var_string else '',
        data_dir=data_dir,
        task_arguments=' '.join(arguments.launch_task_arguments),
        log_dir=log_dir,
        bg='nohup ' if not arguments.wait else '',
        end_bg=' &' if not arguments.wait else '',
        sudo_user=sudo_user,
    )

    result = run_remote_shell(inventory, arguments, command)
    if arguments.wait and arguments.log_path:
        host_group = get_ansible_inventory_host(arguments)
        fetch_arguments = [host_group, '-m', 'fetch']
        if arguments.user:
            fetch_arguments.extend(['-u', arguments.user])
        for filename in ('edx_analytics.log', 'launch-task.trace'):
            module_arguments = 'src={src} dest={dest} flat=yes'.format(
                src=os.path.join(log_dir, filename),
                dest=os.path.join(arguments.log_path, filename)
            )
            run_ansible(fetch_arguments + ['-a', module_arguments], arguments)

    return result

def get_ansible_inventory_host(arguments):
    return 'mr_{0}_master'.format(arguments.job_flow_id or arguments.job_flow_name)

def convert_args_to_extra_vars(arguments, uid):
    """
    Generate the set of variables that need to be passed in to ansible since they are expected to be set by the
    playbook.

    Args:
        arguments (argparse.Namespace): The arguments that were passed in on the command line.
        uid (str): A unique identifier for this task execution.
    """
    if arguments.host:
        name = 'all'
    else:
        name = get_ansible_inventory_host(arguments)
    extra_vars = {
        'name': name,
        'branch': arguments.branch,
        'uuid': uid,
        'root_data_dir': REMOTE_DATA_DIR,
        'root_log_dir': REMOTE_LOG_DIR,
    }
    if arguments.repo:
        extra_vars['repo'] = arguments.repo
    if arguments.override_config:
        extra_vars['override_config'] = arguments.override_config
    if arguments.secure_config_repo:
        extra_vars['secure_config_repo'] = arguments.secure_config_repo
    if arguments.secure_config_branch:
        extra_vars['secure_config_branch'] = arguments.secure_config_branch
    if arguments.secure_config:
        extra_vars['secure_config'] = arguments.secure_config
    if arguments.wheel_url:
        extra_vars['install_env'] = {
            'WHEEL_URL': arguments.wheel_url,
            'WHEEL_PYVER': '2.7'
        }
    if arguments.vagrant_path:
        extra_vars['write_luigi_config'] = False
    return json.dumps(extra_vars)

def parse_vagrant_ssh_config(arguments):
    log('Connecting to vagrant container in {0}'.format(arguments.vagrant_path))
    command = 'vagrant ssh-config'
    log('Running command = {0}'.format(command))
    with open('/dev/null', 'r+') as devnull:
        proc = Popen(
            command,
            stdin=devnull,
            stdout=PIPE,
            cwd=arguments.vagrant_path,
            shell=True
        )
        stdout = proc.communicate()[0]

    if proc.returncode != 0:
        raise RuntimeError('Unable to determine vagrant connectivity parameters.')

    hostname = '127.0.0.1'
    port = '2222'
    for line in stdout.split('\n'):
        split_line = line.strip().split(' ')
        if len(split_line) != 2:
            continue

        key, value = split_line
        if key == "HostName":
            hostname = value
        elif key == "User":
            arguments.user = value
        elif key == "Port":
            port = value
        elif key == "IdentityFile":
            arguments.private_key = value

    arguments.host = hostname + ':' + port

def get_ansible_inventory():
    """
    Ensure the EC2 inventory cache is cleared before running ansible.

    Otherwise new resources will not be present in the inventory which will cause ansible to fail to connect to them.

    """
    command = [EC2_INVENTORY_PATH, '--refresh-cache']
    log('Running command = {0}'.format(command))
    with open('/dev/null', 'r+') as devnull:
        proc = Popen(
            command,
            stdin=devnull,
            stdout=PIPE,
            cwd=STATIC_FILES_PATH
        )
        stdout = proc.communicate()[0]

    if proc.returncode != 0:
        raise RuntimeError('Unable to refresh ansible inventory cache.')

    return json.loads(stdout)

def run_ansible(args, arguments, executable='ansible'):
    """
    Execute ansible passing in the provided arguments.

    Args:
        args (iterable): A collection of arguments to pass to ansible on the command line.
        verbose (bool): Tell ansible to produce verbose output detailing exactly what commands it is executing.
        executable (str): The executable script to invoke on the command line.  Defaults to "ansible".

    """
    if arguments.host:
        inventory_file_path = arguments.host + ','
    else:
        inventory_file_path = EC2_INVENTORY_PATH
    executable_path = os.path.join(sys.prefix, 'bin', executable)
    command = [executable_path, '-i', inventory_file_path] + list(args)
    if arguments.user:
        command.extend(['-u', arguments.user])
    if arguments.private_key:
        command.extend(['--private-key', arguments.private_key])
    if arguments.verbose:
        command.append('-vvvv')

    env = dict(os.environ)
    env.update({
        # Ansible may be pulling down private git repos on the remote machine.  Forward the local agent so that the
        # remote machine can access any repos this one can. These machines are dynamically created, so we don't know
        # their host key.  In an ideal world we would store the host key at provisioning time, however, that doesn't
        # happen, so just trust we have the right machine.
        'ANSIBLE_SSH_ARGS': '-o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
    })
    log('Running command = {0}'.format(command))
    with open('/dev/null', 'r+') as devnull:
        proc = Popen(
            command,
            stdin=devnull,
            env=env,
            cwd=STATIC_FILES_PATH
        )
        proc.wait()

    return proc.returncode

def run_remote_shell(inventory, arguments, shell_command):
    """Run a shell command on a hadoop cluster."""
    port = None
    if not arguments.host:
        ansible_group_name = get_ansible_inventory_host(arguments)
        hostname = inventory[ansible_group_name][0]
    else:
        split_host = arguments.host.split(':')
        hostname = split_host[0]
        if len(split_host) > 1:
            port = split_host[1]
    sudo_user = arguments.sudo_user
    if sudo_user:
        shell_command = 'sudo -Hu {0} /bin/bash -c {1}'.format(sudo_user, pipes.quote(shell_command))
    command = [
        'ssh',
        '-tt',
        '-o', 'ForwardAgent=yes',
        '-o', 'StrictHostKeyChecking=no',
        '-o', 'UserKnownHostsFile=/dev/null',
        '-o', 'KbdInteractiveAuthentication=no',
        '-o', 'PasswordAuthentication=no',
        '-o', 'User=' + arguments.user,
        '-o', 'ConnectTimeout=10',
    ]
    if port:
        command.extend(['-p', port])
    if arguments.private_key:
        command.extend(['-i', arguments.private_key])
    command.extend([hostname, shell_command])
    log('Running command = {0}'.format(command))
    proc = Popen(
        command,
        cwd=STATIC_FILES_PATH
    )
    proc.wait()
    return proc.returncode

def log(message):
    """Writes debugging information to stderr."""
    sys.stderr.write(message)
    sys.stderr.write('\n')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Functions for reading NBT (Named Binary Tags) files.
"""
from __future__ import print_function
import gzip
import struct

_TYPE_NAMES = {}
_TYPE_IDS = {}

for i, name in enumerate(['end',
                          'byte',
                          'short',
                          'int',
                          'long',
                          'float',
                          'double',
                          'bytearray',
                          'string',
                          'list',
                          'compound',
                          'intarray']):

    _TYPE_NAMES[i] = name
    _TYPE_IDS[name] = i
del i, name

class TagFile(object):
    def __init__(self, data):
        self.pos = 0
        self.data = data

    def tell(self):
        return self.pos

    def read(self, length):
        data = self.data[self.pos:self.pos+length]
        self.pos += length
        return data

class TagFileDebugger(object):
    """Wraps around TagFile."""
    def __init__(self, file):
        self.file = file

    def _read_byte(self):
        char = self.file.read(1)
        pos = self.file.tell()
        print('    {:08x}: {:02x} {!r}'.format(pos,
                                               ord(char),
                                               char))
        pos += 1
        return char

    def read(self, length):
        data = b''

        print('  {READ')
        for i in range(length):
            data += self._read_byte()
        print('  }')

        return data

class Parser(object):
    def __init__(self, data, hexdata=False):
        self.file = TagFile(data)
        # self.file = TagFileDebugger(self.file)

    def parse(self):
        # This assumed that the outer tag is a compound.
        return self._read_tag()[1]

    def _read_tag(self):
        tagtype = ord(self.file.read(1))
        typename = _TYPE_NAMES[tagtype]
        if typename == 'end':
            raise StopIteration

        name = self._read_string()

        if typename == 'list':
            datatype = tag_names[self._read_byte()]
            name = '{}:{}list'.format(typename, datatype)
            value = self._read_list(datatype)
        else:
            name = '{}:{}'.format(name, typename)
            value = getattr(self, '_read_' + typename)()

        return (name, value)

    def _read_byte(self):
        return ord(self.file.read(1))

    def _read_short(self):
        # 4 byte signed.
        return struct.unpack('>h', self.file.read(2))[0]

    def _read_int(self):
        # 4 byte signed.
        return struct.unpack('>i', self.file.read(4))[0]

    def _read_long(self):
        # 8 byte signed.
        return struct.unpack('>q', self.file.read(8))[0]

    def _read_float(self):
        # 8 byte signed.
        return struct.unpack('>f', self.file.read(4))[0]

    def _read_double(self):
        # 8 byte signed.
        return struct.unpack('>d', self.file.read(8))[0]

    def _read_bytearray(self):
        # Todo: implement hexdata.
        length = self._read_int()
        return bytearray(self.file.read(length))

    def _read_string(self):
        length = struct.unpack('>h', self.file.read(2))[0]
        if length:
            return self.file.read(length).decode('UTF-8')
        else:
            return ''

    def _read_compound(self):
        compound = {}
        try:
            while True:
                name, value = self._read_tag()
                compound[name] = value
        except StopIteration:
            # Why do we have to do this?
            # Shouldn't that happen automatically?
            pass
        return compound

    def _read_list(self, datatype):
        length = self._read_int()
        read = getattr(self, '_read_' + datatype)
        return [read() for _ in range(length)]
        print(lst)
        return lst

    def _read_intarray(self):
        length = self._read_int()
        value = []
        for i in range(length):
            value.append(self._read_int())
        return value

def decode(data):
    return Parser(data).parse()

def read(filename):
    return decode(gzip.GzipFile(filename, 'rb').read())
<commit_msg>Bugfix: forgot to rename a variable.<commit_after>"""
Functions for reading NBT (Named Binary Tags) files.
"""
from __future__ import print_function
import gzip
import struct

_TYPE_NAMES = {}
_TYPE_IDS = {}

for i, name in enumerate(['end',
                          'byte',
                          'short',
                          'int',
                          'long',
                          'float',
                          'double',
                          'bytearray',
                          'string',
                          'list',
                          'compound',
                          'intarray']):

    _TYPE_NAMES[i] = name
    _TYPE_IDS[name] = i
del i, name

class TagFile(object):
    def __init__(self, data):
        self.pos = 0
        self.data = data

    def tell(self):
        return self.pos

    def read(self, length):
        data = self.data[self.pos:self.pos+length]
        self.pos += length
        return data

class TagFileDebugger(object):
    """Wraps around TagFile."""
    def __init__(self, file):
        self.file = file

    def _read_byte(self):
        char = self.file.read(1)
        pos = self.file.tell()
        print('    {:08x}: {:02x} {!r}'.format(pos,
                                               ord(char),
                                               char))
        pos += 1
        return char

    def read(self, length):
        data = b''

        print('  {READ')
        for i in range(length):
            data += self._read_byte()
        print('  }')

        return data

class Parser(object):
    def __init__(self, data, hexdata=False):
        self.file = TagFile(data)
        # self.file = TagFileDebugger(self.file)

    def parse(self):
        # This assumed that the outer tag is a compound.
        return self._read_tag()[1]

    def _read_tag(self):
        tagtype = ord(self.file.read(1))
        typename = _TYPE_NAMES[tagtype]
        if typename == 'end':
            raise StopIteration

        name = self._read_string()

        if typename == 'list':
            datatype = _TYPE_NAMES[self._read_byte()]
            name = '{}:{}list'.format(typename, datatype)
            value = self._read_list(datatype)
        else:
            name = '{}:{}'.format(name, typename)
            value = getattr(self, '_read_' + typename)()

        return (name, value)

    def _read_byte(self):
        return ord(self.file.read(1))

    def _read_short(self):
        # 4 byte signed.
        return struct.unpack('>h', self.file.read(2))[0]

    def _read_int(self):
        # 4 byte signed.
        return struct.unpack('>i', self.file.read(4))[0]

    def _read_long(self):
        # 8 byte signed.
        return struct.unpack('>q', self.file.read(8))[0]

    def _read_float(self):
        # 8 byte signed.
        return struct.unpack('>f', self.file.read(4))[0]

    def _read_double(self):
        # 8 byte signed.
        return struct.unpack('>d', self.file.read(8))[0]

    def _read_bytearray(self):
        # Todo: implement hexdata.
        length = self._read_int()
        return bytearray(self.file.read(length))

    def _read_string(self):
        length = struct.unpack('>h', self.file.read(2))[0]
        if length:
            return self.file.read(length).decode('UTF-8')
        else:
            return ''

    def _read_compound(self):
        compound = {}
        try:
            while True:
                name, value = self._read_tag()
                compound[name] = value
        except StopIteration:
            # Why do we have to do this?
            # Shouldn't that happen automatically?
            pass
        return compound

    def _read_list(self, datatype):
        length = self._read_int()
        read = getattr(self, '_read_' + datatype)
        return [read() for _ in range(length)]
        print(lst)
        return lst

    def _read_intarray(self):
        length = self._read_int()
        value = []
        for i in range(length):
            value.append(self._read_int())
        return value

def decode(data):
    return Parser(data).parse()

def read(filename):
    return decode(gzip.GzipFile(filename, 'rb').read())
<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.sites.models import Site
from django.contrib.sites.requests import RequestSite
from django import forms
from django.http import HttpRequest
from django.utils.translation import ugettext_lazy as _

from .models import RegistrationProfile

#! need to test settings
# view https://docs.djangoproject.com/en/1.6/topics/testing/tools/#overriding-settings
EXTRA_FIELDS = getattr(settings,"REGISTRATION_EXTRA_FIELDS",['password2'])

class RegistrationForm(forms.Form):
    """
    A form to create inactive users and create a registration profile. Uses the following settings:

    REGISTRATION_EXTRA_FIELDS - a list with 'pasword2' and/or 'tos'

    REGISTRATION_UNIQUE_EMAIL - if True, clean_email will validate whether or not a users email is unique

    REGISTRATION_NO_FREE_EMAIL - if True, clean email will reject emails using free email services

    REGISTRATION_IGNORE_DOTS - if True, clean email with detect a.b.c@gmail.com as being the same as abc@gmail.com
    """
    required_css_class = 'required'
    _e = {'invalid': _("This value may contain only letters, numbers and @/./+/-/_ characters.")}
    username = forms.RegexField(regex=r'^[\w.@+-]+$',max_length=30,label=_("Username"),error_messages=_e)
    email = forms.EmailField(label=_("E-mail"))
    password1 = forms.CharField(widget=forms.PasswordInput,label=_("Password"))
    if 'password2' in EXTRA_FIELDS: #! needs test
        password2 = forms.CharField(widget=forms.PasswordInput,label=_("Password (again)"))
    if 'tos' in EXTRA_FIELDS: #! needs test
        tos = forms.BooleanField(widget=forms.CheckboxInput,
                                 label=_(u'I have read and agree to the Terms of Service'),
                                 error_messages={'required': _("You must agree to the terms to register")})

    def __init__(self,request,*args,**kwargs):
        if not isinstance(request,HttpRequest):
            raise ValueError("The RegistrationForm must take a request as it's first argument")
        self.request = request
        return super(RegistrationForm,self).__init__(*args,**kwargs)

    def clean_email(self):
        User = get_user_model()
        e = _("This email address is already in use. Please supply a different email address.")
        if getattr(settings,"REGISTRATION_UNIQUE_EMAIL",False): #! needs test
            if User.objects.filter(email__iexact=self.cleaned_data['email']):
                raise forms.ValidationError(e)

        if getattr(settings,"REGISTRATION_IGNORE_DOTS",False): #! needs test
            name,domain = self.cleaned_data['email'].split('@')
            name = name.lower().replace('.','')
            for user in User.objects.filter(email__iendswith=domain):
                if name == user.email.split('@')[0].lower().replace('.',''):
                    raise forms.ValidationError(e)

        if getattr(settings,"REGISTRATION_NO_FREE_EMAIL",False): #! needs test
            bad_domains = ['aim.com', 'aol.com', 'email.com', 'gmail.com',
                           'googlemail.com', 'hotmail.com', 'hushmail.com',
                           'msn.com', 'mail.ru', 'mailinator.com', 'live.com',
                           'yahoo.com']
            email_domain = self.cleaned_data['email'].split('@')[1]
            e = _("Registration using free email addresses is prohibited. Please supply a different email address.")
            if email_domain in bad_domains:
                raise forms.ValidationError(e)

        return self.cleaned_data['email']

    def clean_username(self):
        User = get_user_model()
        if User.objects.filter(username__iexact=self.cleaned_data['username']):
            raise forms.ValidationError(_("A user with that username already exists."))
        return self.cleaned_data['username']

    def clean_password2(self):
        if 'password1' in self.cleaned_data and 'password2' in self.cleaned_data:
            if self.cleaned_data['password1'] != self.cleaned_data['password2']:
                raise forms.ValidationError(_("The two password fields didn't match."))
        return self.cleaned_data.get('password2','')

    def save(self):
        d = self.cleaned_data
        if Site._meta.installed:
            site = Site.objects.get_current()
        else:
            site = RequestSite(request)
        RegistrationProfile.objects.create_inactive_user(d['username'],d['email'],d['password1'],site)
<commit_msg>using lablackeys request form<commit_after>from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.sites.models import Site
from django.contrib.sites.requests import RequestSite
from django import forms
from django.http import HttpRequest
from django.utils.translation import ugettext_lazy as _

from .models import RegistrationProfile

from lablackey.forms import RequestForm

#! need to test settings
# view https://docs.djangoproject.com/en/1.6/topics/testing/tools/#overriding-settings
EXTRA_FIELDS = getattr(settings,"REGISTRATION_EXTRA_FIELDS",['password2'])

class RegistrationForm(RequestForm):
    """
    A form to create inactive users and create a registration profile. Uses the following settings:

    REGISTRATION_EXTRA_FIELDS - a list with 'pasword2' and/or 'tos'

    REGISTRATION_UNIQUE_EMAIL - if True, clean_email will validate whether or not a users email is unique

    REGISTRATION_NO_FREE_EMAIL - if True, clean email will reject emails using free email services

    REGISTRATION_IGNORE_DOTS - if True, clean email with detect a.b.c@gmail.com as being the same as abc@gmail.com
    """
    required_css_class = 'required'
    _e = {'invalid': _("This value may contain only letters, numbers and @/./+/-/_ characters.")}
    username = forms.RegexField(regex=r'^[\w.@+-]+$',max_length=30,label=_("Username"),error_messages=_e)
    email = forms.EmailField(label=_("E-mail"))
    password1 = forms.CharField(widget=forms.PasswordInput,label=_("Password"))
    if 'password2' in EXTRA_FIELDS: #! needs test
        password2 = forms.CharField(widget=forms.PasswordInput,label=_("Password (again)"))
    if 'tos' in EXTRA_FIELDS: #! needs test
        tos = forms.BooleanField(widget=forms.CheckboxInput,
                                 label=_(u'I have read and agree to the Terms of Service'),
                                 error_messages={'required': _("You must agree to the terms to register")})

    def clean_email(self):
        User = get_user_model()
        e = _("This email address is already in use. Please supply a different email address.")
        if getattr(settings,"REGISTRATION_UNIQUE_EMAIL",False): #! needs test
            if User.objects.filter(email__iexact=self.cleaned_data['email']):
                raise forms.ValidationError(e)

        if getattr(settings,"REGISTRATION_IGNORE_DOTS",False): #! needs test
            name,domain = self.cleaned_data['email'].split('@')
            name = name.lower().replace('.','')
            for user in User.objects.filter(email__iendswith=domain):
                if name == user.email.split('@')[0].lower().replace('.',''):
                    raise forms.ValidationError(e)

        if getattr(settings,"REGISTRATION_NO_FREE_EMAIL",False): #! needs test
            bad_domains = ['aim.com', 'aol.com', 'email.com', 'gmail.com',
                           'googlemail.com', 'hotmail.com', 'hushmail.com',
                           'msn.com', 'mail.ru', 'mailinator.com', 'live.com',
                           'yahoo.com']
            email_domain = self.cleaned_data['email'].split('@')[1]
            e = _("Registration using free email addresses is prohibited. Please supply a different email address.")
            if email_domain in bad_domains:
                raise forms.ValidationError(e)

        return self.cleaned_data['email']

    def clean_username(self):
        User = get_user_model()
        if User.objects.filter(username__iexact=self.cleaned_data['username']):
            raise forms.ValidationError(_("A user with that username already exists."))
        return self.cleaned_data['username']

    def clean_password2(self):
        if 'password1' in self.cleaned_data and 'password2' in self.cleaned_data:
            if self.cleaned_data['password1'] != self.cleaned_data['password2']:
                raise forms.ValidationError(_("The two password fields didn't match."))
        return self.cleaned_data.get('password2','')

    def save(self):
        d = self.cleaned_data
        if Site._meta.installed:
            site = Site.objects.get_current()
        else:
            site = RequestSite(request)
        RegistrationProfile.objects.create_inactive_user(d['username'],d['email'],d['password1'],site)
<|endoftext|>


<commit_before>"""Command-line utility for using (and testing) s3 utility methods."""

import argparse
import os

from boto import connect_s3
from edx.analytics.tasks.s3_util import generate_s3_sources, join_as_s3_url, get_s3_key

def list_s3_files(source_url, patterns):
    """List remote s3 files that match one of the patterns."""
    s3_conn = connect_s3()
    for _bucket, _root, path in generate_s3_sources(s3_conn, source_url, patterns):
        print path

def get_s3_files(source_url, dest_root, patterns):
    """Copy remote s3 files that match one of the patterns to a local destination."""
    s3_conn = connect_s3()
    for bucket, root, path in generate_s3_sources(s3_conn, source_url, patterns):
        source = join_as_s3_url(bucket, root, path)
        dest_name = path.replace('/', '_')
        destination = os.path.join(dest_root, dest_name)
        src_key = get_s3_key(s3_conn, source)
        if src_key is not None:
            src_key.get_contents_to_filename(destination)
        else:
            print "No key for source " + source

def main():
    """Command-line utility for using (and testing) s3 utility methods."""
    arg_parser = argparse.ArgumentParser(description='Perform s3 utility operations.')
    arg_parser.add_argument(
        'command',
        help='Commands supported by this utility.',
        choices=['ls', 'get'],
    )
    arg_parser.add_argument(
        'input',
        help='Read s3 files from this location in s3.',
    )
    arg_parser.add_argument(
        '-o', '--output',
        help='Write s3 files to this location in the local file system.',
    )
    arg_parser.add_argument(
        '-p', '--patterns',
        help='Apply patterns to match the s3 file paths being fetched.',
        nargs='*',
        default=['*'],
    )
    args = arg_parser.parse_args()
    if args.command == 'ls':
        list_s3_files(args.input, args.patterns)
    elif args.command == 'get':
        get_s3_files(args.input, args.output, args.patterns)

if __name__ == '__main__':
    main()
<commit_msg>Add filesize output to s3util ls.<commit_after>"""Command-line utility for using (and testing) s3 utility methods."""

import argparse
import os

from boto import connect_s3
from edx.analytics.tasks.s3_util import generate_s3_sources, join_as_s3_url, get_s3_key

def list_s3_files(source_url, patterns):
    """List remote s3 files that match one of the patterns."""
    s3_conn = connect_s3()
    for bucket, root, path in generate_s3_sources(s3_conn, source_url, patterns):
        source = join_as_s3_url(bucket, root, path)
        src_key = get_s3_key(s3_conn, source)
        print "%10d %s" % (src_key.size if src_key is not None else -1, path)

def get_s3_files(source_url, dest_root, patterns):
    """Copy remote s3 files that match one of the patterns to a local destination."""
    s3_conn = connect_s3()
    for bucket, root, path in generate_s3_sources(s3_conn, source_url, patterns):
        source = join_as_s3_url(bucket, root, path)
        dest_name = path.replace('/', '_')
        destination = os.path.join(dest_root, dest_name)
        src_key = get_s3_key(s3_conn, source)
        if src_key is not None:
            src_key.get_contents_to_filename(destination)
        else:
            print "No key for source " + source

def main():
    """Command-line utility for using (and testing) s3 utility methods."""
    arg_parser = argparse.ArgumentParser(description='Perform s3 utility operations.')
    arg_parser.add_argument(
        'command',
        help='Commands supported by this utility.',
        choices=['ls', 'get'],
    )
    arg_parser.add_argument(
        'input',
        help='Read s3 files from this location in s3.',
    )
    arg_parser.add_argument(
        '-o', '--output',
        help='Write s3 files to this location in the local file system.',
    )
    arg_parser.add_argument(
        '-p', '--patterns',
        help='Apply patterns to match the s3 file paths being fetched.',
        nargs='*',
        default=['*'],
    )
    args = arg_parser.parse_args()
    if args.command == 'ls':
        list_s3_files(args.input, args.patterns)
    elif args.command == 'get':
        get_s3_files(args.input, args.output, args.patterns)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Downloads Go binaries from Google Cloud Storage and extracts them to
INSTALL_DIR, updating INSTALL_DIR/VERSION stamp file with current version.
Does nothing if INSTALL_DIR/VERSION is already up to date.
"""

import os
import shutil
import subprocess
import sys
import tarfile

# Path constants. (All of these should be absolute paths.)
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
MOJO_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..'))
# Should be the same as in upload.py.
INSTALL_DIR = os.path.join(MOJO_DIR, 'third_party', 'go', 'tool')

sys.path.insert(0, os.path.join(MOJO_DIR, 'tools'))
import find_depot_tools

DEPOT_PATH = find_depot_tools.add_depot_tools_to_path()
GSUTIL_PATH = os.path.join(DEPOT_PATH, 'gsutil.py')

def RunCommand(command):
  """Run command and return success (True) or failure."""

  print 'Running %s' % (str(command))
  if subprocess.call(command, shell=False) == 0:
    return True
  print 'Failed.'
  return False

def VersionFileName():
  if sys.platform.startswith('linux'):
    platform_suffix = 'LINUX'
  elif sys.platform == 'darwin':
    platform_suffix = 'MACOSX'
  else:
    raise Exception('unsupported platform: ' + sys.platform)
  return 'VERSION_' + platform_suffix

def GetInstalledVersion():
  version_file = os.path.join(INSTALL_DIR, VersionFileName())
  if not os.path.exists(version_file):
    return None
  with open(version_file) as f:
    return f.read().strip()

def InstallGoBinaries(version):
  """Downloads zipped go binaries from Google Cloud Storage and extracts them,
     stamping current version."""

  # Remove current installation.
  if os.path.exists(INSTALL_DIR):
    shutil.rmtree(INSTALL_DIR)
  os.mkdir(INSTALL_DIR)
  # Download go tool binaries from GCS.
  archive_path = os.path.join(INSTALL_DIR, 'go.tar.gz')
  download_cmd = ['python', GSUTIL_PATH, '-b', 'cp',
                  'gs://mojo/go/tool/%s.tar.gz' % version,
                  archive_path]
  if not RunCommand(download_cmd):
    print ('WARNING: Failed to download Go tool binaries.')
    return

  print "Extracting Go binaries."
  with tarfile.open(archive_path) as arch:
    arch.extractall(INSTALL_DIR)
  os.remove(archive_path)
  # Write version as the last step.
  with open(os.path.join(INSTALL_DIR, VersionFileName()), 'w+') as f:
    f.write('%s\n' % version)

def main():
  # Read latest version.
  version = ''
  with open(os.path.join(THIS_DIR, VersionFileName())) as f:
    version = f.read().strip()
  # Return if installed binaries are up to date.
  if version == GetInstalledVersion():
    return
  InstallGoBinaries(version)

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>Don't pass "-b" to gsutil when downloading go tool binaries.<commit_after>#!/usr/bin/python
# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Downloads Go binaries from Google Cloud Storage and extracts them to
INSTALL_DIR, updating INSTALL_DIR/VERSION stamp file with current version.
Does nothing if INSTALL_DIR/VERSION is already up to date.
"""

import os
import shutil
import subprocess
import sys
import tarfile

# Path constants. (All of these should be absolute paths.)
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
MOJO_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..'))
# Should be the same as in upload.py.
INSTALL_DIR = os.path.join(MOJO_DIR, 'third_party', 'go', 'tool')

sys.path.insert(0, os.path.join(MOJO_DIR, 'tools'))
import find_depot_tools

DEPOT_PATH = find_depot_tools.add_depot_tools_to_path()
GSUTIL_PATH = os.path.join(DEPOT_PATH, 'gsutil.py')

def RunCommand(command):
  """Run command and return success (True) or failure."""

  print 'Running %s' % (str(command))
  if subprocess.call(command, shell=False) == 0:
    return True
  print 'Failed.'
  return False

def VersionFileName():
  if sys.platform.startswith('linux'):
    platform_suffix = 'LINUX'
  elif sys.platform == 'darwin':
    platform_suffix = 'MACOSX'
  else:
    raise Exception('unsupported platform: ' + sys.platform)
  return 'VERSION_' + platform_suffix

def GetInstalledVersion():
  version_file = os.path.join(INSTALL_DIR, VersionFileName())
  if not os.path.exists(version_file):
    return None
  with open(version_file) as f:
    return f.read().strip()

def InstallGoBinaries(version):
  """Downloads zipped go binaries from Google Cloud Storage and extracts them,
     stamping current version."""

  # Remove current installation.
  if os.path.exists(INSTALL_DIR):
    shutil.rmtree(INSTALL_DIR)
  os.mkdir(INSTALL_DIR)
  # Download go tool binaries from GCS.
  archive_path = os.path.join(INSTALL_DIR, 'go.tar.gz')
  download_cmd = ['python', GSUTIL_PATH, 'cp',
                  'gs://mojo/go/tool/%s.tar.gz' % version,
                  archive_path]
  if not RunCommand(download_cmd):
    print ('WARNING: Failed to download Go tool binaries.')
    return

  print "Extracting Go binaries."
  with tarfile.open(archive_path) as arch:
    arch.extractall(INSTALL_DIR)
  os.remove(archive_path)
  # Write version as the last step.
  with open(os.path.join(INSTALL_DIR, VersionFileName()), 'w+') as f:
    f.write('%s\n' % version)

def main():
  # Read latest version.
  version = ''
  with open(os.path.join(THIS_DIR, VersionFileName())) as f:
    version = f.read().strip()
  # Return if installed binaries are up to date.
  if version == GetInstalledVersion():
    return
  InstallGoBinaries(version)

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before>"""
Forms and validation code for user registration.

"""

from django.contrib.auth.models import User
from django import forms
from django.utils.translation import ugettext_lazy as _

# I put this on all required fields, because it's easier to pick up
# on them with CSS or JavaScript if they have a class of "required"
# in the HTML. Your mileage may vary. If/when Django ticket #3515
# lands in trunk, this will no longer be necessary.
attrs_dict = { 'class': 'required' }

class RegistrationForm(forms.Form):
    """
    Form for registering a new user account.

    Validates that the requested username is not already in use, and
    requires the password to be entered twice to catch typos.

    Subclasses should feel free to add any additional validation they
    need, but should avoid defining a ``save()`` method -- the actual
    saving of collected user data is delegated to the active
    registration backend.

    """
    username = forms.RegexField(regex=r'^\w+$',
                                max_length=30,
                                widget=forms.TextInput(attrs=attrs_dict),
                                label=_(u'username'),
                                error_messages={ 'invalid': _("This value must contain only letters, numbers and underscores.")) })
    email = forms.EmailField(widget=forms.TextInput(attrs=dict(attrs_dict,
                                                               maxlength=75)),
                             label=_("Email address"))
    password1 = forms.CharField(widget=forms.PasswordInput(attrs=attrs_dict, render_value=False),
                                label=_("Password"))
    password2 = forms.CharField(widget=forms.PasswordInput(attrs=attrs_dict, render_value=False),
                                label=_("Password (again)"))

    def clean_username(self):
        """
        Validate that the username is alphanumeric and is not already
        in use.

        """
        try:
            user = User.objects.get(username__iexact=self.cleaned_data['username'])
        except User.DoesNotExist:
            return self.cleaned_data['username']
        raise forms.ValidationError(_("A user with that username already exists."))

    def clean(self):
        """
        Verifiy that the values entered into the two password fields
        match. Note that an error here will end up in
        ``non_field_errors()`` because it doesn't apply to a single
        field.

        """
        if 'password1' in self.cleaned_data and 'password2' in self.cleaned_data:
            if self.cleaned_data['password1'] != self.cleaned_data['password2']:
                raise forms.ValidationError(_("The two password fields didn't match."))
        return self.cleaned_data

class RegistrationFormTermsOfService(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which adds a required checkbox
    for agreeing to a site's Terms of Service.

    """
    tos = forms.BooleanField(widget=forms.CheckboxInput(attrs=attrs_dict),
                             label=_(u'I have read and agree to the Terms of Service'),
                             error_messages={ 'required': _("You must agree to the terms to register") })

class RegistrationFormUniqueEmail(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which enforces uniqueness of
    email addresses.

    """
    def clean_email(self):
        """
        Validate that the supplied email address is unique for the
        site.

        """
        if User.objects.filter(email__iexact=self.cleaned_data['email']):
            raise forms.ValidationError(_("This email address is already in use. Please supply a different email address."))
        return self.cleaned_data['email']

class RegistrationFormNoFreeEmail(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which disallows registration with
    email addresses from popular free webmail services; moderately
    useful for preventing automated spam registrations.

    To change the list of banned domains, subclass this form and
    override the attribute ``bad_domains``.

    """
    bad_domains = ['aim.com', 'aol.com', 'email.com', 'gmail.com',
                   'googlemail.com', 'hotmail.com', 'hushmail.com',
                   'msn.com', 'mail.ru', 'mailinator.com', 'live.com',
                   'yahoo.com']

    def clean_email(self):
        """
        Check the supplied email address against a list of known free
        webmail domains.

        """
        email_domain = self.cleaned_data['email'].split('@')[1]
        if email_domain in self.bad_domains:
            raise forms.ValidationError(_("Registration using free email addresses is prohibited. Please supply a different email address."))
        return self.cleaned_data['email']
<commit_msg>Missed one.<commit_after>"""
Forms and validation code for user registration.

"""

from django.contrib.auth.models import User
from django import forms
from django.utils.translation import ugettext_lazy as _

# I put this on all required fields, because it's easier to pick up
# on them with CSS or JavaScript if they have a class of "required"
# in the HTML. Your mileage may vary. If/when Django ticket #3515
# lands in trunk, this will no longer be necessary.
attrs_dict = { 'class': 'required' }

class RegistrationForm(forms.Form):
    """
    Form for registering a new user account.

    Validates that the requested username is not already in use, and
    requires the password to be entered twice to catch typos.

    Subclasses should feel free to add any additional validation they
    need, but should avoid defining a ``save()`` method -- the actual
    saving of collected user data is delegated to the active
    registration backend.

    """
    username = forms.RegexField(regex=r'^\w+$',
                                max_length=30,
                                widget=forms.TextInput(attrs=attrs_dict),
                                label=_("Username"),
                                error_messages={ 'invalid': _("This value must contain only letters, numbers and underscores.")) })
    email = forms.EmailField(widget=forms.TextInput(attrs=dict(attrs_dict,
                                                               maxlength=75)),
                             label=_("Email address"))
    password1 = forms.CharField(widget=forms.PasswordInput(attrs=attrs_dict, render_value=False),
                                label=_("Password"))
    password2 = forms.CharField(widget=forms.PasswordInput(attrs=attrs_dict, render_value=False),
                                label=_("Password (again)"))

    def clean_username(self):
        """
        Validate that the username is alphanumeric and is not already
        in use.

        """
        try:
            user = User.objects.get(username__iexact=self.cleaned_data['username'])
        except User.DoesNotExist:
            return self.cleaned_data['username']
        raise forms.ValidationError(_("A user with that username already exists."))

    def clean(self):
        """
        Verifiy that the values entered into the two password fields
        match. Note that an error here will end up in
        ``non_field_errors()`` because it doesn't apply to a single
        field.

        """
        if 'password1' in self.cleaned_data and 'password2' in self.cleaned_data:
            if self.cleaned_data['password1'] != self.cleaned_data['password2']:
                raise forms.ValidationError(_("The two password fields didn't match."))
        return self.cleaned_data

class RegistrationFormTermsOfService(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which adds a required checkbox
    for agreeing to a site's Terms of Service.

    """
    tos = forms.BooleanField(widget=forms.CheckboxInput(attrs=attrs_dict),
                             label=_(u'I have read and agree to the Terms of Service'),
                             error_messages={ 'required': _("You must agree to the terms to register") })

class RegistrationFormUniqueEmail(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which enforces uniqueness of
    email addresses.

    """
    def clean_email(self):
        """
        Validate that the supplied email address is unique for the
        site.

        """
        if User.objects.filter(email__iexact=self.cleaned_data['email']):
            raise forms.ValidationError(_("This email address is already in use. Please supply a different email address."))
        return self.cleaned_data['email']

class RegistrationFormNoFreeEmail(RegistrationForm):
    """
    Subclass of ``RegistrationForm`` which disallows registration with
    email addresses from popular free webmail services; moderately
    useful for preventing automated spam registrations.

    To change the list of banned domains, subclass this form and
    override the attribute ``bad_domains``.

    """
    bad_domains = ['aim.com', 'aol.com', 'email.com', 'gmail.com',
                   'googlemail.com', 'hotmail.com', 'hushmail.com',
                   'msn.com', 'mail.ru', 'mailinator.com', 'live.com',
                   'yahoo.com']

    def clean_email(self):
        """
        Check the supplied email address against a list of known free
        webmail domains.

        """
        email_domain = self.cleaned_data['email'].split('@')[1]
        if email_domain in self.bad_domains:
            raise forms.ValidationError(_("Registration using free email addresses is prohibited. Please supply a different email address."))
        return self.cleaned_data['email']
<|endoftext|>


<commit_before>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Unit Tests for remote procedure calls using queue
"""

import mock
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_db import exception as db_exc

from karbor import context
from karbor import db
from karbor import exception
from karbor import manager
from karbor import rpc
from karbor import service
from karbor.tests import base

test_service_opts = [
    cfg.StrOpt("fake_manager",
               default="karbor.tests.unit.test_service.FakeManager",
               help="Manager for testing"),
    cfg.StrOpt("test_service_listen",
               help="Host to bind test service to"),
    cfg.IntOpt("test_service_listen_port",
               default=0,
               help="Port number to bind test service to"), ]

CONF = cfg.CONF
CONF.register_opts(test_service_opts)

class FakeManager(manager.Manager):
    """Fake manager for tests."""
    def __init__(self, host=None,
                 db_driver=None, service_name=None):
        super(FakeManager, self).__init__(host=host,
                                          db_driver=db_driver)

    def test_method(self):
        return 'manager'

class ExtendedService(service.Service):
    def test_method(self):
        return 'service'

class ServiceManagerTestCase(base.TestCase):
    """Test cases for Services."""

    def test_message_gets_to_manager(self):
        serv = service.Service('test',
                               'test',
                               'test',
                               'karbor.tests.unit.test_service.FakeManager')
        serv.start()
        self.assertEqual('manager', serv.test_method())

    def test_override_manager_method(self):
        serv = ExtendedService('test',
                               'test',
                               'test',
                               'karbor.tests.unit.test_service.FakeManager')
        serv.start()
        self.assertEqual('service', serv.test_method())

class ServiceFlagsTestCase(base.TestCase):
    def test_service_enabled_on_create_based_on_flag(self):
        self.flags(enable_new_services=True)
        host = 'foo'
        binary = 'karbor-fake'
        app = service.Service.create(host=host, binary=binary)
        app.start()
        app.stop()
        ref = db.service_get(context.get_admin_context(), app.service_id)
        db.service_destroy(context.get_admin_context(), app.service_id)
        self.assertFalse(ref['disabled'])

    def test_service_disabled_on_create_based_on_flag(self):
        self.flags(enable_new_services=False)
        host = 'foo'
        binary = 'karbor-fake'
        app = service.Service.create(host=host, binary=binary)
        app.start()
        app.stop()
        ref = db.service_get(context.get_admin_context(), app.service_id)
        db.service_destroy(context.get_admin_context(), app.service_id)
        self.assertTrue(ref['disabled'])

class ServiceTestCase(base.TestCase):
    """Test cases for Services."""

    def setUp(self):
        super(ServiceTestCase, self).setUp()
        self.host = 'foo'
        self.binary = 'karbor-fake'
        self.topic = 'fake'

    def test_create(self):
        app = service.Service.create(host=self.host,
                                     binary=self.binary,
                                     topic=self.topic)

        self.assertTrue(app)

    def test_report_state_newly_disconnected(self):
        service_ref = {'host': self.host,
                       'binary': self.binary,
                       'topic': self.topic,
                       'report_count': 0,
                       'id': 1}
        with mock.patch.object(service, 'db') as mock_db:
            mock_db.service_get_by_args.side_effect = exception.NotFound()
            mock_db.service_create.return_value = service_ref
            mock_db.service_get.side_effect = db_exc.DBConnectionError()

            serv = service.Service(
                self.host,
                self.binary,
                self.topic,
                'karbor.tests.unit.test_service.FakeManager'
            )
            serv.start()
            serv.report_state()
            self.assertTrue(serv.model_disconnected)
            self.assertFalse(mock_db.service_update.called)

    def test_report_state_disconnected_DBError(self):
        service_ref = {'host': self.host,
                       'binary': self.binary,
                       'topic': self.topic,
                       'report_count': 0,
                       'id': 1}
        with mock.patch.object(service, 'db') as mock_db:
            mock_db.service_get_by_args.side_effect = exception.NotFound()
            mock_db.service_create.return_value = service_ref
            mock_db.service_get.side_effect = db_exc.DBError()

            serv = service.Service(
                self.host,
                self.binary,
                self.topic,
                'karbor.tests.unit.test_service.FakeManager'
            )
            serv.start()
            serv.report_state()
            self.assertTrue(serv.model_disconnected)
            self.assertFalse(mock_db.service_update.called)

    def test_report_state_newly_connected(self):
        service_ref = {'host': self.host,
                       'binary': self.binary,
                       'topic': self.topic,
                       'report_count': 0,
                       'id': 1}
        with mock.patch.object(service, 'db') as mock_db:
            mock_db.service_get_by_args.side_effect = exception.NotFound()
            mock_db.service_create.return_value = service_ref
            mock_db.service_get.return_value = service_ref

            serv = service.Service(
                self.host,
                self.binary,
                self.topic,
                'karbor.tests.unit.test_service.FakeManager'
            )
            serv.start()
            serv.model_disconnected = True
            serv.report_state()

            self.assertFalse(serv.model_disconnected)
            self.assertTrue(mock_db.service_update.called)

    def test_report_state_manager_not_working(self):
        service_ref = {'host': self.host,
                       'binary': self.binary,
                       'topic': self.topic,
                       'report_count': 0,
                       'id': 1}
        with mock.patch('karbor.db') as mock_db:
            mock_db.service_get.return_value = service_ref

            serv = service.Service(
                self.host,
                self.binary,
                self.topic,
                'karbor.tests.unit.test_service.FakeManager'
            )
            serv.manager.is_working = mock.Mock(return_value=False)
            serv.start()
            serv.report_state()

            serv.manager.is_working.assert_called_once_with()
            self.assertFalse(mock_db.service_update.called)

    def test_service_with_long_report_interval(self):
        self.override_config('service_down_time', 10)
        self.override_config('report_interval', 10)
        service.Service.create(
            binary="test_service",
            manager="karbor.tests.unit.test_service.FakeManager")
        self.assertEqual(25, CONF.service_down_time)

    @mock.patch.object(rpc, 'get_server')
    @mock.patch('karbor.db')
    def test_service_stop_waits_for_rpcserver(self, mock_db, mock_rpc):
        serv = service.Service(
            self.host,
            self.binary,
            self.topic,
            'karbor.tests.unit.test_service.FakeManager'
        )
        serv.start()
        serv.stop()
        serv.wait()
        serv.rpcserver.start.assert_called_once_with()
        serv.rpcserver.stop.assert_called_once_with()
        serv.rpcserver.wait.assert_called_once_with()

class TestWSGIService(base.TestCase):

    def setUp(self):
        super(TestWSGIService, self).setUp()

    @mock.patch('oslo_service.wsgi.Loader')
    def test_service_random_port(self, mock_loader):
        test_service = service.WSGIService("test_service")
        self.assertEqual(0, test_service.port)
        test_service.start()
        self.assertNotEqual(0, test_service.port)
        test_service.stop()
        self.assertTrue(mock_loader.called)

    @mock.patch('oslo_service.wsgi.Loader')
    def test_reset_pool_size_to_default(self, mock_loader):
        test_service = service.WSGIService("test_service")
        test_service.start()

        # Stopping the service, which in turn sets pool size to 0
        test_service.stop()
        self.assertEqual(0, test_service.server._pool.size)

        # Resetting pool size to default
        test_service.reset()
        test_service.start()
        self.assertEqual(cfg.CONF.wsgi_default_pool_size,
                         test_service.server._pool.size)
        self.assertTrue(mock_loader.called)

    @mock.patch('oslo_service.wsgi.Loader')
    def test_workers_set_default(self, mock_loader):
        self.override_config('osapi_karbor_listen_port',
                             CONF.test_service_listen_port)
        test_service = service.WSGIService("osapi_karbor")
        self.assertEqual(processutils.get_worker_count(),
                         test_service.workers)
        self.assertTrue(mock_loader.called)

    @mock.patch('oslo_service.wsgi.Loader')
    def test_workers_set_good_user_setting(self, mock_loader):
        self.override_config('osapi_karbor_listen_port',
                             CONF.test_service_listen_port)
        self.override_config('osapi_karbor_workers', 8)
        test_service = service.WSGIService("osapi_karbor")
        self.assertEqual(8, test_service.workers)
        self.assertTrue(mock_loader.called)

    @mock.patch('oslo_service.wsgi.Loader')
    def test_workers_set_zero_user_setting(self, mock_loader):
        self.override_config('osapi_karbor_listen_port',
                             CONF.test_service_listen_port)
        self.override_config('osapi_karbor_workers', 0)
        test_service = service.WSGIService("osapi_karbor")
        # If a value less than 1 is used, defaults to number of procs
        # available
        self.assertEqual(processutils.get_worker_count(),
                         test_service.workers)
        self.assertTrue(mock_loader.called)

    @mock.patch('oslo_service.wsgi.Loader')
    def test_workers_set_negative_user_setting(self, mock_loader):
        self.override_config('osapi_karbor_workers', -1)
        self.assertRaises(exception.InvalidInput,
                          service.WSGIService, "osapi_karbor")
        self.assertTrue(mock_loader.called)
<commit_msg>Use PortOpt for port options<commit_after>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Unit Tests for remote procedure calls using queue
"""

import mock
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_db import exception as db_exc

from karbor import context
from karbor import db
from karbor import exception
from karbor import manager
from karbor import rpc
from karbor import service
from karbor.tests import base

test_service_opts = [
    cfg.StrOpt("fake_manager",
               default="karbor.tests.unit.test_service.FakeManager",
               help="Manager for testing"),
    cfg.StrOpt("test_service_listen",
               help="Host to bind test service to"),
    cfg.PortOpt("test_service_listen_port",
                default=8799,
                help="Port number to bind test service to"), ]

CONF = cfg.CONF
CONF.register_opts(test_service_opts)

class FakeManager(manager.Manager):
    """Fake manager for tests."""
    def __init__(self, host=None,
                 db_driver=None, service_name=None):
        super(FakeManager, self).__init__(host=host,
                                          db_driver=db_driver)

    def test_method(self):
        return 'manager'

class ExtendedService(service.Service):
    def test_method(self):
        return 'service'

class ServiceManagerTestCase(base.TestCase):
    """Test cases for Services."""

    def test_message_gets_to_manager(self):
        serv = service.Service('test',
                               'test',
                               'test',
                               'karbor.tests.unit.test_service.FakeManager')
        serv.start()
        self.assertEqual('manager', serv.test_method())

    def test_override_manager_method(self):
        serv = ExtendedService('test',
                               'test',
                               'test',
                               'karbor.tests.unit.test_service.FakeManager')
        serv.start()
        self.assertEqual('service', serv.test_method())

class ServiceFlagsTestCase(base.TestCase):
    def test_service_enabled_on_create_based_on_flag(self):
        self.flags(enable_new_services=True)
        host = 'foo'
        binary = 'karbor-fake'
        app = service.Service.create(host=host, binary=binary)
        app.start()
        app.stop()
        ref = db.service_get(context.get_admin_context(), app.service_id)
        db.service_destroy(context.get_admin_context(), app.service_id)
        self.assertFalse(ref['disabled'])

    def test_service_disabled_on_create_based_on_flag(self):
        self.flags(enable_new_services=False)
        host = 'foo'
        binary = 'karbor-fake'
        app = service.Service.create(host=host, binary=binary)
        app.start()
        app.stop()
        ref = db.service_get(context.get_admin_context(), app.service_id)
        db.service_destroy(context.get_admin_context(), app.service_id)
        self.assertTrue(ref['disabled'])

class ServiceTestCase(base.TestCase):
    """Test cases for Services."""

    def setUp(self):
        super(ServiceTestCase, self).setUp()
        self.host = 'foo'
        self.binary = 'karbor-fake'
        self.topic = 'fake'

    def test_create(self):
        app = service.Service.create(host=self.host,
                                     binary=self.binary,
                                     topic=self.topic)

        self.assertTrue(app)

    def test_report_state_newly_disconnected(self):
        service_ref = {'host': self.host,
                       'binary': self.binary,
                       'topic': self.topic,
                       'report_count': 0,
                       'id': 1}
        with mock.patch.object(service, 'db') as mock_db:
            mock_db.service_get_by_args.side_effect = exception.NotFound()
            mock_db.service_create.return_value = service_ref
            mock_db.service_get.side_effect = db_exc.DBConnectionError()

            serv = service.Service(
                self.host,
                self.binary,
                self.topic,
                'karbor.tests.unit.test_service.FakeManager'
            )
            serv.start()
            serv.report_state()
            self.assertTrue(serv.model_disconnected)
            self.assertFalse(mock_db.service_update.called)

    def test_report_state_disconnected_DBError(self):
        service_ref = {'host': self.host,
                       'binary': self.binary,
                       'topic': self.topic,
                       'report_count': 0,
                       'id': 1}
        with mock.patch.object(service, 'db') as mock_db:
            mock_db.service_get_by_args.side_effect = exception.NotFound()
            mock_db.service_create.return_value = service_ref
            mock_db.service_get.side_effect = db_exc.DBError()

            serv = service.Service(
                self.host,
                self.binary,
                self.topic,
                'karbor.tests.unit.test_service.FakeManager'
            )
            serv.start()
            serv.report_state()
            self.assertTrue(serv.model_disconnected)
            self.assertFalse(mock_db.service_update.called)

    def test_report_state_newly_connected(self):
        service_ref = {'host': self.host,
                       'binary': self.binary,
                       'topic': self.topic,
                       'report_count': 0,
                       'id': 1}
        with mock.patch.object(service, 'db') as mock_db:
            mock_db.service_get_by_args.side_effect = exception.NotFound()
            mock_db.service_create.return_value = service_ref
            mock_db.service_get.return_value = service_ref

            serv = service.Service(
                self.host,
                self.binary,
                self.topic,
                'karbor.tests.unit.test_service.FakeManager'
            )
            serv.start()
            serv.model_disconnected = True
            serv.report_state()

            self.assertFalse(serv.model_disconnected)
            self.assertTrue(mock_db.service_update.called)

    def test_report_state_manager_not_working(self):
        service_ref = {'host': self.host,
                       'binary': self.binary,
                       'topic': self.topic,
                       'report_count': 0,
                       'id': 1}
        with mock.patch('karbor.db') as mock_db:
            mock_db.service_get.return_value = service_ref

            serv = service.Service(
                self.host,
                self.binary,
                self.topic,
                'karbor.tests.unit.test_service.FakeManager'
            )
            serv.manager.is_working = mock.Mock(return_value=False)
            serv.start()
            serv.report_state()

            serv.manager.is_working.assert_called_once_with()
            self.assertFalse(mock_db.service_update.called)

    def test_service_with_long_report_interval(self):
        self.override_config('service_down_time', 10)
        self.override_config('report_interval', 10)
        service.Service.create(
            binary="test_service",
            manager="karbor.tests.unit.test_service.FakeManager")
        self.assertEqual(25, CONF.service_down_time)

    @mock.patch.object(rpc, 'get_server')
    @mock.patch('karbor.db')
    def test_service_stop_waits_for_rpcserver(self, mock_db, mock_rpc):
        serv = service.Service(
            self.host,
            self.binary,
            self.topic,
            'karbor.tests.unit.test_service.FakeManager'
        )
        serv.start()
        serv.stop()
        serv.wait()
        serv.rpcserver.start.assert_called_once_with()
        serv.rpcserver.stop.assert_called_once_with()
        serv.rpcserver.wait.assert_called_once_with()

class TestWSGIService(base.TestCase):

    def setUp(self):
        super(TestWSGIService, self).setUp()

    @mock.patch('oslo_service.wsgi.Loader')
    def test_service_random_port(self, mock_loader):
        test_service = service.WSGIService("test_service")
        self.assertEqual(8799, test_service.port)
        test_service.start()
        self.assertNotEqual(0, test_service.port)
        test_service.stop()
        self.assertTrue(mock_loader.called)

    @mock.patch('oslo_service.wsgi.Loader')
    def test_reset_pool_size_to_default(self, mock_loader):
        test_service = service.WSGIService("test_service")
        test_service.start()

        # Stopping the service, which in turn sets pool size to 0
        test_service.stop()
        self.assertEqual(0, test_service.server._pool.size)

        # Resetting pool size to default
        test_service.reset()
        test_service.start()
        self.assertEqual(cfg.CONF.wsgi_default_pool_size,
                         test_service.server._pool.size)
        self.assertTrue(mock_loader.called)

    @mock.patch('oslo_service.wsgi.Loader')
    def test_workers_set_default(self, mock_loader):
        self.override_config('osapi_karbor_listen_port',
                             CONF.test_service_listen_port)
        test_service = service.WSGIService("osapi_karbor")
        self.assertEqual(processutils.get_worker_count(),
                         test_service.workers)
        self.assertTrue(mock_loader.called)

    @mock.patch('oslo_service.wsgi.Loader')
    def test_workers_set_good_user_setting(self, mock_loader):
        self.override_config('osapi_karbor_listen_port',
                             CONF.test_service_listen_port)
        self.override_config('osapi_karbor_workers', 8)
        test_service = service.WSGIService("osapi_karbor")
        self.assertEqual(8, test_service.workers)
        self.assertTrue(mock_loader.called)

    @mock.patch('oslo_service.wsgi.Loader')
    def test_workers_set_zero_user_setting(self, mock_loader):
        self.override_config('osapi_karbor_listen_port',
                             CONF.test_service_listen_port)
        self.override_config('osapi_karbor_workers', 0)
        test_service = service.WSGIService("osapi_karbor")
        # If a value less than 1 is used, defaults to number of procs
        # available
        self.assertEqual(processutils.get_worker_count(),
                         test_service.workers)
        self.assertTrue(mock_loader.called)

    @mock.patch('oslo_service.wsgi.Loader')
    def test_workers_set_negative_user_setting(self, mock_loader):
        self.override_config('osapi_karbor_workers', -1)
        self.assertRaises(exception.InvalidInput,
                          service.WSGIService, "osapi_karbor")
        self.assertTrue(mock_loader.called)
<|endoftext|>