<commit_before># (C) Datadog, Inc. 2018
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
import os

import click
import pyperclip

from ....utils import dir_exists, file_exists, path_join
from ...e2e import E2E_SUPPORTED_TYPES, derive_interface, start_environment, stop_environment
from ...e2e.agent import DEFAULT_PYTHON_VERSION
from ...testing import get_available_tox_envs, get_tox_env_python_version
from ...utils import get_tox_file
from ..console import CONTEXT_SETTINGS, abort, echo_failure, echo_info, echo_success, echo_waiting, echo_warning

@click.command(context_settings=CONTEXT_SETTINGS, short_help='Start an environment')
@click.argument('check')
@click.argument('env')
@click.option(
    '--agent',
    '-a',
    help=(
        'The agent build to use e.g. a Docker image like `datadog/agent:6.5.2`. For '
        'Docker environments you can use an integer corresponding to fields in the '
        'config (agent5, agent6, etc.)'
    ),
)
@click.option(
    '--python',
    '-py',
    type=click.INT,
    help='The version of Python to use. Defaults to {} if no tox Python is specified.'.format(DEFAULT_PYTHON_VERSION),
)
@click.option('--dev/--prod', help='Whether to use the latest version of a check or what is shipped')
@click.option('--base', is_flag=True, help='Whether to use the latest version of the base check or what is shipped')
@click.option(
    '--env-vars',
    '-e',
    multiple=True,
    help=(
        'ENV Variable that should be passed to the Agent container. '
        'Ex: -e DD_URL=app.datadoghq.com -e DD_API_KEY=123456'
    ),
)
@click.pass_context
def start(ctx, check, env, agent, python, dev, base, env_vars):
    """Start an environment."""
    if not file_exists(get_tox_file(check)):
        abort('`{}` is not a testable check.'.format(check))

    base_package = None
    if base:
        core_dir = os.path.expanduser(ctx.obj.get('core', ''))
        if not dir_exists(core_dir):
            if core_dir:
                abort('`{}` directory does not exist.'.format(core_dir))
            else:
                abort('`core` config setting does not exist.')

        base_package = path_join(core_dir, 'datadog_checks_base')
        if not dir_exists(base_package):
            abort('`datadog_checks_base` directory does not exist.')

    envs = get_available_tox_envs(check, e2e_only=True)

    if env not in envs:
        echo_failure('`{}` is not an available environment.'.format(env))
        echo_info('Available environments for {}:\n    {}'.format(check, '\n    '.join(envs)))
        echo_info('You can also use `ddev env ls {}` to see available environments.'.format(check))
        abort()

    env_python_version = get_tox_env_python_version(env)
    if not python:
        # Make the tox environment Python specifier influence the Agent
        python = env_python_version or DEFAULT_PYTHON_VERSION
    elif env_python_version and env_python_version != int(python):
        echo_warning(
            'The local environment `{}` does not match the expected Python. The Agent will use Python {}. '
            'To influence the Agent Python version, use the `-py/--python` option.'.format(env, python)
        )

    api_key = ctx.obj['dd_api_key']
    if api_key is None:
        echo_warning(
            'Environment variable DD_API_KEY does not exist; a well-formatted '
            'fake API key will be used instead. You can also set the API key '
            'by doing `ddev config set dd_api_key`.'
        )

    echo_waiting('Setting up environment `{}`... '.format(env), nl=False)
    config, metadata, error = start_environment(check, env)

    if error:
        if 'does not support this platform' in error:
            echo_warning(error)
            abort(code=0)
        else:
            echo_failure('failed!')
            echo_waiting('Stopping the environment...')
            stop_environment(check, env, metadata=metadata)
            abort(error)
    echo_success('success!')

    env_type = metadata['env_type']

    agent_ver = agent or os.getenv('DDEV_E2E_AGENT', '6')
    agent_build = ctx.obj.get('agent{}'.format(agent_ver), agent_ver)
    if isinstance(agent_build, dict):
        agent_build = agent_build.get(env_type, env_type)

    interface = derive_interface(env_type)
    if interface is None:
        echo_failure('`{}` is an unsupported environment type.'.format(env_type))
        echo_waiting('Stopping the environment...')
        stop_environment(check, env, metadata=metadata)
        abort()

    if env_type not in E2E_SUPPORTED_TYPES and agent_ver.isdigit():
        echo_failure('Configuration for default Agents are only for Docker. You must specify the full build.')
        echo_waiting('Stopping the environment...')
        stop_environment(check, env, metadata=metadata)
        abort()

    env_vars = dict(ev.split('=', 1) for ev in env_vars)
    for key, value in metadata.get('env_vars', {}).items():
        env_vars.setdefault(key, value)

    environment = interface(check, env, base_package, config, env_vars, metadata, agent_build, api_key, python)

    echo_waiting('Updating `{}`... '.format(agent_build), nl=False)
    environment.update_agent()
    echo_success('success!')

    echo_waiting('Detecting the major version... ', nl=False)
    environment.detect_agent_version()
    echo_info('Agent {} detected'.format(environment.agent_version))

    echo_waiting('Writing configuration for `{}`... '.format(env), nl=False)
    environment.write_config()
    echo_success('success!')

    echo_waiting('Starting the Agent... ', nl=False)
    result = environment.start_agent()
    if result.code:
        click.echo()
        echo_info(result.stdout + result.stderr)
        echo_failure('An error occurred.')
        echo_waiting('Stopping the environment...')
        stop_environment(check, env, metadata=metadata)
        environment.remove_config()
        abort()
    echo_success('success!')

    start_commands = metadata.get('start_commands', [])
    if start_commands:
        echo_waiting('Running extra start-up commands... ', nl=False)

        for command in start_commands:
            result = environment.exec_command(command, capture=True)
            if result.code:
                click.echo()
                echo_info(result.stdout + result.stderr)
                echo_failure('An error occurred.')
                echo_waiting('Stopping the environment...')
                stop_environment(check, env, metadata=metadata)
                echo_waiting('Stopping the Agent...')
                environment.stop_agent()
                environment.remove_config()
                abort()

        echo_success('success!')

    if base and not dev:
        dev = True
        echo_info(
            'Will install the development version of the check too so the base package can import it (in editable mode)'
        )

    editable_warning = (
        '\nEnv will started with an editable check install for the {} package. '
        'This check will remain in an editable install after '
        'the environment is torn down. Would you like to proceed?'
    )

    if base:
        echo_waiting('Upgrading the base package to the development version... ', nl=False)
        if environment.ENV_TYPE == 'local' and not click.confirm(editable_warning.format('base')):
            echo_success('skipping')
        else:
            environment.update_base_package()
            echo_success('success!')

    if dev:
        echo_waiting('Upgrading `{}` check to the development version... '.format(check), nl=False)
        if environment.ENV_TYPE == 'local' and not click.confirm(editable_warning.format(environment.check)):
            echo_success('skipping')
        else:
            environment.update_check()
            echo_success('success!')

    click.echo()

    try:
        pyperclip.copy(environment.config_file)
    except Exception:
        config_message = 'Config file: '
    else:
        config_message = 'Config file (copied to your clipboard): '

    echo_success(config_message, nl=False)
    echo_info(environment.config_file)

    echo_success('To run this check, do: ', nl=False)
    echo_info('ddev env check {} {}'.format(check, env))

    echo_success('To stop this check, do: ', nl=False)
    if ctx.obj['repo_choice'] == 'extras' and not ctx.obj.get('repo') == 'extras':
        echo_info('ddev -e env stop {} {}'.format(check, env))
    else:
        echo_info('ddev env stop {} {}'.format(check, env))
<commit_msg>Reload environments if there are extra startup commands (#4614)<commit_after># (C) Datadog, Inc. 2018
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
import os

import click
import pyperclip

from ....utils import dir_exists, file_exists, path_join
from ...e2e import E2E_SUPPORTED_TYPES, derive_interface, start_environment, stop_environment
from ...e2e.agent import DEFAULT_PYTHON_VERSION
from ...testing import get_available_tox_envs, get_tox_env_python_version
from ...utils import get_tox_file
from ..console import CONTEXT_SETTINGS, abort, echo_failure, echo_info, echo_success, echo_waiting, echo_warning

@click.command(context_settings=CONTEXT_SETTINGS, short_help='Start an environment')
@click.argument('check')
@click.argument('env')
@click.option(
    '--agent',
    '-a',
    help=(
        'The agent build to use e.g. a Docker image like `datadog/agent:6.5.2`. For '
        'Docker environments you can use an integer corresponding to fields in the '
        'config (agent5, agent6, etc.)'
    ),
)
@click.option(
    '--python',
    '-py',
    type=click.INT,
    help='The version of Python to use. Defaults to {} if no tox Python is specified.'.format(DEFAULT_PYTHON_VERSION),
)
@click.option('--dev/--prod', help='Whether to use the latest version of a check or what is shipped')
@click.option('--base', is_flag=True, help='Whether to use the latest version of the base check or what is shipped')
@click.option(
    '--env-vars',
    '-e',
    multiple=True,
    help=(
        'ENV Variable that should be passed to the Agent container. '
        'Ex: -e DD_URL=app.datadoghq.com -e DD_API_KEY=123456'
    ),
)
@click.pass_context
def start(ctx, check, env, agent, python, dev, base, env_vars):
    """Start an environment."""
    if not file_exists(get_tox_file(check)):
        abort('`{}` is not a testable check.'.format(check))

    base_package = None
    if base:
        core_dir = os.path.expanduser(ctx.obj.get('core', ''))
        if not dir_exists(core_dir):
            if core_dir:
                abort('`{}` directory does not exist.'.format(core_dir))
            else:
                abort('`core` config setting does not exist.')

        base_package = path_join(core_dir, 'datadog_checks_base')
        if not dir_exists(base_package):
            abort('`datadog_checks_base` directory does not exist.')

    envs = get_available_tox_envs(check, e2e_only=True)

    if env not in envs:
        echo_failure('`{}` is not an available environment.'.format(env))
        echo_info('Available environments for {}:\n    {}'.format(check, '\n    '.join(envs)))
        echo_info('You can also use `ddev env ls {}` to see available environments.'.format(check))
        abort()

    env_python_version = get_tox_env_python_version(env)
    if not python:
        # Make the tox environment Python specifier influence the Agent
        python = env_python_version or DEFAULT_PYTHON_VERSION
    elif env_python_version and env_python_version != int(python):
        echo_warning(
            'The local environment `{}` does not match the expected Python. The Agent will use Python {}. '
            'To influence the Agent Python version, use the `-py/--python` option.'.format(env, python)
        )

    api_key = ctx.obj['dd_api_key']
    if api_key is None:
        echo_warning(
            'Environment variable DD_API_KEY does not exist; a well-formatted '
            'fake API key will be used instead. You can also set the API key '
            'by doing `ddev config set dd_api_key`.'
        )

    echo_waiting('Setting up environment `{}`... '.format(env), nl=False)
    config, metadata, error = start_environment(check, env)

    if error:
        if 'does not support this platform' in error:
            echo_warning(error)
            abort(code=0)
        else:
            echo_failure('failed!')
            echo_waiting('Stopping the environment...')
            stop_environment(check, env, metadata=metadata)
            abort(error)
    echo_success('success!')

    env_type = metadata['env_type']

    agent_ver = agent or os.getenv('DDEV_E2E_AGENT', '6')
    agent_build = ctx.obj.get('agent{}'.format(agent_ver), agent_ver)
    if isinstance(agent_build, dict):
        agent_build = agent_build.get(env_type, env_type)

    interface = derive_interface(env_type)
    if interface is None:
        echo_failure('`{}` is an unsupported environment type.'.format(env_type))
        echo_waiting('Stopping the environment...')
        stop_environment(check, env, metadata=metadata)
        abort()

    if env_type not in E2E_SUPPORTED_TYPES and agent_ver.isdigit():
        echo_failure('Configuration for default Agents are only for Docker. You must specify the full build.')
        echo_waiting('Stopping the environment...')
        stop_environment(check, env, metadata=metadata)
        abort()

    env_vars = dict(ev.split('=', 1) for ev in env_vars)
    for key, value in metadata.get('env_vars', {}).items():
        env_vars.setdefault(key, value)

    environment = interface(check, env, base_package, config, env_vars, metadata, agent_build, api_key, python)

    echo_waiting('Updating `{}`... '.format(agent_build), nl=False)
    environment.update_agent()
    echo_success('success!')

    echo_waiting('Detecting the major version... ', nl=False)
    environment.detect_agent_version()
    echo_info('Agent {} detected'.format(environment.agent_version))

    echo_waiting('Writing configuration for `{}`... '.format(env), nl=False)
    environment.write_config()
    echo_success('success!')

    echo_waiting('Starting the Agent... ', nl=False)
    result = environment.start_agent()
    if result.code:
        click.echo()
        echo_info(result.stdout + result.stderr)
        echo_failure('An error occurred.')
        echo_waiting('Stopping the environment...')
        stop_environment(check, env, metadata=metadata)
        environment.remove_config()
        abort()
    echo_success('success!')

    start_commands = metadata.get('start_commands', [])
    if start_commands:
        echo_waiting('Running extra start-up commands... ', nl=False)

        for command in start_commands:
            result = environment.exec_command(command, capture=True)
            if result.code:
                click.echo()
                echo_info(result.stdout + result.stderr)
                echo_failure('An error occurred.')
                echo_waiting('Stopping the environment...')
                stop_environment(check, env, metadata=metadata)
                echo_waiting('Stopping the Agent...')
                environment.stop_agent()
                environment.remove_config()
                abort()

        echo_success('success!')

        echo_waiting('Reloading the environment to reflect changes... ', nl=False)
        result = environment.restart_agent()

        if result.code:
            click.echo()
            echo_info(result.stdout + result.stderr)
            echo_failure('An error occurred.')
            echo_waiting('Stopping the environment...')
            stop_environment(check, env, metadata=metadata)
            echo_waiting('Stopping the Agent...')
            environment.stop_agent()
            environment.remove_config()
        else:
            echo_success('success!')

    if base and not dev:
        dev = True
        echo_info(
            'Will install the development version of the check too so the base package can import it (in editable mode)'
        )

    editable_warning = (
        '\nEnv will started with an editable check install for the {} package. '
        'This check will remain in an editable install after '
        'the environment is torn down. Would you like to proceed?'
    )

    if base:
        echo_waiting('Upgrading the base package to the development version... ', nl=False)
        if environment.ENV_TYPE == 'local' and not click.confirm(editable_warning.format('base')):
            echo_success('skipping')
        else:
            environment.update_base_package()
            echo_success('success!')

    if dev:
        echo_waiting('Upgrading `{}` check to the development version... '.format(check), nl=False)
        if environment.ENV_TYPE == 'local' and not click.confirm(editable_warning.format(environment.check)):
            echo_success('skipping')
        else:
            environment.update_check()
            echo_success('success!')

    click.echo()

    try:
        pyperclip.copy(environment.config_file)
    except Exception:
        config_message = 'Config file: '
    else:
        config_message = 'Config file (copied to your clipboard): '

    echo_success(config_message, nl=False)
    echo_info(environment.config_file)

    echo_success('To run this check, do: ', nl=False)
    echo_info('ddev env check {} {}'.format(check, env))

    echo_success('To stop this check, do: ', nl=False)
    if ctx.obj['repo_choice'] == 'extras' and not ctx.obj.get('repo') == 'extras':
        echo_info('ddev -e env stop {} {}'.format(check, env))
    else:
        echo_info('ddev env stop {} {}'.format(check, env))
<|endoftext|>


<commit_before># This script is used to import information from OS Boundary-Line,
# which contains digital boundaries for administrative areas within
# Great Britain. Northern Ireland is handled separately, during the
# postcode import phase.

import re
from django.core.management.base import LabelCommand
# Not using LayerMapping as want more control, but what it does is what this does
#from django.contrib.gis.utils import LayerMapping
from django.contrib.gis.gdal import *
from django.contrib.gis.geos import MultiPolygon
from mapit.areas.models import Area, Generation

class Command(LabelCommand):
    help = 'Import OS Boundary-Line'
    args = '<Boundary-Line SHP files>'

    def handle_label(self,  filename, **options):
        print filename
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception, "No new generation to be used for import!"

        ds = DataSource(filename)
        layer = ds[0]
        ons_code_to_shape = {}
        unit_id_to_shape = {}
        for feat in layer:
            name = unicode(feat['NAME'].value, 'iso-8859-1')
            print " ", name

            name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
            #name = re.sub('\s*\(B\)$', '', name)

            ons_code = feat['CODE'].value if feat['CODE'].value != '999999' else None
            unit_id = str(feat['UNIT_ID'].value)
            area_code = feat['AREA_CODE'].value

            if area_code == 'NCP': continue # Ignore Non Parished Areas

            if ons_code in ons_code_to_shape:
                m = ons_code_to_shape[ons_code]
                m_name = m.names.get(type='O').name
                if name != m_name:
                    raise Exception, "ONS code %s is used for %s and %s" % (ons_code, name, m_name)
                # Otherwise, combine the two shapes for one area
                print "    Adding subsequent shape to ONS code %s" % ons_code
                new_poly = [ shape for shape in m.polygon ]
                new_poly.append(feat.geom.geos)
                m.polygon = MultiPolygon(new_poly)
                continue

            if unit_id in unit_id_to_shape:
                m = unit_id_to_shape[unit_id]
                m_name = m.names.get(type='O').name
                if name != m_name:
                    raise Exception, "Unit ID code %s is used for %s and %s" % (unit_id, name, m_name)
                # Otherwise, combine the two shapes for one area
                print "    Adding subsequent shape to unit ID %s" % unit_id
                new_poly = [ shape for shape in m.polygon ]
                new_poly.append(feat.geom.geos)
                m.polygon = MultiPolygon(new_poly)
                continue

            try:
                if ons_code:
                    m = Area.objects.get(codes__type='ons', codes__code=ons_code)
                elif unit_id:
                    m = Area.objects.get(codes__type='unit_id', codes__code=unit_id)
                else:
                    # UK Parliamentary Constituencies don't have any code in Boundary-Line
                    # (although they will have a code in GSS, looks like).
                    # Let us assume if there's one with the right name, we'll use that.
                    assert area_code == 'WMC'
                    m = Area.objects.get(type=area_code, names__type='O', names__name=name)
            except Area.DoesNotExist:
                g = OGRGeometry(OGRGeomType('MultiPolygon'))
                g.add(feat.geom)
                country = ''
                if area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'):
                    country = 'E'
                elif (area_code == 'EUR' and 'Scotland' in name) or area_code in ('SPC', 'SPE') or (ons_code and ons_code[0:3] in ('00Q', '00R')):
                    country = 'S'
                elif (area_code == 'EUR' and 'Wales' in name) or area_code in ('WAC', 'WAE') or (ons_code and ons_code[0:3] in ('00N', '00P')):
                    country = 'W'
                elif area_code in ('EUR', 'UTA', 'UTE', 'UTW', 'CPC'):
                    country = 'E'
                # That leaves WMC
                # Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks
                # Do parents and WMC countries in separate PinP code after this is done.
                m = Area(
                    type = area_code,
                    country = country,
                    polygon = g.wkt,
                    generation_low = new_generation,
                    generation_high = new_generation,
                )
                m.save()

            if m.generation_high and m.generation_high < current_generation:
                raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
            if m.generation_high < new_generation:
                m.generation_high = new_generation
                m.save()

            m.names.update_or_create({ 'type': 'O' }, { 'name': name })
            if ons_code:
                ons_code_to_shape[ons_code] = m
                m.codes.update_or_create({ 'type': 'ons' }, { 'code': ons_code })
            if unit_id:
                unit_id_to_shape[unit_id] = m
                m.codes.update_or_create({ 'type': 'unit_id' }, { 'code': unit_id })

<commit_msg>Save 41429 (Scottish EUR) to the end.<commit_after># This script is used to import information from OS Boundary-Line,
# which contains digital boundaries for administrative areas within
# Great Britain. Northern Ireland is handled separately, during the
# postcode import phase.

import re
from django.core.management.base import LabelCommand
# Not using LayerMapping as want more control, but what it does is what this does
#from django.contrib.gis.utils import LayerMapping
from django.contrib.gis.gdal import *
from django.contrib.gis.geos import MultiPolygon
from mapit.areas.models import Area, Generation

class Command(LabelCommand):
    help = 'Import OS Boundary-Line'
    args = '<Boundary-Line SHP files>'

    def handle_label(self,  filename, **options):
        print filename
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception, "No new generation to be used for import!"

        ds = DataSource(filename)
        layer = ds[0]
        ons_code_to_shape = {}
        unit_id_to_shape = {}
        for feat in layer:
            name = unicode(feat['NAME'].value, 'iso-8859-1')
            print " ", name

            name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
            #name = re.sub('\s*\(B\)$', '', name)

            ons_code = feat['CODE'].value if feat['CODE'].value != '999999' else None
            unit_id = str(feat['UNIT_ID'].value)
            area_code = feat['AREA_CODE'].value

            if area_code == 'NCP': continue # Ignore Non Parished Areas

            if ons_code in ons_code_to_shape:
                m = ons_code_to_shape[ons_code]
                m_name = m.names.get(type='O').name
                if name != m_name:
                    raise Exception, "ONS code %s is used for %s and %s" % (ons_code, name, m_name)
                # Otherwise, combine the two shapes for one area
                print "    Adding subsequent shape to ONS code %s" % ons_code
                new_poly = [ shape for shape in m.polygon ]
                new_poly.append(feat.geom.geos)
                m.polygon = MultiPolygon(new_poly)
                m.save()
                continue

            if unit_id in unit_id_to_shape:
                m = unit_id_to_shape[unit_id]
                m_name = m.names.get(type='O').name
                if name != m_name:
                    raise Exception, "Unit ID code %s is used for %s and %s" % (unit_id, name, m_name)
                # Otherwise, combine the two shapes for one area
                print "    Adding subsequent shape to unit ID %s" % unit_id
                new_poly = [ shape for shape in m.polygon ]
                new_poly.append(feat.geom.geos)
                m.polygon = MultiPolygon(new_poly)
                if unit_id != 41429:
                    m.save()
                continue

            try:
                if ons_code:
                    m = Area.objects.get(codes__type='ons', codes__code=ons_code)
                elif unit_id:
                    m = Area.objects.get(codes__type='unit_id', codes__code=unit_id)
                else:
                    # UK Parliamentary Constituencies don't have any code in Boundary-Line
                    # (although they will have a code in GSS, looks like).
                    # Let us assume if there's one with the right name, we'll use that.
                    assert area_code == 'WMC'
                    m = Area.objects.get(type=area_code, names__type='O', names__name=name)
            except Area.DoesNotExist:
                g = OGRGeometry(OGRGeomType('MultiPolygon'))
                g.add(feat.geom)
                country = ''
                if area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'):
                    country = 'E'
                elif (area_code == 'EUR' and 'Scotland' in name) or area_code in ('SPC', 'SPE') or (ons_code and ons_code[0:3] in ('00Q', '00R')):
                    country = 'S'
                elif (area_code == 'EUR' and 'Wales' in name) or area_code in ('WAC', 'WAE') or (ons_code and ons_code[0:3] in ('00N', '00P')):
                    country = 'W'
                elif area_code in ('EUR', 'UTA', 'UTE', 'UTW', 'CPC'):
                    country = 'E'
                # That leaves WMC
                # Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks
                # Do parents and WMC countries in separate PinP code after this is done.
                m = Area(
                    type = area_code,
                    country = country,
                    polygon = g.wkt,
                    generation_low = new_generation,
                    generation_high = new_generation,
                )
                m.save()

            if m.generation_high and m.generation_high < current_generation:
                raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
            if m.generation_high < new_generation:
                m.generation_high = new_generation
                m.save()

            m.names.update_or_create({ 'type': 'O' }, { 'name': name })
            if ons_code:
                ons_code_to_shape[ons_code] = m
                m.codes.update_or_create({ 'type': 'ons' }, { 'code': ons_code })
            if unit_id:
                unit_id_to_shape[unit_id] = m
                m.codes.update_or_create({ 'type': 'unit_id' }, { 'code': unit_id })

        # So many shapes, leave the save until the end
        if 41429 in unit_id_to_shape:
            unit_id_to_shape[41429].save()

<|endoftext|>


<commit_before>import unittest

from urllib3.filepost import encode_multipart_formdata, iter_fields
from urllib3.fields import RequestField
from urllib3.packages.six import b, u

BOUNDARY = '!! test boundary !!'

class TestIterfields(unittest.TestCase):

    def test_dict(self):
        for fieldname, value in iter_fields(dict(a='b')):
            self.assertEqual((fieldname, value), ('a', 'b'))

        self.assertEqual(
            list(sorted(iter_fields(dict(a='b', c='d')))),
            [('a', 'b'), ('c', 'd')])

    def test_tuple_list(self):
        for fieldname, value in iter_fields([('a', 'b')]):
            self.assertEqual((fieldname, value), ('a', 'b'))

        self.assertEqual(
            list(iter_fields([('a', 'b'), ('c', 'd')])),
            [('a', 'b'), ('c', 'd')])

class TestMultipartEncoding(unittest.TestCase):

    def test_input_datastructures(self):
        fieldsets = [
            dict(k='v', k2='v2'),
            [('k', 'v'), ('k2', 'v2')],
        ]

        for fields in fieldsets:
            encoded, _ = encode_multipart_formdata(fields, boundary=BOUNDARY)
            self.assertEqual(encoded.count(b(BOUNDARY)), 3)

    def test_field_encoding(self):
        fieldsets = [
            [('k', 'v'), ('k2', 'v2')],
            [('k', b'v'), (u('k2'), b'v2')],
            [('k', b'v'), (u('k2'), 'v2')],
        ]

        for fields in fieldsets:
            encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
            expected = (b'--' + b(BOUNDARY) + b'\r\n'
                        b'Content-Disposition: form-data; name="k"\r\n'
                        b'\r\n'
                        b'v\r\n'
                        b'--' + b(BOUNDARY) + b'\r\n'
                        b'Content-Disposition: form-data; name="k2"\r\n'
                        b'\r\n'
                        b'v2\r\n'
                        b'--' + b(BOUNDARY) + b'--\r\n')

            self.assertEqual(encoded, expected, fields)

            self.assertEqual(content_type,
                             'multipart/form-data; boundary=' + str(BOUNDARY))

    def test_filename(self):
        fields = [('k', ('somename', b'v'))]

        encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
        expected = (b'--' + b(BOUNDARY) + b'\r\n'
                    b'Content-Disposition: form-data; name="k"; filename="somename"\r\n'
                    b'Content-Type: application/octet-stream\r\n'
                    b'\r\n'
                    b'v\r\n'
                    b'--' + b(BOUNDARY) + b'--\r\n')

        self.assertEqual(encoded, expected)

        self.assertEqual(content_type,
                         'multipart/form-data; boundary=' + str(BOUNDARY))

    def test_textplain(self):
        fields = [('k', ('somefile.txt', b'v'))]

        encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
        expected = (b'--' + b(BOUNDARY) + b'\r\n'
                    b'Content-Disposition: form-data; name="k"; filename="somefile.txt"\r\n'
                    b'Content-Type: text/plain\r\n'
                    b'\r\n'
                    b'v\r\n'
                    b'--' + b(BOUNDARY) + b'--\r\n')

        self.assertEqual(encoded, expected)

        self.assertEqual(content_type,
                         'multipart/form-data; boundary=' + str(BOUNDARY))

    def test_explicit(self):
        fields = [('k', ('somefile.txt', b'v', 'image/jpeg'))]

        encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
        expected = (b'--' + b(BOUNDARY) + b'\r\n'
                    b'Content-Disposition: form-data; name="k"; filename="somefile.txt"\r\n'
                    b'Content-Type: image/jpeg\r\n'
                    b'\r\n'
                    b'v\r\n'
                    b'--' + b(BOUNDARY) + b'--\r\n')

        self.assertEqual(encoded, expected)

        self.assertEqual(content_type,
                         'multipart/form-data; boundary=' + str(BOUNDARY))

    def test_request_fields(self):
        fields = [RequestField('k', b'v',
                               filename='somefile.txt',
                               headers={'Content-Type': 'image/jpeg'})]

        encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
        expected = (b'--' + b(BOUNDARY) + b'\r\n'
                    b'Content-Type: image/jpeg\r\n'
                    b'\r\n'
                    b'v\r\n'
                    b'--' + b(BOUNDARY) + b'--\r\n')

        self.assertEqual(encoded, expected)
<commit_msg>Rewrite test_filepost.py to be pytest-style<commit_after>import pytest

from urllib3.filepost import encode_multipart_formdata, iter_fields
from urllib3.fields import RequestField
from urllib3.packages.six import b, u

BOUNDARY = '!! test boundary !!'

class TestIterfields(object):

    def test_dict(self):
        for fieldname, value in iter_fields(dict(a='b')):
            assert (fieldname, value) == ('a', 'b')

        assert (
            list(sorted(iter_fields(dict(a='b', c='d')))) ==
            [('a', 'b'), ('c', 'd')])

    def test_tuple_list(self):
        for fieldname, value in iter_fields([('a', 'b')]):
            assert (fieldname, value) == ('a', 'b')

        assert (
            list(iter_fields([('a', 'b'), ('c', 'd')])) ==
            [('a', 'b'), ('c', 'd')])

class TestMultipartEncoding(object):

    @pytest.mark.parametrize('fields', [
        dict(k='v', k2='v2'),
        [('k', 'v'), ('k2', 'v2')],
    ])
    def test_input_datastructures(self, fields):
        encoded, _ = encode_multipart_formdata(fields, boundary=BOUNDARY)
        assert encoded.count(b(BOUNDARY)) == 3

    @pytest.mark.parametrize('fields', [
        [('k', 'v'), ('k2', 'v2')],
        [('k', b'v'), (u('k2'), b'v2')],
        [('k', b'v'), (u('k2'), 'v2')],
    ])
    def test_field_encoding(self, fields):
        encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
        expected = (b'--' + b(BOUNDARY) + b'\r\n'
                    b'Content-Disposition: form-data; name="k"\r\n'
                    b'\r\n'
                    b'v\r\n'
                    b'--' + b(BOUNDARY) + b'\r\n'
                    b'Content-Disposition: form-data; name="k2"\r\n'
                    b'\r\n'
                    b'v2\r\n'
                    b'--' + b(BOUNDARY) + b'--\r\n')

        assert encoded == expected

        assert content_type == 'multipart/form-data; boundary=' + str(BOUNDARY)

    def test_filename(self):
        fields = [('k', ('somename', b'v'))]

        encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
        expected = (b'--' + b(BOUNDARY) + b'\r\n'
                    b'Content-Disposition: form-data; name="k"; filename="somename"\r\n'
                    b'Content-Type: application/octet-stream\r\n'
                    b'\r\n'
                    b'v\r\n'
                    b'--' + b(BOUNDARY) + b'--\r\n')

        assert encoded == expected

        assert content_type == 'multipart/form-data; boundary=' + str(BOUNDARY)

    def test_textplain(self):
        fields = [('k', ('somefile.txt', b'v'))]

        encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
        expected = (b'--' + b(BOUNDARY) + b'\r\n'
                    b'Content-Disposition: form-data; name="k"; filename="somefile.txt"\r\n'
                    b'Content-Type: text/plain\r\n'
                    b'\r\n'
                    b'v\r\n'
                    b'--' + b(BOUNDARY) + b'--\r\n')

        assert encoded == expected

        assert content_type == 'multipart/form-data; boundary=' + str(BOUNDARY)

    def test_explicit(self):
        fields = [('k', ('somefile.txt', b'v', 'image/jpeg'))]

        encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
        expected = (b'--' + b(BOUNDARY) + b'\r\n'
                    b'Content-Disposition: form-data; name="k"; filename="somefile.txt"\r\n'
                    b'Content-Type: image/jpeg\r\n'
                    b'\r\n'
                    b'v\r\n'
                    b'--' + b(BOUNDARY) + b'--\r\n')

        assert encoded == expected

        assert content_type == 'multipart/form-data; boundary=' + str(BOUNDARY)

    def test_request_fields(self):
        fields = [RequestField('k', b'v',
                               filename='somefile.txt',
                               headers={'Content-Type': 'image/jpeg'})]

        encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
        expected = (b'--' + b(BOUNDARY) + b'\r\n'
                    b'Content-Type: image/jpeg\r\n'
                    b'\r\n'
                    b'v\r\n'
                    b'--' + b(BOUNDARY) + b'--\r\n')

        assert encoded == expected
<|endoftext|>


<commit_before>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import pytest
import numpy as np

from bigdl.dataset.base import maybe_download
from test.zoo.pipeline.utils.test_utils import ZooTestCase
from zoo.pipeline.inference import InferenceModel

import tarfile

np.random.seed(1337)  # for reproducibility

resource_path = os.path.join(os.path.split(__file__)[0], "../../resources")
property_path = os.path.join(os.path.split(__file__)[0],
                             "../../../../../zoo/target/classes/app.properties")
data_url = "http://download.tensorflow.org"
with open(property_path) as f:
    for _ in range(2):  # skip the first two lines
        next(f)
    for line in f:
        if "data-store-url" in line:
            line = line.strip()
            data_url = line.split("=")[1].replace("\\", "")

class TestInferenceModel(ZooTestCase):

    def test_load_bigdl(self):
        model = InferenceModel(3)
        model.load_bigdl(os.path.join(resource_path, "models/bigdl/bigdl_lenet.model"))
        input_data = np.random.random([4, 28, 28, 1])
        output_data = model.predict(input_data)

    def test_load_caffe(self):
        model = InferenceModel(10)
        model.load_caffe(os.path.join(resource_path, "models/caffe/test_persist.prototxt"),
                         os.path.join(resource_path, "models/caffe/test_persist.caffemodel"))
        input_data = np.random.random([4, 3, 8, 8])
        output_data = model.predict(input_data)

    # def test_load_tf_openvino(self):
    #     local_path = self.create_temp_dir()
    #     url = data_url + "/models/object_detection/faster_rcnn_resnet101_coco_2018_01_28.tar.gz"
    #     file_abs_path = maybe_download("faster_rcnn_resnet101_coco_2018_01_28.tar.gz",
    #                                    local_path, url)
    #     tar = tarfile.open(file_abs_path, "r:gz")
    #     extracted_to = os.path.join(local_path, "faster_rcnn_resnet101_coco_2018_01_28")
    #     if not os.path.exists(extracted_to):
    #         print("Extracting %s to %s" % (file_abs_path, extracted_to))
    #         tar.extractall(local_path)
    #         tar.close()
    #     model = InferenceModel(3)
    #     model.load_tf(model_path=extracted_to + "/frozen_inference_graph.pb",
    #                   backend="openvino",
    #                   model_type="faster_rcnn_resnet101_coco",
    #                   ov_pipeline_config_path=extracted_to + "/pipeline.config",
    #                   ov_extensions_config_path=None)
    #     input_data = np.random.random([4, 1, 3, 600, 600])
    #     output_data = model.predict(input_data)
    #     model2 = InferenceModel(3)
    #     model2.load_tf_object_detection_as_openvino(
    #         model_path=extracted_to + "/frozen_inference_graph.pb",
    #         object_detection_model_type="faster_rcnn_resnet101_coco",
    #         pipeline_config_path=extracted_to + "/pipeline.config",
    #         extensions_config_path=None)
    #     model2.predict(input_data)

    # def test_load_tf_openvino_ic(self):
    #     local_path = self.create_temp_dir()
    #     print(local_path)
    #     url = data_url + "/models/resnet_v1_50_2016_08_28.tar.gz"
    #     file_abs_path = maybe_download("resnet_v1_50_2016_08_28.tar.gz", local_path, url)
    #     tar = tarfile.open(file_abs_path, "r:gz")
    #     print("Extracting %s to %s" % (file_abs_path, local_path))
    #     tar.extractall(local_path)
    #     tar.close()
    #     model = InferenceModel(3)
    #     model.load_tf_image_classification_as_openvino(
    #         model_path=None,
    #         image_classification_model_type="resnet_v1_50",
    #         checkpoint_path=local_path + "/resnet_v1_50.ckpt",
    #         input_shape=[4, 224, 224, 3],
    #         if_reverse_input_channels=True,
    #         mean_values=[123.68, 116.78, 103.94],
    #         scale=1)
    #     print(model)
    #     input_data = np.random.random([4, 1, 224, 224, 3])
    #     model.predict(input_data)

if __name__ == "__main__":
    pytest.main([__file__])
<commit_msg>Refine OpenVINO Python unit test (#2239)<commit_after>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import pytest
import numpy as np

from bigdl.dataset.base import maybe_download
from test.zoo.pipeline.utils.test_utils import ZooTestCase
from zoo.pipeline.inference import InferenceModel

import tarfile

np.random.seed(1337)  # for reproducibility

resource_path = os.path.join(os.path.split(__file__)[0], "../../resources")
property_path = os.path.join(os.path.split(__file__)[0],
                             "../../../../../zoo/target/classes/app.properties")
data_url = "http://download.tensorflow.org"
s3_url = "https://s3-ap-southeast-1.amazonaws.com"
with open(property_path) as f:
    for _ in range(2):  # skip the first two lines
        next(f)
    for line in f:
        if "data-store-url" in line:
            line = line.strip()
            data_url = line.split("=")[1].replace("\\", "")

class TestInferenceModel(ZooTestCase):

    def test_load_bigdl(self):
        model = InferenceModel(3)
        model.load_bigdl(os.path.join(resource_path, "models/bigdl/bigdl_lenet.model"))
        input_data = np.random.random([4, 28, 28, 1])
        output_data = model.predict(input_data)

    def test_load_caffe(self):
        model = InferenceModel(10)
        model.load_caffe(os.path.join(resource_path, "models/caffe/test_persist.prototxt"),
                         os.path.join(resource_path, "models/caffe/test_persist.caffemodel"))
        input_data = np.random.random([4, 3, 8, 8])
        output_data = model.predict(input_data)

    def test_load_openvino(self):
        local_path = self.create_temp_dir()
        model = InferenceModel(1)
        model_url = s3_url + "/analytics-zoo-models/openvino/2018_R5/resnet_v1_50.xml"
        weight_url = s3_url + "/analytics-zoo-models/openvino/2018_R5/resnet_v1_50.bin"
        model_path = maybe_download("resnet_v1_50.xml",
                                    local_path, model_url)
        weight_path = maybe_download("resnet_v1_50.bin",
                                     local_path, weight_url)
        model.load_openvino(model_path, weight_path)
        input_data = np.random.random([4, 1, 224, 224, 3])
        model.predict(input_data)

    def test_load_tf_openvino_od(self):
        local_path = self.create_temp_dir()
        url = data_url + "/models/object_detection/faster_rcnn_resnet101_coco_2018_01_28.tar.gz"
        file_abs_path = maybe_download("faster_rcnn_resnet101_coco_2018_01_28.tar.gz",
                                       local_path, url)
        tar = tarfile.open(file_abs_path, "r:gz")
        extracted_to = os.path.join(local_path, "faster_rcnn_resnet101_coco_2018_01_28")
        if not os.path.exists(extracted_to):
            print("Extracting %s to %s" % (file_abs_path, extracted_to))
            tar.extractall(local_path)
            tar.close()
        model = InferenceModel(3)
        model.load_tf(model_path=extracted_to + "/frozen_inference_graph.pb",
                      backend="openvino",
                      model_type="faster_rcnn_resnet101_coco",
                      ov_pipeline_config_path=extracted_to + "/pipeline.config",
                      ov_extensions_config_path=None)
        input_data = np.random.random([4, 1, 3, 600, 600])
        output_data = model.predict(input_data)
        model2 = InferenceModel(3)
        model2.load_tf_object_detection_as_openvino(
            model_path=extracted_to + "/frozen_inference_graph.pb",
            object_detection_model_type="faster_rcnn_resnet101_coco",
            pipeline_config_path=extracted_to + "/pipeline.config",
            extensions_config_path=None)
        model2.predict(input_data)

    # def test_load_tf_openvino_ic(self):
    #     local_path = self.create_temp_dir()
    #     print(local_path)
    #     url = data_url + "/models/resnet_v1_50_2016_08_28.tar.gz"
    #     file_abs_path = maybe_download("resnet_v1_50_2016_08_28.tar.gz", local_path, url)
    #     tar = tarfile.open(file_abs_path, "r:gz")
    #     print("Extracting %s to %s" % (file_abs_path, local_path))
    #     tar.extractall(local_path)
    #     tar.close()
    #     model = InferenceModel(3)
    #     model.load_tf_image_classification_as_openvino(
    #         model_path=None,
    #         image_classification_model_type="resnet_v1_50",
    #         checkpoint_path=local_path + "/resnet_v1_50.ckpt",
    #         input_shape=[4, 224, 224, 3],
    #         if_reverse_input_channels=True,
    #         mean_values=[123.68, 116.78, 103.94],
    #         scale=1)
    #     print(model)
    #     input_data = np.random.random([4, 1, 224, 224, 3])
    #     model.predict(input_data)

if __name__ == "__main__":
    pytest.main([__file__])
<|endoftext|>


<commit_before>#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import six
import numpy as np
import warnings

from paddle import framework
import paddle
from paddle.fluid import core
import paddle.distributed as dist
from paddle.fluid.dygraph.parallel import _split_tensors, sync_params_buffers, build_groups
from collections import OrderedDict
from .log_util import logger

__all__ = []

def _apply_collective_grads(parameters, comm_group):
    grad_var_set = set()
    grad_vars = []
    sparse_grad_vars = []

    for param in parameters:
        if param.trainable and (param._grad_ivar() is not None):
            g_var = param._grad_ivar()
            assert not g_var._is_sparse(
            ), "Now, it doesn't support sparse parameters"
            grad_vars.append(g_var)
            assert g_var not in grad_var_set
            grad_var_set.add(g_var)

    coalesced_grads_and_vars = build_groups(grad_vars, 128 * 1024 * 1024)

    for coalesced_grad, _, _ in coalesced_grads_and_vars:
        # need to div nranks
        nranks = dist.get_world_size(
        ) if comm_group is None else comm_group.nranks
        div_factor = paddle.to_tensor(nranks, dtype=coalesced_grad.dtype)
        paddle.fluid.framework._dygraph_tracer().trace_op(
            type="elementwise_div",
            inputs={'X': coalesced_grad,
                    'Y': div_factor},
            outputs={'Out': coalesced_grad},
            attrs={'axis': -1})

        paddle.distributed.all_reduce(coalesced_grad, group=comm_group)

    _split_tensors(coalesced_grads_and_vars)

def _broadcast_data_help(data, shape, dtype, hcg):
    model_parallel_group = hcg.get_model_parallel_group()
    src_rank = hcg.get_model_parallel_group_src_rank()
    mp_rank = hcg.get_model_parallel_rank()

    shape_gpu = paddle.to_tensor(shape, dtype="int32")
    paddle.distributed.broadcast(
        shape_gpu,
        src=src_rank,
        group=model_parallel_group,
        use_calc_stream=True)

    if mp_rank != 0:
        input_data = paddle.zeros(shape_gpu, dtype=dtype)
    else:
        input_data = data

    paddle.distributed.broadcast(
        input_data,
        src=src_rank,
        group=model_parallel_group,
        use_calc_stream=True)

def broadcast_input_data(hcg, *inputs, **kwargs):
    for v in inputs:
        if isinstance(v, core.VarBase):
            with framework.no_grad():
                _broadcast_data_help(v, v.shape, v.dtype, hcg)
        else:
            logger.error("it doesn't support data type {}".format(type(v)))

    for k, v in kwargs.items():
        if isinstance(v, core.VarBase):
            with framework.no_grad():
                _broadcast_data_help(v, v.shape, v.dtype, hcg)
            kwargs[k] = v
        else:
            logger.error("it doesn't support data type {}".format(type(v)))
    return inputs, kwargs

def broadcast_mp_parameters(model, hcg):
    model_parallel_group = hcg.get_model_parallel_group()
    src_rank = hcg.get_model_parallel_group_src_rank()
    sync_params_buffers(
        model, model_parallel_group, src_rank, is_model_parallel=True)

def broadcast_dp_parameters(model, hcg):
    data_parallel_group = hcg.get_data_parallel_group()
    src_rank = hcg.get_data_parallel_group_src_rank()
    sync_params_buffers(
        model, data_parallel_group, src_rank, is_model_parallel=False)

def fused_allreduce_gradients(parameter_list, hcg):
    data_parallel_group = None if hcg is None else hcg.get_data_parallel_group()
    logger.debug("dp start fuse allreduce gradients")
    with framework.no_grad():
        _apply_collective_grads(parameter_list, data_parallel_group)

def sharding_reduce_gradients(parameter_list, hcg):
    # TODO allreduce --> reduce
    # TODO merge grad / nrank with dp
    logger.debug("sharding start gradients sync")
    with framework.no_grad():

        sharding_nrank = hcg.get_sharding_parallel_group().nranks
        for param in parameter_list:
            if param.trainable and (param._grad_ivar() is not None):

                g_var = param._grad_ivar()

                # need use trace_op to allreduce
                # paddle.distributed.all_reduce(
                #     g_var, group=hcg.get_sharding_parallel_group(), use_calc_stream=True)
                paddle.fluid.framework._dygraph_tracer().trace_op(
                    type="c_allreduce_sum",
                    inputs={'X': g_var},
                    outputs={'Out': g_var},
                    attrs={
                        'ring_id': hcg.get_sharding_parallel_group().id,
                        'use_calc_stream': True
                    })

                # grad / sharding_rank
                div_factor = paddle.to_tensor(sharding_nrank, dtype=g_var.dtype)
                paddle.fluid.framework._dygraph_tracer().trace_op(
                    type="elementwise_div",
                    inputs={'X': g_var,
                            'Y': div_factor},
                    outputs={'Out': g_var},
                    attrs={'axis': -1})

def broadcast_sharding_parameters(model, hcg):
    # TODO TO save memory, use un-fused broadcast to avoid potentional OOM
    logger.debug("sharding start init parameters sync")
    sharding_parallel_group = hcg.get_sharding_parallel_group()
    src_rank = hcg.get_sharding_parallel_group_src_rank()
    sync_params_buffers(
        model, sharding_parallel_group, src_rank, is_model_parallel=False)
<commit_msg>fix bugs of PR 35401 (#35746)<commit_after>#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import six
import numpy as np
import warnings

from paddle import framework
import paddle
from paddle.fluid import core
from paddle.fluid.dygraph.parallel import _split_tensors, sync_params_buffers, build_groups
from collections import OrderedDict
from .log_util import logger

__all__ = []

def _apply_collective_grads(parameters, comm_group):
    grad_var_set = set()
    grad_vars = []
    sparse_grad_vars = []

    for param in parameters:
        if param.trainable and (param._grad_ivar() is not None):
            g_var = param._grad_ivar()
            assert not g_var._is_sparse(
            ), "Now, it doesn't support sparse parameters"
            grad_vars.append(g_var)
            assert g_var not in grad_var_set
            grad_var_set.add(g_var)

    coalesced_grads_and_vars = build_groups(grad_vars, 128 * 1024 * 1024)

    for coalesced_grad, _, _ in coalesced_grads_and_vars:
        # need to div nranks
        nranks = paddle.distributed.get_world_size(
        ) if comm_group is None else comm_group.nranks
        div_factor = paddle.to_tensor(nranks, dtype=coalesced_grad.dtype)
        paddle.fluid.framework._dygraph_tracer().trace_op(
            type="elementwise_div",
            inputs={'X': coalesced_grad,
                    'Y': div_factor},
            outputs={'Out': coalesced_grad},
            attrs={'axis': -1})

        paddle.distributed.all_reduce(coalesced_grad, group=comm_group)

    _split_tensors(coalesced_grads_and_vars)

def _broadcast_data_help(data, shape, dtype, hcg):
    model_parallel_group = hcg.get_model_parallel_group()
    src_rank = hcg.get_model_parallel_group_src_rank()
    mp_rank = hcg.get_model_parallel_rank()

    shape_gpu = paddle.to_tensor(shape, dtype="int32")
    paddle.distributed.broadcast(
        shape_gpu,
        src=src_rank,
        group=model_parallel_group,
        use_calc_stream=True)

    if mp_rank != 0:
        input_data = paddle.zeros(shape_gpu, dtype=dtype)
    else:
        input_data = data

    paddle.distributed.broadcast(
        input_data,
        src=src_rank,
        group=model_parallel_group,
        use_calc_stream=True)

def broadcast_input_data(hcg, *inputs, **kwargs):
    for v in inputs:
        if isinstance(v, core.VarBase):
            with framework.no_grad():
                _broadcast_data_help(v, v.shape, v.dtype, hcg)
        else:
            logger.error("it doesn't support data type {}".format(type(v)))

    for k, v in kwargs.items():
        if isinstance(v, core.VarBase):
            with framework.no_grad():
                _broadcast_data_help(v, v.shape, v.dtype, hcg)
            kwargs[k] = v
        else:
            logger.error("it doesn't support data type {}".format(type(v)))
    return inputs, kwargs

def broadcast_mp_parameters(model, hcg):
    model_parallel_group = hcg.get_model_parallel_group()
    src_rank = hcg.get_model_parallel_group_src_rank()
    sync_params_buffers(
        model, model_parallel_group, src_rank, is_model_parallel=True)

def broadcast_dp_parameters(model, hcg):
    data_parallel_group = hcg.get_data_parallel_group()
    src_rank = hcg.get_data_parallel_group_src_rank()
    sync_params_buffers(
        model, data_parallel_group, src_rank, is_model_parallel=False)

def fused_allreduce_gradients(parameter_list, hcg):
    data_parallel_group = None if hcg is None else hcg.get_data_parallel_group()
    logger.debug("dp start fuse allreduce gradients")
    with framework.no_grad():
        _apply_collective_grads(parameter_list, data_parallel_group)

def sharding_reduce_gradients(parameter_list, hcg):
    # TODO allreduce --> reduce
    # TODO merge grad / nrank with dp
    logger.debug("sharding start gradients sync")
    with framework.no_grad():

        sharding_nrank = hcg.get_sharding_parallel_group().nranks
        for param in parameter_list:
            if param.trainable and (param._grad_ivar() is not None):

                g_var = param._grad_ivar()

                # need use trace_op to allreduce
                # paddle.distributed.all_reduce(
                #     g_var, group=hcg.get_sharding_parallel_group(), use_calc_stream=True)
                paddle.fluid.framework._dygraph_tracer().trace_op(
                    type="c_allreduce_sum",
                    inputs={'X': g_var},
                    outputs={'Out': g_var},
                    attrs={
                        'ring_id': hcg.get_sharding_parallel_group().id,
                        'use_calc_stream': True
                    })

                # grad / sharding_rank
                div_factor = paddle.to_tensor(sharding_nrank, dtype=g_var.dtype)
                paddle.fluid.framework._dygraph_tracer().trace_op(
                    type="elementwise_div",
                    inputs={'X': g_var,
                            'Y': div_factor},
                    outputs={'Out': g_var},
                    attrs={'axis': -1})

def broadcast_sharding_parameters(model, hcg):
    # TODO TO save memory, use un-fused broadcast to avoid potentional OOM
    logger.debug("sharding start init parameters sync")
    sharding_parallel_group = hcg.get_sharding_parallel_group()
    src_rank = hcg.get_sharding_parallel_group_src_rank()
    sync_params_buffers(
        model, sharding_parallel_group, src_rank, is_model_parallel=False)
<|endoftext|>


<commit_before>from eventify.event import Event
from eventify.producer import Producer

class TestProducer:

    def test_create_producer(self):
        producer = Producer(config_file='test/config-test.json')

    def test_emit_event(self):
        producer = Producer(config_file='test/config-test.json')
        event = Event('TestEvent', {'foo': 'bar'})
        response = producer.emit_event(event)
        assert response.result.status_code == 200
<commit_msg>skip one test<commit_after>import unittest

from eventify.event import Event
from eventify.producer import Producer

class TestProducer:

    def test_create_producer(self):
        producer = Producer(config_file='test/config-test.json')

    @unittest.skip('no crossbar available')
    def test_emit_event(self):
        producer = Producer(config_file='test/config-test.json')
        event = Event('TestEvent', {'foo': 'bar'})
        response = producer.emit_event(event)
        assert response.result.status_code == 200
<|endoftext|>


<commit_before>#! python3
# lucky.py - Opens several Google search results.

import requests, sys, webbrowser, bs4, time

print("Googling...")  # display text while downloading the Google page
res = requests.get("http://google.com/search?q=" + ' '.join(sys.argv[1:]))
res.raise_for_status()

# Retrieve top search result links.
soup = bs4.BeautifulSoup(res.text, "lxml")

# Open a browser tab for each result.
linkElems = soup.select(".r a")
numOpen = min(5, len(linkElems))
webbrowser.open("http://google.com" + linkElems[0].get("href"))
time.sleep(10)  # add delay for slow virtual machine
for i in range(1, numOpen):
    webbrowser.open_new_tab("http://google.com" + linkElems[i].get("href"))
    time.sleep(7)
<commit_msg>Update P3_lucky.py added docstring and wrapped in main() function<commit_after>#! python3
"""Lucky

Opens top Google search results for given query.

"""

def main():
    import requests, sys, webbrowser, bs4, time

    print("Googling...")  # display text while downloading the Google page
    res = requests.get("http://google.com/search?q=" + ' '.join(sys.argv[1:]))
    res.raise_for_status()

    # Retrieve top search result links.
    soup = bs4.BeautifulSoup(res.text, "lxml")

    # Open a browser tab for each result.
    linkElems = soup.select(".r a")
    numOpen = min(5, len(linkElems))
    webbrowser.open("http://google.com" + linkElems[0].get("href"))
    time.sleep(10)  # add delay for slow virtual machine
    for i in range(1, numOpen):
        webbrowser.open_new_tab("http://google.com" + linkElems[i].get("href"))
        time.sleep(7)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.11.1 on 2017-08-27 22:15
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion
from django.core.exceptions import ObjectDoesNotExist

from cbe.customer.models import Customer

@property
def set_my_defaults():
    try:
        return Customer.objects.first().pk
    except ObjectDoesNotExist:
        return 0

class Migration(migrations.Migration):

    dependencies = [
        ('customer', '0004_auto_20170824_1650'),
        ('customer_bill', '0005_auto_20170828_1013'),
    ]

    operations = [
        migrations.AddField(
            model_name='customerbill',
            name='customer',
            field=models.ForeignKey(default=set_my_defaults, on_delete=django.db.models.deletion.CASCADE, to='customer.Customer'),
            preserve_default=False,
        ),
    ]
<commit_msg>Changed migraton for customer bill to use function as default<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.11.1 on 2017-08-27 22:15
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion
from django.core.exceptions import ObjectDoesNotExist

from cbe.customer.models import Customer

class Migration(migrations.Migration):

    dependencies = [
        ('customer', '0004_auto_20170824_1650'),
        ('customer_bill', '0005_auto_20170828_1013'),
    ]

    operations = [
        migrations.AddField(
            model_name='customerbill',
            name='customer',
            field=models.ForeignKey(default=self.set_my_defaults, on_delete=django.db.models.deletion.CASCADE, to='customer.Customer'),
            preserve_default=False,
        ),
    ]

    @property
    def set_my_defaults(self):
        try:
            return Customer.objects.first().pk
        except ObjectDoesNotExist:
            return 0
            <|endoftext|>


<commit_before># Copyright 2015 ETH Zurich
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
:mod:`topology_test` --- SCION topology tests
=============================================
"""
# Stdlib
#from unittest.mock import patch
from ipaddress import IPv4Address, IPv6Address

# External packages
import nose
import nose.tools as ntools
#import bitstring
#from bitstring import BitArray

# SCION
from lib.topology import (
    Element
)

class TestElementInit(object):
    """
    Unit tests for lib.topology.Element construction.
    """
    def test_basic(self):
        elem = Element()
        ntools.assert_is_none(elem.addr)
        ntools.assert_is_none(elem.addr_type)
        ntools.assert_is_none(elem.to_addr)
        ntools.assert_is_none(elem.name)

    def test_ipv4(self):
        elem = Element('192.168.0.1', 'ipv4')
        ntools.assert_equal(elem.addr, IPv4Address('192.168.0.1'))
        ntools.assert_is_instance(elem.addr, IPv4Address)
        ntools.assert_equal(elem.addr_type, 'ipv4')
        ntools.assert_is_none(elem.to_addr)
        ntools.assert_is_none(elem.name)

    def test_ipv6(self):
        elem = Element('2001:db8::', 'ipv6')
        ntools.assert_equal(elem.addr, IPv6Address('2001:db8::'))
        ntools.assert_is_instance(elem.addr, IPv6Address)
        ntools.assert_equal(elem.addr_type, 'ipv6')
        ntools.assert_is_none(elem.to_addr)
        ntools.assert_is_none(elem.name)

    def test_name_basic(self):
        elem = Element(name='localhost')
        ntools.assert_is_none(elem.addr)
        ntools.assert_is_none(elem.addr_type)
        ntools.assert_is_none(elem.to_addr)
        ntools.assert_is_not_none(elem.name)
        ntools.assert_equal(elem.name, 'localhost')

    def test_name_numeric(self):
        elem = Element(name=42)
        ntools.assert_is_none(elem.addr)
        ntools.assert_is_none(elem.addr_type)
        ntools.assert_is_none(elem.to_addr)
        ntools.assert_is_not_none(elem.name)
        ntools.assert_equal(elem.name, '42')

    def test_to_addr_ipv4(self):
        elem = Element('192.168.0.1', 'ipv4', '192.168.1.1')
        ntools.assert_equal(elem.addr, IPv4Address('192.168.0.1'))
        ntools.assert_is_instance(elem.addr, IPv4Address)
        ntools.assert_equal(elem.addr_type, 'ipv4')
        ntools.assert_equal(elem.to_addr, IPv4Address('192.168.1.1'))
        ntools.assert_is_none(elem.name)

    def test_to_addr_ipv6(self):
        elem = Element('2001:db8::', 'ipv6', '::1')
        ntools.assert_equal(elem.addr, IPv6Address('2001:db8::'))
        ntools.assert_is_instance(elem.addr, IPv6Address)
        ntools.assert_equal(elem.addr_type, 'ipv6')
        ntools.assert_equal(elem.to_addr, IPv6Address('::1'))
        ntools.assert_is_none(elem.name)

if __name__ == "__main__":
    nose.run(defaultTest=__name__)
<commit_msg>Added test for bogus address type<commit_after># Copyright 2015 ETH Zurich
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
:mod:`topology_test` --- SCION topology tests
=============================================
"""
# Stdlib
#from unittest.mock import patch
from ipaddress import IPv4Address, IPv6Address

# External packages
import nose
import nose.tools as ntools
#import bitstring
#from bitstring import BitArray

# SCION
from lib.topology import (
    Element
)

class TestElementInit(object):
    """
    Unit tests for lib.topology.Element construction.
    """
    def test_basic(self):
        elem = Element()
        ntools.assert_is_none(elem.addr)
        ntools.assert_is_none(elem.addr_type)
        ntools.assert_is_none(elem.to_addr)
        ntools.assert_is_none(elem.name)

    def test_ipv4(self):
        elem = Element('192.168.0.1', 'ipv4')
        ntools.assert_equal(elem.addr, IPv4Address('192.168.0.1'))
        ntools.assert_is_instance(elem.addr, IPv4Address)
        ntools.assert_equal(elem.addr_type, 'ipv4')
        ntools.assert_is_none(elem.to_addr)
        ntools.assert_is_none(elem.name)

    def test_ipv6(self):
        elem = Element('2001:db8::', 'ipv6')
        ntools.assert_equal(elem.addr, IPv6Address('2001:db8::'))
        ntools.assert_is_instance(elem.addr, IPv6Address)
        ntools.assert_equal(elem.addr_type, 'ipv6')
        ntools.assert_is_none(elem.to_addr)
        ntools.assert_is_none(elem.name)

    def test_name_basic(self):
        elem = Element(name='localhost')
        ntools.assert_is_none(elem.addr)
        ntools.assert_is_none(elem.addr_type)
        ntools.assert_is_none(elem.to_addr)
        ntools.assert_is_not_none(elem.name)
        ntools.assert_equal(elem.name, 'localhost')

    def test_name_numeric(self):
        elem = Element(name=42)
        ntools.assert_is_none(elem.addr)
        ntools.assert_is_none(elem.addr_type)
        ntools.assert_is_none(elem.to_addr)
        ntools.assert_is_not_none(elem.name)
        ntools.assert_equal(elem.name, '42')

    def test_to_addr_ipv4(self):
        elem = Element('192.168.0.1', 'ipv4', '192.168.1.1')
        ntools.assert_equal(elem.addr, IPv4Address('192.168.0.1'))
        ntools.assert_is_instance(elem.addr, IPv4Address)
        ntools.assert_equal(elem.addr_type, 'ipv4')
        ntools.assert_equal(elem.to_addr, IPv4Address('192.168.1.1'))
        ntools.assert_is_none(elem.name)

    def test_to_addr_ipv6(self):
        elem = Element('2001:db8::', 'ipv6', '::1')
        ntools.assert_equal(elem.addr, IPv6Address('2001:db8::'))
        ntools.assert_is_instance(elem.addr, IPv6Address)
        ntools.assert_equal(elem.addr_type, 'ipv6')
        ntools.assert_equal(elem.to_addr, IPv6Address('::1'))
        ntools.assert_is_none(elem.name)

    def test_invalid_addr_type(self):
        elem = Element('42.42.42.42.42', 'ipv5')
        ntools.assert_is_none(elem.addr)
        ntools.assert_is_none(elem.addr_type)
        ntools.assert_is_none(elem.to_addr)
        ntools.assert_is_none(elem.name)

if __name__ == "__main__":
    nose.run(defaultTest=__name__)
<|endoftext|>