<commit_before># Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from telemetry.page import test_expectations

# Valid expectation conditions are:
#
# Operating systems:
#     win, xp, vista, win7, mac, leopard, snowleopard, lion, mountainlion,
#     linux, chromeos, android
#
# GPU vendors:
#     amd, arm, broadcom, hisilicon, intel, imagination, nvidia, qualcomm,
#     vivante
#
# Specific GPUs can be listed as a tuple with vendor name and device ID.
# Examples: ('nvidia', 0x1234), ('arm', 'Mali-T604')
# Device IDs must be paired with a GPU vendor.

class WebGLConformanceExpectations(test_expectations.TestExpectations):
  def SetExpectations(self):
    # Sample Usage:
    # self.Fail('gl-enable-vertex-attrib.html',
    #     ['mac', 'amd', ('nvidia', 0x1234)], bug=123)

    # Fails everywhere.
    self.Skip('conformance/glsl/misc/large-loop-compile.html',
        bug=322764)

    # Windows/Intel failures
    self.Fail('conformance/textures/texture-size.html',
        ['win', 'intel'], bug=121139)
    self.Fail('conformance/rendering/gl-scissor-test.html',
        ['win', 'intel'], bug=314997)

    # Windows/AMD failures
    self.Fail('conformance/rendering/more-than-65536-indices.html',
        ['win', 'amd'], bug=314997)

    # Windows 7/Intel failures
    self.Fail('conformance/context/context-lost-restored.html',
        ['win7', 'intel'])
    self.Fail('conformance/context/premultiplyalpha-test.html',
        ['win7', 'intel'])
    self.Fail('conformance/extensions/oes-texture-float-with-image-data.html',
        ['win7', 'intel'])
    self.Fail('conformance/extensions/oes-texture-float.html',
        ['win7', 'intel'])
    self.Fail('conformance/limits/gl-min-attribs.html',
        ['win7', 'intel'])
    self.Fail('conformance/limits/gl-max-texture-dimensions.html',
        ['win7', 'intel'])
    self.Fail('conformance/limits/gl-min-textures.html',
        ['win7', 'intel'])
    self.Fail('conformance/limits/gl-min-uniforms.html',
        ['win7', 'intel'])
    self.Fail('conformance/rendering/gl-clear.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/copy-tex-image-and-sub-image-2d.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/gl-teximage.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-and-sub-image-2d-with-array-buffer-view.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-and-sub-image-2d-with-image-data.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgb565.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgba4444.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgba5551.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-with-format-and-type.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-sub-image-2d.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texparameter-test.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-active-bind-2.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-active-bind.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-complete.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-formats-test.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-mips.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-npot.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-size-cube-maps.html',
        ['win7', 'intel'])

    # Mac failures.
    self.Fail('conformance/glsl/misc/shaders-with-varyings.html',
        ['mac'], bug=322760)

    # Mac/Intel failures
    self.Fail('conformance/rendering/gl-scissor-test.html',
        ['mac', 'intel'], bug=314997)
    # The following two tests hang the WindowServer.
    self.Skip('conformance/canvas/drawingbuffer-static-canvas-test.html',
        ['mac', 'intel'], bug=303915)
    self.Skip('conformance/canvas/drawingbuffer-test.html',
        ['mac', 'intel'], bug=303915)
    # The following three tests only fail.
    # Radar 13499677
    self.Fail(
        'conformance/glsl/functions/glsl-function-smoothstep-gentype.html',
        ['mac', 'intel'], bug=225642)
    # Radar 13499466
    self.Fail('conformance/limits/gl-max-texture-dimensions.html',
        ['mac', 'intel'], bug=225642)
    # Radar 13499623
    self.Fail('conformance/textures/texture-size.html',
        ['mac', 'intel'], bug=225642)

    # Mac/Intel failures on 10.7
    self.Skip('conformance/glsl/functions/glsl-function-asin.html',
        ['lion', 'intel'])
    self.Skip('conformance/glsl/functions/glsl-function-dot.html',
        ['lion', 'intel'])
    self.Skip('conformance/glsl/functions/glsl-function-faceforward.html',
        ['lion', 'intel'])
    self.Skip('conformance/glsl/functions/glsl-function-length.html',
        ['lion', 'intel'])
    self.Skip('conformance/glsl/functions/glsl-function-normalize.html',
        ['lion', 'intel'])
    self.Skip('conformance/glsl/functions/glsl-function-reflect.html',
        ['lion', 'intel'])
    self.Skip(
        'conformance/glsl/functions/glsl-function-smoothstep-gentype.html',
        ['lion', 'intel'])
    self.Skip('conformance/limits/gl-max-texture-dimensions.html',
        ['lion', 'intel'])
    self.Skip('conformance/rendering/line-loop-tri-fan.html',
        ['lion', 'intel'])

    # Mac/ATI failures
    self.Skip('conformance/extensions/oes-texture-float-with-image-data.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/rendering/gl-clear.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/textures/tex-image-and-sub-image-2d-with-array-buffer-view.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/textures/tex-image-and-sub-image-2d-with-image-data.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgb565.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgba4444.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgba5551.html',
        ['mac', 'amd'], bug=308328)
    self.Fail('conformance/canvas/drawingbuffer-test.html',
        ['mac', 'amd'], bug=314997)

    # Android failures
    self.Fail('conformance/textures/texture-npot-video.html',
        ['android'], bug=306485)
    # The following test is very slow and therefore times out on Android bot.
    self.Skip('conformance/rendering/multisample-corruption.html',
        ['android'])
    self.Fail('conformance/canvas/drawingbuffer-test.html',
        ['android'], bug=314997)
    self.Fail('conformance/glsl/misc/empty_main.vert.html',
        ['android'], bug=315976)
    self.Fail('conformance/glsl/misc/gl_position_unset.vert.html',
        ['android'], bug=315976)

    # Skip slow tests.
    self.Skip('conformance/context/context-creation-and-destruction.html',
        bug=322689)
    self.Skip('conformance/rendering/multisample-corruption.html',
        bug=322689)
<commit_msg>WebGL conformance test expectations update<commit_after># Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from telemetry.page import test_expectations

# Valid expectation conditions are:
#
# Operating systems:
#     win, xp, vista, win7, mac, leopard, snowleopard, lion, mountainlion,
#     linux, chromeos, android
#
# GPU vendors:
#     amd, arm, broadcom, hisilicon, intel, imagination, nvidia, qualcomm,
#     vivante
#
# Specific GPUs can be listed as a tuple with vendor name and device ID.
# Examples: ('nvidia', 0x1234), ('arm', 'Mali-T604')
# Device IDs must be paired with a GPU vendor.

class WebGLConformanceExpectations(test_expectations.TestExpectations):
  def SetExpectations(self):
    # Sample Usage:
    # self.Fail('gl-enable-vertex-attrib.html',
    #     ['mac', 'amd', ('nvidia', 0x1234)], bug=123)

    # Fails everywhere.
    self.Skip('conformance/glsl/misc/large-loop-compile.html',
        bug=322764)

    # Windows/Intel failures
    self.Fail('conformance/textures/texture-size.html',
        ['win', 'intel'], bug=121139)
    self.Fail('conformance/rendering/gl-scissor-test.html',
        ['win', 'intel'], bug=314997)

    # Windows/AMD failures
    self.Fail('conformance/rendering/more-than-65536-indices.html',
        ['win', 'amd'], bug=314997)

    # Windows 7/Intel failures
    self.Fail('conformance/context/context-lost-restored.html',
        ['win7', 'intel'])
    self.Fail('conformance/context/premultiplyalpha-test.html',
        ['win7', 'intel'])
    self.Fail('conformance/extensions/oes-texture-float-with-image-data.html',
        ['win7', 'intel'])
    self.Fail('conformance/extensions/oes-texture-float.html',
        ['win7', 'intel'])
    self.Fail('conformance/limits/gl-min-attribs.html',
        ['win7', 'intel'])
    self.Fail('conformance/limits/gl-max-texture-dimensions.html',
        ['win7', 'intel'])
    self.Fail('conformance/limits/gl-min-textures.html',
        ['win7', 'intel'])
    self.Fail('conformance/limits/gl-min-uniforms.html',
        ['win7', 'intel'])
    self.Fail('conformance/rendering/gl-clear.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/copy-tex-image-and-sub-image-2d.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/gl-teximage.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-and-sub-image-2d-with-array-buffer-view.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-and-sub-image-2d-with-image-data.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgb565.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgba4444.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgba5551.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-image-with-format-and-type.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/tex-sub-image-2d.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texparameter-test.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-active-bind-2.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-active-bind.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-complete.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-formats-test.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-mips.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-npot.html',
        ['win7', 'intel'])
    self.Fail('conformance/textures/texture-size-cube-maps.html',
        ['win7', 'intel'])
    self.Fail('conformance/context/context-attribute-preserve-drawing-buffer.html',
        ['win7', 'intel'], bug=322770)

    # Mac failures.
    self.Fail('conformance/glsl/misc/shaders-with-varyings.html',
        ['mac'], bug=322760)
    self.Fail('conformance/context/context-attribute-preserve-drawing-buffer.html',
        ['mac'], bug=322770)

    # Mac/Intel failures
    self.Fail('conformance/rendering/gl-scissor-test.html',
        ['mac', 'intel'], bug=314997)
    # The following two tests hang the WindowServer.
    self.Skip('conformance/canvas/drawingbuffer-static-canvas-test.html',
        ['mac', 'intel'], bug=303915)
    self.Skip('conformance/canvas/drawingbuffer-test.html',
        ['mac', 'intel'], bug=303915)
    # The following three tests only fail.
    # Radar 13499677
    self.Fail(
        'conformance/glsl/functions/glsl-function-smoothstep-gentype.html',
        ['mac', 'intel'], bug=225642)
    # Radar 13499466
    self.Fail('conformance/limits/gl-max-texture-dimensions.html',
        ['mac', 'intel'], bug=225642)
    # Radar 13499623
    self.Fail('conformance/textures/texture-size.html',
        ['mac', 'intel'], bug=225642)

    # Mac/Intel failures on 10.7
    self.Skip('conformance/glsl/functions/glsl-function-asin.html',
        ['lion', 'intel'])
    self.Skip('conformance/glsl/functions/glsl-function-dot.html',
        ['lion', 'intel'])
    self.Skip('conformance/glsl/functions/glsl-function-faceforward.html',
        ['lion', 'intel'])
    self.Skip('conformance/glsl/functions/glsl-function-length.html',
        ['lion', 'intel'])
    self.Skip('conformance/glsl/functions/glsl-function-normalize.html',
        ['lion', 'intel'])
    self.Skip('conformance/glsl/functions/glsl-function-reflect.html',
        ['lion', 'intel'])
    self.Skip(
        'conformance/glsl/functions/glsl-function-smoothstep-gentype.html',
        ['lion', 'intel'])
    self.Skip('conformance/limits/gl-max-texture-dimensions.html',
        ['lion', 'intel'])
    self.Skip('conformance/rendering/line-loop-tri-fan.html',
        ['lion', 'intel'])

    # Mac/ATI failures
    self.Skip('conformance/extensions/oes-texture-float-with-image-data.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/rendering/gl-clear.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/textures/tex-image-and-sub-image-2d-with-array-buffer-view.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/textures/tex-image-and-sub-image-2d-with-image-data.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgb565.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgba4444.html',
        ['mac', 'amd'], bug=308328)
    self.Skip('conformance/textures/tex-image-and-sub-image-2d-with-image-data-rgba5551.html',
        ['mac', 'amd'], bug=308328)
    self.Fail('conformance/canvas/drawingbuffer-test.html',
        ['mac', 'amd'], bug=314997)

    # Android failures
    self.Fail('conformance/textures/texture-npot-video.html',
        ['android'], bug=306485)
    # The following test is very slow and therefore times out on Android bot.
    self.Skip('conformance/rendering/multisample-corruption.html',
        ['android'])
    self.Fail('conformance/canvas/drawingbuffer-test.html',
        ['android'], bug=314997)
    self.Fail('conformance/glsl/misc/empty_main.vert.html',
        ['android'], bug=315976)
    self.Fail('conformance/glsl/misc/gl_position_unset.vert.html',
        ['android'], bug=315976)

    # Skip slow tests.
    self.Skip('conformance/context/context-creation-and-destruction.html',
        bug=322689)
    self.Skip('conformance/rendering/multisample-corruption.html',
        bug=322689)
<|endoftext|>


<commit_before>import os
import re
import shutil
from django.contrib.auth.models import Group
from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
from django.db import models
from django.db.models import Q
from django.db.models.signals import post_save, post_delete
from django.dispatch import receiver
from django.utils.translation import ugettext_lazy as _
from guardian.shortcuts import assign_perm
from archer.core import exceptions

def validator_dir_exists(path):
    if not os.path.exists(path):
        raise ValidationError('%s does not exist!' % path, code='dir_not_exist')
    if not os.path.isdir(path):
        raise ValidationError('%s is not a directory!' % path, code='dir_not_dir')
    if not os.access(path, os.W_OK | os.X_OK):
        raise ValidationError('%s is not writable!' % path, code='dir_not_writable')

def validator_relative_path(path):
    rel_path = os.path.normpath(path)
    if re.match('^(\.\./.+|/.+|\.\.)$', rel_path):
        raise ValidationError(_('Directory path name must be relative path beneath the file system.'), code='dir_name')

class FileSystem(models.Model):
    alias = models.CharField(max_length=2000, blank=True)
    mount_point = models.CharField(max_length=2000, null=False, blank=False, unique=True,
                                   validators=[validator_dir_exists])

    class Meta:
        permissions = (
            ('setup_filesystem', 'Setup file system'),
        )

    def validate_unique(self, *args, **kwargs):
        """
            Validates if "alias" is unique within scope of "mount_point".
            "alias" mustn't be equal to any other "alias" or "mount_point".
            Same applies to "mount_point" which mustn't equal to any other "alias" except for the same entity.
            """

        def validate_uniqueness_of_alias_and_mount_point(rethrow_error=None):
            uq_mount = self.__class__.objects.filter(
                Q(alias=self.mount_point)
            ).exclude(alias='')
            uq_alias = self.__class__.objects.filter(
                Q(mount_point=self.alias)
            )

            errors = {}
            if not self._state.adding and self.pk is not None:
                uq_mount = uq_mount.exclude(pk=self.pk)
                uq_alias = uq_alias.exclude(pk=self.pk)

            if uq_mount.exists():
                errors['mount_point'] = (
                    'Field is the same as "Alias" for other file system (alias="%s", mount_point="%s")!' % (
                        uq_mount.get().alias, uq_mount.get().mount_point), )
            if uq_alias.exists():
                errors['alias'] = (
                    'Field is the same as "Mount point" for other file system (alias="%s", mount_point="%s")!' % (
                        uq_alias.get().alias, uq_alias.get().mount_point), )
            if len(errors) > 0:
                errors[NON_FIELD_ERRORS] = ('To avoid confusion choose different name!',)
                if rethrow_error is None:
                    rethrow_error = ValidationError(errors)
                else:
                    rethrow_error.message_dict.update(errors)
            if rethrow_error is not None:
                return rethrow_error

        try:
            super(FileSystem, self).validate_unique(*args, **kwargs)
            e = validate_uniqueness_of_alias_and_mount_point()
            if e is not None:
                raise e
        except ValidationError as e:
            raise validate_uniqueness_of_alias_and_mount_point(e)

    def alias_path(self):
        if self.alias is None or len(self.alias) == 0:
            return self.mount_point
        else:
            return self.alias

    @classmethod
    def can_setup(cls, user):
        return user is not None and user.is_superuser and user.is_staff

    def __unicode__(self):
        return self.alias_path()

class Project(models.Model):
    file_system = models.ForeignKey(FileSystem, null=False)
    directory = models.CharField(max_length=200, null=False, blank=False, validators=[validator_relative_path])

    class Meta:
        unique_together = ('file_system', 'directory')
        permissions = (
            ('view_project', 'View project'),
            ('upload_package', 'Upload package'),
            ('deploy_package', 'Deploy package'),
            ('make_directory', 'Make directory'),
            ('remove_directory', 'Remove directory'),
            ('remove_file', 'Remove file'),
            ('setup_project', 'Setup project'),
        )

    def full_path(self):
        return os.path.join(self.file_system.mount_point, self.directory)

    def alias_path(self):
        return os.path.join(self.file_system.alias_path(), self.directory)

    def subdir(self, subdir=''):
        """
        Returns absolute path to subdirectory of `self.directory`.
        Raises archer.core.exceptions.ValidationError if `subdir` is not a offspring of `self.directory`.
        """
        if subdir is None or len(subdir) == 0:
            return self.full_path()
        if subdir.startswith('/'):
            raise exceptions.ValidationError('Cannot provide root path "%s" as subdir!' % subdir)
        if os.path.normpath(subdir).startswith('..'):
            raise exceptions.ValidationError('Cannot privide a path "%s" pointing to parent directory!' % subdir)
        return os.path.abspath(os.path.join(self.full_path(), subdir))

    def clear_dir(self, subdir=''):
        """
        Removes all files in project's directory.
        """
        if not os.path.isdir(self.file_system.mount_point):
            raise exceptions.ValidationError("mount_point %s is not a directory" % self.file_system.mount_point)
        if not os.path.isdir(self.full_path()):
            raise exceptions.ValidationError("project %s is not a directory" % self.full_path())
        path = self.subdir(subdir)
        if not os.path.isdir(path):
            raise exceptions.ValidationError("%s is not a directory" % path)
        for root, dirs, files in os.walk(path):
            for f in files:
                os.unlink(os.path.join(root, f))
            for d in dirs:
                shutil.rmtree(os.path.join(root, d))
        return True

    @classmethod
    def can_setup(cls, user):
        return user is not None and user.is_superuser and user.is_staff and FileSystem.objects.count() > 0

    def __unicode__(self):
        return self.alias_path()

class MyModel(models.Model):
    field1 = models.TextField()
    field2 = models.IntegerField()

@receiver(post_save, sender=Project)
def project_create_directory(sender, **kwargs):
    path = kwargs['instance'].full_path()
    if not os.path.exists(path):
        os.makedirs(path, mode=0755)

@receiver(post_delete, sender=Project)
def project_delete_directory(sender, **kwargs):
    path = kwargs['instance'].full_path()
    if os.path.exists(path):
        shutil.rmtree(path, ignore_errors=True)

@receiver(post_save, sender=Project)
def project_create_project_group(sender, **kwargs):
    instance = kwargs['instance']
    group, created = Group.objects.get_or_create(name=instance.alias_path())
    for permission in ['view_project', 'upload_package', 'deploy_package', 'make_directory', 'remove_directory']:
        assign_perm('projects.%s' % permission, group, instance)

@receiver(post_delete, sender=Project)
def project_delete_project_group(sender, **kwargs):
    instance = kwargs['instance']
    try:
        group = Group.objects.get(name=instance.alias_path())
        group.delete()
    except Group.DoesNotExist:
        pass

<commit_msg>allow to remove files by default<commit_after>import os
import re
import shutil
from django.contrib.auth.models import Group
from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
from django.db import models
from django.db.models import Q
from django.db.models.signals import post_save, post_delete
from django.dispatch import receiver
from django.utils.translation import ugettext_lazy as _
from guardian.shortcuts import assign_perm
from archer.core import exceptions

def validator_dir_exists(path):
    if not os.path.exists(path):
        raise ValidationError('%s does not exist!' % path, code='dir_not_exist')
    if not os.path.isdir(path):
        raise ValidationError('%s is not a directory!' % path, code='dir_not_dir')
    if not os.access(path, os.W_OK | os.X_OK):
        raise ValidationError('%s is not writable!' % path, code='dir_not_writable')

def validator_relative_path(path):
    rel_path = os.path.normpath(path)
    if re.match('^(\.\./.+|/.+|\.\.)$', rel_path):
        raise ValidationError(_('Directory path name must be relative path beneath the file system.'), code='dir_name')

class FileSystem(models.Model):
    alias = models.CharField(max_length=2000, blank=True)
    mount_point = models.CharField(max_length=2000, null=False, blank=False, unique=True,
                                   validators=[validator_dir_exists])

    class Meta:
        permissions = (
            ('setup_filesystem', 'Setup file system'),
        )

    def validate_unique(self, *args, **kwargs):
        """
            Validates if "alias" is unique within scope of "mount_point".
            "alias" mustn't be equal to any other "alias" or "mount_point".
            Same applies to "mount_point" which mustn't equal to any other "alias" except for the same entity.
            """

        def validate_uniqueness_of_alias_and_mount_point(rethrow_error=None):
            uq_mount = self.__class__.objects.filter(
                Q(alias=self.mount_point)
            ).exclude(alias='')
            uq_alias = self.__class__.objects.filter(
                Q(mount_point=self.alias)
            )

            errors = {}
            if not self._state.adding and self.pk is not None:
                uq_mount = uq_mount.exclude(pk=self.pk)
                uq_alias = uq_alias.exclude(pk=self.pk)

            if uq_mount.exists():
                errors['mount_point'] = (
                    'Field is the same as "Alias" for other file system (alias="%s", mount_point="%s")!' % (
                        uq_mount.get().alias, uq_mount.get().mount_point), )
            if uq_alias.exists():
                errors['alias'] = (
                    'Field is the same as "Mount point" for other file system (alias="%s", mount_point="%s")!' % (
                        uq_alias.get().alias, uq_alias.get().mount_point), )
            if len(errors) > 0:
                errors[NON_FIELD_ERRORS] = ('To avoid confusion choose different name!',)
                if rethrow_error is None:
                    rethrow_error = ValidationError(errors)
                else:
                    rethrow_error.message_dict.update(errors)
            if rethrow_error is not None:
                return rethrow_error

        try:
            super(FileSystem, self).validate_unique(*args, **kwargs)
            e = validate_uniqueness_of_alias_and_mount_point()
            if e is not None:
                raise e
        except ValidationError as e:
            raise validate_uniqueness_of_alias_and_mount_point(e)

    def alias_path(self):
        if self.alias is None or len(self.alias) == 0:
            return self.mount_point
        else:
            return self.alias

    @classmethod
    def can_setup(cls, user):
        return user is not None and user.is_superuser and user.is_staff

    def __unicode__(self):
        return self.alias_path()

class Project(models.Model):
    file_system = models.ForeignKey(FileSystem, null=False)
    directory = models.CharField(max_length=200, null=False, blank=False, validators=[validator_relative_path])

    class Meta:
        unique_together = ('file_system', 'directory')
        permissions = (
            ('view_project', 'View project'),
            ('upload_package', 'Upload package'),
            ('deploy_package', 'Deploy package'),
            ('make_directory', 'Make directory'),
            ('remove_directory', 'Remove directory'),
            ('remove_file', 'Remove file'),
            ('setup_project', 'Setup project'),
        )

    def full_path(self):
        return os.path.join(self.file_system.mount_point, self.directory)

    def alias_path(self):
        return os.path.join(self.file_system.alias_path(), self.directory)

    def subdir(self, subdir=''):
        """
        Returns absolute path to subdirectory of `self.directory`.
        Raises archer.core.exceptions.ValidationError if `subdir` is not a offspring of `self.directory`.
        """
        if subdir is None or len(subdir) == 0:
            return self.full_path()
        if subdir.startswith('/'):
            raise exceptions.ValidationError('Cannot provide root path "%s" as subdir!' % subdir)
        if os.path.normpath(subdir).startswith('..'):
            raise exceptions.ValidationError('Cannot privide a path "%s" pointing to parent directory!' % subdir)
        return os.path.abspath(os.path.join(self.full_path(), subdir))

    def clear_dir(self, subdir=''):
        """
        Removes all files in project's directory.
        """
        if not os.path.isdir(self.file_system.mount_point):
            raise exceptions.ValidationError("mount_point %s is not a directory" % self.file_system.mount_point)
        if not os.path.isdir(self.full_path()):
            raise exceptions.ValidationError("project %s is not a directory" % self.full_path())
        path = self.subdir(subdir)
        if not os.path.isdir(path):
            raise exceptions.ValidationError("%s is not a directory" % path)
        for root, dirs, files in os.walk(path):
            for f in files:
                os.unlink(os.path.join(root, f))
            for d in dirs:
                shutil.rmtree(os.path.join(root, d))
        return True

    @classmethod
    def can_setup(cls, user):
        return user is not None and user.is_superuser and user.is_staff and FileSystem.objects.count() > 0

    def __unicode__(self):
        return self.alias_path()

class MyModel(models.Model):
    field1 = models.TextField()
    field2 = models.IntegerField()

@receiver(post_save, sender=Project)
def project_create_directory(sender, **kwargs):
    path = kwargs['instance'].full_path()
    if not os.path.exists(path):
        os.makedirs(path, mode=0755)

@receiver(post_delete, sender=Project)
def project_delete_directory(sender, **kwargs):
    path = kwargs['instance'].full_path()
    if os.path.exists(path):
        shutil.rmtree(path, ignore_errors=True)

@receiver(post_save, sender=Project)
def project_create_project_group(sender, **kwargs):
    instance = kwargs['instance']
    group, created = Group.objects.get_or_create(name=instance.alias_path())
    for permission in ['view_project', 'upload_package', 'deploy_package',
                       'make_directory', 'remove_directory', 'remove_file']:
        assign_perm('projects.%s' % permission, group, instance)

@receiver(post_delete, sender=Project)
def project_delete_project_group(sender, **kwargs):
    instance = kwargs['instance']
    try:
        group = Group.objects.get(name=instance.alias_path())
        group.delete()
    except Group.DoesNotExist:
        pass

<|endoftext|>


<commit_before># Tim Cornwell <realtimcornwell@gmail.com>
#
# Synthesis imaging functions
#

import functools
import numpy
import pylru
import copy
from astropy import constants as const
from astropy import units as units
from astropy import wcs
from astropy.coordinates import CartesianRepresentation

from crocodile.simulate import simulate_point, skycoord_to_lmn
from crocodile.synthesis import w_cache_imaging, w_cache_predict, w_kernel, w_conj_kernel_fn, do_imaging, do_predict

from arl.data_models import *
from arl.image_operations import create_image_from_array
from arl.parameters import *

import logging
log = logging.getLogger("arl.fourier_transforms")

"""
Functions that perform imaging i.e. conversion of an Image to/from a Visibility
"""

def create_wcs_from_visibility(vis: Visibility, params={}):
    """Make a world coordinate system from params and Visibility

    :param vis:
    :type Visibility: Visibility to be processed
    :param params: keyword=value parameters
    :returns: WCS
    """
    log_parameters(params)
    log.debug("create_wcs_from_visibility: Parsing parameters to get definition of WCS")
    imagecentre = get_parameter(params, "imagecentre", vis.phasecentre)
    phasecentre = get_parameter(params, "phasecentre", vis.phasecentre)
    reffrequency = get_parameter(params, "reffrequency", numpy.max(vis.frequency)) * units.Hz
    deffaultbw = vis.frequency[0]
    if len(vis.frequency) > 1:
        deffaultbw = vis.frequency[1] - vis.frequency[0]
    channelwidth = get_parameter(params, "channelwidth", deffaultbw) * units.Hz
    log.debug("create_wcs_from_visibility: Defining Image at %s, frequency %s, and bandwidth %s"
          % (imagecentre, reffrequency, channelwidth))

    npixel = get_parameter(params, "npixel", 512)
    uvmax = (numpy.abs(vis.data['uvw']).max() * reffrequency / const.c).value
    log.debug("create_wcs_from_visibility: uvmax = %f lambda" % uvmax)
    criticalcellsize = 1.0 / (uvmax * 2.0)
    log.debug("create_wcs_from_visibility: Critical cellsize = %f radians, %f degrees" % (
        criticalcellsize, criticalcellsize * 180.0 / numpy.pi))
    cellsize = get_parameter(params, "cellsize", 0.5 * criticalcellsize)
    log.debug("create_wcs_from_visibility: Cellsize          = %f radians, %f degrees" % (cellsize,
                                                                                cellsize * 180.0 / numpy.pi))
    if cellsize > criticalcellsize:
        log.debug("Resetting cellsize %f radians to criticalcellsize %f radians" % (cellsize, criticalcellsize))
        cellsize = criticalcellsize

    npol = 4
    # Beware of python indexing order! wcs and the array have opposite ordering
    shape = [len(vis.frequency), npol, npixel, npixel]
    w = wcs.WCS(naxis=4)
    w.wcs.cdelt = [cellsize * 180.0 / numpy.pi, cellsize * 180.0 / numpy.pi, 1.0, channelwidth.value]
    w.wcs.crpix = [npixel // 2 + 1, npixel // 2 + 1, 1.0, 1.0]
    w.wcs.ctype = ["RA---SIN", "DEC--SIN", 'STOKES', 'FREQ']
    w.wcs.crval = [phasecentre.ra.value, phasecentre.dec.value, 1.0, reffrequency.value]
    w.naxis = 4

    w.wcs.radesys = get_parameter(params, 'frame', 'ICRS')
    w.wcs.equinox = get_parameter(params, 'equinox', 2000.0)

    return shape, reffrequency, cellsize, w, imagecentre

def w_cache_size(vis:Visibility, wstep, frequency=None):
    """
    Determine optimal w-kernel cache size for de/gridding the
    given visibilities.
    """

    if frequency is None:
        frequency = max(vis.frequency)

    wmax = numpy.max(numpy.abs(vis.w))
    wmax *= float(frequency) / const.c.value
    return int(numpy.ceil(wmax / wstep))

def invert_visibility(vis: Visibility, params={}):
    """Invert to make dirty Image and PSF

    :param vis:
    :type Visibility: Visibility to be processed
    :returns: (dirty image, psf)
    """
    log_parameters(params)
    log.debug("invert_visibility: Inverting Visibility to make dirty and psf")
    shape, reffrequency, cellsize, w, imagecentre = create_wcs_from_visibility(vis, params=params)

    npixel = shape[3]
    theta = npixel * cellsize

    log.debug("invert_visibility: Specified npixel=%d, cellsize = %f rad, FOV = %f rad" %
          (npixel, cellsize, theta))

    # Set up the gridding kernel. We try to use a cached version
    gridding_algorithm = get_parameter(params, 'gridding_algorithm', 'wprojection')

    if gridding_algorithm == 'wprojection':
        log.debug("invert_visibility: Gridding by w projection")

        wstep = get_parameter(params, "wstep", 10000.0)
        wcachesize = w_cache_size(vis, wstep)
        log.debug("invert_visibility: Making w-kernel cache of %d kernels" % wcachesize)

        cache_fn = w_conj_kernel_fn(pylru.FunctionCacheManager(w_kernel, wcachesize))
        imgfn = functools.partial(w_cache_imaging,
                                  wstep=wstep, kernel_cache=cache_fn,
                                  NpixFF=256, NpixKern=15, Qpx=4)
    else:
        raise NotImplementedError("gridding algorithm %s not supported" % gridding_algorithm)

    # Apply a phase rotation from the visibility phase centre to the image phase centre
    #    visphaserotate = phaserotate(vis, imagecentre)

    d = numpy.zeros(shape)
    p = numpy.zeros(shape)

    spectral_mode = get_parameter(params, 'spectral_mode', 'channel')
    log.debug('invert_visibility: spectral mode is %s' % spectral_mode)

    if spectral_mode == 'channel':
        pmax = 0.0
        nchan = shape[0]
        npol = shape[1]
        for channel in range(nchan):
            for pol in range(npol):
                log.debug('invert_visibility: Inverting channel %d, polarisation %d' % (channel, pol))
                d[channel, pol, :, :], p[channel, 0, :, :], pmax = \
                    do_imaging(theta, 1.0 / cellsize, vis.uvw_lambda(channel),
                               vis.vis[:, channel, pol], imgfn=imgfn)
            assert pmax > 0.0, ("No data gridded for channel %d" % channel)
    else:
        raise NotImplementedError("mode %s not supported" % spectral_mode)

    dirty = create_image_from_array(d, w)
    psf = create_image_from_array(p, w)
    log.debug("invert_visibility: Finished making dirty and psf")

    return dirty, psf, pmax

def predict_visibility(vis: Visibility, sm: SkyModel, params={}) -> Visibility:
    """Predict the visibility from a SkyModel including both components and images

    :param vis:
    :type Visibility: Visibility to be processed
    :param sm:
    :type SkyModel:
    :returns: Visibility
    """
    shape, reffrequency, cellsize, w, imagecentre = create_wcs_from_visibility(vis, params=params)

    # Create copy of visibilities
    vis = copy.copy(vis)
    vis.data = copy.copy(vis.data)
    vis.data['vis'] = numpy.zeros(vis.vis.shape)

    spectral_mode = get_parameter(params, 'spectral_mode', 'channel')
    log.debug('predict_visibility: spectral mode is %s' % spectral_mode)

    if len(sm.images):
        log.debug("predict_visibility: Predicting Visibility from sky model images")

        for im in sm.images:
            assert_same_chan_pol(vis, im)

            # Determine image size
            cellsize = abs(im.wcs.wcs.cdelt[0]) * numpy.pi / 180.0
            theta = im.npixel * cellsize
            log.debug("predict_visibility: Image cellsize %f radians" % cellsize)
            log.debug("predict_visibility: Field of view %f radians" % theta)
            assert (theta / numpy.sqrt(2) < 1.0), "Field of view larger than celestial sphere"

            # Parameterise imaging
            wstep = get_parameter(params, "wstep", 10000.0)
            wcachesize = w_cache_size(vis, wstep)
            log.debug("predict_visibility: Making w-kernel cache of %d kernels" % wcachesize)

            cache_fn = w_conj_kernel_fn(pylru.FunctionCacheManager(w_kernel, wcachesize))
            predfn = functools.partial(w_cache_predict,
                                       wstep=wstep, kernel_cache=cache_fn,
                                       NpixFF=256, NpixKern=15, Qpx=4)

            spectral_mode = get_parameter(params, 'spectral_mode', 'channel')
            log.debug('predict_visibility: spectral mode is %s' % spectral_mode)

            if spectral_mode == 'channel':
                for channel in range(im.nchan):
                    uvw = vis.uvw_lambda(channel)
                    for pol in range(im.npol):
                        log.debug('predict_visibility: Predicting from image channel %d, polarisation %d' % (
                        channel, pol))
                        img = sm.images[0].data[channel, pol, :, :]
                        dv = do_predict(theta, 1.0 / cellsize, uvw, img, predfn)
                        vis.vis[:, channel, pol] += dv
            else:
                raise NotImplementedError("mode %s not supported" % spectral_mode)

            log.debug("predict_visibility: Finished predicting Visibility from sky model images")

    if len(sm.components):
        log.debug("predict_visibility: Predicting Visibility from sky model components")

        for icomp, comp in enumerate(sm.components):

            log.debug("predict_visibility: visibility shape = %s" % str(vis.vis.shape))
            assert_same_chan_pol(vis, comp)

            l,m,n = skycoord_to_lmn(comp.direction, vis.phasecentre)
            log.debug('fourier_transforms.predict_visibility: Cartesian representation of component %d = (%f, %f, %f)'
                  % (icomp, l,m,n))

            if spectral_mode =='channel':
                for channel in range(comp.nchan):
                    uvw = vis.uvw_lambda(channel)
                    phasor = simulate_point(uvw, l, m)
                    for pol in range(comp.npol):
                        log.debug(
                            'predict_visibility: Predicting from component %d channel %d, polarisation %d' % (
                            icomp, channel,
                            pol))
                        vis.vis[:, channel, pol] += comp.flux[channel, pol] * phasor
            else:
                raise NotImplementedError("mode %s not supported" % spectral_mode)

        log.debug("predict_visibility: Finished predicting Visibility from sky model components")

    return vis

def weight_visibility(vis, im, params={}):
    """ Reweight the visibility data in place a selected algorithm

    :param vis:
    :type Visibility: Visibility to be processed
    :param im:
    :type Image:
    :param params: Dictionary containing parameters
    :returns: Configuration
    """
    # TODO: implement

    log_parameters(params)
    log.error("visibility_operations.weight_visibility: not yet implemented")
    return vis

<commit_msg>Cast UVW to numpy array before passing to to do_predict<commit_after># Tim Cornwell <realtimcornwell@gmail.com>
#
# Synthesis imaging functions
#

import functools
import numpy
import pylru
import copy
from astropy import constants as const
from astropy import units as units
from astropy import wcs
from astropy.coordinates import CartesianRepresentation

from crocodile.simulate import simulate_point, skycoord_to_lmn
from crocodile.synthesis import w_cache_imaging, w_cache_predict, w_kernel, w_conj_kernel_fn, do_imaging, do_predict

from arl.data_models import *
from arl.image_operations import create_image_from_array
from arl.parameters import *

import logging
log = logging.getLogger("arl.fourier_transforms")

"""
Functions that perform imaging i.e. conversion of an Image to/from a Visibility
"""

def create_wcs_from_visibility(vis: Visibility, params={}):
    """Make a world coordinate system from params and Visibility

    :param vis:
    :type Visibility: Visibility to be processed
    :param params: keyword=value parameters
    :returns: WCS
    """
    log_parameters(params)
    log.debug("create_wcs_from_visibility: Parsing parameters to get definition of WCS")
    imagecentre = get_parameter(params, "imagecentre", vis.phasecentre)
    phasecentre = get_parameter(params, "phasecentre", vis.phasecentre)
    reffrequency = get_parameter(params, "reffrequency", numpy.max(vis.frequency)) * units.Hz
    deffaultbw = vis.frequency[0]
    if len(vis.frequency) > 1:
        deffaultbw = vis.frequency[1] - vis.frequency[0]
    channelwidth = get_parameter(params, "channelwidth", deffaultbw) * units.Hz
    log.debug("create_wcs_from_visibility: Defining Image at %s, frequency %s, and bandwidth %s"
          % (imagecentre, reffrequency, channelwidth))

    npixel = get_parameter(params, "npixel", 512)
    uvmax = (numpy.abs(vis.data['uvw']).max() * reffrequency / const.c).value
    log.debug("create_wcs_from_visibility: uvmax = %f lambda" % uvmax)
    criticalcellsize = 1.0 / (uvmax * 2.0)
    log.debug("create_wcs_from_visibility: Critical cellsize = %f radians, %f degrees" % (
        criticalcellsize, criticalcellsize * 180.0 / numpy.pi))
    cellsize = get_parameter(params, "cellsize", 0.5 * criticalcellsize)
    log.debug("create_wcs_from_visibility: Cellsize          = %f radians, %f degrees" % (cellsize,
                                                                                cellsize * 180.0 / numpy.pi))
    if cellsize > criticalcellsize:
        log.debug("Resetting cellsize %f radians to criticalcellsize %f radians" % (cellsize, criticalcellsize))
        cellsize = criticalcellsize

    npol = 4
    # Beware of python indexing order! wcs and the array have opposite ordering
    shape = [len(vis.frequency), npol, npixel, npixel]
    w = wcs.WCS(naxis=4)
    w.wcs.cdelt = [cellsize * 180.0 / numpy.pi, cellsize * 180.0 / numpy.pi, 1.0, channelwidth.value]
    w.wcs.crpix = [npixel // 2 + 1, npixel // 2 + 1, 1.0, 1.0]
    w.wcs.ctype = ["RA---SIN", "DEC--SIN", 'STOKES', 'FREQ']
    w.wcs.crval = [phasecentre.ra.value, phasecentre.dec.value, 1.0, reffrequency.value]
    w.naxis = 4

    w.wcs.radesys = get_parameter(params, 'frame', 'ICRS')
    w.wcs.equinox = get_parameter(params, 'equinox', 2000.0)

    return shape, reffrequency, cellsize, w, imagecentre

def w_cache_size(vis:Visibility, wstep, frequency=None):
    """
    Determine optimal w-kernel cache size for de/gridding the
    given visibilities.
    """

    if frequency is None:
        frequency = max(vis.frequency)

    wmax = numpy.max(numpy.abs(vis.w))
    wmax *= float(frequency) / const.c.value
    return int(numpy.ceil(wmax / wstep))

def invert_visibility(vis: Visibility, params={}):
    """Invert to make dirty Image and PSF

    :param vis:
    :type Visibility: Visibility to be processed
    :returns: (dirty image, psf)
    """
    log_parameters(params)
    log.debug("invert_visibility: Inverting Visibility to make dirty and psf")
    shape, reffrequency, cellsize, w, imagecentre = create_wcs_from_visibility(vis, params=params)

    npixel = shape[3]
    theta = npixel * cellsize

    log.debug("invert_visibility: Specified npixel=%d, cellsize = %f rad, FOV = %f rad" %
          (npixel, cellsize, theta))

    # Set up the gridding kernel. We try to use a cached version
    gridding_algorithm = get_parameter(params, 'gridding_algorithm', 'wprojection')

    if gridding_algorithm == 'wprojection':
        log.debug("invert_visibility: Gridding by w projection")

        wstep = get_parameter(params, "wstep", 10000.0)
        wcachesize = w_cache_size(vis, wstep)
        log.debug("invert_visibility: Making w-kernel cache of %d kernels" % wcachesize)

        cache_fn = w_conj_kernel_fn(pylru.FunctionCacheManager(w_kernel, wcachesize))
        imgfn = functools.partial(w_cache_imaging,
                                  wstep=wstep, kernel_cache=cache_fn,
                                  NpixFF=256, NpixKern=15, Qpx=4)
    else:
        raise NotImplementedError("gridding algorithm %s not supported" % gridding_algorithm)

    # Apply a phase rotation from the visibility phase centre to the image phase centre
    #    visphaserotate = phaserotate(vis, imagecentre)

    d = numpy.zeros(shape)
    p = numpy.zeros(shape)

    spectral_mode = get_parameter(params, 'spectral_mode', 'channel')
    log.debug('invert_visibility: spectral mode is %s' % spectral_mode)

    if spectral_mode == 'channel':
        pmax = 0.0
        nchan = shape[0]
        npol = shape[1]
        for channel in range(nchan):
            for pol in range(npol):
                log.debug('invert_visibility: Inverting channel %d, polarisation %d' % (channel, pol))
                d[channel, pol, :, :], p[channel, 0, :, :], pmax = \
                    do_imaging(theta, 1.0 / cellsize, vis.uvw_lambda(channel),
                               vis.vis[:, channel, pol], imgfn=imgfn)
            assert pmax > 0.0, ("No data gridded for channel %d" % channel)
    else:
        raise NotImplementedError("mode %s not supported" % spectral_mode)

    dirty = create_image_from_array(d, w)
    psf = create_image_from_array(p, w)
    log.debug("invert_visibility: Finished making dirty and psf")

    return dirty, psf, pmax

def predict_visibility(vis: Visibility, sm: SkyModel, params={}) -> Visibility:
    """Predict the visibility from a SkyModel including both components and images

    :param vis:
    :type Visibility: Visibility to be processed
    :param sm:
    :type SkyModel:
    :returns: Visibility
    """
    shape, reffrequency, cellsize, w, imagecentre = create_wcs_from_visibility(vis, params=params)

    # Create copy of visibilities
    vis = copy.copy(vis)
    vis.data = copy.copy(vis.data)
    vis.data['vis'] = numpy.zeros(vis.vis.shape)

    spectral_mode = get_parameter(params, 'spectral_mode', 'channel')
    log.debug('predict_visibility: spectral mode is %s' % spectral_mode)

    if len(sm.images):
        log.debug("predict_visibility: Predicting Visibility from sky model images")

        for im in sm.images:
            assert_same_chan_pol(vis, im)

            # Determine image size
            cellsize = abs(im.wcs.wcs.cdelt[0]) * numpy.pi / 180.0
            theta = im.npixel * cellsize
            log.debug("predict_visibility: Image cellsize %f radians" % cellsize)
            log.debug("predict_visibility: Field of view %f radians" % theta)
            assert (theta / numpy.sqrt(2) < 1.0), "Field of view larger than celestial sphere"

            # Parameterise imaging
            wstep = get_parameter(params, "wstep", 10000.0)
            wcachesize = w_cache_size(vis, wstep)
            log.debug("predict_visibility: Making w-kernel cache of %d kernels" % wcachesize)

            cache_fn = w_conj_kernel_fn(pylru.FunctionCacheManager(w_kernel, wcachesize))
            predfn = functools.partial(w_cache_predict,
                                       wstep=wstep, kernel_cache=cache_fn,
                                       NpixFF=256, NpixKern=15, Qpx=4)

            spectral_mode = get_parameter(params, 'spectral_mode', 'channel')
            log.debug('predict_visibility: spectral mode is %s' % spectral_mode)

            if spectral_mode == 'channel':
                for channel in range(im.nchan):
                    uvw = vis.uvw_lambda(channel)
                    for pol in range(im.npol):
                        log.debug('predict_visibility: Predicting from image channel %d, polarisation %d' % (
                        channel, pol))
                        img = sm.images[0].data[channel, pol, :, :]
                        dv = do_predict(theta, 1.0 / cellsize, numpy.array(uvw), img, predfn)
                        vis.vis[:, channel, pol] += dv
            else:
                raise NotImplementedError("mode %s not supported" % spectral_mode)

            log.debug("predict_visibility: Finished predicting Visibility from sky model images")

    if len(sm.components):
        log.debug("predict_visibility: Predicting Visibility from sky model components")

        for icomp, comp in enumerate(sm.components):

            log.debug("predict_visibility: visibility shape = %s" % str(vis.vis.shape))
            assert_same_chan_pol(vis, comp)

            l,m,n = skycoord_to_lmn(comp.direction, vis.phasecentre)
            log.debug('fourier_transforms.predict_visibility: Cartesian representation of component %d = (%f, %f, %f)'
                  % (icomp, l,m,n))

            if spectral_mode =='channel':
                for channel in range(comp.nchan):
                    uvw = vis.uvw_lambda(channel)
                    phasor = simulate_point(uvw, l, m)
                    for pol in range(comp.npol):
                        log.debug(
                            'predict_visibility: Predicting from component %d channel %d, polarisation %d' % (
                            icomp, channel,
                            pol))
                        vis.vis[:, channel, pol] += comp.flux[channel, pol] * phasor
            else:
                raise NotImplementedError("mode %s not supported" % spectral_mode)

        log.debug("predict_visibility: Finished predicting Visibility from sky model components")

    return vis

def weight_visibility(vis, im, params={}):
    """ Reweight the visibility data in place a selected algorithm

    :param vis:
    :type Visibility: Visibility to be processed
    :param im:
    :type Image:
    :param params: Dictionary containing parameters
    :returns: Configuration
    """
    # TODO: implement

    log_parameters(params)
    log.error("visibility_operations.weight_visibility: not yet implemented")
    return vis

<|endoftext|>


<commit_before>"""

conversions.py

@author: derricw

Bunch of random conversion functions.

"""

from PIL import Image, ImageOps, ImageDraw, ImageFont
import numpy as np
from bisect import bisect
import random
import os
import cv2
from misc import *

RESOURCE_DIR = os.path.join(os.path.dirname(__file__),'res')

GREYSCALE_RANDOM = [
    " ",
    ". ",
    ",-",
    "_ivc=!/|\\~",
    "gjezt*+",
    "2](YL)[T7Vf",
    "mdK4",
    "mdK4ZGbN",
    "DXY5P"
    "#%$"
    "W8KMA",

    ]

GREYSCALE_UNIFORM = [
    " ",
    ".",
    "'",
    "-",
    ":",
    ";",
    "!",
    "~",
    "*",
    "+",
    "e",
    "m",
    "6",
    "8",
    "g",
    "#",
    "W",
    "M",
    "@",
]

BINS = [15, 25, 45, 60, 75, 90, 100, 115, 135, 155, 170, 185, 205, 220, 235,
        245, 250]

ASPECTCORRECTIONFACTOR = 6.0/11.0  # because text pixels are rectangular

def image_to_ascii(img, scalefactor=0.2, invert=False, equalize=True):
    """
    Generates and ascii string from an image of some kind.

    Parameters
    ----------
    img : str, ndarray, PIL.Image
        Image to convert
    scalefactor : float
        ASCII chars per pixel
    invert : bool
        Invert luminance?
    equalize : bool
        Equalize histogram?

    Returns
    -------
    str

    Examples
    --------

    >>> ascii_img = image_to_ascii("http://i.imgur.com/l2FU2J0.jpg", scalefactor=0.3)
    >>> print(ascii_img)

    """
    if type(img) == str:
        img = open_pil_img(img)
        text = pil_to_ascii(img, scalefactor, invert, equalize)
    elif type(img) == np.ndarray:
        #text = numpy_to_ascii(img, scalefactor, invert, equalize)  #WHY IS THIS SLOWER?
        text = pil_to_ascii(numpy_to_pil(img), scalefactor, invert, equalize)
    else:
        try:
            pil_to_ascii(img, scalefactor, invert, equalize)
        except:
            raise TypeError("That image type doesn't work.  Try PIL, Numpy, or file path...")
    return text

def pil_to_ascii(img, scalefactor=0.2, invert=False, equalize=True):
    """
    Generates an ascii string from a PIL image.

    Parameters
    ----------
    img : PIL.Image
        PIL image to transform.
    scalefactor : float
        ASCII characters per pixel
    invert : bool
        Invert luminance?
    equalize : bool
        equalize histogram (for best results do this)

    Returns
    -------
    str

    Examples
    --------

    >>> from asciisciit.misc import open_pil_img
    >>> img = open_pil_img("http://i.imgur.com/l2FU2J0.jpg")
    >>> text_img = pil_to_ascii(img, scalefactor=0.3)
    >>> print(text_img)

    >>> from PIL import Image
    >>> img = Image.open("some_image.png")
    >>> text_img = pil_to_ascii(img)
    >>> print(text_img)

    """
    img = img.resize((int(img.size[0]*scalefactor),
        int(img.size[1]*scalefactor*ASPECTCORRECTIONFACTOR)),
        Image.BILINEAR)
    img = img.convert("L")  # convert to mono
    if equalize:
        img = ImageOps.equalize(img)

    if invert:
        img = ImageOps.invert(img)

    text = "\n"

    ##TODO: custom LUT
    lut = GREYSCALE_UNIFORM

    #SLOW ##TODO: USE Image.point(lut) instead
    for y in range(0, img.size[1]):
        for x in range(0, img.size[0]):
            lum = img.getpixel((x, y))
            row = bisect(BINS, lum)
            character = lut[row]
            text += character
        text += "\n"

    return text

def ascii_to_pil(text, font_size=10, bg_color=(20, 20, 20),
                 fg_color=(255, 255, 255), font_path=None):
    """
    Renders Ascii text to an Image of the appropriate size, using text of the
        specified font size.

    Parameters
    ----------
    text : str
        Ascii text to render.
    font_size : int (10)
        Font size for rendered image.
    bg_color : tuple (20,20,20)
        (R,G,B) values for image background.
    fg_color : tuple (255,255,255)
        (R,G,B) values for text color.
    font_path : str
        Use a custom font .ttf file.

    Returns
    -------
    PIL.Image

    Examples
    --------

    >>> ascii = AsciiImage("http://i.imgur.com/l2FU2J0.jpg", scalefactor=0.4)
    >>> pil = ascii_to_pil(ascii.data)
    >>> pil.show()

    """
    #font = ImageFont.load_default()
    if not font_path:
        font_path = os.path.join(RESOURCE_DIR, "Cousine-Regular.ttf")

    font = ImageFont.truetype(font_path, font_size)
    font_width, font_height = font.getsize(" ")  # shape of 1 char

    img_height, img_width = get_ascii_image_size(text)

    out_img = np.zeros((font_height*img_height, font_width*img_width, 3),
                       dtype=np.uint8)
    out_img[:, :, 0] += bg_color[0]
    out_img[:, :, 1] += bg_color[1]
    out_img[:, :, 2] += bg_color[2]

    img = Image.fromarray(out_img)
    draw = ImageDraw.Draw(img)

    for index, line in enumerate(text.split("\n")):
        y = (font_height)*index
        draw.text((0, y), line, fg_color, font=font)
    return img

def ascii_seq_to_gif(seq, output_path, fps=15.0, font_size=10):
    try:
        from images2gif import writeGif
    except ImportError as e:
        raise RuntimeError("Writing gifs requires images2gif library.\nTry 'pip install images2gif' %s" % e)

    images = []

    status = StatusBar(len(seq), text="Generating frames: ",)

    for index, ascii_img in enumerate(seq):
        if type(ascii_img) == str:
            #raw text
            text = ascii_img
        else:
            #AsciiImage instance
            text = ascii_img.data
        images.append(ascii_to_pil(text, font_size=font_size))
        status.update(index)

    status.complete()

    duration = 1.0/fps

    writeGif(output_path, images, duration)

def numpy_to_ascii(img, scalefactor=0.2, invert=False, equalize=True):
    """
    Generates and ascii string from a numpy image.

    SLOW FOR SOME REASON SO I DONT USE IT.

    """
    size = img.shape

    img=cv2.resize(img, (int(size[1]*scalefactor),
        int(size[0]*scalefactor*ASPECTCORRECTIONFACTOR)))

    img = cv2.cvtColor(img,cv2.cv.CV_RGB2GRAY)

    if equalize:
        img=cv2.equalizeHist(img)

    if not invert:
        img = 255-img

    text="\n"

    lut = GREYSCALE_UNIFORM

    #SLOW REWRITE USING ONLY NUMPY
    for y in range(img.shape[0]):
        for x in range(img.shape[1]):
            lum=img[y,x]
            row=bisect(BINS,lum)
            character=lut[row]
            text+=character
        text+="\n"

    return text

def image_to_numpy(path):
    """
    Image file to numpy matrix.
    """
    img = open_pil_img(path)
    return np.array(img, dtype=np.uint8)

def numpy_to_pil(nparray):
    """
    Numpy matrix to PIL Image.
    """
    return Image.fromarray(nparray)

def gif_to_numpy(gif_path):
    """
    Converts a GIF into a numpy movie.
    """
    gif = open_pil_img(gif_path)
    if hasattr(gif, 'info'):
        frame_duration = gif.info.get('duration', None)
    else:
        frame_duration = None
    length = get_length_of_gif(gif)

    size = gif.size

    status = StatusBar(length, "Reading frames: ")

    frames = []
    frame_count = 0
    while gif:
        new_img = Image.new("RGBA",size)
        new_img.paste(gif)
        frames.append(new_img)
        frame_count += 1
        try:
            gif.seek(frame_count)
        except EOFError:
            break
        status.update(frame_count)

    status.complete()

    assert(length==len(frames))

    final_frame_count = len(frames)
    frame1 = np.array(frames[0])
    shape = frame1.shape
    matrix = np.zeros((final_frame_count,shape[0],shape[1],shape[2]), dtype=np.uint8)
    for i, frame in enumerate(frames):
        img = np.asarray(frame)
        matrix[i] = img
    return matrix, frame_duration

def figure_to_numpy(mpl_figure):
    """
    Converts a matplotlib figure to numpy matrix.
    """
    mpl_figure.tight_layout(pad=0.1)
    mpl_figure.canvas.draw()
    data = np.fromstring(mpl_figure.canvas.tostring_rgb(), dtype=np.uint8, sep='')
    data = data.reshape(mpl_figure.canvas.get_width_height()[::-1]+ (3,))
    return data

def figure_to_ascii(mpl_figure):
    """
    Converts a matplotlib figure to ascii image.
    """
    npy_fig = figure_to_numpy(mpl_figure)
    return image_to_ascii(npy_fig, scalefactor=0.15, invert=False, equalize=False)

if __name__ == '__main__':
    pass
<commit_msg>lut converted to regular str<commit_after>"""

conversions.py

@author: derricw

Bunch of random conversion functions.

"""

from PIL import Image, ImageOps, ImageDraw, ImageFont
import numpy as np
from bisect import bisect
import random
import os
import cv2
from misc import *

RESOURCE_DIR = os.path.join(os.path.dirname(__file__),'res')

GREYSCALE_UNIFORM = " .'-:;!~*+em68g#WM@"

BINS = [15, 25, 45, 60, 75, 90, 100, 115, 135, 155, 170, 185, 205, 220, 235,
        245, 250]

ASPECTCORRECTIONFACTOR = 6.0/11.0  # because text pixels are rectangular

def image_to_ascii(img, scalefactor=0.2, invert=False, equalize=True):
    """
    Generates and ascii string from an image of some kind.

    Parameters
    ----------
    img : str, ndarray, PIL.Image
        Image to convert
    scalefactor : float
        ASCII chars per pixel
    invert : bool
        Invert luminance?
    equalize : bool
        Equalize histogram?

    Returns
    -------
    str

    Examples
    --------

    >>> ascii_img = image_to_ascii("http://i.imgur.com/l2FU2J0.jpg", scalefactor=0.3)
    >>> print(ascii_img)

    """
    if type(img) == str:
        img = open_pil_img(img)
        text = pil_to_ascii(img, scalefactor, invert, equalize)
    elif type(img) == np.ndarray:
        #text = numpy_to_ascii(img, scalefactor, invert, equalize)  #WHY IS THIS SLOWER?
        text = pil_to_ascii(numpy_to_pil(img), scalefactor, invert, equalize)
    else:
        try:
            pil_to_ascii(img, scalefactor, invert, equalize)
        except:
            raise TypeError("That image type doesn't work.  Try PIL, Numpy, or file path...")
    return text

def pil_to_ascii(img, scalefactor=0.2, invert=False, equalize=True):
    """
    Generates an ascii string from a PIL image.

    Parameters
    ----------
    img : PIL.Image
        PIL image to transform.
    scalefactor : float
        ASCII characters per pixel
    invert : bool
        Invert luminance?
    equalize : bool
        equalize histogram (for best results do this)

    Returns
    -------
    str

    Examples
    --------

    >>> from asciisciit.misc import open_pil_img
    >>> img = open_pil_img("http://i.imgur.com/l2FU2J0.jpg")
    >>> text_img = pil_to_ascii(img, scalefactor=0.3)
    >>> print(text_img)

    >>> from PIL import Image
    >>> img = Image.open("some_image.png")
    >>> text_img = pil_to_ascii(img)
    >>> print(text_img)

    """
    img = img.resize((int(img.size[0]*scalefactor),
        int(img.size[1]*scalefactor*ASPECTCORRECTIONFACTOR)),
        Image.BILINEAR)
    img = img.convert("L")  # convert to mono
    if equalize:
        img = ImageOps.equalize(img)

    if invert:
        img = ImageOps.invert(img)

    text = "\n"

    ##TODO: custom LUT
    lut = list(GREYSCALE_UNIFORM)

    #SLOW ##TODO: USE Image.point(lut) instead
    for y in range(0, img.size[1]):
        for x in range(0, img.size[0]):
            lum = img.getpixel((x, y))
            row = bisect(BINS, lum)
            character = lut[row]
            text += character
        text += "\n"

    return text

def ascii_to_pil(text, font_size=10, bg_color=(20, 20, 20),
                 fg_color=(255, 255, 255), font_path=None):
    """
    Renders Ascii text to an Image of the appropriate size, using text of the
        specified font size.

    Parameters
    ----------
    text : str
        Ascii text to render.
    font_size : int (10)
        Font size for rendered image.
    bg_color : tuple (20,20,20)
        (R,G,B) values for image background.
    fg_color : tuple (255,255,255)
        (R,G,B) values for text color.
    font_path : str
        Use a custom font .ttf file.

    Returns
    -------
    PIL.Image

    Examples
    --------

    >>> ascii = AsciiImage("http://i.imgur.com/l2FU2J0.jpg", scalefactor=0.4)
    >>> pil = ascii_to_pil(ascii.data)
    >>> pil.show()

    """
    #font = ImageFont.load_default()
    if not font_path:
        font_path = os.path.join(RESOURCE_DIR, "Cousine-Regular.ttf")

    font = ImageFont.truetype(font_path, font_size)
    font_width, font_height = font.getsize(" ")  # shape of 1 char

    img_height, img_width = get_ascii_image_size(text)

    out_img = np.zeros((font_height*img_height, font_width*img_width, 3),
                       dtype=np.uint8)
    out_img[:, :, 0] += bg_color[0]
    out_img[:, :, 1] += bg_color[1]
    out_img[:, :, 2] += bg_color[2]

    img = Image.fromarray(out_img)
    draw = ImageDraw.Draw(img)

    for index, line in enumerate(text.split("\n")):
        y = (font_height)*index
        draw.text((0, y), line, fg_color, font=font)
    return img

def ascii_seq_to_gif(seq, output_path, fps=15.0, font_size=10):
    try:
        from images2gif import writeGif
    except ImportError as e:
        raise RuntimeError("Writing gifs requires images2gif library.\nTry 'pip install images2gif' %s" % e)

    images = []

    status = StatusBar(len(seq), text="Generating frames: ",)

    for index, ascii_img in enumerate(seq):
        if type(ascii_img) == str:
            #raw text
            text = ascii_img
        else:
            #AsciiImage instance
            text = ascii_img.data
        images.append(ascii_to_pil(text, font_size=font_size))
        status.update(index)

    status.complete()

    duration = 1.0/fps

    writeGif(output_path, images, duration)

def numpy_to_ascii(img, scalefactor=0.2, invert=False, equalize=True):
    """
    Generates and ascii string from a numpy image.

    SLOW FOR SOME REASON SO I DONT USE IT.

    """
    size = img.shape

    img=cv2.resize(img, (int(size[1]*scalefactor),
        int(size[0]*scalefactor*ASPECTCORRECTIONFACTOR)))

    img = cv2.cvtColor(img,cv2.cv.CV_RGB2GRAY)

    if equalize:
        img=cv2.equalizeHist(img)

    if not invert:
        img = 255-img

    text="\n"

    lut = GREYSCALE_UNIFORM

    #SLOW REWRITE USING ONLY NUMPY
    for y in range(img.shape[0]):
        for x in range(img.shape[1]):
            lum=img[y,x]
            row=bisect(BINS,lum)
            character=lut[row]
            text+=character
        text+="\n"

    return text

def image_to_numpy(path):
    """
    Image file to numpy matrix.
    """
    img = open_pil_img(path)
    return np.array(img, dtype=np.uint8)

def numpy_to_pil(nparray):
    """
    Numpy matrix to PIL Image.
    """
    return Image.fromarray(nparray)

def gif_to_numpy(gif_path):
    """
    Converts a GIF into a numpy movie.
    """
    gif = open_pil_img(gif_path)
    if hasattr(gif, 'info'):
        frame_duration = gif.info.get('duration', None)
    else:
        frame_duration = None
    length = get_length_of_gif(gif)

    size = gif.size

    status = StatusBar(length, "Reading frames: ")

    frames = []
    frame_count = 0
    while gif:
        new_img = Image.new("RGBA",size)
        new_img.paste(gif)
        frames.append(new_img)
        frame_count += 1
        try:
            gif.seek(frame_count)
        except EOFError:
            break
        status.update(frame_count)

    status.complete()

    assert(length==len(frames))

    final_frame_count = len(frames)
    frame1 = np.array(frames[0])
    shape = frame1.shape
    matrix = np.zeros((final_frame_count,shape[0],shape[1],shape[2]), dtype=np.uint8)
    for i, frame in enumerate(frames):
        img = np.asarray(frame)
        matrix[i] = img
    return matrix, frame_duration

def figure_to_numpy(mpl_figure):
    """
    Converts a matplotlib figure to numpy matrix.
    """
    mpl_figure.tight_layout(pad=0.1)
    mpl_figure.canvas.draw()
    data = np.fromstring(mpl_figure.canvas.tostring_rgb(), dtype=np.uint8, sep='')
    data = data.reshape(mpl_figure.canvas.get_width_height()[::-1]+ (3,))
    return data

def figure_to_ascii(mpl_figure):
    """
    Converts a matplotlib figure to ascii image.
    """
    npy_fig = figure_to_numpy(mpl_figure)
    return image_to_ascii(npy_fig, scalefactor=0.15, invert=False, equalize=False)

if __name__ == '__main__':
    pass
<|endoftext|>


<commit_before>"""An aspect which extracts the runtime classpath from a java target."""

def _runtime_classpath_impl(target, ctx):
    """The top level aspect implementation function.

    Args:
      target: Essentially a struct representing a BUILD target.

      ctx: The context object that can be used to access attributes and generate
      outputs and actions.

    Returns:
      A struct with only the output_groups provider.
    """
    ctx = ctx  # unused argument
    return struct(output_groups = {
        "runtime_classpath": _get_runtime_jars(target),
    })

def _get_runtime_jars(target):
    # JavaInfo constructor doesn't fill in compilation info
    # https://github.com/bazelbuild/bazel/issues/10170
    if JavaInfo in target and target[JavaInfo].compilation_info:
        return target[JavaInfo].compilation_info.runtime_classpath
    return depset()

def _aspect_def(impl):
    return aspect(implementation = impl)

java_classpath_aspect = _aspect_def(_runtime_classpath_impl)
<commit_msg>Fall back to full transitive set of runtime jars if JavaInfo compilation info isn't filled in.<commit_after>"""An aspect which extracts the runtime classpath from a java target."""

def _runtime_classpath_impl(target, ctx):
    """The top level aspect implementation function.

    Args:
      target: Essentially a struct representing a BUILD target.

      ctx: The context object that can be used to access attributes and generate
      outputs and actions.

    Returns:
      A struct with only the output_groups provider.
    """
    ctx = ctx  # unused argument
    return struct(output_groups = {
        "runtime_classpath": _get_runtime_jars(target),
    })

def _get_runtime_jars(target):
    if JavaInfo not in target:
        return depset()
    if target[JavaInfo].compilation_info:
        return target[JavaInfo].compilation_info.runtime_classpath

    # JavaInfo constructor doesn't fill in compilation info, so just return the
    # full transitive set of runtime jars
    # https://github.com/bazelbuild/bazel/issues/10170
    return target[JavaInfo].transitive_runtime_jars

def _aspect_def(impl):
    return aspect(implementation = impl)

java_classpath_aspect = _aspect_def(_runtime_classpath_impl)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Tests for the Model Migration feature"""

from __future__ import print_function

import argparse
import logging
import os
from subprocess import CalledProcessError
import sys
from time import sleep

from assess_user_grant_revoke import User
from deploy_stack import BootstrapManager
from utility import (
    JujuAssertionError,
    add_basic_testing_arguments,
    configure_logging,
    temp_dir,
    until_timeout,
)

__metaclass__ = type

log = logging.getLogger("assess_model_migration")

def assess_model_migration(bs1, bs2, upload_tools):
    with bs1.booted_context(upload_tools):
        bs1.client.enable_feature('migration')

        bs2.client.env.juju_home = bs1.client.env.juju_home
        with bs2.existing_booted_context(upload_tools):
            ensure_able_to_migrate_model_between_controllers(
                bs1, bs2, upload_tools)

            with temp_dir() as temp:
                ensure_migrating_with_user_permissions(
                    bs1, bs2, upload_tools, temp)

def parse_args(argv):
    """Parse all arguments."""
    parser = argparse.ArgumentParser(
        description="Test model migration feature"
    )
    add_basic_testing_arguments(parser)
    return parser.parse_args(argv)

def get_bootstrap_managers(args):
    """Create 2 bootstrap managers from the provided args.

    Need to make a couple of elements uniqe (e.g. environment name) so we can
    have 2 bootstrapped at the same time.

    """
    bs_1 = BootstrapManager.from_args(args)
    bs_2 = BootstrapManager.from_args(args)

    # Need to be able to upgrade this controller.
    bs_1.client.env.config['enable-os-upgrade'] = True

    # Give the second a separate/unique name.
    bs_2.temp_env_name = '{}-b'.format(bs_1.temp_env_name)

    bs_1.log_dir = _new_log_dir(args.logs, 'a')
    bs_2.log_dir = _new_log_dir(args.logs, 'b')

    return bs_1, bs_2

def _new_log_dir(log_dir, post_fix):
    new_log_dir = os.path.join(log_dir, 'env-{}'.format(post_fix))
    os.mkdir(new_log_dir)
    return new_log_dir

def wait_for_model(client, model_name, timeout=60):
    """Wait for a given `timeout` for a model of `model_name` to appear within
    `client`.

    Defaults to 10 seconds timeout.
    :raises AssertionError: If the named model does not appear in the specified
      timeout.

    """
    for _ in until_timeout(timeout):
        models = client.get_models()
        if model_name in [m['name'] for m in models['models']]:
            return
        sleep(1)
    raise JujuAssertionError(
        'Model \'{}\' failed to appear after {} seconds'.format(
            model_name, timeout
        ))

def test_deployed_mongo_is_up(client):
    """Ensure the mongo service is running as expected."""
    try:
        output = client.get_juju_output(
            'run', '--unit', 'mongodb/0', 'mongo --eval "db.getMongo()"')
        if 'connecting to: test' in output:
            return
    except CalledProcessError as e:
        # Pass through to assertion error
        log.error('Mongodb check command failed: {}'.format(e))
    raise AssertionError('Mongo db is not in an expected state.')

def ensure_able_to_migrate_model_between_controllers(
        source_environ, dest_environ, upload_tools):
    """Test simple migration of a model to another controller.

    Ensure that migration a model that has an application deployed upon it is
    able to continue it's operation after the migration process.

    Given 2 bootstrapped environments:
      - Deploy an application
        - ensure it's operating as expected
      - Migrate that model to the other environment
        - Ensure it's operating as expected
        - Add a new unit to the application to ensure the model is functional
      - Migrate the model back to the original environment
        - Note: Test for lp:1607457
        - Ensure it's operating as expected
        - Add a new unit to the application to ensure the model is functional

    """
    bundle = 'mongodb'
    application = 'mongodb'

    log.info('Deploying charm')
    # Don't move the default model so we can reuse it in later tests.
    test_model = source_environ.client.add_model(
        source_environ.client.env.clone('example-model'))
    test_model.juju("deploy", (bundle))
    test_model.wait_for_started()
    test_model.wait_for_workloads()
    test_deployed_mongo_is_up(test_model)

    log.info('Initiating migration process')

    migration_target_client = migrate_model_to_controller(
        test_model, dest_environ.client)

    test_deployed_mongo_is_up(migration_target_client)
    ensure_model_is_functional(migration_target_client, application)

def migrate_model_to_controller(source_client, dest_client):
    source_client.controller_juju(
        'migrate',
        (source_client.env.environment,
         dest_client.env.controller.name))

    migration_target_client = dest_client.clone(
        dest_client.env.clone(
            source_client.env.environment))

    wait_for_model(
        migration_target_client, source_client.env.environment)

    # For logging purposes
    migration_target_client.show_status()
    migration_target_client.wait_for_started()

    return migration_target_client

def ensure_model_is_functional(client, application):
    """Ensures that the migrated model is functional

    Add unit to application to ensure the model is contactable and working.
    Ensure that added unit is created on a new machine (check for bug
    LP:1607599)

    """
    client.juju('add-unit', (application,))
    client.wait_for_started()

    assert_units_on_different_machines(client, application)

def assert_units_on_different_machines(client, application):
    status = client.get_status()
    unit_machines = [u[1]['machine'] for u in status.iter_units()]

    raise_if_shared_machines(unit_machines)

def raise_if_shared_machines(unit_machines):
    """Raise an exception if `unit_machines` contain double ups of machine ids.

    A unique list of machine ids will be equal in length to the set of those
    machine ids.

    :raises ValueError: if an empty list is passed in.
    :raises JujuAssertionError: if any double-ups of machine ids are detected.

    """
    if not unit_machines:
        raise ValueError('Cannot share 0 machines. Empty list provided.')
    if len(unit_machines) != len(set(unit_machines)):
        raise JujuAssertionError('Appliction units reside on the same machine')

def ensure_migrating_with_user_permissions(
        source_bs, dest_bs, upload_tools, temp_dir):
    """Ensure migration fails when a user does not have the right permissions.

    A non-superuser on a controller cannot migrate their models between
    controllers.

    """
    # Create a user for both controllers that only has addmodel
    # permissions not superuser.
    new_user_home = os.path.join(temp_dir, 'example_user')
    os.makedirs(new_user_home)
    new_user = User('testuser', 'write', [])
    normal_user_client_1 = source_bs.client.register_user(
        new_user, new_user_home)
    source_bs.client.grant(new_user.name, 'addmodel')

    second_controller_name = '{}_controllerb'.format(new_user.name)
    normal_user_client_2 = dest_bs.client.register_user(
        new_user,
        new_user_home,
        controller_name=second_controller_name)
    dest_bs.client.grant(new_user.name, 'addmodel')

    user_new_model_client = normal_user_client_1.add_model(
        normal_user_client_1.env.clone('model-a'))

    user_new_model_client.juju('deploy', ('ubuntu'))
    user_new_model_client.wait_for_started()

    log.info('Attempting migration process')

    expect_migration_attempt_to_fail(
        user_new_model_client,
        normal_user_client_2)

def expect_migration_attempt_to_fail(source_client, dest_client):
    try:
        args = ['-c', source_client.env.controller.name,
                source_client.env.environment,
                dest_client.env.controller.name]
        log_output = source_client.get_juju_output(
            'migrate', *args, merge_stderr=True, include_e=False)
    except CalledProcessError as e:
        if 'permission denied' not in e.output:
            raise
        log.info('SUCCESS: Migrate command failed as expected.')
    else:
        raise JujuAssertionError('Migration did not fail as expected.')
    finally:
        print(log_output, file=sys.stderr)

def main(argv=None):
    args = parse_args(argv)
    configure_logging(args.verbose)

    bs1, bs2 = get_bootstrap_managers(args)

    assess_model_migration(bs1, bs2, args.upload_tools)

    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Use dummy charm for multi-user test.<commit_after>#!/usr/bin/env python
"""Tests for the Model Migration feature"""

from __future__ import print_function

import argparse
import logging
import os
from subprocess import CalledProcessError
import sys
from time import sleep

from assess_user_grant_revoke import User
from deploy_stack import BootstrapManager
from jujucharm import local_charm_path
from utility import (
    JujuAssertionError,
    add_basic_testing_arguments,
    configure_logging,
    temp_dir,
    until_timeout,
)

__metaclass__ = type

log = logging.getLogger("assess_model_migration")

def assess_model_migration(bs1, bs2, upload_tools):
    with bs1.booted_context(upload_tools):
        bs1.client.enable_feature('migration')

        bs2.client.env.juju_home = bs1.client.env.juju_home
        with bs2.existing_booted_context(upload_tools):
            ensure_able_to_migrate_model_between_controllers(
                bs1, bs2, upload_tools)

            with temp_dir() as temp:
                ensure_migrating_with_user_permissions(
                    bs1, bs2, upload_tools, temp)

def parse_args(argv):
    """Parse all arguments."""
    parser = argparse.ArgumentParser(
        description="Test model migration feature"
    )
    add_basic_testing_arguments(parser)
    return parser.parse_args(argv)

def get_bootstrap_managers(args):
    """Create 2 bootstrap managers from the provided args.

    Need to make a couple of elements uniqe (e.g. environment name) so we can
    have 2 bootstrapped at the same time.

    """
    bs_1 = BootstrapManager.from_args(args)
    bs_2 = BootstrapManager.from_args(args)

    # Need to be able to upgrade this controller.
    bs_1.client.env.config['enable-os-upgrade'] = True

    # Give the second a separate/unique name.
    bs_2.temp_env_name = '{}-b'.format(bs_1.temp_env_name)

    bs_1.log_dir = _new_log_dir(args.logs, 'a')
    bs_2.log_dir = _new_log_dir(args.logs, 'b')

    return bs_1, bs_2

def _new_log_dir(log_dir, post_fix):
    new_log_dir = os.path.join(log_dir, 'env-{}'.format(post_fix))
    os.mkdir(new_log_dir)
    return new_log_dir

def wait_for_model(client, model_name, timeout=60):
    """Wait for a given `timeout` for a model of `model_name` to appear within
    `client`.

    Defaults to 10 seconds timeout.
    :raises AssertionError: If the named model does not appear in the specified
      timeout.

    """
    for _ in until_timeout(timeout):
        models = client.get_models()
        if model_name in [m['name'] for m in models['models']]:
            return
        sleep(1)
    raise JujuAssertionError(
        'Model \'{}\' failed to appear after {} seconds'.format(
            model_name, timeout
        ))

def test_deployed_mongo_is_up(client):
    """Ensure the mongo service is running as expected."""
    try:
        output = client.get_juju_output(
            'run', '--unit', 'mongodb/0', 'mongo --eval "db.getMongo()"')
        if 'connecting to: test' in output:
            return
    except CalledProcessError as e:
        # Pass through to assertion error
        log.error('Mongodb check command failed: {}'.format(e))
    raise AssertionError('Mongo db is not in an expected state.')

def ensure_able_to_migrate_model_between_controllers(
        source_environ, dest_environ, upload_tools):
    """Test simple migration of a model to another controller.

    Ensure that migration a model that has an application deployed upon it is
    able to continue it's operation after the migration process.

    Given 2 bootstrapped environments:
      - Deploy an application
        - ensure it's operating as expected
      - Migrate that model to the other environment
        - Ensure it's operating as expected
        - Add a new unit to the application to ensure the model is functional
      - Migrate the model back to the original environment
        - Note: Test for lp:1607457
        - Ensure it's operating as expected
        - Add a new unit to the application to ensure the model is functional

    """
    bundle = 'mongodb'
    application = 'mongodb'

    log.info('Deploying charm')
    # Don't move the default model so we can reuse it in later tests.
    test_model = source_environ.client.add_model(
        source_environ.client.env.clone('example-model'))
    test_model.juju("deploy", (bundle))
    test_model.wait_for_started()
    test_model.wait_for_workloads()
    test_deployed_mongo_is_up(test_model)

    log.info('Initiating migration process')

    migration_target_client = migrate_model_to_controller(
        test_model, dest_environ.client)

    test_deployed_mongo_is_up(migration_target_client)
    ensure_model_is_functional(migration_target_client, application)

def migrate_model_to_controller(source_client, dest_client):
    source_client.controller_juju(
        'migrate',
        (source_client.env.environment,
         dest_client.env.controller.name))

    migration_target_client = dest_client.clone(
        dest_client.env.clone(
            source_client.env.environment))

    wait_for_model(
        migration_target_client, source_client.env.environment)

    # For logging purposes
    migration_target_client.show_status()
    migration_target_client.wait_for_started()

    return migration_target_client

def ensure_model_is_functional(client, application):
    """Ensures that the migrated model is functional

    Add unit to application to ensure the model is contactable and working.
    Ensure that added unit is created on a new machine (check for bug
    LP:1607599)

    """
    client.juju('add-unit', (application,))
    client.wait_for_started()

    assert_units_on_different_machines(client, application)

def assert_units_on_different_machines(client, application):
    status = client.get_status()
    unit_machines = [u[1]['machine'] for u in status.iter_units()]

    raise_if_shared_machines(unit_machines)

def raise_if_shared_machines(unit_machines):
    """Raise an exception if `unit_machines` contain double ups of machine ids.

    A unique list of machine ids will be equal in length to the set of those
    machine ids.

    :raises ValueError: if an empty list is passed in.
    :raises JujuAssertionError: if any double-ups of machine ids are detected.

    """
    if not unit_machines:
        raise ValueError('Cannot share 0 machines. Empty list provided.')
    if len(unit_machines) != len(set(unit_machines)):
        raise JujuAssertionError('Appliction units reside on the same machine')

def ensure_migrating_with_user_permissions(
        source_bs, dest_bs, upload_tools, temp_dir):
    """Ensure migration fails when a user does not have the right permissions.

    A non-superuser on a controller cannot migrate their models between
    controllers.

    """
    # Create a user for both controllers that only has addmodel
    # permissions not superuser.
    new_user_home = os.path.join(temp_dir, 'example_user')
    os.makedirs(new_user_home)
    new_user = User('testuser', 'write', [])
    normal_user_client_1 = source_bs.client.register_user(
        new_user, new_user_home)
    source_bs.client.grant(new_user.name, 'addmodel')

    second_controller_name = '{}_controllerb'.format(new_user.name)
    normal_user_client_2 = dest_bs.client.register_user(
        new_user,
        new_user_home,
        controller_name=second_controller_name)
    dest_bs.client.grant(new_user.name, 'addmodel')

    user_new_model_client = normal_user_client_1.add_model(
        normal_user_client_1.env.clone('model-a'))

    charm_path = local_charm_path(
        charm='dummy-source', juju_ver=user_new_model_client.version)
    user_new_model_client.deploy(charm_path)
    user_new_model_client.wait_for_started()

    log.info('Attempting migration process')

    expect_migration_attempt_to_fail(
        user_new_model_client,
        normal_user_client_2)

def expect_migration_attempt_to_fail(source_client, dest_client):
    try:
        args = ['-c', source_client.env.controller.name,
                source_client.env.environment,
                dest_client.env.controller.name]
        log_output = source_client.get_juju_output(
            'migrate', *args, merge_stderr=True, include_e=False)
    except CalledProcessError as e:
        if 'permission denied' not in e.output:
            raise
        log.info('SUCCESS: Migrate command failed as expected.')
    else:
        raise JujuAssertionError('Migration did not fail as expected.')
    finally:
        print(log_output, file=sys.stderr)

def main(argv=None):
    args = parse_args(argv)
    configure_logging(args.verbose)

    bs1, bs2 = get_bootstrap_managers(args)

    assess_model_migration(bs1, bs2, args.upload_tools)

    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>