<commit_before># Copyright (c) 2017 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

import os.path #To find the test files.
import pytest #This module contains unit tests.
import unittest.mock #To monkeypatch some mocks in place of dependencies.

import cura.Settings.GlobalStack #The module we're testing.
from UM.Settings.DefinitionContainer import DefinitionContainer #To test against the class DefinitionContainer.
import UM.Settings.ContainerRegistry
import UM.Settings.ContainerStack

##  Fake container registry that always provides all containers you ask of.
@pytest.fixture()
def container_registry():
    registry = unittest.mock.MagicMock()
    def findContainers(id = None):
        if not id:
            return [UM.Settings.ContainerRegistry._EmptyInstanceContainer("test_container")]
        else:
            return [UM.Settings.ContainerRegistry._EmptyInstanceContainer(id)]
    registry.findContainers = findContainers
    return registry

#An empty global stack to test with.
@pytest.fixture()
def global_stack():
    return cura.Settings.GlobalStack.GlobalStack("TestStack")

##  Place-in function for findContainer that finds only containers that start
#   with "some_".
def findSomeContainers(container_id = "*", container_type = None, type = None, category = "*"):
    if container_id.startswith("some_"):
        return UM.Settings.ContainerRegistry._EmptyInstanceContainer(container_id)
    if container_type == DefinitionContainer:
        definition_mock = unittest.mock.MagicMock()
        definition_mock.getId = unittest.mock.MagicMock(return_value = "some_definition") #getId returns some_definition.
        return definition_mock

##  Helper function to read the contents of a container stack in the test
#   stack folder.
#
#   \param filename The name of the file in the "stacks" folder to read from.
#   \return The contents of that file.
def readStack(filename):
    with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "stacks", filename)) as file_handle:
        serialized = file_handle.read()
    return serialized

##  Tests whether the user changes are being read properly from a global stack.
@pytest.mark.parametrize("filename,                 user_changes_id", [
                        ("Global.global.cfg",       "empty"),
                        ("Global.stack.cfg",        "empty"),
                        ("MachineLegacy.stack.cfg", "empty"),
                        ("OnlyUser.global.cfg",     "some_instance"), #This one does have a user profile.
                        ("Complete.global.cfg",     "some_user_changes")
])
def test_deserializeUserChanges(filename, user_changes_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.userChanges.getId() == user_changes_id

##  Tests whether the quality changes are being read properly from a global
#   stack.
@pytest.mark.parametrize("filename,                       quality_changes_id", [
                        ("Global.global.cfg",             "empty"),
                        ("Global.stack.cfg",              "empty"),
                        ("MachineLegacy.stack.cfg",       "empty"),
                        ("OnlyQualityChanges.global.cfg", "some_instance"),
                        ("Complete.global.cfg",           "some_quality_changes")
])
def test_deserializeQualityChanges(filename, quality_changes_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.qualityChanges.getId() == quality_changes_id

##  Tests whether the quality profile is being read properly from a global
#   stack.
@pytest.mark.parametrize("filename,                 quality_id", [
                        ("Global.global.cfg",       "empty"),
                        ("Global.stack.cfg",        "empty"),
                        ("MachineLegacy.stack.cfg", "empty"),
                        ("OnlyQuality.global.cfg",  "some_instance"),
                        ("Complete.global.cfg",     "some_quality")
])
def test_deserializeQuality(filename, quality_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.quality.getId() == quality_id

##  Tests whether the material profile is being read properly from a global
#   stack.
@pytest.mark.parametrize("filename,                   material_id", [
                        ("Global.global.cfg",         "some_instance"),
                        ("Global.stack.cfg",          "some_instance"),
                        ("MachineLegacy.stack.cfg",   "some_instance"),
                        ("OnlyDefinition.global.cfg", "empty"),
                        ("OnlyMaterial.global.cfg",   "some_instance"),
                        ("Complete.global.cfg",       "some_material")
])
def test_deserializeMaterial(filename, material_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.material.getId() == material_id

##  Tests whether the variant profile is being read properly from a global
#   stack.
@pytest.mark.parametrize("filename,                 variant_id", [
                        ("Global.global.cfg",       "empty"),
                        ("Global.stack.cfg",        "empty"),
                        ("MachineLegacy.stack.cfg", "empty"),
                        ("OnlyVariant.global.cfg",  "some_instance"),
                        ("Complete.global.cfg",     "some_variant")
])
def test_deserializeVariant(filename, variant_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.variant.getId() == variant_id

##  Tests whether the definition changes profile is being read properly from a
#   global stack.
@pytest.mark.parametrize("filename,                          definition_changes_id", [
                        ("Global.global.cfg",                "empty"),
                        ("Global.stack.cfg",                 "empty"),
                        ("MachineLegacy.stack.cfg",          "empty"),
                        ("OnlyDefinitionChanges.global.cfg", "some_instance"),
                        ("Complete.global.cfg",              "some_material")
])
def test_deserializeDefinitionChanges(filename, definition_changes_id, container_registry, global_stack):
    serialized = readStack(filename)
    global_stack = cura.Settings.GlobalStack.GlobalStack("TestStack")

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.definitionChanges.getId() == definition_changes_id

##  Tests whether the definition is being read properly from a global stack.
@pytest.mark.parametrize("filename,                   definition_id", [
                        ("Global.global.cfg",         "some_definition"),
                        ("Global.stack.cfg",          "some_definition"),
                        ("MachineLegacy.stack.cfg",   "some_definition"),
                        ("OnlyDefinition.global.cfg", "some_definition"),
                        ("Complete.global.cfg",       "some_definition")
])
def test_deserializeDefinition(filename, definition_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.definition.getId() == definition_id

##  Tests whether the hasUserValue returns true for settings that are changed in
#   the user-changes container.
def test_hasUserValueUserChanges(global_stack):
    user_changes = unittest.mock.MagicMock()
    def hasProperty(key, property):
        return key == "layer_height" and property == "value" #Only have the layer_height property set.
    user_changes.hasProperty = hasProperty

    global_stack.userChanges = user_changes

    assert not global_stack.hasUserValue("infill_sparse_density")
    assert global_stack.hasUserValue("layer_height")
    assert not global_stack.hasUserValue("")

##  Tests whether the hasUserValue returns true for settings that are changed in
#   the quality-changes container.
def test_hasUserValueQualityChanges(global_stack):
    quality_changes = unittest.mock.MagicMock()
    def hasProperty(key, property):
        return key == "layer_height" and property == "value" #Only have the layer_height property set.
    quality_changes.hasProperty = hasProperty

    global_stack.qualityChanges = quality_changes

    assert not global_stack.hasUserValue("infill_sparse_density")
    assert global_stack.hasUserValue("layer_height")
    assert not global_stack.hasUserValue("")<commit_msg>Restore the original container registry after each test<commit_after># Copyright (c) 2017 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

import os.path #To find the test files.
import pytest #This module contains unit tests.
import unittest.mock #To monkeypatch some mocks in place of dependencies.

import cura.Settings.GlobalStack #The module we're testing.
from UM.Settings.DefinitionContainer import DefinitionContainer #To test against the class DefinitionContainer.
import UM.Settings.ContainerRegistry
import UM.Settings.ContainerStack

##  Fake container registry that always provides all containers you ask of.
@pytest.fixture()
def container_registry():
    registry = unittest.mock.MagicMock()
    def findContainers(id = None):
        if not id:
            return [UM.Settings.ContainerRegistry._EmptyInstanceContainer("test_container")]
        else:
            return [UM.Settings.ContainerRegistry._EmptyInstanceContainer(id)]
    registry.findContainers = findContainers
    return registry

#An empty global stack to test with.
@pytest.fixture()
def global_stack():
    return cura.Settings.GlobalStack.GlobalStack("TestStack")

##  Place-in function for findContainer that finds only containers that start
#   with "some_".
def findSomeContainers(container_id = "*", container_type = None, type = None, category = "*"):
    if container_id.startswith("some_"):
        return UM.Settings.ContainerRegistry._EmptyInstanceContainer(container_id)
    if container_type == DefinitionContainer:
        definition_mock = unittest.mock.MagicMock()
        definition_mock.getId = unittest.mock.MagicMock(return_value = "some_definition") #getId returns some_definition.
        return definition_mock

##  Helper function to read the contents of a container stack in the test
#   stack folder.
#
#   \param filename The name of the file in the "stacks" folder to read from.
#   \return The contents of that file.
def readStack(filename):
    with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "stacks", filename)) as file_handle:
        serialized = file_handle.read()
    return serialized

##  Tests whether the user changes are being read properly from a global stack.
@pytest.mark.parametrize("filename,                 user_changes_id", [
                        ("Global.global.cfg",       "empty"),
                        ("Global.stack.cfg",        "empty"),
                        ("MachineLegacy.stack.cfg", "empty"),
                        ("OnlyUser.global.cfg",     "some_instance"), #This one does have a user profile.
                        ("Complete.global.cfg",     "some_user_changes")
])
def test_deserializeUserChanges(filename, user_changes_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    original_container_registry = UM.Settings.ContainerStack._containerRegistry
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.userChanges.getId() == user_changes_id

    #Restore.
    UM.Settings.ContainerStack._containerRegistry = original_container_registry

##  Tests whether the quality changes are being read properly from a global
#   stack.
@pytest.mark.parametrize("filename,                       quality_changes_id", [
                        ("Global.global.cfg",             "empty"),
                        ("Global.stack.cfg",              "empty"),
                        ("MachineLegacy.stack.cfg",       "empty"),
                        ("OnlyQualityChanges.global.cfg", "some_instance"),
                        ("Complete.global.cfg",           "some_quality_changes")
])
def test_deserializeQualityChanges(filename, quality_changes_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    original_container_registry = UM.Settings.ContainerStack._containerRegistry
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.qualityChanges.getId() == quality_changes_id

    #Restore.
    UM.Settings.ContainerStack._containerRegistry = original_container_registry

##  Tests whether the quality profile is being read properly from a global
#   stack.
@pytest.mark.parametrize("filename,                 quality_id", [
                        ("Global.global.cfg",       "empty"),
                        ("Global.stack.cfg",        "empty"),
                        ("MachineLegacy.stack.cfg", "empty"),
                        ("OnlyQuality.global.cfg",  "some_instance"),
                        ("Complete.global.cfg",     "some_quality")
])
def test_deserializeQuality(filename, quality_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    original_container_registry = UM.Settings.ContainerStack._containerRegistry
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.quality.getId() == quality_id

    #Restore.
    UM.Settings.ContainerStack._containerRegistry = original_container_registry

##  Tests whether the material profile is being read properly from a global
#   stack.
@pytest.mark.parametrize("filename,                   material_id", [
                        ("Global.global.cfg",         "some_instance"),
                        ("Global.stack.cfg",          "some_instance"),
                        ("MachineLegacy.stack.cfg",   "some_instance"),
                        ("OnlyDefinition.global.cfg", "empty"),
                        ("OnlyMaterial.global.cfg",   "some_instance"),
                        ("Complete.global.cfg",       "some_material")
])
def test_deserializeMaterial(filename, material_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    original_container_registry = UM.Settings.ContainerStack._containerRegistry
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.material.getId() == material_id

    #Restore.
    UM.Settings.ContainerStack._containerRegistry = original_container_registry

##  Tests whether the variant profile is being read properly from a global
#   stack.
@pytest.mark.parametrize("filename,                 variant_id", [
                        ("Global.global.cfg",       "empty"),
                        ("Global.stack.cfg",        "empty"),
                        ("MachineLegacy.stack.cfg", "empty"),
                        ("OnlyVariant.global.cfg",  "some_instance"),
                        ("Complete.global.cfg",     "some_variant")
])
def test_deserializeVariant(filename, variant_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    original_container_registry = UM.Settings.ContainerStack._containerRegistry
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.variant.getId() == variant_id

    #Restore.
    UM.Settings.ContainerStack._containerRegistry = original_container_registry

##  Tests whether the definition changes profile is being read properly from a
#   global stack.
@pytest.mark.parametrize("filename,                          definition_changes_id", [
                        ("Global.global.cfg",                "empty"),
                        ("Global.stack.cfg",                 "empty"),
                        ("MachineLegacy.stack.cfg",          "empty"),
                        ("OnlyDefinitionChanges.global.cfg", "some_instance"),
                        ("Complete.global.cfg",              "some_material")
])
def test_deserializeDefinitionChanges(filename, definition_changes_id, container_registry, global_stack):
    serialized = readStack(filename)
    global_stack = cura.Settings.GlobalStack.GlobalStack("TestStack")

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    original_container_registry = UM.Settings.ContainerStack._containerRegistry
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.definitionChanges.getId() == definition_changes_id

    #Restore.
    UM.Settings.ContainerStack._containerRegistry = original_container_registry

##  Tests whether the definition is being read properly from a global stack.
@pytest.mark.parametrize("filename,                   definition_id", [
                        ("Global.global.cfg",         "some_definition"),
                        ("Global.stack.cfg",          "some_definition"),
                        ("MachineLegacy.stack.cfg",   "some_definition"),
                        ("OnlyDefinition.global.cfg", "some_definition"),
                        ("Complete.global.cfg",       "some_definition")
])
def test_deserializeDefinition(filename, definition_id, container_registry, global_stack):
    serialized = readStack(filename)

    #Mock the loading of the instance containers.
    global_stack.findContainer = findSomeContainers
    original_container_registry = UM.Settings.ContainerStack._containerRegistry
    UM.Settings.ContainerStack._containerRegistry = container_registry #Always has all the profiles you ask of.

    global_stack.deserialize(serialized)

    assert global_stack.definition.getId() == definition_id

    #Restore.
    UM.Settings.ContainerStack._containerRegistry = original_container_registry

##  Tests whether the hasUserValue returns true for settings that are changed in
#   the user-changes container.
def test_hasUserValueUserChanges(global_stack):
    user_changes = unittest.mock.MagicMock()
    def hasProperty(key, property):
        return key == "layer_height" and property == "value" #Only have the layer_height property set.
    user_changes.hasProperty = hasProperty

    global_stack.userChanges = user_changes

    assert not global_stack.hasUserValue("infill_sparse_density")
    assert global_stack.hasUserValue("layer_height")
    assert not global_stack.hasUserValue("")

##  Tests whether the hasUserValue returns true for settings that are changed in
#   the quality-changes container.
def test_hasUserValueQualityChanges(global_stack):
    quality_changes = unittest.mock.MagicMock()
    def hasProperty(key, property):
        return key == "layer_height" and property == "value" #Only have the layer_height property set.
    quality_changes.hasProperty = hasProperty

    global_stack.qualityChanges = quality_changes

    assert not global_stack.hasUserValue("infill_sparse_density")
    assert global_stack.hasUserValue("layer_height")
    assert not global_stack.hasUserValue("")<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                        unicode_literals, with_statement)

import os
import unittest
from contextlib import contextmanager

from pants.cache.artifact_cache import (NonfatalArtifactCacheError, call_insert,
                                        call_use_cached_files)
from pants.cache.local_artifact_cache import LocalArtifactCache, TempLocalArtifactCache
from pants.cache.pinger import BestUrlSelector, InvalidRESTfulCacheProtoError
from pants.cache.restful_artifact_cache import RESTfulArtifactCache
from pants.invalidation.build_invalidator import CacheKey
from pants.util.contextutil import temporary_dir, temporary_file, temporary_file_path
from pants.util.dirutil import safe_mkdir
from pants_test.cache.cache_server import cache_server

TEST_CONTENT1 = b'muppet'
TEST_CONTENT2 = b'kermit'

class TestArtifactCache(unittest.TestCase):
  @contextmanager
  def setup_local_cache(self):
    with temporary_dir() as artifact_root:
      with temporary_dir() as cache_root:
        yield LocalArtifactCache(artifact_root, cache_root, compression=1)

  @contextmanager
  def setup_server(self, return_failed=False, cache_root=None):
    with cache_server(return_failed=return_failed, cache_root=cache_root) as server:
      yield server

  @contextmanager
  def setup_rest_cache(self, local=None, return_failed=False):
    with temporary_dir() as artifact_root:
      local = local or TempLocalArtifactCache(artifact_root, 0)
      with self.setup_server(return_failed=return_failed) as server:
        yield RESTfulArtifactCache(artifact_root, BestUrlSelector([server.url]), local)

  @contextmanager
  def setup_test_file(self, parent):
    with temporary_file(parent) as f:
      # Write the file.
      f.write(TEST_CONTENT1)
      path = f.name
      f.close()
      yield path

  def test_local_cache(self):
    with self.setup_local_cache() as artifact_cache:
      self.do_test_artifact_cache(artifact_cache)

  def test_restful_cache(self):
    with self.assertRaises(InvalidRESTfulCacheProtoError):
      RESTfulArtifactCache('foo', BestUrlSelector(['ftp://localhost/bar']), 'foo')

    with self.setup_rest_cache() as artifact_cache:
      self.do_test_artifact_cache(artifact_cache)

  def test_restful_cache_failover(self):
    bad_url = 'http://badhost:123'

    with temporary_dir() as artifact_root:
      local = TempLocalArtifactCache(artifact_root, 0)

      # With fail-over, rest call second time will succeed
      with self.setup_server() as good_server:
        artifact_cache = RESTfulArtifactCache(artifact_root,
                                              BestUrlSelector([bad_url, good_server.url], max_failures=0),
                                              local)
        with self.assertRaises(NonfatalArtifactCacheError) as ex:
          self.do_test_artifact_cache(artifact_cache)
        self.assertIn('Connection aborted', str(ex.exception))

        self.do_test_artifact_cache(artifact_cache)

  def do_test_artifact_cache(self, artifact_cache):
    key = CacheKey('muppet_key', 'fake_hash')
    with self.setup_test_file(artifact_cache.artifact_root) as path:
      # Cache it.
      self.assertFalse(artifact_cache.has(key))
      self.assertFalse(bool(artifact_cache.use_cached_files(key)))
      artifact_cache.insert(key, [path])
      self.assertTrue(artifact_cache.has(key))

      # Stomp it.
      with open(path, 'w') as outfile:
        outfile.write(TEST_CONTENT2)

      # Recover it from the cache.
      self.assertTrue(bool(artifact_cache.use_cached_files(key)))

      # Check that it was recovered correctly.
      with open(path, 'r') as infile:
        content = infile.read()
      self.assertEquals(content, TEST_CONTENT1)

      # Delete it.
      artifact_cache.delete(key)
      self.assertFalse(artifact_cache.has(key))

  def test_local_backed_remote_cache(self):
    """make sure that the combined cache finds what it should and that it backfills"""
    with self.setup_server() as server:
      with self.setup_local_cache() as local:
        tmp = TempLocalArtifactCache(local.artifact_root, 0)
        remote = RESTfulArtifactCache(local.artifact_root, BestUrlSelector([server.url]), tmp)
        combined = RESTfulArtifactCache(local.artifact_root, BestUrlSelector([server.url]), local)

        key = CacheKey('muppet_key', 'fake_hash')

        with self.setup_test_file(local.artifact_root) as path:
          # No cache has key.
          self.assertFalse(local.has(key))
          self.assertFalse(remote.has(key))
          self.assertFalse(combined.has(key))

          # No cache returns key.
          self.assertFalse(bool(local.use_cached_files(key)))
          self.assertFalse(bool(remote.use_cached_files(key)))
          self.assertFalse(bool(combined.use_cached_files(key)))

          # Attempting to use key that no cache had should not change anything.
          self.assertFalse(local.has(key))
          self.assertFalse(remote.has(key))
          self.assertFalse(combined.has(key))

          # Add to only remote cache.
          remote.insert(key, [path])

          # After insertion to remote, remote and only remote should have key
          self.assertFalse(local.has(key))
          self.assertTrue(remote.has(key))
          self.assertTrue(combined.has(key))

          # Successfully using via remote should NOT change local.
          self.assertTrue(bool(remote.use_cached_files(key)))
          self.assertFalse(local.has(key))

          # Successfully using via combined SHOULD backfill local.
          self.assertTrue(bool(combined.use_cached_files(key)))
          self.assertTrue(local.has(key))
          self.assertTrue(bool(local.use_cached_files(key)))

  def test_local_backed_remote_cache_corrupt_artifact(self):
    """Ensure that a combined cache clears outputs after a failure to extract an artifact."""
    with temporary_dir() as remote_cache_dir:
      with self.setup_server(cache_root=remote_cache_dir) as server:
        with self.setup_local_cache() as local:
          tmp = TempLocalArtifactCache(local.artifact_root, compression=1)
          remote = RESTfulArtifactCache(local.artifact_root, BestUrlSelector([server.url]), tmp)
          combined = RESTfulArtifactCache(local.artifact_root, BestUrlSelector([server.url]), local)

          key = CacheKey('muppet_key', 'fake_hash')

          results_dir = os.path.join(local.artifact_root, 'a/sub/dir')
          safe_mkdir(results_dir)
          self.assertTrue(os.path.exists(results_dir))

          with self.setup_test_file(results_dir) as path:
            # Add to only the remote cache.
            remote.insert(key, [path])

            # Corrupt the artifact in the remote storage.
            self.assertTrue(server.corrupt_artifacts(r'.*muppet_key.*') == 1)

            # An attempt to read the corrupt artifact should fail.
            self.assertFalse(combined.use_cached_files(key, results_dir=results_dir))

            # The local artifact should not have been stored, and the results_dir should exist,
            # but be empty.
            self.assertFalse(local.has(key))
            self.assertTrue(os.path.exists(results_dir))
            self.assertTrue(len(os.listdir(results_dir)) == 0)

  def test_multiproc(self):
    key = CacheKey('muppet_key', 'fake_hash')

    with self.setup_local_cache() as cache:
      self.assertEquals(map(call_use_cached_files, [(cache, key, None)]), [False])
      with self.setup_test_file(cache.artifact_root) as path:
        map(call_insert, [(cache, key, [path], False)])
      self.assertEquals(map(call_use_cached_files, [(cache, key, None)]), [True])

    with self.setup_rest_cache() as cache:
      self.assertEquals(map(call_use_cached_files, [(cache, key, None)]), [False])
      with self.setup_test_file(cache.artifact_root) as path:
        map(call_insert, [(cache, key, [path], False)])
      self.assertEquals(map(call_use_cached_files, [(cache, key, None)]), [True])

  def test_failed_multiproc(self):
    key = CacheKey('muppet_key', 'fake_hash')

    # Failed requests should return failure status, but not raise exceptions
    with self.setup_rest_cache(return_failed=True) as cache:
      self.assertFalse(map(call_use_cached_files, [(cache, key, None)])[0])
      with self.setup_test_file(cache.artifact_root) as path:
        map(call_insert, [(cache, key, [path], False)])
      self.assertFalse(map(call_use_cached_files, [(cache, key, None)])[0])

  def test_successful_request_cleans_result_dir(self):
    key = CacheKey('muppet_key', 'fake_hash')

    with self.setup_local_cache() as cache:
      self._do_test_successful_request_cleans_result_dir(cache, key)

    with self.setup_rest_cache() as cache:
      self._do_test_successful_request_cleans_result_dir(cache, key)

  def _do_test_successful_request_cleans_result_dir(self, cache, key):
    with self.setup_test_file(cache.artifact_root) as path:
      with temporary_dir() as results_dir:
        with temporary_file_path(root_dir=results_dir) as canary:
          map(call_insert, [(cache, key, [path], False)])
          map(call_use_cached_files, [(cache, key, results_dir)])
          # Results content should have been deleted.
          self.assertFalse(os.path.exists(canary))

  def test_failed_request_doesnt_clean_result_dir(self):
    key = CacheKey('muppet_key', 'fake_hash')
    with temporary_dir() as results_dir:
      with temporary_file_path(root_dir=results_dir) as canary:
        with self.setup_local_cache() as cache:
          self.assertEquals(
            map(call_use_cached_files, [(cache, key, results_dir)]),
            [False])
          self.assertTrue(os.path.exists(canary))

        with self.setup_rest_cache() as cache:
          self.assertEquals(
            map(call_use_cached_files, [(cache, key, results_dir)]),
            [False])
          self.assertTrue(os.path.exists(canary))

  def test_corrupted_cached_file_cleaned_up(self):
    key = CacheKey('muppet_key', 'fake_hash')

    with self.setup_local_cache() as artifact_cache:
      with self.setup_test_file(artifact_cache.artifact_root) as path:
        artifact_cache.insert(key, [path])
        tarfile = artifact_cache._cache_file_for_key(key)

        self.assertTrue(artifact_cache.use_cached_files(key))
        self.assertTrue(os.path.exists(tarfile))

        with open(tarfile, 'w') as outfile:
          outfile.write(b'not a valid tgz any more')

        self.assertFalse(artifact_cache.use_cached_files(key))
        self.assertFalse(os.path.exists(tarfile))
<commit_msg>Use more generic portion of `requests` exception message in tests. (#4981)<commit_after># coding=utf-8
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                        unicode_literals, with_statement)

import os
import unittest
from contextlib import contextmanager

from pants.cache.artifact_cache import (NonfatalArtifactCacheError, call_insert,
                                        call_use_cached_files)
from pants.cache.local_artifact_cache import LocalArtifactCache, TempLocalArtifactCache
from pants.cache.pinger import BestUrlSelector, InvalidRESTfulCacheProtoError
from pants.cache.restful_artifact_cache import RESTfulArtifactCache
from pants.invalidation.build_invalidator import CacheKey
from pants.util.contextutil import temporary_dir, temporary_file, temporary_file_path
from pants.util.dirutil import safe_mkdir
from pants_test.cache.cache_server import cache_server

TEST_CONTENT1 = b'muppet'
TEST_CONTENT2 = b'kermit'

class TestArtifactCache(unittest.TestCase):
  @contextmanager
  def setup_local_cache(self):
    with temporary_dir() as artifact_root:
      with temporary_dir() as cache_root:
        yield LocalArtifactCache(artifact_root, cache_root, compression=1)

  @contextmanager
  def setup_server(self, return_failed=False, cache_root=None):
    with cache_server(return_failed=return_failed, cache_root=cache_root) as server:
      yield server

  @contextmanager
  def setup_rest_cache(self, local=None, return_failed=False):
    with temporary_dir() as artifact_root:
      local = local or TempLocalArtifactCache(artifact_root, 0)
      with self.setup_server(return_failed=return_failed) as server:
        yield RESTfulArtifactCache(artifact_root, BestUrlSelector([server.url]), local)

  @contextmanager
  def setup_test_file(self, parent):
    with temporary_file(parent) as f:
      # Write the file.
      f.write(TEST_CONTENT1)
      path = f.name
      f.close()
      yield path

  def test_local_cache(self):
    with self.setup_local_cache() as artifact_cache:
      self.do_test_artifact_cache(artifact_cache)

  def test_restful_cache(self):
    with self.assertRaises(InvalidRESTfulCacheProtoError):
      RESTfulArtifactCache('foo', BestUrlSelector(['ftp://localhost/bar']), 'foo')

    with self.setup_rest_cache() as artifact_cache:
      self.do_test_artifact_cache(artifact_cache)

  def test_restful_cache_failover(self):
    bad_url = 'http://badhost:123'

    with temporary_dir() as artifact_root:
      local = TempLocalArtifactCache(artifact_root, 0)

      # With fail-over, rest call second time will succeed
      with self.setup_server() as good_server:
        artifact_cache = RESTfulArtifactCache(artifact_root,
                                              BestUrlSelector([bad_url, good_server.url], max_failures=0),
                                              local)
        with self.assertRaises(NonfatalArtifactCacheError) as ex:
          self.do_test_artifact_cache(artifact_cache)
        self.assertIn('Failed to HEAD', str(ex.exception))

        self.do_test_artifact_cache(artifact_cache)

  def do_test_artifact_cache(self, artifact_cache):
    key = CacheKey('muppet_key', 'fake_hash')
    with self.setup_test_file(artifact_cache.artifact_root) as path:
      # Cache it.
      self.assertFalse(artifact_cache.has(key))
      self.assertFalse(bool(artifact_cache.use_cached_files(key)))
      artifact_cache.insert(key, [path])
      self.assertTrue(artifact_cache.has(key))

      # Stomp it.
      with open(path, 'w') as outfile:
        outfile.write(TEST_CONTENT2)

      # Recover it from the cache.
      self.assertTrue(bool(artifact_cache.use_cached_files(key)))

      # Check that it was recovered correctly.
      with open(path, 'r') as infile:
        content = infile.read()
      self.assertEquals(content, TEST_CONTENT1)

      # Delete it.
      artifact_cache.delete(key)
      self.assertFalse(artifact_cache.has(key))

  def test_local_backed_remote_cache(self):
    """make sure that the combined cache finds what it should and that it backfills"""
    with self.setup_server() as server:
      with self.setup_local_cache() as local:
        tmp = TempLocalArtifactCache(local.artifact_root, 0)
        remote = RESTfulArtifactCache(local.artifact_root, BestUrlSelector([server.url]), tmp)
        combined = RESTfulArtifactCache(local.artifact_root, BestUrlSelector([server.url]), local)

        key = CacheKey('muppet_key', 'fake_hash')

        with self.setup_test_file(local.artifact_root) as path:
          # No cache has key.
          self.assertFalse(local.has(key))
          self.assertFalse(remote.has(key))
          self.assertFalse(combined.has(key))

          # No cache returns key.
          self.assertFalse(bool(local.use_cached_files(key)))
          self.assertFalse(bool(remote.use_cached_files(key)))
          self.assertFalse(bool(combined.use_cached_files(key)))

          # Attempting to use key that no cache had should not change anything.
          self.assertFalse(local.has(key))
          self.assertFalse(remote.has(key))
          self.assertFalse(combined.has(key))

          # Add to only remote cache.
          remote.insert(key, [path])

          # After insertion to remote, remote and only remote should have key
          self.assertFalse(local.has(key))
          self.assertTrue(remote.has(key))
          self.assertTrue(combined.has(key))

          # Successfully using via remote should NOT change local.
          self.assertTrue(bool(remote.use_cached_files(key)))
          self.assertFalse(local.has(key))

          # Successfully using via combined SHOULD backfill local.
          self.assertTrue(bool(combined.use_cached_files(key)))
          self.assertTrue(local.has(key))
          self.assertTrue(bool(local.use_cached_files(key)))

  def test_local_backed_remote_cache_corrupt_artifact(self):
    """Ensure that a combined cache clears outputs after a failure to extract an artifact."""
    with temporary_dir() as remote_cache_dir:
      with self.setup_server(cache_root=remote_cache_dir) as server:
        with self.setup_local_cache() as local:
          tmp = TempLocalArtifactCache(local.artifact_root, compression=1)
          remote = RESTfulArtifactCache(local.artifact_root, BestUrlSelector([server.url]), tmp)
          combined = RESTfulArtifactCache(local.artifact_root, BestUrlSelector([server.url]), local)

          key = CacheKey('muppet_key', 'fake_hash')

          results_dir = os.path.join(local.artifact_root, 'a/sub/dir')
          safe_mkdir(results_dir)
          self.assertTrue(os.path.exists(results_dir))

          with self.setup_test_file(results_dir) as path:
            # Add to only the remote cache.
            remote.insert(key, [path])

            # Corrupt the artifact in the remote storage.
            self.assertTrue(server.corrupt_artifacts(r'.*muppet_key.*') == 1)

            # An attempt to read the corrupt artifact should fail.
            self.assertFalse(combined.use_cached_files(key, results_dir=results_dir))

            # The local artifact should not have been stored, and the results_dir should exist,
            # but be empty.
            self.assertFalse(local.has(key))
            self.assertTrue(os.path.exists(results_dir))
            self.assertTrue(len(os.listdir(results_dir)) == 0)

  def test_multiproc(self):
    key = CacheKey('muppet_key', 'fake_hash')

    with self.setup_local_cache() as cache:
      self.assertEquals(map(call_use_cached_files, [(cache, key, None)]), [False])
      with self.setup_test_file(cache.artifact_root) as path:
        map(call_insert, [(cache, key, [path], False)])
      self.assertEquals(map(call_use_cached_files, [(cache, key, None)]), [True])

    with self.setup_rest_cache() as cache:
      self.assertEquals(map(call_use_cached_files, [(cache, key, None)]), [False])
      with self.setup_test_file(cache.artifact_root) as path:
        map(call_insert, [(cache, key, [path], False)])
      self.assertEquals(map(call_use_cached_files, [(cache, key, None)]), [True])

  def test_failed_multiproc(self):
    key = CacheKey('muppet_key', 'fake_hash')

    # Failed requests should return failure status, but not raise exceptions
    with self.setup_rest_cache(return_failed=True) as cache:
      self.assertFalse(map(call_use_cached_files, [(cache, key, None)])[0])
      with self.setup_test_file(cache.artifact_root) as path:
        map(call_insert, [(cache, key, [path], False)])
      self.assertFalse(map(call_use_cached_files, [(cache, key, None)])[0])

  def test_successful_request_cleans_result_dir(self):
    key = CacheKey('muppet_key', 'fake_hash')

    with self.setup_local_cache() as cache:
      self._do_test_successful_request_cleans_result_dir(cache, key)

    with self.setup_rest_cache() as cache:
      self._do_test_successful_request_cleans_result_dir(cache, key)

  def _do_test_successful_request_cleans_result_dir(self, cache, key):
    with self.setup_test_file(cache.artifact_root) as path:
      with temporary_dir() as results_dir:
        with temporary_file_path(root_dir=results_dir) as canary:
          map(call_insert, [(cache, key, [path], False)])
          map(call_use_cached_files, [(cache, key, results_dir)])
          # Results content should have been deleted.
          self.assertFalse(os.path.exists(canary))

  def test_failed_request_doesnt_clean_result_dir(self):
    key = CacheKey('muppet_key', 'fake_hash')
    with temporary_dir() as results_dir:
      with temporary_file_path(root_dir=results_dir) as canary:
        with self.setup_local_cache() as cache:
          self.assertEquals(
            map(call_use_cached_files, [(cache, key, results_dir)]),
            [False])
          self.assertTrue(os.path.exists(canary))

        with self.setup_rest_cache() as cache:
          self.assertEquals(
            map(call_use_cached_files, [(cache, key, results_dir)]),
            [False])
          self.assertTrue(os.path.exists(canary))

  def test_corrupted_cached_file_cleaned_up(self):
    key = CacheKey('muppet_key', 'fake_hash')

    with self.setup_local_cache() as artifact_cache:
      with self.setup_test_file(artifact_cache.artifact_root) as path:
        artifact_cache.insert(key, [path])
        tarfile = artifact_cache._cache_file_for_key(key)

        self.assertTrue(artifact_cache.use_cached_files(key))
        self.assertTrue(os.path.exists(tarfile))

        with open(tarfile, 'w') as outfile:
          outfile.write(b'not a valid tgz any more')

        self.assertFalse(artifact_cache.use_cached_files(key))
        self.assertFalse(os.path.exists(tarfile))
<|endoftext|>


<commit_before>from __future__ import unicode_literals
import boto
from boto.ec2.autoscale.launchconfig import LaunchConfiguration
from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping

import sure  # noqa

from moto import mock_autoscaling
from tests.helpers import requires_boto_gte

@mock_autoscaling
def test_create_launch_configuration():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        instance_type='m1.small',
        key_name='the_keys',
        security_groups=["default", "default2"],
        user_data=b"This is some user_data",
        instance_monitoring=True,
        instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing',
        spot_price=0.1,
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.name.should.equal('tester')
    launch_config.image_id.should.equal('ami-abcd1234')
    launch_config.instance_type.should.equal('m1.small')
    launch_config.key_name.should.equal('the_keys')
    set(launch_config.security_groups).should.equal(set(['default', 'default2']))
    launch_config.user_data.should.equal(b"This is some user_data")
    launch_config.instance_monitoring.enabled.should.equal('true')
    launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing')
    launch_config.spot_price.should.equal(0.1)

@requires_boto_gte("2.27.0")
@mock_autoscaling
def test_create_launch_configuration_with_block_device_mappings():
    block_device_mapping = BlockDeviceMapping()

    ephemeral_drive = BlockDeviceType()
    ephemeral_drive.ephemeral_name = 'ephemeral0'
    block_device_mapping['/dev/xvdb'] = ephemeral_drive

    snapshot_drive = BlockDeviceType()
    snapshot_drive.snapshot_id = "snap-1234abcd"
    snapshot_drive.volume_type = "standard"
    block_device_mapping['/dev/xvdp'] = snapshot_drive

    ebs_drive = BlockDeviceType()
    ebs_drive.volume_type = "io1"
    ebs_drive.size = 100
    ebs_drive.iops = 1000
    ebs_drive.delete_on_termination = False
    block_device_mapping['/dev/xvdh'] = ebs_drive

    conn = boto.connect_autoscale(use_block_device_types=True)
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        instance_type='m1.small',
        key_name='the_keys',
        security_groups=["default", "default2"],
        user_data=b"This is some user_data",
        instance_monitoring=True,
        instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing',
        spot_price=0.1,
        block_device_mappings=[block_device_mapping]
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.name.should.equal('tester')
    launch_config.image_id.should.equal('ami-abcd1234')
    launch_config.instance_type.should.equal('m1.small')
    launch_config.key_name.should.equal('the_keys')
    set(launch_config.security_groups).should.equal(set(['default', 'default2']))
    launch_config.user_data.should.equal(b"This is some user_data")
    launch_config.instance_monitoring.enabled.should.equal('true')
    launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing')
    launch_config.spot_price.should.equal(0.1)
    len(launch_config.block_device_mappings).should.equal(3)

    returned_mapping = launch_config.block_device_mappings

    set(returned_mapping.keys()).should.equal(set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh']))

    returned_mapping['/dev/xvdh'].iops.should.equal(1000)
    returned_mapping['/dev/xvdh'].size.should.equal(100)
    returned_mapping['/dev/xvdh'].volume_type.should.equal("io1")
    returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false

    returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd")
    returned_mapping['/dev/xvdp'].volume_type.should.equal("standard")

    returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')

@requires_boto_gte("2.12")
@mock_autoscaling
def test_create_launch_configuration_for_2_12():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        ebs_optimized=True,
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.ebs_optimized.should.equal(True)

@requires_boto_gte("2.25.0")
@mock_autoscaling
def test_create_launch_configuration_using_ip_association():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        associate_public_ip_address=True,
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.associate_public_ip_address.should.equal(True)

@requires_boto_gte("2.25.0")
@mock_autoscaling
def test_create_launch_configuration_using_ip_association_should_default_to_false():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.associate_public_ip_address.should.equal(False)

@mock_autoscaling
def test_create_launch_configuration_defaults():
    """ Test with the minimum inputs and check that all of the proper defaults
    are assigned for the other attributes """
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        instance_type='m1.small',
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.name.should.equal('tester')
    launch_config.image_id.should.equal('ami-abcd1234')
    launch_config.instance_type.should.equal('m1.small')

    # Defaults
    launch_config.key_name.should.equal('')
    list(launch_config.security_groups).should.equal([])
    launch_config.user_data.should.equal(b"")
    launch_config.instance_monitoring.enabled.should.equal('false')
    launch_config.instance_profile_name.should.equal(None)
    launch_config.spot_price.should.equal(None)
    launch_config.ebs_optimized.should.equal(False)

@requires_boto_gte("2.12")
@mock_autoscaling
def test_create_launch_configuration_defaults_for_2_12():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.ebs_optimized.should.equal(False)

@mock_autoscaling
def test_launch_configuration_describe_filter():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        instance_type='m1.small',
    )
    conn.create_launch_configuration(config)
    config.name = 'tester2'
    conn.create_launch_configuration(config)
    config.name = 'tester3'
    conn.create_launch_configuration(config)

    conn.get_all_launch_configurations(names=['tester', 'tester2']).should.have.length_of(2)
    conn.get_all_launch_configurations().should.have.length_of(3)

@mock_autoscaling
def test_launch_configuration_delete():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        instance_type='m1.small',
    )
    conn.create_launch_configuration(config)

    conn.get_all_launch_configurations().should.have.length_of(1)

    conn.delete_launch_configuration('tester')
    conn.get_all_launch_configurations().should.have.length_of(0)
<commit_msg>Tests: Tweaked a general test to be compatible with Boto 2.7, attribute check is redundant with a specific version-gated test.<commit_after>from __future__ import unicode_literals
import boto
from boto.ec2.autoscale.launchconfig import LaunchConfiguration
from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping

import sure  # noqa

from moto import mock_autoscaling
from tests.helpers import requires_boto_gte

@mock_autoscaling
def test_create_launch_configuration():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        instance_type='m1.small',
        key_name='the_keys',
        security_groups=["default", "default2"],
        user_data=b"This is some user_data",
        instance_monitoring=True,
        instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing',
        spot_price=0.1,
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.name.should.equal('tester')
    launch_config.image_id.should.equal('ami-abcd1234')
    launch_config.instance_type.should.equal('m1.small')
    launch_config.key_name.should.equal('the_keys')
    set(launch_config.security_groups).should.equal(set(['default', 'default2']))
    launch_config.user_data.should.equal(b"This is some user_data")
    launch_config.instance_monitoring.enabled.should.equal('true')
    launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing')
    launch_config.spot_price.should.equal(0.1)

@requires_boto_gte("2.27.0")
@mock_autoscaling
def test_create_launch_configuration_with_block_device_mappings():
    block_device_mapping = BlockDeviceMapping()

    ephemeral_drive = BlockDeviceType()
    ephemeral_drive.ephemeral_name = 'ephemeral0'
    block_device_mapping['/dev/xvdb'] = ephemeral_drive

    snapshot_drive = BlockDeviceType()
    snapshot_drive.snapshot_id = "snap-1234abcd"
    snapshot_drive.volume_type = "standard"
    block_device_mapping['/dev/xvdp'] = snapshot_drive

    ebs_drive = BlockDeviceType()
    ebs_drive.volume_type = "io1"
    ebs_drive.size = 100
    ebs_drive.iops = 1000
    ebs_drive.delete_on_termination = False
    block_device_mapping['/dev/xvdh'] = ebs_drive

    conn = boto.connect_autoscale(use_block_device_types=True)
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        instance_type='m1.small',
        key_name='the_keys',
        security_groups=["default", "default2"],
        user_data=b"This is some user_data",
        instance_monitoring=True,
        instance_profile_name='arn:aws:iam::123456789012:instance-profile/testing',
        spot_price=0.1,
        block_device_mappings=[block_device_mapping]
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.name.should.equal('tester')
    launch_config.image_id.should.equal('ami-abcd1234')
    launch_config.instance_type.should.equal('m1.small')
    launch_config.key_name.should.equal('the_keys')
    set(launch_config.security_groups).should.equal(set(['default', 'default2']))
    launch_config.user_data.should.equal(b"This is some user_data")
    launch_config.instance_monitoring.enabled.should.equal('true')
    launch_config.instance_profile_name.should.equal('arn:aws:iam::123456789012:instance-profile/testing')
    launch_config.spot_price.should.equal(0.1)
    len(launch_config.block_device_mappings).should.equal(3)

    returned_mapping = launch_config.block_device_mappings

    set(returned_mapping.keys()).should.equal(set(['/dev/xvdb', '/dev/xvdp', '/dev/xvdh']))

    returned_mapping['/dev/xvdh'].iops.should.equal(1000)
    returned_mapping['/dev/xvdh'].size.should.equal(100)
    returned_mapping['/dev/xvdh'].volume_type.should.equal("io1")
    returned_mapping['/dev/xvdh'].delete_on_termination.should.be.false

    returned_mapping['/dev/xvdp'].snapshot_id.should.equal("snap-1234abcd")
    returned_mapping['/dev/xvdp'].volume_type.should.equal("standard")

    returned_mapping['/dev/xvdb'].ephemeral_name.should.equal('ephemeral0')

@requires_boto_gte("2.12")
@mock_autoscaling
def test_create_launch_configuration_for_2_12():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        ebs_optimized=True,
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.ebs_optimized.should.equal(True)

@requires_boto_gte("2.25.0")
@mock_autoscaling
def test_create_launch_configuration_using_ip_association():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        associate_public_ip_address=True,
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.associate_public_ip_address.should.equal(True)

@requires_boto_gte("2.25.0")
@mock_autoscaling
def test_create_launch_configuration_using_ip_association_should_default_to_false():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.associate_public_ip_address.should.equal(False)

@mock_autoscaling
def test_create_launch_configuration_defaults():
    """ Test with the minimum inputs and check that all of the proper defaults
    are assigned for the other attributes """
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        instance_type='m1.small',
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.name.should.equal('tester')
    launch_config.image_id.should.equal('ami-abcd1234')
    launch_config.instance_type.should.equal('m1.small')

    # Defaults
    launch_config.key_name.should.equal('')
    list(launch_config.security_groups).should.equal([])
    launch_config.user_data.should.equal(b"")
    launch_config.instance_monitoring.enabled.should.equal('false')
    launch_config.instance_profile_name.should.equal(None)
    launch_config.spot_price.should.equal(None)

@requires_boto_gte("2.12")
@mock_autoscaling
def test_create_launch_configuration_defaults_for_2_12():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
    )
    conn.create_launch_configuration(config)

    launch_config = conn.get_all_launch_configurations()[0]
    launch_config.ebs_optimized.should.equal(False)

@mock_autoscaling
def test_launch_configuration_describe_filter():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        instance_type='m1.small',
    )
    conn.create_launch_configuration(config)
    config.name = 'tester2'
    conn.create_launch_configuration(config)
    config.name = 'tester3'
    conn.create_launch_configuration(config)

    conn.get_all_launch_configurations(names=['tester', 'tester2']).should.have.length_of(2)
    conn.get_all_launch_configurations().should.have.length_of(3)

@mock_autoscaling
def test_launch_configuration_delete():
    conn = boto.connect_autoscale()
    config = LaunchConfiguration(
        name='tester',
        image_id='ami-abcd1234',
        instance_type='m1.small',
    )
    conn.create_launch_configuration(config)

    conn.get_all_launch_configurations().should.have.length_of(1)

    conn.delete_launch_configuration('tester')
    conn.get_all_launch_configurations().should.have.length_of(0)
<|endoftext|>


<commit_before>import bredis
import utils
from config import creator

def creator_only(func):
    def isCreator(bot, update, *args, **kwargs):
        if update.message.from_user.id in creator:
            func(bot, update, *args, **kwargs)
        else:
            update.message.reply_text('who are you? my lordy told me never to talk to strangers... *runs away*', quote=False)
    return isCreator

###Creator Only
@creator_only
def add(bot, update, args):
    bredis.superadmin.add(args)
    chat_id = update.message.chat.id
    user_id = args[0]
    try:
        member = bot.get_chat_member(chat_id, user_id)
        name = member.user.first_name
        msg = "*shyly peeks out from behind lord* hi {} youre no longer a stranger"
    except:
        name = args[0]
        msg = "{} is now a superadmin"
    update.message.reply_text(msg.format(name), quote=False)

@creator_only
def adminsend(bot, update, args):
    for uid in bredis.superadmin.get():
        bot.sendMessage(uid, ' '.join(args).format(u=update.message.from_user.id))

def sAdmin_only(func):
    def isAdmin(bot, update, *args, **kwargs):
        adminId = update.message.from_user.id
        isAdmin = bredis.superadmin.check(adminId)
        args = []
        if isAdmin is True:
            func(bot, update, *args, **kwargs)
        else:
            update.message.reply_text('who are you? my lordy told me never to talk to strangers... *runs away*', quote=False)
    return isAdmin

###SuperAdmin Only
@sAdmin_only
def block_user(bot, update, args):
    blockedId = args[0]
    bredis.blocked.add(str(blockedId))
    update.message.reply_text('`{}` has been blocked from setting their own welcome'.format(blockedId), quote=False, parse_mode='Markdown')

@sAdmin_only
def list_blocked_users(bot, update):
    blockedUsers = bredis.blocked.get()
    resp = []
    for i  in blockedUsers:
        resp.append(str(i))
    update.message.reply_text('\n'.join(resp), quote=False, parse_mode='Markdown')

@sAdmin_only
def unblock_user(bot, update, args):
    blockedId = args[0]
    bredis.blocked.rem(str(blockedId))
    update.message.reply_text('`{}` has been unblocked'.format(blockedId), quote=False, parse_mode='Markdown')

@sAdmin_only
def send(bot, update, args):
    gid = args[0]
    del args[0]
    bot.sendMessage(gid, msg, parse_mode='Markdown')

<commit_msg>Fixed an issue with /bsend"<commit_after>import bredis
import utils
from config import creator

def creator_only(func):
    def isCreator(bot, update, *args, **kwargs):
        if update.message.from_user.id in creator:
            func(bot, update, *args, **kwargs)
        else:
            update.message.reply_text('who are you? my lordy told me never to talk to strangers... *runs away*', quote=False)
    return isCreator

###Creator Only
@creator_only
def add(bot, update, args):
    bredis.superadmin.add(args)
    chat_id = update.message.chat.id
    user_id = args[0]
    try:
        member = bot.get_chat_member(chat_id, user_id)
        name = member.user.first_name
        msg = "*shyly peeks out from behind lord* hi {} youre no longer a stranger"
    except:
        name = args[0]
        msg = "{} is now a superadmin"
    update.message.reply_text(msg.format(name), quote=False)

@creator_only
def adminsend(bot, update, args):
    for uid in bredis.superadmin.get():
        bot.sendMessage(uid, ' '.join(args).format(u=update.message.from_user.id))

def sAdmin_only(func):
    def isAdmin(bot, update, *args, **kwargs):
        adminId = update.message.from_user.id
        isAdmin = bredis.superadmin.check(adminId)
        args = []
        if isAdmin is True:
            func(bot, update, *args, **kwargs)
        else:
            update.message.reply_text('who are you? my lordy told me never to talk to strangers... *runs away*', quote=False)
    return isAdmin

###SuperAdmin Only
@sAdmin_only
def block_user(bot, update, args):
    blockedId = args[0]
    bredis.blocked.add(str(blockedId))
    update.message.reply_text('`{}` has been blocked from setting their own welcome'.format(blockedId), quote=False, parse_mode='Markdown')

@sAdmin_only
def list_blocked_users(bot, update):
    blockedUsers = bredis.blocked.get()
    resp = []
    for i  in blockedUsers:
        resp.append(str(i))
    update.message.reply_text('\n'.join(resp), quote=False, parse_mode='Markdown')

@sAdmin_only
def unblock_user(bot, update, args):
    blockedId = args[0]
    bredis.blocked.rem(str(blockedId))
    update.message.reply_text('`{}` has been unblocked'.format(blockedId), quote=False, parse_mode='Markdown')

@sAdmin_only
def send(bot, update, args):
    gid = args[0]
    del args[0]
    msg = ' '.join(args)
    bot.sendMessage(gid, msg, parse_mode='Markdown')

<|endoftext|>


<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Unit tests for the clamodel module."""

import unittest2 as unittest

from nupic.frameworks.opf.clamodel import CLAModel

class CLAModelTest(unittest.TestCase):
  """CLAModel unit tests."""

  def testRemoveUnlikelyPredictionsEmpty(self):
    result = CLAModel._removeUnlikelyPredictions({}, 0.01, 3)
    self.assertDictEqual(result, {})

  def testRemoveUnlikelyPredictionsSingleValues(self):
    result = CLAModel._removeUnlikelyPredictions({1: 0.1}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.1})
    result = CLAModel._removeUnlikelyPredictions({1: 0.001}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.001})

  def testRemoveUnlikelyPredictionsLikelihoodThresholds(self):
    result = CLAModel._removeUnlikelyPredictions({1: 0.1, 2: 0.001}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.1})
    result = CLAModel._removeUnlikelyPredictions({1: 0.001, 2: 0.002}, 0.01, 3)
    self.assertDictEqual(result, {2: 0.002})
    result = CLAModel._removeUnlikelyPredictions({1: 0.002, 2: 0.001}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.002})

  def testRemoveUnlikelyPredictionsMaxPredictions(self):
    result = CLAModel._removeUnlikelyPredictions({1: 0.1, 2: 0.2, 3: 0.3},
                                                 0.01, 3)
    self.assertDictEqual(result, {1: 0.1, 2: 0.2, 3: 0.3})
    result = CLAModel._removeUnlikelyPredictions(
        {1: 0.1, 2: 0.2, 3: 0.3, 4: 0.4}, 0.01, 3)
    self.assertDictEqual(result, {2: 0.2, 3: 0.3, 4: 0.4})

  def testRemoveUnlikelyPredictionsComplex(self):
    result = CLAModel._removeUnlikelyPredictions(
        {1: 0.1, 2: 0.2, 3: 0.3, 4: 0.004}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.1, 2: 0.2, 3: 0.3})
    result = CLAModel._removeUnlikelyPredictions(
        {1: 0.1, 2: 0.2, 3: 0.3, 4: 0.4, 5: 0.005}, 0.01, 3)
    self.assertDictEqual(result, {2: 0.2, 3: 0.3, 4: 0.4})
    result = CLAModel._removeUnlikelyPredictions(
        {1: 0.1, 2: 0.2, 3: 0.3, 4: 0.004, 5: 0.005}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.1, 2: 0.2, 3: 0.3})

if __name__ == "__main__":
  unittest.main()
<commit_msg>add test for CLAModel() init without any parameters<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2013, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Unit tests for the clamodel module."""

import unittest2 as unittest

from nupic.frameworks.opf.clamodel import CLAModel

class CLAModelTest(unittest.TestCase):
  """CLAModel unit tests."""

  def testRemoveUnlikelyPredictionsEmpty(self):
    result = CLAModel._removeUnlikelyPredictions({}, 0.01, 3)
    self.assertDictEqual(result, {})

  def testRemoveUnlikelyPredictionsSingleValues(self):
    result = CLAModel._removeUnlikelyPredictions({1: 0.1}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.1})
    result = CLAModel._removeUnlikelyPredictions({1: 0.001}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.001})

  def testRemoveUnlikelyPredictionsLikelihoodThresholds(self):
    result = CLAModel._removeUnlikelyPredictions({1: 0.1, 2: 0.001}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.1})
    result = CLAModel._removeUnlikelyPredictions({1: 0.001, 2: 0.002}, 0.01, 3)
    self.assertDictEqual(result, {2: 0.002})
    result = CLAModel._removeUnlikelyPredictions({1: 0.002, 2: 0.001}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.002})

  def testRemoveUnlikelyPredictionsMaxPredictions(self):
    result = CLAModel._removeUnlikelyPredictions({1: 0.1, 2: 0.2, 3: 0.3},
                                                 0.01, 3)
    self.assertDictEqual(result, {1: 0.1, 2: 0.2, 3: 0.3})
    result = CLAModel._removeUnlikelyPredictions(
        {1: 0.1, 2: 0.2, 3: 0.3, 4: 0.4}, 0.01, 3)
    self.assertDictEqual(result, {2: 0.2, 3: 0.3, 4: 0.4})

  def testRemoveUnlikelyPredictionsComplex(self):
    result = CLAModel._removeUnlikelyPredictions(
        {1: 0.1, 2: 0.2, 3: 0.3, 4: 0.004}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.1, 2: 0.2, 3: 0.3})
    result = CLAModel._removeUnlikelyPredictions(
        {1: 0.1, 2: 0.2, 3: 0.3, 4: 0.4, 5: 0.005}, 0.01, 3)
    self.assertDictEqual(result, {2: 0.2, 3: 0.3, 4: 0.4})
    result = CLAModel._removeUnlikelyPredictions(
        {1: 0.1, 2: 0.2, 3: 0.3, 4: 0.004, 5: 0.005}, 0.01, 3)
    self.assertDictEqual(result, {1: 0.1, 2: 0.2, 3: 0.3})

  def testInitBasicCLAModel(self):
    self.assertTrue(CLAModel(), CLAModel)

if __name__ == "__main__":
  unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env conda-execute

# conda execute
# env:
#  - python
#  - conda-smithy >=1.1
#  - gitpython
#  - pygithub
# channels:
#  - conda-forge
# run_with: python

import argparse
from contextlib import contextmanager
import os
import random
import re
import textwrap
import time

import git
import github

import conda_smithy.github
import conda_smithy.feedstocks as feedstocks

pinned = {
          'boost': 'boost 1.61.*',
          'bzip2': 'bzip2 1.0.*',
          'ffmpeg': 'ffmpeg 2.8.*',
          'fontconfig': 'fontconfig 2.11.*',
          'freetype': 'freetype 2.6.*',
          'geos': 'geos 3.4.*',
          'giflib': 'giflib 5.1.*',
          'hdf5': 'hdf5 1.8.17|1.8.17.*',
          'icu': 'icu 56.*',
          'jpeg': 'jpeg 9*',
          'libblitz': 'libblitz 0.10|0.10.*',
          'libmatio': 'libmatio 1.5.*',
          'libnetcdf': 'libnetcdf 4.4.*',
          'libpng': 'libpng >=1.6.21,<1.7',
          'libsvm': 'libsvm 3.21|3.21.*',
          'libtiff': 'libtiff 4.0.*',
          'ncurses': 'ncurses 5.9*',
          'openblas': 'openblas 0.2.18|0.2.18.*',
          'openssl': 'openssl 1.0.*',
          'proj.4': 'proj.4 4.9.3',
          'readline': 'readline 6.2*',
          'sox': 'sox 14.4.2',
          'sqlite': 'sqlite 3.13.*',
          'tk': 'tk 8.5.*',
          'vlfeat': 'vlfeat 0.9.20',
          'xz': 'xz 5.2.*',
          'zlib': 'zlib 1.2.*',
        }

parser = argparse.ArgumentParser(description='Propose a feedstock update.')
parser.add_argument('--feedstocks-dir', help="The location of the feedstocks.",
                    default="~/dev/conda-forge/feedstocks")
regexp_or_package = parser.add_mutually_exclusive_group()
regexp_or_package.add_argument('--regexp', help="Regexp of feedstocks to consider.")
regexp_or_package.add_argument('--package', help="A specific package to check.")
parser.add_argument('--local', help="Whether to use the existing local checkouts of the feedstocks, or to fetch from upstream", action="store_true", default=False)
parser.add_argument('--limit', help="Limit the number of packages to propose changes for (0 is unlimited).",
                    default=1, type=int)
args = parser.parse_args()

feedstocks_dir = os.path.expanduser(args.feedstocks_dir)
change_limit = args.limit

gh_token = conda_smithy.github.gh_token()
gh = github.Github(gh_token)

gh_me = gh.get_user()

if gh_me.login != 'conda-forge-admin':
    raise ValueError("The github token isn't that of conda-forge-admin (it's "
                     "for {}), I'm going to have to bail.".format(gh_me.login))

gh_forge = gh.get_organization('conda-forge')

if args.package:
    package_name = args.package
    package_feedstock = '{}-feedstock'.format(package_name)
    args.regexp = package_name

need_to_clone = args.local == False
feedstock_gen = feedstocks.feedstocks_yaml('conda-forge', feedstocks_dir, use_local=args.local,
                                           randomise=True, pull_up_to_date=need_to_clone, regexp=args.regexp)

def my_repos(gh_user):
    """
    List all of my repos.
    See https://github.com/PyGithub/PyGithub/issues/390 for rationale.

    """
    return github.PaginatedList.PaginatedList(
                github.Repository.Repository,
                gh_user._requester,
                gh_user.url + "/repos",
                dict(affiliation="owner"))

def list_pulls(repo, state='open', head=None):
    """
    List all of the pull requests that match the given critera.

    At the time of writing, pygithub doesn't allow you to specify the head,
    so I had to create this function.

    """
    url_parameters = dict(state=state)
    if head:
        url_parameters['head'] = head
    return github.PaginatedList.PaginatedList(
        github.PullRequest.PullRequest,
        repo._requester,
        repo.url + "/pulls",
        url_parameters
    )

if args.package:
    try:
        my_repos = {package_feedstock: gh_me.get_repo(package_feedstock)}
    except github.UnknownObjectException:
        # We haven't forked it yet!
        my_repos = {}
    forge_repos = {package_feedstock: gh_forge.get_repo(package_feedstock)}
else:
    print("Collecting list of conda-forge-admin repos...")
    my_repos = {repo.name: repo for repo in my_repos(gh_me)}
    print("Collecting list of conda-forge repos...")
    forge_repos = {repo.name: repo for repo in gh_forge.get_repos()}

@contextmanager
def tmp_remote(repo, remote_name, url):
    if remote_name in [remote.name for remote in repo.remotes]:
        repo.delete_remote(remote_name)
    remote = repo.create_remote(remote_name, url)
    yield remote
    repo.delete_remote(remote_name)

@contextmanager
def create_update_pr(clone, remote_head, fork_remote, upstream_remote):
    target_branch = 'feedstock_version_pin_{}'.format(remote_head)
    if target_branch in clone.heads:
        # Detatch the head
        clone.head.reference = clone.commit('upstream/master')
        clone.delete_head(target_branch, '-D')
    clone.create_head(target_branch, upstream_remote.refs[remote_head]).set_tracking_branch(upstream_remote.refs[remote_head])

    # Reset the working tree to a clean state.
    clone.head.reset(index=True, working_tree=True)
    clone.heads[target_branch].checkout()

    # It is at this point we pass context back to the caller so that they can
    # do whatever they like to the repo (like rerender the feedstock).
    context = []
    yield context

    # If nothing was done, don't need a PR!
    has_change = True
    if not clone.is_dirty():
        # We don't need this feedstock - it is slap-bang up to date. :)
        print("{} was checked, and is up-to-date".format(feedstock.name))
        has_change = False

    if has_change:
        clone.git.add('-A')
        author = git.Actor(gh_me.login, gh_me.email)
        commit = clone.index.commit("MNT: Updated some of the pinned versions",
                                    author=author)

        change_from_remote_branch = True
        full_ref = '{}/{}'.format(fork_remote.name, target_branch)

        if full_ref in [ref.name for ref in fork_remote.refs]:
            diff = commit.diff(fork_remote.refs[target_branch])
            if not diff:
                # There were no differences between this and the remote targt branch, so just continue.
                print("{} was checked, and whilst there are changes needed, the branch ({}) is up-to-date".format(feedstock.name, target_branch))
                change_from_remote_branch = False

        fork_remote.push('+{}'.format(target_branch))

        if change_from_remote_branch:
            fork_remote.push('+{}'.format(target_branch))

            pull_requests = list(list_pulls(forge_feedstock, state='open', head='{}:{}'.format(gh_me.login, target_branch)))

            if pull_requests:
                pull = pull_requests[0]
                msg = textwrap.dedent("""
    It's the friendly automated conda-forge-admin here again.

    Just to let you know, I've updated this PR so that it has the latest pinned versions.

    If there are no problems with it, please consider merging this PR.
    If there are concerns about it, please ping the 'conda-forge/core' team (using the @ notation in a comment).

    Thanks!
                       """)
                pull.create_issue_comment(msg)
                print('Updated PR on {}'.format(pull.html_url))
            else:
                msg = textwrap.dedent("""
    Hi! This is the friendly conda-forge-admin automated user.

    I've bumped some of the conda-forge pinned versions, and noticed that it impacts this feedstock.
    If the changes look good, then please go ahead and merge this PR.
    If you have any questions about the changes though, please feel free to ping the 'conda-forge/core' team (using the @ notation in a comment).

    Thanks!

                """)

                pull = forge_feedstock.create_pull(title='MNT: Update pinned versions.',
                                                   body=msg,
                                                   head="{}:{}".format(gh_me.login, target_branch), base=remote_head)
                print('Opened PR on {}'.format(pull.html_url))
            context.append(pull)

count = 0
for feedstock, git_ref, meta_content, recipe in feedstock_gen:
    if feedstock.name not in forge_repos:
        raise ValueError("There exists a feedstock ({}) which isn't in the "
                         "conda-forge org.".format(feedstock.name))

    if feedstock.name not in my_repos:
        forge_repo = gh_forge.get_repo(feedstock.name)
        print('Forking {}'.format(feedstock.name))
        my_repos[feedstock.name] = gh_me.create_fork(forge_repo)

    clone = git.Repo(feedstock.directory)
    admin_fork = my_repos[feedstock.name]
    forge_feedstock = forge_repos[feedstock.name]

    skip_after_package = False

    # Put an appropriate conda-forge-admin remote in place.
    with tmp_remote(clone, gh_me.login,
                    admin_fork.clone_url.replace('https://',
                                                 'https://{}@'.format(gh_token))) as remote:
        try:
            remote.fetch()
        except git.exc.GitCommandError as ex:
            # Periodically this fetch fails with a "git remote error: access denied
            # or repository not exported". Presumably this is because we have only just
            # forked it and GitHub is taking time to catch up, so we wait a few seconds
            # and try again.
            print("Sleeping for 10s before retrying due to the following error: \n", str(ex))
            time.sleep(10)
            remote.fetch()

        remote_branch = git_ref.remote_head.replace('{}/'.format(gh_me.login), '')
        with create_update_pr(clone, remote_branch, remote, clone.remotes['upstream']) as pr:
            replacements = {}
            for section_name in ['run', 'build']:
                requirements = recipe.get('requirements')
                if requirements is None:
                    break
                section = requirements.get(section_name)
                if not section:
                    continue

                for pos, dep in enumerate(section):
                    for name, pin in pinned.items():
                        if dep.startswith(name) and dep != pin:
                            replacements['- ' + str(dep)] = '- ' + pin
            if replacements:
                current_build_number = recipe['build']['number']
                replacements['number: {}'.format(current_build_number)] = 'number: {}'.format(current_build_number + 1)
            content = meta_content
            for orig, new in replacements.items():
                content = content.replace(orig, new)
            forge_yaml = os.path.join(feedstock.directory, 'recipe', 'meta.yaml')
            with open(forge_yaml, 'w') as fh:
                fh.write(content)
        if pr:
            skip_after_package = True
    # Stop processing any more feedstocks until the next time the script is run.
    if skip_after_package:
        count += 1

    if change_limit > 0 and count >= change_limit:
        break
<commit_msg>Pin qt/pyqt<commit_after>#!/usr/bin/env conda-execute

# conda execute
# env:
#  - python
#  - conda-smithy >=1.1
#  - gitpython
#  - pygithub
# channels:
#  - conda-forge
# run_with: python

import argparse
from contextlib import contextmanager
import os
import random
import re
import textwrap
import time

import git
import github

import conda_smithy.github
import conda_smithy.feedstocks as feedstocks

pinned = {
          'boost': 'boost 1.61.*',
          'bzip2': 'bzip2 1.0.*',
          'ffmpeg': 'ffmpeg 2.8.*',
          'fontconfig': 'fontconfig 2.11.*',
          'freetype': 'freetype 2.6.*',
          'geos': 'geos 3.4.*',
          'giflib': 'giflib 5.1.*',
          'hdf5': 'hdf5 1.8.17|1.8.17.*',
          'icu': 'icu 56.*',
          'jpeg': 'jpeg 9*',
          'libblitz': 'libblitz 0.10|0.10.*',
          'libmatio': 'libmatio 1.5.*',
          'libnetcdf': 'libnetcdf 4.4.*',
          'libpng': 'libpng >=1.6.21,<1.7',
          'libsvm': 'libsvm 3.21|3.21.*',
          'libtiff': 'libtiff 4.0.*',
          'ncurses': 'ncurses 5.9*',
          'openblas': 'openblas 0.2.18|0.2.18.*',
          'openssl': 'openssl 1.0.*',
          'proj.4': 'proj.4 4.9.3',
          'pyqt': 'pyqt 4.11.*',
          'qt': 'qt 4.8.*',
          'readline': 'readline 6.2*',
          'sox': 'sox 14.4.2',
          'sqlite': 'sqlite 3.13.*',
          'tk': 'tk 8.5.*',
          'vlfeat': 'vlfeat 0.9.20',
          'xz': 'xz 5.2.*',
          'zlib': 'zlib 1.2.*',
        }

parser = argparse.ArgumentParser(description='Propose a feedstock update.')
parser.add_argument('--feedstocks-dir', help="The location of the feedstocks.",
                    default="~/dev/conda-forge/feedstocks")
regexp_or_package = parser.add_mutually_exclusive_group()
regexp_or_package.add_argument('--regexp', help="Regexp of feedstocks to consider.")
regexp_or_package.add_argument('--package', help="A specific package to check.")
parser.add_argument('--local', help="Whether to use the existing local checkouts of the feedstocks, or to fetch from upstream", action="store_true", default=False)
parser.add_argument('--limit', help="Limit the number of packages to propose changes for (0 is unlimited).",
                    default=1, type=int)
args = parser.parse_args()

feedstocks_dir = os.path.expanduser(args.feedstocks_dir)
change_limit = args.limit

gh_token = conda_smithy.github.gh_token()
gh = github.Github(gh_token)

gh_me = gh.get_user()

if gh_me.login != 'conda-forge-admin':
    raise ValueError("The github token isn't that of conda-forge-admin (it's "
                     "for {}), I'm going to have to bail.".format(gh_me.login))

gh_forge = gh.get_organization('conda-forge')

if args.package:
    package_name = args.package
    package_feedstock = '{}-feedstock'.format(package_name)
    args.regexp = package_name

need_to_clone = args.local == False
feedstock_gen = feedstocks.feedstocks_yaml('conda-forge', feedstocks_dir, use_local=args.local,
                                           randomise=True, pull_up_to_date=need_to_clone, regexp=args.regexp)

def my_repos(gh_user):
    """
    List all of my repos.
    See https://github.com/PyGithub/PyGithub/issues/390 for rationale.

    """
    return github.PaginatedList.PaginatedList(
                github.Repository.Repository,
                gh_user._requester,
                gh_user.url + "/repos",
                dict(affiliation="owner"))

def list_pulls(repo, state='open', head=None):
    """
    List all of the pull requests that match the given critera.

    At the time of writing, pygithub doesn't allow you to specify the head,
    so I had to create this function.

    """
    url_parameters = dict(state=state)
    if head:
        url_parameters['head'] = head
    return github.PaginatedList.PaginatedList(
        github.PullRequest.PullRequest,
        repo._requester,
        repo.url + "/pulls",
        url_parameters
    )

if args.package:
    try:
        my_repos = {package_feedstock: gh_me.get_repo(package_feedstock)}
    except github.UnknownObjectException:
        # We haven't forked it yet!
        my_repos = {}
    forge_repos = {package_feedstock: gh_forge.get_repo(package_feedstock)}
else:
    print("Collecting list of conda-forge-admin repos...")
    my_repos = {repo.name: repo for repo in my_repos(gh_me)}
    print("Collecting list of conda-forge repos...")
    forge_repos = {repo.name: repo for repo in gh_forge.get_repos()}

@contextmanager
def tmp_remote(repo, remote_name, url):
    if remote_name in [remote.name for remote in repo.remotes]:
        repo.delete_remote(remote_name)
    remote = repo.create_remote(remote_name, url)
    yield remote
    repo.delete_remote(remote_name)

@contextmanager
def create_update_pr(clone, remote_head, fork_remote, upstream_remote):
    target_branch = 'feedstock_version_pin_{}'.format(remote_head)
    if target_branch in clone.heads:
        # Detatch the head
        clone.head.reference = clone.commit('upstream/master')
        clone.delete_head(target_branch, '-D')
    clone.create_head(target_branch, upstream_remote.refs[remote_head]).set_tracking_branch(upstream_remote.refs[remote_head])

    # Reset the working tree to a clean state.
    clone.head.reset(index=True, working_tree=True)
    clone.heads[target_branch].checkout()

    # It is at this point we pass context back to the caller so that they can
    # do whatever they like to the repo (like rerender the feedstock).
    context = []
    yield context

    # If nothing was done, don't need a PR!
    has_change = True
    if not clone.is_dirty():
        # We don't need this feedstock - it is slap-bang up to date. :)
        print("{} was checked, and is up-to-date".format(feedstock.name))
        has_change = False

    if has_change:
        clone.git.add('-A')
        author = git.Actor(gh_me.login, gh_me.email)
        commit = clone.index.commit("MNT: Updated some of the pinned versions",
                                    author=author)

        change_from_remote_branch = True
        full_ref = '{}/{}'.format(fork_remote.name, target_branch)

        if full_ref in [ref.name for ref in fork_remote.refs]:
            diff = commit.diff(fork_remote.refs[target_branch])
            if not diff:
                # There were no differences between this and the remote targt branch, so just continue.
                print("{} was checked, and whilst there are changes needed, the branch ({}) is up-to-date".format(feedstock.name, target_branch))
                change_from_remote_branch = False

        fork_remote.push('+{}'.format(target_branch))

        if change_from_remote_branch:
            fork_remote.push('+{}'.format(target_branch))

            pull_requests = list(list_pulls(forge_feedstock, state='open', head='{}:{}'.format(gh_me.login, target_branch)))

            if pull_requests:
                pull = pull_requests[0]
                msg = textwrap.dedent("""
    It's the friendly automated conda-forge-admin here again.

    Just to let you know, I've updated this PR so that it has the latest pinned versions.

    If there are no problems with it, please consider merging this PR.
    If there are concerns about it, please ping the 'conda-forge/core' team (using the @ notation in a comment).

    Thanks!
                       """)
                pull.create_issue_comment(msg)
                print('Updated PR on {}'.format(pull.html_url))
            else:
                msg = textwrap.dedent("""
    Hi! This is the friendly conda-forge-admin automated user.

    I've bumped some of the conda-forge pinned versions, and noticed that it impacts this feedstock.
    If the changes look good, then please go ahead and merge this PR.
    If you have any questions about the changes though, please feel free to ping the 'conda-forge/core' team (using the @ notation in a comment).

    Thanks!

                """)

                pull = forge_feedstock.create_pull(title='MNT: Update pinned versions.',
                                                   body=msg,
                                                   head="{}:{}".format(gh_me.login, target_branch), base=remote_head)
                print('Opened PR on {}'.format(pull.html_url))
            context.append(pull)

count = 0
for feedstock, git_ref, meta_content, recipe in feedstock_gen:
    if feedstock.name not in forge_repos:
        raise ValueError("There exists a feedstock ({}) which isn't in the "
                         "conda-forge org.".format(feedstock.name))

    if feedstock.name not in my_repos:
        forge_repo = gh_forge.get_repo(feedstock.name)
        print('Forking {}'.format(feedstock.name))
        my_repos[feedstock.name] = gh_me.create_fork(forge_repo)

    clone = git.Repo(feedstock.directory)
    admin_fork = my_repos[feedstock.name]
    forge_feedstock = forge_repos[feedstock.name]

    skip_after_package = False

    # Put an appropriate conda-forge-admin remote in place.
    with tmp_remote(clone, gh_me.login,
                    admin_fork.clone_url.replace('https://',
                                                 'https://{}@'.format(gh_token))) as remote:
        try:
            remote.fetch()
        except git.exc.GitCommandError as ex:
            # Periodically this fetch fails with a "git remote error: access denied
            # or repository not exported". Presumably this is because we have only just
            # forked it and GitHub is taking time to catch up, so we wait a few seconds
            # and try again.
            print("Sleeping for 10s before retrying due to the following error: \n", str(ex))
            time.sleep(10)
            remote.fetch()

        remote_branch = git_ref.remote_head.replace('{}/'.format(gh_me.login), '')
        with create_update_pr(clone, remote_branch, remote, clone.remotes['upstream']) as pr:
            replacements = {}
            for section_name in ['run', 'build']:
                requirements = recipe.get('requirements')
                if requirements is None:
                    break
                section = requirements.get(section_name)
                if not section:
                    continue

                for pos, dep in enumerate(section):
                    for name, pin in pinned.items():
                        if dep.startswith(name) and dep != pin:
                            replacements['- ' + str(dep)] = '- ' + pin
            if replacements:
                current_build_number = recipe['build']['number']
                replacements['number: {}'.format(current_build_number)] = 'number: {}'.format(current_build_number + 1)
            content = meta_content
            for orig, new in replacements.items():
                content = content.replace(orig, new)
            forge_yaml = os.path.join(feedstock.directory, 'recipe', 'meta.yaml')
            with open(forge_yaml, 'w') as fh:
                fh.write(content)
        if pr:
            skip_after_package = True
    # Stop processing any more feedstocks until the next time the script is run.
    if skip_after_package:
        count += 1

    if change_limit > 0 and count >= change_limit:
        break
<|endoftext|>