<commit_before># Copyright 2019 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A set of standard TFX Artifact types.

Note: the artifact definitions here are expected to change.
"""

import decimal
import math

import absl
from tfx.types.artifact import Artifact
from tfx.types.artifact import Property
from tfx.types.artifact import PropertyType
from tfx.types.system_artifacts import Dataset
from tfx.types.system_artifacts import Model as SystemModel
from tfx.types.system_artifacts import Statistics
from tfx.types.value_artifact import ValueArtifact
from tfx.utils import json_utils

# Span for an artifact.
SPAN_PROPERTY = Property(type=PropertyType.INT)
# Version for an artifact.
VERSION_PROPERTY = Property(type=PropertyType.INT)
# Comma separated of splits for an artifact. Empty string means artifact
# has no split.
SPLIT_NAMES_PROPERTY = Property(type=PropertyType.STRING)
# Value for a string-typed artifact.
STRING_VALUE_PROPERTY = Property(type=PropertyType.STRING)

class _TfxArtifact(Artifact):
  """TFX first-party component artifact definition.

  Do not construct directly, used for creating Channel, e.g.,
  ```
    Channel(type=standard_artifacts.Model)
  ```
  """

  def __init__(self, *args, **kwargs):
    """Construct TFX first-party component artifact."""
    # TODO(b/176795331): Refactor directory structure to make it clearer that
    # TFX-specific artifacts require the full "tfx" package be installed.
    #
    # Do not allow usage of TFX-specific artifact if only the core pipeline
    # SDK package is installed.
    try:
      import setuptools as _  # pytype: disable=import-error  # pylint: disable=g-import-not-at-top
      # Test import only when setuptools is available.
      try:
        # `extensions` is not included in ml_pipelines_sdk and doesn't have any
        # transitive import.
        import tfx.extensions as _  # type: ignore  # pylint: disable=g-import-not-at-top
      except ModuleNotFoundError as err:
        # The following condition detects exactly whether only the DSL package
        # is installed, and is bypassed when tests run in Bazel.
        raise RuntimeError('The "tfx" and all dependent packages need to be '
                           'installed to use this functionality.') from err
    except ModuleNotFoundError:
      pass

    super().__init__(*args, **kwargs)

class Examples(_TfxArtifact):
  TYPE_NAME = 'Examples'
  TYPE_ANNOTATION = Dataset
  PROPERTIES = {
      'span': SPAN_PROPERTY,
      'version': VERSION_PROPERTY,
      'split_names': SPLIT_NAMES_PROPERTY,
  }

class ExampleAnomalies(_TfxArtifact):
  TYPE_NAME = 'ExampleAnomalies'
  PROPERTIES = {
      'span': SPAN_PROPERTY,
      'split_names': SPLIT_NAMES_PROPERTY,
  }

class ExampleStatistics(_TfxArtifact):
  TYPE_NAME = 'ExampleStatistics'
  TYPE_ANNOTATION = Statistics
  PROPERTIES = {
      'span': SPAN_PROPERTY,
      'split_names': SPLIT_NAMES_PROPERTY,
  }

# TODO(b/158334890): deprecate ExternalArtifact.
class ExternalArtifact(_TfxArtifact):
  TYPE_NAME = 'ExternalArtifact'

class InferenceResult(_TfxArtifact):
  TYPE_NAME = 'InferenceResult'

class InfraBlessing(_TfxArtifact):
  TYPE_NAME = 'InfraBlessing'

class Model(_TfxArtifact):
  TYPE_NAME = 'Model'
  TYPE_ANNOTATION = SystemModel

class ModelRun(_TfxArtifact):
  TYPE_NAME = 'ModelRun'

class ModelBlessing(_TfxArtifact):
  TYPE_NAME = 'ModelBlessing'

class ModelEvaluation(_TfxArtifact):
  TYPE_NAME = 'ModelEvaluation'

class PushedModel(_TfxArtifact):
  TYPE_NAME = 'PushedModel'
  TYPE_ANNOTATION = SystemModel

class Schema(_TfxArtifact):
  TYPE_NAME = 'Schema'

class TransformCache(_TfxArtifact):
  TYPE_NAME = 'TransformCache'

class JsonValue(ValueArtifact):
  """Artifacts representing a Jsonable value."""
  TYPE_NAME = 'JsonValue'

  def encode(self, value: json_utils.JsonableType) -> str:
    return json_utils.dumps(value)

  def decode(self, serialized_value: str) -> json_utils.JsonableType:
    return json_utils.loads(serialized_value)

class Bytes(ValueArtifact):
  """Artifacts representing raw bytes."""
  TYPE_NAME = 'Bytes'

  def encode(self, value: bytes):
    if not isinstance(value, bytes):
      raise TypeError('Expecting bytes but got value %s of type %s' %
                      (str(value), type(value)))
    return value

  def decode(self, serialized_value: bytes):
    return serialized_value

class String(ValueArtifact):
  """String-typed artifact."""
  TYPE_NAME = 'String'

  # Note, currently we enforce unicode-encoded string.
  def encode(self, value: str) -> bytes:
    if not isinstance(value, str):
      raise TypeError('Expecting Text but got value %s of type %s' %
                      (str(value), type(value)))
    return value.encode('utf-8')

  def decode(self, serialized_value: bytes) -> str:
    return serialized_value.decode('utf-8')

class Boolean(ValueArtifact):
  """Artifacts representing a boolean.

  Boolean value artifacts are encoded as "1" for True and "0" for False.
  """
  TYPE_NAME = 'Boolean'

  def encode(self, value: bool):
    if not isinstance(value, bool):
      raise TypeError('Expecting bytes but got value %s of type %s' %
                      (str(value), type(value)))
    return b'1' if value else b'0'

  def decode(self, serialized_value: bytes):
    return int(serialized_value) != 0

class Integer(ValueArtifact):
  """Integer-typed artifact."""
  TYPE_NAME = 'Integer'

  def encode(self, value: int) -> bytes:
    if not isinstance(value, int):
      raise TypeError('Expecting int but got value %s of type %s' %
                      (str(value), type(value)))
    return str(value).encode('utf-8')

  def decode(self, serialized_value: bytes) -> int:
    return int(serialized_value)

class Float(ValueArtifact):
  """Float-typed artifact."""
  TYPE_NAME = 'Float'

  _POSITIVE_INFINITY = float('Inf')
  _NEGATIVE_INFINITY = float('-Inf')

  _ENCODED_POSITIVE_INFINITY = 'Infinity'
  _ENCODED_NEGATIVE_INFINITY = '-Infinity'
  _ENCODED_NAN = 'NaN'

  def encode(self, value: float) -> bytes:
    if not isinstance(value, float):
      raise TypeError('Expecting float but got value %s of type %s' %
                      (str(value), type(value)))
    if math.isinf(value) or math.isnan(value):
      absl.logging.warning(
          '! The number "%s" may be unsupported by non-python components.' %
          value)
    str_value = str(value)
    # Special encoding for infinities and NaN to increase comatibility with
    # other languages.
    # Decoding works automatically.
    if math.isinf(value):
      if value >= 0:
        str_value = Float._ENCODED_POSITIVE_INFINITY
      else:
        str_value = Float._ENCODED_NEGATIVE_INFINITY
    if math.isnan(value):
      str_value = Float._ENCODED_NAN

    return str_value.encode('utf-8')

  def decode(self, serialized_value: bytes) -> float:
    result = float(serialized_value)

    # Check that the decoded value exactly matches the encoded string.
    # Note that float() can handle bytes, but Decimal() cannot.
    serialized_string = serialized_value.decode('utf-8')
    reserialized_string = str(result)
    is_exact = (decimal.Decimal(serialized_string) ==
                decimal.Decimal(reserialized_string))
    if not is_exact:
      absl.logging.warning(
          'The number "%s" has lost precision when converted to float "%s"' %
          (serialized_value, reserialized_string))

    return result

class TransformGraph(_TfxArtifact):
  TYPE_NAME = 'TransformGraph'

class HyperParameters(_TfxArtifact):
  TYPE_NAME = 'HyperParameters'

# WIP and subject to change.
class DataView(_TfxArtifact):
  TYPE_NAME = 'DataView'
<commit_msg>Explains Examples and Model artifact file structures.<commit_after># Copyright 2019 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A set of standard TFX Artifact types.

Note: the artifact definitions here are expected to change.
"""

import decimal
import math

import absl
from tfx.types.artifact import Artifact
from tfx.types.artifact import Property
from tfx.types.artifact import PropertyType
from tfx.types.system_artifacts import Dataset
from tfx.types.system_artifacts import Model as SystemModel
from tfx.types.system_artifacts import Statistics
from tfx.types.value_artifact import ValueArtifact
from tfx.utils import json_utils

SPAN_PROPERTY = Property(type=PropertyType.INT)
VERSION_PROPERTY = Property(type=PropertyType.INT)
SPLIT_NAMES_PROPERTY = Property(type=PropertyType.STRING)
# (DEPRECATED. Do not use.) Value for a string-typed artifact.
STRING_VALUE_PROPERTY = Property(type=PropertyType.STRING)

class _TfxArtifact(Artifact):
  """TFX first-party component artifact definition.

  Do not construct directly, used for creating Channel, e.g.,
  ```
    Channel(type=standard_artifacts.Model)
  ```
  """

  def __init__(self, *args, **kwargs):
    """Construct TFX first-party component artifact."""
    # TODO(b/176795331): Refactor directory structure to make it clearer that
    # TFX-specific artifacts require the full "tfx" package be installed.
    #
    # Do not allow usage of TFX-specific artifact if only the core pipeline
    # SDK package is installed.
    try:
      import setuptools as _  # pytype: disable=import-error  # pylint: disable=g-import-not-at-top
      # Test import only when setuptools is available.
      try:
        # `extensions` is not included in ml_pipelines_sdk and doesn't have any
        # transitive import.
        import tfx.extensions as _  # type: ignore  # pylint: disable=g-import-not-at-top
      except ModuleNotFoundError as err:
        # The following condition detects exactly whether only the DSL package
        # is installed, and is bypassed when tests run in Bazel.
        raise RuntimeError('The "tfx" and all dependent packages need to be '
                           'installed to use this functionality.') from err
    except ModuleNotFoundError:
      pass

    super().__init__(*args, **kwargs)

class Examples(_TfxArtifact):
  """Artifact that contains the training data.

  Training data should be brought in to the TFX pipeline using components
  like ExampleGen. Data in Examples artifact is split and stored separately.
  The file and payload format must be specified as optional custom properties
  if not using default formats.
  Please see
  https://www.tensorflow.org/tfx/guide/examplegen#span_version_and_split to
  understand about span, version and splits.

  * Properties:
     - `span`: Integer to distinguish group of Examples.
     - `version`: Integer to represent updated data.
     - `split_names`: JSON string of the list of split names. For example,
        '["train", "test"]'. Empty string means artifact has no split.

  * File structure:
     - `{uri}/`
        - `Split-{split_name1}/`: Files for split
           - All direct children files are recognized as the data.
           - File format and payload format are determined by custom properties.
        - `Split-{split_name2}/`: Another split...

  * Commonly used custom properties of the Examples artifact:
    - `file_format`: a string that represents the file format. See
      tfx/components/util/tfxio_utils.py:make_tfxio for
      available values.
    - `payload_format`: int (enum) value of the data payload format.
      See tfx/proto/example_gen.proto:PayloadFormat for available formats.
  """
  TYPE_NAME = 'Examples'
  TYPE_ANNOTATION = Dataset
  PROPERTIES = {
      'span': SPAN_PROPERTY,
      'version': VERSION_PROPERTY,
      'split_names': SPLIT_NAMES_PROPERTY,
  }

class ExampleAnomalies(_TfxArtifact):
  TYPE_NAME = 'ExampleAnomalies'
  PROPERTIES = {
      'span': SPAN_PROPERTY,
      'split_names': SPLIT_NAMES_PROPERTY,
  }

class ExampleStatistics(_TfxArtifact):
  TYPE_NAME = 'ExampleStatistics'
  TYPE_ANNOTATION = Statistics
  PROPERTIES = {
      'span': SPAN_PROPERTY,
      'split_names': SPLIT_NAMES_PROPERTY,
  }

# TODO(b/158334890): deprecate ExternalArtifact.
class ExternalArtifact(_TfxArtifact):
  TYPE_NAME = 'ExternalArtifact'

class InferenceResult(_TfxArtifact):
  TYPE_NAME = 'InferenceResult'

class InfraBlessing(_TfxArtifact):
  TYPE_NAME = 'InfraBlessing'

class Model(_TfxArtifact):
  """Artifact that contains the actual persisted model.

  Training components stores the trained model like a saved model in this
  artifact. A `Model` artifact contains serialization of the trained model in
  one or more formats, each suitable for different usage (e.g. serving,
  evaluation), and serving environments.

  * File structure:
     - `{uri}/`
        - `Format-Serving/`: Model exported for serving.
           - `saved_model.pb`
           - Other actual model files.
        - `Format-TFMA/`: Model exported for evaluation.
           - `saved_model.pb`
           - Other actual model files.

  * Commonly used custom properties of the Model artifact:
  """
  TYPE_NAME = 'Model'
  TYPE_ANNOTATION = SystemModel

class ModelRun(_TfxArtifact):
  TYPE_NAME = 'ModelRun'

class ModelBlessing(_TfxArtifact):
  TYPE_NAME = 'ModelBlessing'

class ModelEvaluation(_TfxArtifact):
  TYPE_NAME = 'ModelEvaluation'

class PushedModel(_TfxArtifact):
  TYPE_NAME = 'PushedModel'
  TYPE_ANNOTATION = SystemModel

class Schema(_TfxArtifact):
  TYPE_NAME = 'Schema'

class TransformCache(_TfxArtifact):
  TYPE_NAME = 'TransformCache'

class JsonValue(ValueArtifact):
  """Artifacts representing a Jsonable value."""
  TYPE_NAME = 'JsonValue'

  def encode(self, value: json_utils.JsonableType) -> str:
    return json_utils.dumps(value)

  def decode(self, serialized_value: str) -> json_utils.JsonableType:
    return json_utils.loads(serialized_value)

class Bytes(ValueArtifact):
  """Artifacts representing raw bytes."""
  TYPE_NAME = 'Bytes'

  def encode(self, value: bytes):
    if not isinstance(value, bytes):
      raise TypeError('Expecting bytes but got value %s of type %s' %
                      (str(value), type(value)))
    return value

  def decode(self, serialized_value: bytes):
    return serialized_value

class String(ValueArtifact):
  """String-typed artifact."""
  TYPE_NAME = 'String'

  # Note, currently we enforce unicode-encoded string.
  def encode(self, value: str) -> bytes:
    if not isinstance(value, str):
      raise TypeError('Expecting Text but got value %s of type %s' %
                      (str(value), type(value)))
    return value.encode('utf-8')

  def decode(self, serialized_value: bytes) -> str:
    return serialized_value.decode('utf-8')

class Boolean(ValueArtifact):
  """Artifacts representing a boolean.

  Boolean value artifacts are encoded as "1" for True and "0" for False.
  """
  TYPE_NAME = 'Boolean'

  def encode(self, value: bool):
    if not isinstance(value, bool):
      raise TypeError('Expecting bytes but got value %s of type %s' %
                      (str(value), type(value)))
    return b'1' if value else b'0'

  def decode(self, serialized_value: bytes):
    return int(serialized_value) != 0

class Integer(ValueArtifact):
  """Integer-typed artifact."""
  TYPE_NAME = 'Integer'

  def encode(self, value: int) -> bytes:
    if not isinstance(value, int):
      raise TypeError('Expecting int but got value %s of type %s' %
                      (str(value), type(value)))
    return str(value).encode('utf-8')

  def decode(self, serialized_value: bytes) -> int:
    return int(serialized_value)

class Float(ValueArtifact):
  """Float-typed artifact."""
  TYPE_NAME = 'Float'

  _POSITIVE_INFINITY = float('Inf')
  _NEGATIVE_INFINITY = float('-Inf')

  _ENCODED_POSITIVE_INFINITY = 'Infinity'
  _ENCODED_NEGATIVE_INFINITY = '-Infinity'
  _ENCODED_NAN = 'NaN'

  def encode(self, value: float) -> bytes:
    if not isinstance(value, float):
      raise TypeError('Expecting float but got value %s of type %s' %
                      (str(value), type(value)))
    if math.isinf(value) or math.isnan(value):
      absl.logging.warning(
          '! The number "%s" may be unsupported by non-python components.' %
          value)
    str_value = str(value)
    # Special encoding for infinities and NaN to increase comatibility with
    # other languages.
    # Decoding works automatically.
    if math.isinf(value):
      if value >= 0:
        str_value = Float._ENCODED_POSITIVE_INFINITY
      else:
        str_value = Float._ENCODED_NEGATIVE_INFINITY
    if math.isnan(value):
      str_value = Float._ENCODED_NAN

    return str_value.encode('utf-8')

  def decode(self, serialized_value: bytes) -> float:
    result = float(serialized_value)

    # Check that the decoded value exactly matches the encoded string.
    # Note that float() can handle bytes, but Decimal() cannot.
    serialized_string = serialized_value.decode('utf-8')
    reserialized_string = str(result)
    is_exact = (decimal.Decimal(serialized_string) ==
                decimal.Decimal(reserialized_string))
    if not is_exact:
      absl.logging.warning(
          'The number "%s" has lost precision when converted to float "%s"' %
          (serialized_value, reserialized_string))

    return result

class TransformGraph(_TfxArtifact):
  TYPE_NAME = 'TransformGraph'

class HyperParameters(_TfxArtifact):
  TYPE_NAME = 'HyperParameters'

# WIP and subject to change.
class DataView(_TfxArtifact):
  TYPE_NAME = 'DataView'
<|endoftext|>


<commit_before>from awacs.aws import (
    Action,
    Allow,
    Bool,
    Condition,
    Policy,
    Principal,
    AWSPrincipal,
    Statement,
    StringEquals,
)
import awacs.logs
import awacs.s3
import awacs.kms
import awacs.firehose
from awacs import sts
from stacker.blueprints.base import Blueprint
from troposphere import (
    iam,
    kms,
    s3,
    GetAtt,
    Join,
    Output,
    Ref,
)

BUCKET = 'S3Bucket'
IAM_ROLE = 'IAMRole'
ROLE_POLICY = 'RolePolicy'
FIREHOSE_WRITE_POLICY = 'FirehoseWriteAccess'
LOGS_POLICY = 'LogsPolicy'
S3_WRITE_POLICY = 'S3WriteAccess'
LOGS_WRITE_POLICY = 'LogsWriteAccess'
KMS_KEY = "EncryptionKey"

class FirehoseAction(Action):
    def __init__(self, action=None):
        self.prefix = "firehose"
        self.action = action

def s3_arn(bucket):
    return Join('', ['arn:aws:s3:::', bucket])

def logs_policy():
    statements = [
        Statement(
            Effect=Allow,
            Action=[
                awacs.logs.CreateLogStream,
                awacs.logs.CreateLogGroup,
            ],
            Resource=['*'],
        ),
    ]
    return Policy(Statement=statements)

def firehose_write_policy():
    statements = [
        Statement(
            Effect=Allow,
            Action=[
                awacs.firehose.CreateDeliveryStream,
                awacs.firehose.DeleteDeliveryStream,
                awacs.firehose.DescribeDeliveryStream,
                awacs.firehose.PutRecord,
                awacs.firehose.PutRecordBatch,
            ],
            Resource=['*'],
        ),
    ]
    return Policy(Statement=statements)

def logs_write_policy():
    statements = [
        Statement(
            Effect=Allow,
            Action=[
                awacs.logs.PutLogEvents,
            ],
            Resource=['*'],
        ),
    ]
    return Policy(Statement=statements)

def s3_write_policy(bucket):
    statements = [
        Statement(
            Effect=Allow,
            Action=[
                awacs.s3.AbortMultipartUpload,
                awacs.s3.GetBucketLocation,
                awacs.s3.GetObject,
                awacs.s3.ListBucket,
                awacs.s3.ListBucketMultipartUploads,
                awacs.s3.PutObject,
            ],
            Resource=[
                s3_arn(bucket),
                s3_arn(Join("/", [bucket, "*"]))
            ],
        ),
    ]
    return Policy(Statement=statements)

def kms_key_policy(key_use_arns, key_admin_arns):
    """ Creates a key policy for use of a KMS Key.

    key_use_arns is a list of arns that should have access to use the KMS
    key.
    """

    root_arn = Join(":", ["arn:aws:iam:", Ref("AWS::AccountId"), "root"])

    statements = []
    statements.append(
        Statement(
            Sid="Enable IAM User Permissions",
            Effect=Allow,
            Principal=AWSPrincipal(root_arn),
            Action=[
                Action("kms", "*"),
            ],
            Resource=["*"]
        )
    )
    statements.append(
        Statement(
            Sid="Allow use of the key",
            Effect=Allow,
            Principal=AWSPrincipal(key_use_arns),
            Action=[
                awacs.kms.Encrypt,
                awacs.kms.Decrypt,
                awacs.kms.ReEncrypt,
                awacs.kms.GenerateDataKey,
                awacs.kms.GenerateDataKeyWithoutPlaintext,
                awacs.kms.DescribeKey,
            ],
            Resource=["*"]
        )
    )
    statements.append(
        Statement(
            Sid="Allow attachment of persistent resources",
            Effect=Allow,
            Principal=AWSPrincipal(key_use_arns),
            Action=[
                awacs.kms.CreateGrant,
                awacs.kms.ListGrants,
                awacs.kms.RevokeGrant,
            ],
            Resource=["*"],
            Condition=Condition(Bool("kms:GrantIsForAWSResource", True))
        )
    )
    statements.append(
        Statement(
            Sid="Allow access for Key Administrators",
            Effect=Allow,
            Principal=AWSPrincipal(key_admin_arns),
            Action=[
                Action("kms", "Create*"),
                Action("kms", "Describe*"),
                Action("kms", "Enable*"),
                Action("kms", "List*"),
                Action("kms", "Put*"),
                Action("kms", "Update*"),
                Action("kms", "Revoke*"),
                Action("kms", "Disable*"),
                Action("kms", "Get*"),
                Action("kms", "Delete*"),
                Action("kms", "ScheduleKeyDeletion"),
                Action("kms", "CancelKeyDeletion"),
            ],
            Resource=["*"],
        )
    )

    return Policy(Version="2012-10-17", Id="key-default-1",
                  Statement=statements)

class Firehose(Blueprint):
    VARIABLES = {
        "RoleNames": {
            "type": list,
            "description": "A list of role names that should have access to "
                           "write to the firehose stream.",
            "default": [],
        },
        "GroupNames": {
            "type": list,
            "description": "A list of group names that should have access to "
                           "write to the firehose stream.",
            "default": [],
        },
        "UserNames": {
            "type": list,
            "description": "A list of user names that should have access to "
                           "write to the firehose stream.",
            "default": [],
        },
        "BucketName": {
            "type": str,
            "description": "Name for the S3 Bucket",
        },
        "EncryptS3Bucket": {
            "type": bool,
            "description": "If set to true, a KMS key will be created to use "
                           "for encrypting the S3 Bucket's contents. If set "
                           "to false, no encryption will occur. Default: true",
            "default": True,
        },
        "EnableKeyRotation": {
            "type": bool,
            "description": "Whether to enable key rotation on the KMS key "
                           "generated if EncryptS3Bucket is set to true. "
                           "Default: true",
            "default": True,
        },
        "KeyUseArns": {
            "type": list,
            "description": "A list of ARNs that need access to the KMS key "
                           "created with EncryptS3Bucket",
            "default": [],
        },
        "KeyAdminArns": {
            "type": list,
            "description": "A list of ARNs that need to admin the KMS key "
                           "created with EncryptS3Bucket",
            "default": [],
        }
    }

    def create_kms_key(self):
        t = self.template
        variables = self.get_variables()

        if not variables["EncryptS3Bucket"]:
            return

        key_description = Join(
            "",
            [
                "S3 Bucket kms encryption key for stack ",
                Ref("AWS::StackName")
            ]
        )

        key_use_arns = variables["KeyUseArns"]
        # auto add the created IAM Role
        key_use_arns.append(GetAtt(IAM_ROLE, "Arn"))

        key_admin_arns = variables["KeyAdminArns"]

        t.add_resource(
            kms.Key(
                KMS_KEY,
                Description=key_description,
                Enabled=True,
                EnableKeyRotation=variables["EnableKeyRotation"],
                KeyPolicy=kms_key_policy(key_use_arns, key_admin_arns),
            )
        )
        key_arn = Join(
            "",
            [
                "arn:aws:kms:",
                Ref("AWS::Region"),
                ":",
                Ref("AWS::AccountId"),
                ":key/",
                Ref(KMS_KEY)
            ]
        )
        t.add_output(Output("KmsKeyArn", Value=key_arn))
        t.add_output(Output("KmsKeyId", Value=Ref(KMS_KEY)))

    def create_bucket(self):
        t = self.template
        variables = self.get_variables()

        bucket_name = variables.get("BucketName", Ref("AWS::NoValue"))

        t.add_resource(
            s3.Bucket(
                BUCKET,
                BucketName=bucket_name,
            )
        )
        t.add_output(Output('Bucket', Value=Ref(BUCKET)))

    def generate_iam_policies(self):
        ns = self.context.namespace
        name_prefix = "%s-%s" % (ns, self.name)
        s3_policy = iam.Policy(
            S3_WRITE_POLICY,
            PolicyName='{}-s3-write'.format(name_prefix),
            PolicyDocument=s3_write_policy(Ref(BUCKET)),
        )
        logs_policy = iam.Policy(
            LOGS_WRITE_POLICY,
            PolicyName='{}-logs-write'.format(name_prefix),
            PolicyDocument=logs_write_policy(),
        )
        return [s3_policy, logs_policy]

    def create_role(self):
        t = self.template

        statements = [
            Statement(
                Principal=Principal('Service', ['firehose.amazonaws.com']),
                Effect=Allow,
                Action=[sts.AssumeRole],
                Condition=Condition(
                    StringEquals('sts:ExternalId', Ref('AWS::AccountId')),
                ),
            ),
        ]
        firehose_role_policy = Policy(Statement=statements)
        t.add_resource(
            iam.Role(
                IAM_ROLE,
                AssumeRolePolicyDocument=firehose_role_policy,
                Path='/',
                Policies=self.generate_iam_policies(),
            ),
        )
        t.add_output(Output('Role', Value=Ref(IAM_ROLE)))
        t.add_output(Output('RoleArn', Value=GetAtt(IAM_ROLE, 'Arn')))

    def create_policy(self):
        ns = self.context.namespace
        name_prefix = "%s-%s" % (ns, self.name)
        t = self.template
        variables = self.get_variables()

        external_roles = variables.get("RoleNames", Ref("AWS::NoValue"))
        external_groups = variables.get("GroupNames", Ref("AWS::NoValue"))
        external_users = variables.get("UserNames", Ref("AWS::NoValue"))

        create_policy = any([
            variables["RoleNames"],
            variables["GroupNames"],
            variables["UserNames"],
        ])

        if create_policy:
            t.add_resource(
                iam.PolicyType(
                    FIREHOSE_WRITE_POLICY,
                    PolicyName='{}-firehose'.format(name_prefix),
                    PolicyDocument=firehose_write_policy(),
                    Roles=external_roles,
                    Groups=external_groups,
                    Users=external_users,
                ),
            )
            t.add_resource(
                iam.PolicyType(
                    LOGS_POLICY,
                    PolicyName='{}-logs'.format(name_prefix),
                    PolicyDocument=logs_policy(),
                    Roles=external_roles,
                    Groups=external_groups,
                    Users=external_users,
                ),
            )

    def create_template(self):
        self.create_kms_key()
        self.create_policy()
        self.create_bucket()
        self.create_role()
<commit_msg>Make admin/user for kms key optional, cleanup<commit_after>from awacs.aws import (
    Action,
    Allow,
    Bool,
    Condition,
    Policy,
    Principal,
    AWSPrincipal,
    Statement,
    StringEquals,
)
import awacs.logs
import awacs.s3
import awacs.kms
import awacs.firehose
from awacs import sts
from stacker.blueprints.base import Blueprint
from troposphere import (
    iam,
    kms,
    s3,
    GetAtt,
    Join,
    Output,
    Ref,
)

BUCKET = 'S3Bucket'
IAM_ROLE = 'IAMRole'
ROLE_POLICY = 'RolePolicy'
FIREHOSE_WRITE_POLICY = 'FirehoseWriteAccess'
LOGS_POLICY = 'LogsPolicy'
S3_WRITE_POLICY = 'S3WriteAccess'
LOGS_WRITE_POLICY = 'LogsWriteAccess'
KMS_KEY = "EncryptionKey"
KEY_ALIAS = "KeyAlias"

class FirehoseAction(Action):
    def __init__(self, action=None):
        self.prefix = "firehose"
        self.action = action

def s3_arn(bucket):
    return Join('', ['arn:aws:s3:::', bucket])

def logs_policy():
    statements = [
        Statement(
            Effect=Allow,
            Action=[
                awacs.logs.CreateLogStream,
                awacs.logs.CreateLogGroup,
            ],
            Resource=['*'],
        ),
    ]
    return Policy(Statement=statements)

def firehose_write_policy():
    statements = [
        Statement(
            Effect=Allow,
            Action=[
                awacs.firehose.CreateDeliveryStream,
                awacs.firehose.DeleteDeliveryStream,
                awacs.firehose.DescribeDeliveryStream,
                awacs.firehose.PutRecord,
                awacs.firehose.PutRecordBatch,
            ],
            Resource=['*'],
        ),
    ]
    return Policy(Statement=statements)

def logs_write_policy():
    statements = [
        Statement(
            Effect=Allow,
            Action=[
                awacs.logs.PutLogEvents,
            ],
            Resource=['*'],
        ),
    ]
    return Policy(Statement=statements)

def s3_write_policy(bucket):
    statements = [
        Statement(
            Effect=Allow,
            Action=[
                awacs.s3.AbortMultipartUpload,
                awacs.s3.GetBucketLocation,
                awacs.s3.GetObject,
                awacs.s3.ListBucket,
                awacs.s3.ListBucketMultipartUploads,
                awacs.s3.PutObject,
            ],
            Resource=[
                s3_arn(bucket),
                s3_arn(Join("/", [bucket, "*"]))
            ],
        ),
    ]
    return Policy(Statement=statements)

def kms_key_policy(key_use_arns, key_admin_arns):
    """ Creates a key policy for use of a KMS Key.

    key_use_arns is a list of arns that should have access to use the KMS
    key.
    """

    root_arn = Join(":", ["arn:aws:iam:", Ref("AWS::AccountId"), "root"])

    statements = []
    statements.append(
        Statement(
            Sid="Enable IAM User Permissions",
            Effect=Allow,
            Principal=AWSPrincipal(root_arn),
            Action=[
                Action("kms", "*"),
            ],
            Resource=["*"]
        )
    )
    if key_use_arns:
        statements.append(
            Statement(
                Sid="Allow use of the key",
                Effect=Allow,
                Principal=AWSPrincipal(key_use_arns),
                Action=[
                    awacs.kms.Encrypt,
                    awacs.kms.Decrypt,
                    awacs.kms.ReEncrypt,
                    awacs.kms.GenerateDataKey,
                    awacs.kms.GenerateDataKeyWithoutPlaintext,
                    awacs.kms.DescribeKey,
                ],
                Resource=["*"]
            )
        )

        statements.append(
            Statement(
                Sid="Allow attachment of persistent resources",
                Effect=Allow,
                Principal=AWSPrincipal(key_use_arns),
                Action=[
                    awacs.kms.CreateGrant,
                    awacs.kms.ListGrants,
                    awacs.kms.RevokeGrant,
                ],
                Resource=["*"],
                Condition=Condition(Bool("kms:GrantIsForAWSResource", True))
            )
        )

    if key_admin_arns:
        statements.append(
            Statement(
                Sid="Allow access for Key Administrators",
                Effect=Allow,
                Principal=AWSPrincipal(key_admin_arns),
                Action=[
                    Action("kms", "Create*"),
                    Action("kms", "Describe*"),
                    Action("kms", "Enable*"),
                    Action("kms", "List*"),
                    Action("kms", "Put*"),
                    Action("kms", "Update*"),
                    Action("kms", "Revoke*"),
                    Action("kms", "Disable*"),
                    Action("kms", "Get*"),
                    Action("kms", "Delete*"),
                    Action("kms", "ScheduleKeyDeletion"),
                    Action("kms", "CancelKeyDeletion"),
                ],
                Resource=["*"],
            )
        )

    return Policy(Version="2012-10-17", Id="key-default-1",
                  Statement=statements)

class Firehose(Blueprint):
    VARIABLES = {
        "RoleNames": {
            "type": list,
            "description": "A list of role names that should have access to "
                           "write to the firehose stream.",
            "default": [],
        },
        "GroupNames": {
            "type": list,
            "description": "A list of group names that should have access to "
                           "write to the firehose stream.",
            "default": [],
        },
        "UserNames": {
            "type": list,
            "description": "A list of user names that should have access to "
                           "write to the firehose stream.",
            "default": [],
        },
        "BucketName": {
            "type": str,
            "description": "Name for the S3 Bucket",
            "default": "",
        },
        "EncryptS3Bucket": {
            "type": bool,
            "description": "If set to true, a KMS key will be created to use "
                           "for encrypting the S3 Bucket's contents. If set "
                           "to false, no encryption will occur. Default: true",
            "default": True,
        },
        "EnableKeyRotation": {
            "type": bool,
            "description": "Whether to enable key rotation on the KMS key "
                           "generated if EncryptS3Bucket is set to true. "
                           "Default: true",
            "default": True,
        },
        "KeyUseArns": {
            "type": list,
            "description": "A list of ARNs that need access to the KMS key "
                           "created with EncryptS3Bucket",
            "default": [],
        },
        "KeyAdminArns": {
            "type": list,
            "description": "A list of ARNs that need to admin the KMS key "
                           "created with EncryptS3Bucket",
            "default": [],
        }
    }

    def create_kms_key(self):
        t = self.template
        variables = self.get_variables()

        if not variables["EncryptS3Bucket"]:
            return

        key_description = Join(
            "",
            [
                "S3 Bucket kms encryption key for stack ",
                Ref("AWS::StackName")
            ]
        )

        key_use_arns = variables["KeyUseArns"]
        # auto add the created IAM Role
        key_use_arns.append(GetAtt(IAM_ROLE, "Arn"))

        key_admin_arns = variables["KeyAdminArns"]

        t.add_resource(
            kms.Key(
                KMS_KEY,
                Description=key_description,
                Enabled=True,
                EnableKeyRotation=variables["EnableKeyRotation"],
                KeyPolicy=kms_key_policy(key_use_arns, key_admin_arns),
            )
        )

        t.add_resource(
            kms.Alias(
                KEY_ALIAS,
                AliasName="alias/%s" % self.context.get_fqn(self.name),
                TargetKeyId=Ref(KMS_KEY)
            )
        )

        key_arn = Join(
            "",
            [
                "arn:aws:kms:",
                Ref("AWS::Region"),
                ":",
                Ref("AWS::AccountId"),
                ":key/",
                Ref(KMS_KEY)
            ]
        )
        t.add_output(Output("KmsKeyArn", Value=key_arn))
        t.add_output(Output("KmsKeyId", Value=Ref(KMS_KEY)))
        t.add_output(Output("KmsKeyAlias", Value=Ref(KEY_ALIAS)))

    def create_bucket(self):
        t = self.template
        variables = self.get_variables()

        bucket_name = variables.get("BucketName") or Ref("AWS::NoValue")

        t.add_resource(
            s3.Bucket(
                BUCKET,
                BucketName=bucket_name,
            )
        )
        t.add_output(Output('Bucket', Value=Ref(BUCKET)))

    def generate_iam_policies(self):
        name_prefix = self.context.get_fqn(self.name)
        s3_policy = iam.Policy(
            S3_WRITE_POLICY,
            PolicyName='{}-s3-write'.format(name_prefix),
            PolicyDocument=s3_write_policy(Ref(BUCKET)),
        )
        logs_policy = iam.Policy(
            LOGS_WRITE_POLICY,
            PolicyName='{}-logs-write'.format(name_prefix),
            PolicyDocument=logs_write_policy(),
        )
        return [s3_policy, logs_policy]

    def create_role(self):
        t = self.template

        statements = [
            Statement(
                Principal=Principal('Service', ['firehose.amazonaws.com']),
                Effect=Allow,
                Action=[sts.AssumeRole],
                Condition=Condition(
                    StringEquals('sts:ExternalId', Ref('AWS::AccountId')),
                ),
            ),
        ]
        firehose_role_policy = Policy(Statement=statements)
        t.add_resource(
            iam.Role(
                IAM_ROLE,
                AssumeRolePolicyDocument=firehose_role_policy,
                Path='/',
                Policies=self.generate_iam_policies(),
            ),
        )
        t.add_output(Output('Role', Value=Ref(IAM_ROLE)))
        t.add_output(Output('RoleArn', Value=GetAtt(IAM_ROLE, 'Arn')))

    def create_policy(self):
        name_prefix = self.context.get_fqn(self.name)
        t = self.template
        variables = self.get_variables()

        external_roles = variables.get("RoleNames") or Ref("AWS::NoValue")
        external_groups = variables.get("GroupNames") or Ref("AWS::NoValue")
        external_users = variables.get("UserNames") or Ref("AWS::NoValue")

        create_policy = any([
            variables["RoleNames"],
            variables["GroupNames"],
            variables["UserNames"],
        ])

        if create_policy:
            t.add_resource(
                iam.PolicyType(
                    FIREHOSE_WRITE_POLICY,
                    PolicyName='{}-firehose'.format(name_prefix),
                    PolicyDocument=firehose_write_policy(),
                    Roles=external_roles,
                    Groups=external_groups,
                    Users=external_users,
                ),
            )
            t.add_resource(
                iam.PolicyType(
                    LOGS_POLICY,
                    PolicyName='{}-logs'.format(name_prefix),
                    PolicyDocument=logs_policy(),
                    Roles=external_roles,
                    Groups=external_groups,
                    Users=external_users,
                ),
            )

    def create_template(self):
        self.create_kms_key()
        self.create_policy()
        self.create_bucket()
        self.create_role()
<|endoftext|>


<commit_before>"""
fs.expose.importhook
====================

Expose an FS object to the python import machinery, via a PEP-302 loader.

This module allows you to import python modules from an arbitrary FS object,
by placing FS urls on sys.path and/or inserting objects into sys.meta_path.

The main class in this module is FSImportHook, which is a PEP-302-compliant
module finder and loader.  If you place an instance of FSImportHook on
sys.meta_path, you will be able to import modules from the exposed filesystem::

    >>> from fs.memoryfs import MemoryFS
    >>> m = MemoryFS()
    >>> m.setcontents("helloworld.py","print 'hello world!")
    >>>
    >>> import sys
    >>> from fs.expose.importhook import FSImportHook
    >>> sys.meta_path.append(FSImportHook(m))
    >>> import helloworld
    hello world!

It is also possible to install FSImportHook as an import path handler. This
allows you to place filesystem URLs on sys.path and have them automagically
opened for importing.  This example would allow modules to be imported from
an SFTP server::

    >>> from fs.expose.importhook import FSImportHook
    >>> FSImportHook.install()
    >>> sys.path.append("sftp://some.remote.machine/mypath/")

"""

import sys
import imp
import marshal

from fs.base import FS
from fs.opener import fsopendir, OpenerError
from fs.errors import *
from fs.path import *

class FSImportHook(object):
    """PEP-302-compliant module finder and loader for FS objects.

    FSImportHook is a module finder and loader that takes its data from an
    arbitrary FS object.  The FS must have .py or .pyc files stored in the
    standard module structure.

    For easy use with sys.path, FSImportHook will also accept a filesystem
    URL, which is automatically opened using fs.opener.
    """

    _VALID_MODULE_TYPES = set((imp.PY_SOURCE,imp.PY_COMPILED))

    def __init__(self,fs_or_url):
        #  If given a string, try to open it as an FS url.
        #  Don't open things on the local filesystem though.
        if isinstance(fs_or_url,basestring):
            if ":" not in fs_or_url:
                raise ImportError
            try:
                self.fs = fsopendir(fs_or_url)
            except OpenerError:
                raise ImportError
            self.path = fs_or_url
        #  Otherwise, it must be an FS object of some sort.
        else:
            if not isinstance(fs_or_url,FS):
                raise ImportError
            self.fs = fs_or_url
            self.path = None

    @classmethod
    def install(cls):
        """Install this class into the import machinery.

        This classmethod installs the custom FSImportHook class into the
        import machinery of the running process, if it is not already
        installed.
        """
        for i,imp in enumerate(sys.path_hooks):
            try:
                if issubclass(cls,imp):
                    break
            except TypeError:
                pass
        else:
            sys.path_hooks.append(cls)
            sys.path_importer_cache.clear()

    @classmethod
    def uninstall(cls):
        """Uninstall this class from the import machinery.

        This classmethod uninstalls the custom FSImportHook class from the
        import machinery of the running process.
        """
        to_rem = []
        for i,imp in enumerate(sys.path_hooks):
            try:
                if issubclass(cls,imp):
                    to_rem.append(imp)
                    break
            except TypeError:
                pass
        for imp in to_rem:
            sys.path_hooks.remove(imp)
            sys.path_importer_cache.clear()

    def find_module(self,fullname,path=None):
        """Find the FS loader for the given module.

        This object is always its own loader, so this really just checks
        whether it's a valid module on the exposed filesystem.
        """
        try:
            self._get_module_info(fullname)
        except ImportError:
            return None
        else:
            return self

    def _get_module_info(self,fullname):
        """Get basic information about the given module.

        If the specified module exists, this method returns a tuple giving
        its filepath, file type and whether it's a package.  Otherwise,
        it raise ImportError.
        """
        prefix = fullname.replace(".","/")
        #  Is it a regular module?
        (path,type) = self._find_module_file(prefix)
        if path is not None:
            return (path,type,False)
        #  Is it a package?
        prefix = pathjoin(prefix,"__init__")
        (path,type) = self._find_module_file(prefix)
        if path is not None:
            return (path,type,True)
        #  No, it's nothing
        raise ImportError(fullname)

    def _find_module_file(self,prefix):
        """Find a module file from the given path prefix.

        This method iterates over the possible module suffixes, checking each
        in turn and returning the first match found.  It returns a two-tuple
        (path,type) or (None,None) if there's no module.
        """
        for (suffix,mode,type) in imp.get_suffixes():
            if type in self._VALID_MODULE_TYPES:
                path = prefix + suffix
                if self.fs.isfile(path):
                    return (path,type)
        return (None,None)

    def load_module(self,fullname):
        """Load the specified module.

        This method locates the file for the specified module, loads and
        executes it and returns the created module object.
        """
        #  Reuse an existing module if present.
        try:
            return sys.modules[fullname]
        except KeyError:
            pass
        #  Try to create from source or bytecode.
        info = self._get_module_info(fullname)
        code = self.get_code(fullname,info)
        if code is None:
            raise ImportError(fullname)
        mod = imp.new_module(fullname)
        mod.__file__ = "<loading>"
        mod.__loader__ = self
        sys.modules[fullname] = mod
        try:
            exec code in mod.__dict__
            mod.__file__ = self.get_filename(fullname,info)
            if self.is_package(fullname,info):
                if self.path is None:
                    mod.__path__ = []
                else:
                    mod.__path__ = [self.path]
            return mod
        except Exception:
            sys.modules.pop(fullname,None)
            raise

    def is_package(self,fullname,info=None):
        """Check whether the specified module is a package."""
        if info is None:
            info = self._get_module_info(fullname)
        (path,type,ispkg) = info
        return ispkg

    def get_code(self,path,info=None):
        """Get the bytecode for the specified module."""
        if info is None:
            info = self._get_module_info(fullname)
        (path,type,ispkg) = info
        code = self.fs.getcontents(path)
        if type == imp.PY_SOURCE:
            code = code.replace("\r\n","\n")
            return compile(code,path,"exec")
        elif type == imp.PY_COMPILED:
            if code[:4] != imp.get_magic():
                return None
            return marshal.loads(code[8:])
        else:
            return None
        return code

    def get_source(self,fullname,info=None):
        """Get the sourcecode for the specified module, if present."""
        if info is None:
            info = self._get_module_info(fullname)
        (path,type,ispkg) = info
        if type != imp.PY_SOURCE:
            return None
        return self.fs.getcontents(path).replace("\r\n","\n")

    def get_data(self,path):
        """Read the specified data file."""
        try:
            return self.fs.getcontents(path)
        except FSError, e:
            raise IOError(str(e))

    def get_filename(self,fullname,info=None):
        """Get the __file__ attribute for the specified module."""
        if info is None:
            info = self._get_module_info(fullname)
        (path,type,ispkg) = info
        return path

<commit_msg>fix silly typo in docstring<commit_after>"""
fs.expose.importhook
====================

Expose an FS object to the python import machinery, via a PEP-302 loader.

This module allows you to import python modules from an arbitrary FS object,
by placing FS urls on sys.path and/or inserting objects into sys.meta_path.

The main class in this module is FSImportHook, which is a PEP-302-compliant
module finder and loader.  If you place an instance of FSImportHook on
sys.meta_path, you will be able to import modules from the exposed filesystem::

    >>> from fs.memoryfs import MemoryFS
    >>> m = MemoryFS()
    >>> m.setcontents("helloworld.py","print 'hello world!'")
    >>>
    >>> import sys
    >>> from fs.expose.importhook import FSImportHook
    >>> sys.meta_path.append(FSImportHook(m))
    >>> import helloworld
    hello world!

It is also possible to install FSImportHook as an import path handler. This
allows you to place filesystem URLs on sys.path and have them automagically
opened for importing.  This example would allow modules to be imported from
an SFTP server::

    >>> from fs.expose.importhook import FSImportHook
    >>> FSImportHook.install()
    >>> sys.path.append("sftp://some.remote.machine/mypath/")

"""

import sys
import imp
import marshal

from fs.base import FS
from fs.opener import fsopendir, OpenerError
from fs.errors import *
from fs.path import *

class FSImportHook(object):
    """PEP-302-compliant module finder and loader for FS objects.

    FSImportHook is a module finder and loader that takes its data from an
    arbitrary FS object.  The FS must have .py or .pyc files stored in the
    standard module structure.

    For easy use with sys.path, FSImportHook will also accept a filesystem
    URL, which is automatically opened using fs.opener.
    """

    _VALID_MODULE_TYPES = set((imp.PY_SOURCE,imp.PY_COMPILED))

    def __init__(self,fs_or_url):
        #  If given a string, try to open it as an FS url.
        #  Don't open things on the local filesystem though.
        if isinstance(fs_or_url,basestring):
            if ":" not in fs_or_url:
                raise ImportError
            try:
                self.fs = fsopendir(fs_or_url)
            except OpenerError:
                raise ImportError
            self.path = fs_or_url
        #  Otherwise, it must be an FS object of some sort.
        else:
            if not isinstance(fs_or_url,FS):
                raise ImportError
            self.fs = fs_or_url
            self.path = None

    @classmethod
    def install(cls):
        """Install this class into the import machinery.

        This classmethod installs the custom FSImportHook class into the
        import machinery of the running process, if it is not already
        installed.
        """
        for i,imp in enumerate(sys.path_hooks):
            try:
                if issubclass(cls,imp):
                    break
            except TypeError:
                pass
        else:
            sys.path_hooks.append(cls)
            sys.path_importer_cache.clear()

    @classmethod
    def uninstall(cls):
        """Uninstall this class from the import machinery.

        This classmethod uninstalls the custom FSImportHook class from the
        import machinery of the running process.
        """
        to_rem = []
        for i,imp in enumerate(sys.path_hooks):
            try:
                if issubclass(cls,imp):
                    to_rem.append(imp)
                    break
            except TypeError:
                pass
        for imp in to_rem:
            sys.path_hooks.remove(imp)
            sys.path_importer_cache.clear()

    def find_module(self,fullname,path=None):
        """Find the FS loader for the given module.

        This object is always its own loader, so this really just checks
        whether it's a valid module on the exposed filesystem.
        """
        try:
            self._get_module_info(fullname)
        except ImportError:
            return None
        else:
            return self

    def _get_module_info(self,fullname):
        """Get basic information about the given module.

        If the specified module exists, this method returns a tuple giving
        its filepath, file type and whether it's a package.  Otherwise,
        it raise ImportError.
        """
        prefix = fullname.replace(".","/")
        #  Is it a regular module?
        (path,type) = self._find_module_file(prefix)
        if path is not None:
            return (path,type,False)
        #  Is it a package?
        prefix = pathjoin(prefix,"__init__")
        (path,type) = self._find_module_file(prefix)
        if path is not None:
            return (path,type,True)
        #  No, it's nothing
        raise ImportError(fullname)

    def _find_module_file(self,prefix):
        """Find a module file from the given path prefix.

        This method iterates over the possible module suffixes, checking each
        in turn and returning the first match found.  It returns a two-tuple
        (path,type) or (None,None) if there's no module.
        """
        for (suffix,mode,type) in imp.get_suffixes():
            if type in self._VALID_MODULE_TYPES:
                path = prefix + suffix
                if self.fs.isfile(path):
                    return (path,type)
        return (None,None)

    def load_module(self,fullname):
        """Load the specified module.

        This method locates the file for the specified module, loads and
        executes it and returns the created module object.
        """
        #  Reuse an existing module if present.
        try:
            return sys.modules[fullname]
        except KeyError:
            pass
        #  Try to create from source or bytecode.
        info = self._get_module_info(fullname)
        code = self.get_code(fullname,info)
        if code is None:
            raise ImportError(fullname)
        mod = imp.new_module(fullname)
        mod.__file__ = "<loading>"
        mod.__loader__ = self
        sys.modules[fullname] = mod
        try:
            exec code in mod.__dict__
            mod.__file__ = self.get_filename(fullname,info)
            if self.is_package(fullname,info):
                if self.path is None:
                    mod.__path__ = []
                else:
                    mod.__path__ = [self.path]
            return mod
        except Exception:
            sys.modules.pop(fullname,None)
            raise

    def is_package(self,fullname,info=None):
        """Check whether the specified module is a package."""
        if info is None:
            info = self._get_module_info(fullname)
        (path,type,ispkg) = info
        return ispkg

    def get_code(self,path,info=None):
        """Get the bytecode for the specified module."""
        if info is None:
            info = self._get_module_info(fullname)
        (path,type,ispkg) = info
        code = self.fs.getcontents(path)
        if type == imp.PY_SOURCE:
            code = code.replace("\r\n","\n")
            return compile(code,path,"exec")
        elif type == imp.PY_COMPILED:
            if code[:4] != imp.get_magic():
                return None
            return marshal.loads(code[8:])
        else:
            return None
        return code

    def get_source(self,fullname,info=None):
        """Get the sourcecode for the specified module, if present."""
        if info is None:
            info = self._get_module_info(fullname)
        (path,type,ispkg) = info
        if type != imp.PY_SOURCE:
            return None
        return self.fs.getcontents(path).replace("\r\n","\n")

    def get_data(self,path):
        """Read the specified data file."""
        try:
            return self.fs.getcontents(path)
        except FSError, e:
            raise IOError(str(e))

    def get_filename(self,fullname,info=None):
        """Get the __file__ attribute for the specified module."""
        if info is None:
            info = self._get_module_info(fullname)
        (path,type,ispkg) = info
        return path

<|endoftext|>


<commit_before>"""

  fs.tests.test_wrapfs:  testcases for FS wrapper implementations

"""

import unittest
from fs.tests import FSTestCases, ThreadingTestCases

import os
import sys
import shutil
import tempfile

from fs import osfs
from fs.errors import *
from fs.path import *
from fs import wrapfs

import six
from six import PY3, b

class TestWrapFS(unittest.TestCase, FSTestCases, ThreadingTestCases):

    #__test__ = False

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp(u"fstest")
        self.fs = wrapfs.WrapFS(osfs.OSFS(self.temp_dir))

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        self.fs.close()

    def check(self, p):
        return os.path.exists(os.path.join(self.temp_dir, relpath(p)))

from fs.wrapfs.lazyfs import LazyFS
class TestLazyFS(unittest.TestCase, FSTestCases, ThreadingTestCases):

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp(u"fstest")
        self.fs = LazyFS((osfs.OSFS,(self.temp_dir,)))

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        self.fs.close()

    def check(self, p):
        return os.path.exists(os.path.join(self.temp_dir, relpath(p)))

from fs.wrapfs.limitsizefs import LimitSizeFS
class TestLimitSizeFS(TestWrapFS):

    _dont_retest = TestWrapFS._dont_retest + ("test_big_file",)

    def setUp(self):
        super(TestLimitSizeFS,self).setUp()
        self.fs = LimitSizeFS(self.fs,1024*1024*2)  # 2MB limit

    def tearDown(self):
        self.fs.removedir("/",force=True)
        self.assertEquals(self.fs.cur_size,0)
        super(TestLimitSizeFS,self).tearDown()
        self.fs.close()

    def test_storage_error(self):
        total_written = 0
        for i in xrange(1024*2):
            try:
                total_written += 1030
                self.fs.setcontents("file %i" % i, b("C")*1030)
            except StorageSpaceError:
                self.assertTrue(total_written > 1024*1024*2)
                self.assertTrue(total_written < 1024*1024*2 + 1030)
                break
        else:
            self.assertTrue(False,"StorageSpaceError not raised")

from fs.wrapfs.hidedotfilesfs import HideDotFilesFS
class TestHideDotFilesFS(unittest.TestCase):

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp(u"fstest")
        open(os.path.join(self.temp_dir, u".dotfile"), 'w').close()
        open(os.path.join(self.temp_dir, u"regularfile"), 'w').close()
        os.mkdir(os.path.join(self.temp_dir, u".dotdir"))
        os.mkdir(os.path.join(self.temp_dir, u"regulardir"))
        self.fs = HideDotFilesFS(osfs.OSFS(self.temp_dir))

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        self.fs.close()

    def test_hidden(self):
        self.assertEquals(len(self.fs.listdir(hidden=False)), 2)
        self.assertEquals(len(list(self.fs.ilistdir(hidden=False))), 2)

    def test_nonhidden(self):
        self.assertEquals(len(self.fs.listdir(hidden=True)), 4)
        self.assertEquals(len(list(self.fs.ilistdir(hidden=True))), 4)

    def test_default(self):
        self.assertEquals(len(self.fs.listdir()), 2)
        self.assertEquals(len(list(self.fs.ilistdir())), 2)

<commit_msg>Stop WrapFS unit tests throwing a RemoveRootError. Fixes Issue 121<commit_after>"""

  fs.tests.test_wrapfs:  testcases for FS wrapper implementations

"""

import unittest
from fs.tests import FSTestCases, ThreadingTestCases

import os
import sys
import shutil
import tempfile

from fs import osfs
from fs.errors import *
from fs.path import *
from fs.utils import remove_all
from fs import wrapfs

import six
from six import PY3, b

class TestWrapFS(unittest.TestCase, FSTestCases, ThreadingTestCases):

    #__test__ = False

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp(u"fstest")
        self.fs = wrapfs.WrapFS(osfs.OSFS(self.temp_dir))

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        self.fs.close()

    def check(self, p):
        return os.path.exists(os.path.join(self.temp_dir, relpath(p)))

from fs.wrapfs.lazyfs import LazyFS
class TestLazyFS(unittest.TestCase, FSTestCases, ThreadingTestCases):

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp(u"fstest")
        self.fs = LazyFS((osfs.OSFS,(self.temp_dir,)))

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        self.fs.close()

    def check(self, p):
        return os.path.exists(os.path.join(self.temp_dir, relpath(p)))

from fs.wrapfs.limitsizefs import LimitSizeFS
class TestLimitSizeFS(TestWrapFS):

    _dont_retest = TestWrapFS._dont_retest + ("test_big_file",)

    def setUp(self):
        super(TestLimitSizeFS,self).setUp()
        self.fs = LimitSizeFS(self.fs,1024*1024*2)  # 2MB limit

    def tearDown(self):
        remove_all(self.fs, "/")
        self.assertEquals(self.fs.cur_size,0)
        super(TestLimitSizeFS,self).tearDown()
        self.fs.close()

    def test_storage_error(self):
        total_written = 0
        for i in xrange(1024*2):
            try:
                total_written += 1030
                self.fs.setcontents("file %i" % i, b("C")*1030)
            except StorageSpaceError:
                self.assertTrue(total_written > 1024*1024*2)
                self.assertTrue(total_written < 1024*1024*2 + 1030)
                break
        else:
            self.assertTrue(False,"StorageSpaceError not raised")

from fs.wrapfs.hidedotfilesfs import HideDotFilesFS
class TestHideDotFilesFS(unittest.TestCase):

    def setUp(self):
        self.temp_dir = tempfile.mkdtemp(u"fstest")
        open(os.path.join(self.temp_dir, u".dotfile"), 'w').close()
        open(os.path.join(self.temp_dir, u"regularfile"), 'w').close()
        os.mkdir(os.path.join(self.temp_dir, u".dotdir"))
        os.mkdir(os.path.join(self.temp_dir, u"regulardir"))
        self.fs = HideDotFilesFS(osfs.OSFS(self.temp_dir))

    def tearDown(self):
        shutil.rmtree(self.temp_dir)
        self.fs.close()

    def test_hidden(self):
        self.assertEquals(len(self.fs.listdir(hidden=False)), 2)
        self.assertEquals(len(list(self.fs.ilistdir(hidden=False))), 2)

    def test_nonhidden(self):
        self.assertEquals(len(self.fs.listdir(hidden=True)), 4)
        self.assertEquals(len(list(self.fs.ilistdir(hidden=True))), 4)

    def test_default(self):
        self.assertEquals(len(self.fs.listdir()), 2)
        self.assertEquals(len(list(self.fs.ilistdir())), 2)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
test_fsed.py
(c) Will Roberts  12 December, 2015

Test the ``fsed`` module.
'''

from __future__ import absolute_import, print_function, unicode_literals
from .. import fsed
try:
    from StringIO import StringIO
except ImportError:
    from io import BytesIO as StringIO
import unittest

class CMStringIO(StringIO):
    '''StringIO object with context manager.'''

    def __init__(self, *args, **kwargs):
        '''Ctor.'''
        StringIO.__init__(self, *args, **kwargs)

    def __enter__(self):
        self.seek(0)
        return self

    def __exit__(self, _type, _value, _tb):
        pass

PATTERN_TSV = b'''\\bMarco Polo\tMarco_Polo
Kublai Khan\tKublai_Khan
Christopher Columbus\tChristopher_Columbus
and uncle\tand_uncle'''

PATTERN_SED = b'''s/\\bMarco Polo/Marco_Polo/
s/Kublai Khan/Kublai_Khan/
s.Christopher Columbus.Christopher_Columbus.
s/and uncle/and_uncle/'''

INPUT_TEXT = '''and uncle
sand uncle
s and uncle
Kublai Khan
bKublai Khan
Marco Polo
bMarco Polo'''

# without --words
WITHOUT_WORDS_OUTPUT = '''and_uncle
sand_uncle
s and_uncle
Kublai_Khan
bKublai_Khan
Marco_Polo
bMarco Polo'''

# with --words
WITH_WORDS_OUTPUT = '''and_uncle
sand uncle
s and_uncle
Kublai_Khan
bKublai Khan
Marco_Polo
bMarco Polo'''

class TestFsed(unittest.TestCase):
    '''
    Unit tests for the `fsed` module.
    '''

    def test_detect_format(self):
        '''
        Checks the fsed.detect_pattern_format function.
        '''
        pattern_file = CMStringIO(PATTERN_TSV)
        self.assertEqual(fsed.detect_pattern_format(pattern_file, 'utf-8', False),
                         (True, True))
        pattern_file = CMStringIO(PATTERN_SED)
        self.assertEqual(fsed.detect_pattern_format(pattern_file, 'utf-8', False),
                         (False, True))

    def test_rewriting_tsv(self):
        '''
        Tests the fsed.rewrite_str_with_trie function.
        '''
        pattern_file = CMStringIO(PATTERN_TSV)
        trie, boundaries = fsed.build_trie(pattern_file, 'tsv', 'utf-8', False)
        self.assertTrue(boundaries)
        self.assertEqual(fsed.rewrite_str_with_trie(INPUT_TEXT, trie, boundaries),
                         WITHOUT_WORDS_OUTPUT)
        trie, boundaries = fsed.build_trie(pattern_file, 'tsv', 'utf-8', True)
        self.assertTrue(boundaries)
        self.assertEqual(fsed.rewrite_str_with_trie(INPUT_TEXT, trie, boundaries),
                         WITH_WORDS_OUTPUT)

    def test_rewriting_sed(self):
        '''
        Tests the fsed.rewrite_str_with_trie function.
        '''
        pattern_file = CMStringIO(PATTERN_SED)
        trie, boundaries = fsed.build_trie(pattern_file, 'sed', 'utf-8', False)
        self.assertTrue('C' in trie.root)
        self.assertTrue(boundaries)
        self.assertEqual(fsed.rewrite_str_with_trie(INPUT_TEXT, trie, boundaries),
                         WITHOUT_WORDS_OUTPUT)
        trie, boundaries = fsed.build_trie(pattern_file, 'sed', 'utf-8', True)
        self.assertTrue(boundaries)
        self.assertEqual(fsed.rewrite_str_with_trie(INPUT_TEXT, trie, boundaries),
                         WITH_WORDS_OUTPUT)

if __name__ == '__main__':
    unittest.main()
<commit_msg>fsed/tests/test_fsed.py: test_slow<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
test_fsed.py
(c) Will Roberts  12 December, 2015

Test the ``fsed`` module.
'''

from __future__ import absolute_import, print_function, unicode_literals
from .. import ahocorasick
from .. import fsed
try:
    from StringIO import StringIO
except ImportError:
    from io import BytesIO as StringIO
import unittest

class CMStringIO(StringIO):
    '''StringIO object with context manager.'''

    def __init__(self, *args, **kwargs):
        '''Ctor.'''
        StringIO.__init__(self, *args, **kwargs)

    def __enter__(self):
        self.seek(0)
        return self

    def __exit__(self, _type, _value, _tb):
        pass

PATTERN_TSV = b'''\\bMarco Polo\tMarco_Polo
Kublai Khan\tKublai_Khan
Christopher Columbus\tChristopher_Columbus
and uncle\tand_uncle'''

PATTERN_SED = b'''s/\\bMarco Polo/Marco_Polo/
s/Kublai Khan/Kublai_Khan/
s.Christopher Columbus.Christopher_Columbus.
s/and uncle/and_uncle/'''

INPUT_TEXT = '''and uncle
sand uncle
s and uncle
Kublai Khan
bKublai Khan
Marco Polo
bMarco Polo'''

# without --words
WITHOUT_WORDS_OUTPUT = '''and_uncle
sand_uncle
s and_uncle
Kublai_Khan
bKublai_Khan
Marco_Polo
bMarco Polo'''

# with --words
WITH_WORDS_OUTPUT = '''and_uncle
sand uncle
s and_uncle
Kublai_Khan
bKublai Khan
Marco_Polo
bMarco Polo'''

class TestFsed(unittest.TestCase):
    '''
    Unit tests for the `fsed` module.
    '''

    def test_detect_format(self):
        '''
        Checks the fsed.detect_pattern_format function.
        '''
        pattern_file = CMStringIO(PATTERN_TSV)
        self.assertEqual(fsed.detect_pattern_format(pattern_file, 'utf-8', False),
                         (True, True))
        pattern_file = CMStringIO(PATTERN_SED)
        self.assertEqual(fsed.detect_pattern_format(pattern_file, 'utf-8', False),
                         (False, True))

    def test_rewriting_tsv(self):
        '''
        Tests the fsed.rewrite_str_with_trie function.
        '''
        pattern_file = CMStringIO(PATTERN_TSV)
        trie, boundaries = fsed.build_trie(pattern_file, 'tsv', 'utf-8', False)
        self.assertTrue(boundaries)
        self.assertEqual(fsed.rewrite_str_with_trie(INPUT_TEXT, trie, boundaries),
                         WITHOUT_WORDS_OUTPUT)
        trie, boundaries = fsed.build_trie(pattern_file, 'tsv', 'utf-8', True)
        self.assertTrue(boundaries)
        self.assertEqual(fsed.rewrite_str_with_trie(INPUT_TEXT, trie, boundaries),
                         WITH_WORDS_OUTPUT)

    def test_rewriting_sed(self):
        '''
        Tests the fsed.rewrite_str_with_trie function.
        '''
        pattern_file = CMStringIO(PATTERN_SED)
        trie, boundaries = fsed.build_trie(pattern_file, 'sed', 'utf-8', False)
        self.assertTrue('C' in trie.root)
        self.assertTrue(boundaries)
        self.assertEqual(fsed.rewrite_str_with_trie(INPUT_TEXT, trie, boundaries),
                         WITHOUT_WORDS_OUTPUT)
        trie, boundaries = fsed.build_trie(pattern_file, 'sed', 'utf-8', True)
        self.assertTrue(boundaries)
        self.assertEqual(fsed.rewrite_str_with_trie(INPUT_TEXT, trie, boundaries),
                         WITH_WORDS_OUTPUT)

    def test_slow(self):
        '''
        Tests the fsed.rewrite_str_with_trie function with the ``slow``
        flag on.
        '''
        trie = ahocorasick.AhoCorasickTrie()
        trie['a'] = '(a)'
        trie['ab'] = '(ab)'
        trie['bab'] = '(bab)'
        trie['bc'] = '(bc)'
        trie['bca'] = '(bca)'
        trie['c'] = '(c)'
        trie['caa'] = '(caa)'
        self.assertEqual(fsed.rewrite_str_with_trie('abccab', trie, slow=False),
                         '(a)(bc)(c)(a)b')
        self.assertEqual(fsed.rewrite_str_with_trie('abccab', trie, slow=True),
                         '(a)(bc)(c)(ab)')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-

# EarwigBot Configuration File
# This file contains information that the bot uses to connect to IRC.

# our server's hostname
HOST = "irc.freenode.net"

# our server's port
PORT = 6667

# our nick
NICK = "EarwigBot|2"

# our ident
IDENT = "earwigbot"

# our real name
REALNAME = "[[w:en:User:EarwigBot]]"

# channel to join on startup
CHANS = ["##earwigbot"]

# hostnames of users who can update/restart the bot with !update
ADMINS = ["wikipedia/The-Earwig"]
<commit_msg>set standard nick and on-connect channel joins<commit_after># -*- coding: utf-8  -*-

# EarwigBot Configuration File
# This file contains information that the bot uses to connect to IRC.

# our server's hostname
HOST = "irc.freenode.net"

# our server's port
PORT = 6667

# our nick
NICK = "EarwigBot"

# our ident
IDENT = "earwigbot"

# our real name
REALNAME = "[[w:en:User:EarwigBot]]"

# channel to join on startup
CHANS = ["##earwigbot", "##earwig", "#wikipedia-en-afc"]

# hostnames of users who can update/restart the bot with !update
ADMINS = ["wikipedia/The-Earwig"]
<|endoftext|>


<commit_before>from distutils.command.install import install
from distutils.core import setup
from distutils import log
import os
import json
import sys

kernel_json = {
    "argv": [sys.executable,
	     "-m", "metakernel_bash",
	     "-f", "{connection_file}"],
    "display_name": "MetaKernel Bash",
    "name": "metakernel_bash",
}

class install_with_kernelspec(install):
    def run(self):
        install.run(self)
        from IPython.kernel.kernelspec import install_kernel_spec
        from IPython.utils.tempdir import TemporaryDirectory
        from metakernel.utils.kernel import install_kernel_resources
        with TemporaryDirectory() as td:
            os.chmod(td, 0o755) # Starts off as 700, not user readable
            with open(os.path.join(td, 'kernel.json'), 'w') as f:
                json.dump(kernel_json, f, sort_keys=True)
            install_kernel_resources(td)
            log.info('Installing kernel spec')
            try:
                install_kernel_spec(td, 'metakernel_bash', user=self.user,
                                    replace=True)
            except:
                install_kernel_spec(td, 'metakernel_bash', user=not self.user,
                                    replace=True)

svem_flag = '--single-version-externally-managed'
if svem_flag in sys.argv:
    # Die, setuptools, die.
    sys.argv.remove(svem_flag)

setup(name='metakernel_bash',
      version='0.5.1',
      description='A Bash kernel for Jupyter/IPython',
      long_description="A Bash kernel for Jupyter/IPython, based on MetaKernel",
      url="https://github.com/blink1073/metakernel/tree/master/metakernel_bash",
      author='Steven Silvester',
      author_email='steven.silvester@ieee.org',
      py_modules=['metakernel_bash'],
      install_requires=["metakernel"],
      cmdclass={'install': install_with_kernelspec},
      classifiers = [
          'Framework :: IPython',
          'License :: OSI Approved :: BSD License',
          'Programming Language :: Python :: 3',
          'Programming Language :: Python :: 2',
          'Topic :: System :: Shells',
      ]
)
<commit_msg>Update setup.py<commit_after>from distutils.command.install import install
from distutils.core import setup
from distutils import log
import os
import json
import sys

kernel_json = {
    "argv": [sys.executable,
	     "-m", "metakernel_bash",
	     "-f", "{connection_file}"],
    "display_name": "MetaKernel Bash",
    "name": "metakernel_bash",
}

class install_with_kernelspec(install):
    def run(self):
        install.run(self)
        from IPython.kernel.kernelspec import install_kernel_spec
        from IPython.utils.tempdir import TemporaryDirectory
        from metakernel.utils.kernel import install_kernel_resources
        with TemporaryDirectory() as td:
            os.chmod(td, 0o755) # Starts off as 700, not user readable
            with open(os.path.join(td, 'kernel.json'), 'w') as f:
                json.dump(kernel_json, f, sort_keys=True)
            install_kernel_resources(td)
            log.info('Installing kernel spec')
            try:
                install_kernel_spec(td, 'metakernel_bash', user=self.user,
                                    replace=True)
            except:
                install_kernel_spec(td, 'metakernel_bash', user=not self.user,
                                    replace=True)

svem_flag = '--single-version-externally-managed'
if svem_flag in sys.argv:
    # Die, setuptools, die.
    sys.argv.remove(svem_flag)

setup(name='metakernel_bash',
      version='0.5.1',
      description='A Bash kernel for Jupyter/IPython',
      long_description="A Bash kernel for Jupyter/IPython, based on MetaKernel",
      url="https://github.com/calysto/metakernel/tree/master/metakernel_bash",
      author='Steven Silvester',
      author_email='steven.silvester@ieee.org',
      py_modules=['metakernel_bash'],
      install_requires=["metakernel"],
      cmdclass={'install': install_with_kernelspec},
      classifiers = [
          'Framework :: IPython',
          'License :: OSI Approved :: BSD License',
          'Programming Language :: Python :: 3',
          'Programming Language :: Python :: 2',
          'Topic :: System :: Shells',
      ]
)
<|endoftext|>