<commit_before>import json
import base64

from datetime import datetime

import pytest

from botocore.exceptions import ClientError

from zmon_aws_agent.aws import get_hash

ACCOUNT = 'aws:1234'
REGION = 'eu-central-1'

class ThrottleError(ClientError):
    def __init__(self, throttling=True):
        self.throttling = throttling
        self.response = {'Error': {'Code': 'Throttling' if throttling else 'BadRequest'}}

def get_elc_cluster():
    cluster = {
        'CacheClusterStatus': 'available',
        'CacheClusterId': 'elc-1',
        'Engine': 'redis',
        'EngineVersion': '1.0.5',
        'NumCacheNodes': 2,
        'CacheNodeType': 'redis',
        'ReplicationGroupId': 'elc-1-replica',
        'CacheNodes': [
            {
                'CacheNodeStatus': 'available', 'CacheNodeId': 'elc-n-1',
                'Endpoint': {'Port': 2727, 'Address': '0.0.0.0'}
            },
            {'CacheNodeStatus': 'unknown'}
        ]
    }

    resp = {
        'CacheClusters': [cluster.copy() for i in range(4)]
    }

    statuses = ('available', 'modifying', 'snapshotting', 'unknown')
    for idx, c in enumerate(resp['CacheClusters']):
        c['CacheClusterStatus'] = statuses[idx]

    node = {
        'id': 'elc-elc-1-elc-n-1[{}:{}]'.format(ACCOUNT, REGION),
        'region': REGION,
        'created_by': 'agent',
        'infrastructure_account': ACCOUNT,
        'type': 'elc',
        'cluster_id': 'elc-1',
        'node_id': 'elc-n-1',
        'engine': 'redis',
        'version': '1.0.5',
        'cluster_num_nodes': 2,
        'host': '0.0.0.0',
        'port': 2727,
        'instance_type': 'redis',
        'replication_group': 'elc-1-replica',
    }

    return resp, [node] * 3

def get_autoscaling():
    asg = {
        'AutoScalingGroupName': 'asg-1',
        'AvailabilityZones': ['zone-1', 'zone-2'],
        'DesiredCapacity': '3',
        'MaxSize': 10,
        'MinSize': 3,
        'Instances': [
            {'InstanceId': 'ins-1', 'LifecycleState': 'InService'},
            {'InstanceId': 'ins-2', 'LifecycleState': 'InService'},
            {'InstanceId': 'ins-3', 'LifecycleState': 'InService'},
            {'InstanceId': 'ins-4', 'LifecycleState': 'unknown'},
        ]
    }

    reservations = {
        'Reservations': [
            {
                'Instances': [
                    {'PrivateIpAddress': '192.168.20.16', 'InstanceId': 'ins-1'},
                    {'InstanceId': 'ins-2'}
                ]
            }
        ]
    }

    instance_ids = ['ins-1', 'ins-2', 'ins-3']

    resp = {
        'AutoScalingGroups': [asg]
    }

    result = [
        {
            'id': 'asg-asg-1[{}:{}]'.format(ACCOUNT, REGION),
            'type': 'asg',
            'infrastructure_account': ACCOUNT,
            'region': REGION,
            'created_by': 'agent',
            'name': 'asg-1',
            'availability_zones': ['zone-1', 'zone-2'],
            'desired_capacity': '3',
            'max_size': 10,
            'min_size': 3,
            'instances': [{'aws_id': 'ins-1', 'ip': '192.168.20.16'}],
        }
    ]

    return resp, reservations, instance_ids, result

def get_elbs():
    resp = {
        'LoadBalancerDescriptions': [
            {
                'LoadBalancerName': 'elb-1',
                'DNSName': 'elb-1.example.org',
                'Scheme': 'https',
                'Instances': ['ins-1', 'ins-2', 'ins-3'],
                'ListenerDescriptions': [{'Listener': {'Protocol': 'HTTPS'}}],
            },
        ]
    }

    tags = {'TagDescriptions': [{'LoadBalancerName': 'elb-1'}]}

    health = {
        'InstanceStates': [
            {'State': 'InService'},
            {'State': 'InService'},
            {'State': 'OutOfService'},
        ]
    }

    result = [
        {
            'id': 'elb-elb-1[{}:{}]'.format(ACCOUNT, REGION),
            'type': 'elb',
            'infrastructure_account': ACCOUNT,
            'region': REGION,
            'created_by': 'agent',
            'elb_type': 'classic',
            'dns_name': 'elb-1.example.org',
            'host': 'elb-1.example.org',
            'name': 'elb-1',
            'scheme': 'https',
            'url': 'https://elb-1.example.org',
            'members': 3,
            'active_members': 2,
        }
    ]

    return resp, tags, health, result

def get_elbs_application():
    resp = {
        'LoadBalancers': [
            {
                'LoadBalancerArn': 'arn-/app/elb-1/123456',
                'LoadBalancerName': 'elb-1',
                'DNSName': 'elb-1.example.org',
                'Scheme': 'internal',
            }
        ]
    }

    listeners = {
        'Listeners': [{'Protocol': 'HTTP'}]
    }

    tags = {'TagDescriptions': [{'ResourceArn': 'arn-/app/elb-1/123456', 'Tags': []}]}

    groups = {'TargetGroups': [{'TargetGroupArn': 'arn-group-1-elb-1'}]}

    health = {
        'TargetHealthDescriptions': [
            {'TargetHealth': {'State': 'healthy'}},
            {'TargetHealth': {'State': 'healthy'}},
            {'TargetHealth': {'State': 'terminated'}},
        ]
    }

    result = [
        {
            'id': 'elb-elb-1[{}:{}]'.format(ACCOUNT, REGION),
            'type': 'elb',
            'infrastructure_account': ACCOUNT,
            'region': REGION,
            'created_by': 'agent',
            'elb_type': 'application',
            'dns_name': 'elb-1.example.org',
            'host': 'elb-1.example.org',
            'cloudwatch_name': 'app/elb-1/123456',
            'name': 'elb-1',
            'scheme': 'internal',
            'url': 'http://elb-1.example.org',
            'members': 3,
            'active_members': 2,
            'target_groups': 1,
            'target_groups_arns': ['arn-group-1-elb-1'],
        }
    ]

    return resp, tags, listeners, groups, health, result

def get_apps():
    resp = {
        'Reservations': [
            {
                'OwnerId': '1234',
                'Instances': [
                    {
                        'State': {'Name': 'running'},
                        'PrivateIpAddress': '192.168.20.16', 'PublicIpAddress': '194.194.20.16',
                        'InstanceType': 't2.medium', 'InstanceId': 'ins-1', 'StateTransitionReason': 'state',
                        'Tags': [
                            {'Key': 'Name', 'Value': 'stack-1'}, {'Key': 'StackVersion', 'Value': 'stack-1-1.0'},
                            {'Key': 'aws:cloudformation:logical-id', 'Value': 'cd-app'}
                        ]
                    },
                    {
                        'State': {'Name': 'running'},
                        'PrivateIpAddress': '192.168.20.16',
                        'InstanceType': 't2.medium', 'InstanceId': 'ins-2', 'StateTransitionReason': 'state'
                    },
                    {
                        'State': {'Name': 'terminated'},
                    }
                ],
            }
        ]
    }

    status_resp = {'InstanceStatuses': [{'Events': ['ev-1', 'ev-2']}]}

    user_data = [
        {
            'application_id': 'app-1', 'source': 'registry/stups/zmon-aws-agent:cd81',
            'ports': [2222], 'runtime': 'docker',
            'application_version': '1.0',
        },
        {
            'no-appliacation-id': 'dummy'
        }
    ]

    user_resp = [{'UserData': {'Value': base64.encodebytes(bytes(json.dumps(u), 'utf-8'))}} for u in user_data]

    result = [
        {
            'id': 'app-1-stack-1-1.0-{}[{}:{}]'.format(get_hash('192.168.20.16'), ACCOUNT, REGION),
            'type': 'instance', 'created_by': 'agent', 'region': REGION, 'infrastructure_account': 'aws:1234',
            'ip': '192.168.20.16', 'host': '192.168.20.16', 'public_ip': '194.194.20.16',
            'instance_type': 't2.medium', 'aws_id': 'ins-1',
            'state_reason': 'state', 'stack': 'stack-1', 'stack_version': 'stack-1-1.0',
            'resource_id': 'cd-app', 'application_id': 'app-1', 'application_version': '1.0',
            'source': 'registry/stups/zmon-aws-agent:cd81', 'source_base': 'registry/stups/zmon-aws-agent',
            'ports': [2222], 'runtime': 'docker', 'aws:cloudformation:logical_id': 'cd-app', 'name': 'stack-1',
            'events': ['ev-1', 'ev-2'],
        },
        {
            'id': 'ins-2-{}[{}:{}]'.format(get_hash('192.168.20.16'), ACCOUNT, REGION),
            'type': 'instance', 'created_by': 'agent', 'region': REGION, 'infrastructure_account': 'aws:1234',
            'ip': '192.168.20.16', 'host': '192.168.20.16',
            'instance_type': 't2.medium', 'aws_id': 'ins-2',
        }
    ]

    return resp, status_resp, user_resp, result

def get_certificates():
    resp_iam = {
        'ServerCertificateMetadataList': [
            {
                'Arn': 'arn-iam-zmon-cert-1',
                'Expiration': datetime(2023, 4, 26, 0, 0),
                'Path': '/',
                'ServerCertificateId': '123456',
                'ServerCertificateName': 'zmon-cert-1',
                'UploadDate': datetime(2016, 4, 27, 11, 8, 50)
            }
        ]
    }

    resp_acm = {
        'CertificateSummaryList': [
            {
                'CertificateArn': 'arn-acm-zmon-cert-2',
                'DomainName': 'zmon-cert-2',
            },
            {
                'CertificateArn': 'arn-acm-zmon-cert-3',
                'DomainName': 'zmon-cert-3',
            },
        ]
    }

    acm_certs = [
        {
            'Certificate': {
                'DomainName': 'zmon-cert-2',
                'CertificateArn': 'arn-acm-zmon-cert-2',
                'Status': 'ISSUED',
                'NotAfter': datetime(2023, 4, 26, 0, 0),
            }
        },
        {
            'Certificate': {
                'DomainName': 'zmon-cert-3',
                'CertificateArn': 'arn-acm-zmon-cert-3',
                'Status': 'VALIDATION_TIMED_OUT',
            }
        }
    ]

    result = [
        {
            'type': 'certificate', 'status': 'ISSUED', 'region': REGION, 'arn': 'arn-iam-zmon-cert-1',
            'certificate_type': 'iam', 'id': 'cert-iam-zmon-cert-1[{}:{}]'.format(ACCOUNT, REGION),
            'infrastructure_account': ACCOUNT, 'expiration': '2023-04-26T00:00:00',
            'created_by': 'agent', 'name': 'zmon-cert-1',
        },
        {
            'type': 'certificate', 'status': 'ISSUED', 'region': REGION, 'arn': 'arn-acm-zmon-cert-2',
            'certificate_type': 'acm', 'id': 'cert-acm-zmon-cert-2[{}:{}]'.format(ACCOUNT, REGION),
            'infrastructure_account': ACCOUNT, 'expiration': '2023-04-26T00:00:00',
            'created_by': 'agent', 'name': 'zmon-cert-2',
        },
        {
            'type': 'certificate', 'status': 'VALIDATION_TIMED_OUT', 'region': REGION, 'arn': 'arn-acm-zmon-cert-3',
            'certificate_type': 'acm', 'id': 'cert-acm-zmon-cert-3[{}:{}]'.format(ACCOUNT, REGION),
            'infrastructure_account': ACCOUNT, 'expiration': '',
            'created_by': 'agent', 'name': 'zmon-cert-3',
        }
    ]

    return resp_iam, resp_acm, acm_certs, result

@pytest.fixture(params=[
    (
        {
            'DBInstances': [
                {'DBInstanceIdentifier': 'db-1', 'Engine': 'e-1', 'Endpoint': {'Port': 5432, 'Address': '0.0.0.0'}},
                {
                    'DBInstanceIdentifier': 'db-2', 'Engine': 'e-1', 'Endpoint': {'Port': 5432, 'Address': '0.0.0.0'},
                    'EngineVersion': '1.0.2', 'DBName': 'db-2-name',
                },
            ]
        },
        [
            {
                'id': 'rds-db-1[{}]', 'name': 'db-1', 'engine': 'e-1', 'port': 5432, 'host': '0.0.0.0',
                'type': 'database', 'shards': {'db-1': '0.0.0.0:5432/db-1'}
            },
            {
                'id': 'rds-db-2[{}]', 'name': 'db-2', 'engine': 'e-1', 'port': 5432, 'host': '0.0.0.0',
                'type': 'database', 'version': '1.0.2', 'shards': {'db-2-name': '0.0.0.0:5432/db-2-name'}
            },
        ]
    ),
    (
        RuntimeError,
        []
    )
])
def fx_rds(request):
    return request.param

@pytest.fixture(params=[
    (
        {
            'TableNames': ['t-1', 't-2', 't-3']  # paginator
        },
        [
            {'Table': {'TableStatus': 'ACTIVE', 'TableName': 't-1', 'TableArn': 'aws.t-1'}},
            {'Table': {'TableStatus': 'UPDATING', 'TableName': 't-2', 'TableArn': 'aws.t-2'}},
            {'Table': {'TableStatus': 'INACTIVE', 'TableName': 't-3', 'TableArn': 'aws.t-3'}},  # describe table
        ],
        [
            {'id': 'dynamodb-t-1[{}:{}]', 'type': 'dynamodb', 'name': 't-1', 'arn': 'aws.t-1'},
            {'id': 'dynamodb-t-2[{}:{}]', 'type': 'dynamodb', 'name': 't-2', 'arn': 'aws.t-2'},  # result
        ]
    ),
    (
        RuntimeError,
        [],
        []
    )
])
def fx_dynamodb(request):
    return request.param
<commit_msg>#67 add test for instance name<commit_after>import json
import base64

from datetime import datetime

import pytest

from botocore.exceptions import ClientError

from zmon_aws_agent.aws import get_hash

ACCOUNT = 'aws:1234'
REGION = 'eu-central-1'

class ThrottleError(ClientError):
    def __init__(self, throttling=True):
        self.throttling = throttling
        self.response = {'Error': {'Code': 'Throttling' if throttling else 'BadRequest'}}

def get_elc_cluster():
    cluster = {
        'CacheClusterStatus': 'available',
        'CacheClusterId': 'elc-1',
        'Engine': 'redis',
        'EngineVersion': '1.0.5',
        'NumCacheNodes': 2,
        'CacheNodeType': 'redis',
        'ReplicationGroupId': 'elc-1-replica',
        'CacheNodes': [
            {
                'CacheNodeStatus': 'available', 'CacheNodeId': 'elc-n-1',
                'Endpoint': {'Port': 2727, 'Address': '0.0.0.0'}
            },
            {'CacheNodeStatus': 'unknown'}
        ]
    }

    resp = {
        'CacheClusters': [cluster.copy() for i in range(4)]
    }

    statuses = ('available', 'modifying', 'snapshotting', 'unknown')
    for idx, c in enumerate(resp['CacheClusters']):
        c['CacheClusterStatus'] = statuses[idx]

    node = {
        'id': 'elc-elc-1-elc-n-1[{}:{}]'.format(ACCOUNT, REGION),
        'region': REGION,
        'created_by': 'agent',
        'infrastructure_account': ACCOUNT,
        'type': 'elc',
        'cluster_id': 'elc-1',
        'node_id': 'elc-n-1',
        'engine': 'redis',
        'version': '1.0.5',
        'cluster_num_nodes': 2,
        'host': '0.0.0.0',
        'port': 2727,
        'instance_type': 'redis',
        'replication_group': 'elc-1-replica',
    }

    return resp, [node] * 3

def get_autoscaling():
    asg = {
        'AutoScalingGroupName': 'asg-1',
        'AvailabilityZones': ['zone-1', 'zone-2'],
        'DesiredCapacity': '3',
        'MaxSize': 10,
        'MinSize': 3,
        'Instances': [
            {'InstanceId': 'ins-1', 'LifecycleState': 'InService'},
            {'InstanceId': 'ins-2', 'LifecycleState': 'InService'},
            {'InstanceId': 'ins-3', 'LifecycleState': 'InService'},
            {'InstanceId': 'ins-4', 'LifecycleState': 'unknown'},
        ]
    }

    reservations = {
        'Reservations': [
            {
                'Instances': [
                    {'PrivateIpAddress': '192.168.20.16', 'InstanceId': 'ins-1'},
                    {'InstanceId': 'ins-2'}
                ]
            }
        ]
    }

    instance_ids = ['ins-1', 'ins-2', 'ins-3']

    resp = {
        'AutoScalingGroups': [asg]
    }

    result = [
        {
            'id': 'asg-asg-1[{}:{}]'.format(ACCOUNT, REGION),
            'type': 'asg',
            'infrastructure_account': ACCOUNT,
            'region': REGION,
            'created_by': 'agent',
            'name': 'asg-1',
            'availability_zones': ['zone-1', 'zone-2'],
            'desired_capacity': '3',
            'max_size': 10,
            'min_size': 3,
            'instances': [{'aws_id': 'ins-1', 'ip': '192.168.20.16'}],
        }
    ]

    return resp, reservations, instance_ids, result

def get_elbs():
    resp = {
        'LoadBalancerDescriptions': [
            {
                'LoadBalancerName': 'elb-1',
                'DNSName': 'elb-1.example.org',
                'Scheme': 'https',
                'Instances': ['ins-1', 'ins-2', 'ins-3'],
                'ListenerDescriptions': [{'Listener': {'Protocol': 'HTTPS'}}],
            },
        ]
    }

    tags = {'TagDescriptions': [{'LoadBalancerName': 'elb-1'}]}

    health = {
        'InstanceStates': [
            {'State': 'InService'},
            {'State': 'InService'},
            {'State': 'OutOfService'},
        ]
    }

    result = [
        {
            'id': 'elb-elb-1[{}:{}]'.format(ACCOUNT, REGION),
            'type': 'elb',
            'infrastructure_account': ACCOUNT,
            'region': REGION,
            'created_by': 'agent',
            'elb_type': 'classic',
            'dns_name': 'elb-1.example.org',
            'host': 'elb-1.example.org',
            'name': 'elb-1',
            'scheme': 'https',
            'url': 'https://elb-1.example.org',
            'members': 3,
            'active_members': 2,
        }
    ]

    return resp, tags, health, result

def get_elbs_application():
    resp = {
        'LoadBalancers': [
            {
                'LoadBalancerArn': 'arn-/app/elb-1/123456',
                'LoadBalancerName': 'elb-1',
                'DNSName': 'elb-1.example.org',
                'Scheme': 'internal',
            }
        ]
    }

    listeners = {
        'Listeners': [{'Protocol': 'HTTP'}]
    }

    tags = {'TagDescriptions': [{'ResourceArn': 'arn-/app/elb-1/123456', 'Tags': []}]}

    groups = {'TargetGroups': [{'TargetGroupArn': 'arn-group-1-elb-1'}]}

    health = {
        'TargetHealthDescriptions': [
            {'TargetHealth': {'State': 'healthy'}},
            {'TargetHealth': {'State': 'healthy'}},
            {'TargetHealth': {'State': 'terminated'}},
        ]
    }

    result = [
        {
            'id': 'elb-elb-1[{}:{}]'.format(ACCOUNT, REGION),
            'type': 'elb',
            'infrastructure_account': ACCOUNT,
            'region': REGION,
            'created_by': 'agent',
            'elb_type': 'application',
            'dns_name': 'elb-1.example.org',
            'host': 'elb-1.example.org',
            'cloudwatch_name': 'app/elb-1/123456',
            'name': 'elb-1',
            'scheme': 'internal',
            'url': 'http://elb-1.example.org',
            'members': 3,
            'active_members': 2,
            'target_groups': 1,
            'target_groups_arns': ['arn-group-1-elb-1'],
        }
    ]

    return resp, tags, listeners, groups, health, result

def get_apps():
    resp = {
        'Reservations': [
            {
                'OwnerId': '1234',
                'Instances': [
                    {
                        'State': {'Name': 'running'},
                        'PrivateIpAddress': '192.168.20.16', 'PublicIpAddress': '194.194.20.16',
                        'InstanceType': 't2.medium', 'InstanceId': 'ins-1', 'StateTransitionReason': 'state',
                        'Tags': [
                            {'Key': 'Name', 'Value': 'stack-1'}, {'Key': 'StackVersion', 'Value': 'stack-1-1.0'},
                            {'Key': 'aws:cloudformation:logical-id', 'Value': 'cd-app'}
                        ]
                    },
                    {
                        'State': {'Name': 'running'},
                        'PrivateIpAddress': '192.168.20.16',
                        'InstanceType': 't2.medium', 'InstanceId': 'ins-2', 'StateTransitionReason': 'state'
                    },
                    {
                        'State': {'Name': 'terminated'},
                    },
                    {
                        'State': {'Name': 'running'},
                        'PrivateIpAddress': '192.168.20.17',
                        'InstanceType': 't2.medium', 'InstanceId': 'ins-3', 'StateTransitionReason': 'state',
                        'Tags': [
                            {'Key': 'Name', 'Value': 'myname'}
                        ]
                    },
                ],
            }
        ]
    }

    status_resp = {'InstanceStatuses': [{'Events': ['ev-1', 'ev-2']}]}

    user_data = [
        {
            'application_id': 'app-1', 'source': 'registry/stups/zmon-aws-agent:cd81',
            'ports': [2222], 'runtime': 'docker',
            'application_version': '1.0',
        },
        {
            'no-appliacation-id': 'dummy'
        }
    ]

    user_resp = [{'UserData': {'Value': base64.encodebytes(bytes(json.dumps(u), 'utf-8'))}} for u in user_data]

    result = [
        {
            'id': 'app-1-stack-1-1.0-{}[{}:{}]'.format(get_hash('192.168.20.16'), ACCOUNT, REGION),
            'type': 'instance', 'created_by': 'agent', 'region': REGION, 'infrastructure_account': 'aws:1234',
            'ip': '192.168.20.16', 'host': '192.168.20.16', 'public_ip': '194.194.20.16',
            'instance_type': 't2.medium', 'aws_id': 'ins-1',
            'state_reason': 'state', 'stack': 'stack-1', 'stack_version': 'stack-1-1.0',
            'resource_id': 'cd-app', 'application_id': 'app-1', 'application_version': '1.0',
            'source': 'registry/stups/zmon-aws-agent:cd81', 'source_base': 'registry/stups/zmon-aws-agent',
            'ports': [2222], 'runtime': 'docker', 'aws:cloudformation:logical_id': 'cd-app', 'name': 'stack-1',
            'events': ['ev-1', 'ev-2'],
        },
        {
            'id': 'ins-2-{}[{}:{}]'.format(get_hash('192.168.20.16'), ACCOUNT, REGION),
            'type': 'instance', 'created_by': 'agent', 'region': REGION, 'infrastructure_account': 'aws:1234',
            'ip': '192.168.20.16', 'host': '192.168.20.16',
            'instance_type': 't2.medium', 'aws_id': 'ins-2',
        },
        {
            'id': 'myname-{}[{}:{}]'.format(get_hash('192.168.20.17'), ACCOUNT, REGION),
            'type': 'instance', 'created_by': 'agent', 'region': REGION, 'infrastructure_account': 'aws:1234',
            'ip': '192.168.20.17', 'host': '192.168.20.17',
            'instance_type': 't2.medium', 'aws_id': 'ins-3',
        }
    ]

    return resp, status_resp, user_resp, result

def get_certificates():
    resp_iam = {
        'ServerCertificateMetadataList': [
            {
                'Arn': 'arn-iam-zmon-cert-1',
                'Expiration': datetime(2023, 4, 26, 0, 0),
                'Path': '/',
                'ServerCertificateId': '123456',
                'ServerCertificateName': 'zmon-cert-1',
                'UploadDate': datetime(2016, 4, 27, 11, 8, 50)
            }
        ]
    }

    resp_acm = {
        'CertificateSummaryList': [
            {
                'CertificateArn': 'arn-acm-zmon-cert-2',
                'DomainName': 'zmon-cert-2',
            },
            {
                'CertificateArn': 'arn-acm-zmon-cert-3',
                'DomainName': 'zmon-cert-3',
            },
        ]
    }

    acm_certs = [
        {
            'Certificate': {
                'DomainName': 'zmon-cert-2',
                'CertificateArn': 'arn-acm-zmon-cert-2',
                'Status': 'ISSUED',
                'NotAfter': datetime(2023, 4, 26, 0, 0),
            }
        },
        {
            'Certificate': {
                'DomainName': 'zmon-cert-3',
                'CertificateArn': 'arn-acm-zmon-cert-3',
                'Status': 'VALIDATION_TIMED_OUT',
            }
        }
    ]

    result = [
        {
            'type': 'certificate', 'status': 'ISSUED', 'region': REGION, 'arn': 'arn-iam-zmon-cert-1',
            'certificate_type': 'iam', 'id': 'cert-iam-zmon-cert-1[{}:{}]'.format(ACCOUNT, REGION),
            'infrastructure_account': ACCOUNT, 'expiration': '2023-04-26T00:00:00',
            'created_by': 'agent', 'name': 'zmon-cert-1',
        },
        {
            'type': 'certificate', 'status': 'ISSUED', 'region': REGION, 'arn': 'arn-acm-zmon-cert-2',
            'certificate_type': 'acm', 'id': 'cert-acm-zmon-cert-2[{}:{}]'.format(ACCOUNT, REGION),
            'infrastructure_account': ACCOUNT, 'expiration': '2023-04-26T00:00:00',
            'created_by': 'agent', 'name': 'zmon-cert-2',
        },
        {
            'type': 'certificate', 'status': 'VALIDATION_TIMED_OUT', 'region': REGION, 'arn': 'arn-acm-zmon-cert-3',
            'certificate_type': 'acm', 'id': 'cert-acm-zmon-cert-3[{}:{}]'.format(ACCOUNT, REGION),
            'infrastructure_account': ACCOUNT, 'expiration': '',
            'created_by': 'agent', 'name': 'zmon-cert-3',
        }
    ]

    return resp_iam, resp_acm, acm_certs, result

@pytest.fixture(params=[
    (
        {
            'DBInstances': [
                {'DBInstanceIdentifier': 'db-1', 'Engine': 'e-1', 'Endpoint': {'Port': 5432, 'Address': '0.0.0.0'}},
                {
                    'DBInstanceIdentifier': 'db-2', 'Engine': 'e-1', 'Endpoint': {'Port': 5432, 'Address': '0.0.0.0'},
                    'EngineVersion': '1.0.2', 'DBName': 'db-2-name',
                },
            ]
        },
        [
            {
                'id': 'rds-db-1[{}]', 'name': 'db-1', 'engine': 'e-1', 'port': 5432, 'host': '0.0.0.0',
                'type': 'database', 'shards': {'db-1': '0.0.0.0:5432/db-1'}
            },
            {
                'id': 'rds-db-2[{}]', 'name': 'db-2', 'engine': 'e-1', 'port': 5432, 'host': '0.0.0.0',
                'type': 'database', 'version': '1.0.2', 'shards': {'db-2-name': '0.0.0.0:5432/db-2-name'}
            },
        ]
    ),
    (
        RuntimeError,
        []
    )
])
def fx_rds(request):
    return request.param

@pytest.fixture(params=[
    (
        {
            'TableNames': ['t-1', 't-2', 't-3']  # paginator
        },
        [
            {'Table': {'TableStatus': 'ACTIVE', 'TableName': 't-1', 'TableArn': 'aws.t-1'}},
            {'Table': {'TableStatus': 'UPDATING', 'TableName': 't-2', 'TableArn': 'aws.t-2'}},
            {'Table': {'TableStatus': 'INACTIVE', 'TableName': 't-3', 'TableArn': 'aws.t-3'}},  # describe table
        ],
        [
            {'id': 'dynamodb-t-1[{}:{}]', 'type': 'dynamodb', 'name': 't-1', 'arn': 'aws.t-1'},
            {'id': 'dynamodb-t-2[{}:{}]', 'type': 'dynamodb', 'name': 't-2', 'arn': 'aws.t-2'},  # result
        ]
    ),
    (
        RuntimeError,
        [],
        []
    )
])
def fx_dynamodb(request):
    return request.param
<|endoftext|>


<commit_before>import functools
import logging
import os
import pytest
import tempfile

import devns
import devns.cli

from mock import MagicMock

@pytest.fixture
def config():
    return devns.Config()

@pytest.yield_fixture
def resolver_dir(config):
    resolvers = []
    config.resolver_dir = os.path.join(
        tempfile.gettempdir(),
        "{}-{}".format(tempfile.gettempprefix(), "resolver")
    )
    resolvers.append(config.resolver_dir)
    yield config.resolver_dir
    resolvers.append(config.resolver_dir)
    for resolver in filter(None, set(resolvers)):
        if os.path.isdir(resolver):
            os.rmdir(resolver)

@pytest.fixture
def logger(request):
    return logging.getLogger(request.node.nodeid)

@pytest.fixture
def parse_args(config):
    return functools.partial(devns.cli.parse_args, config=config)

@pytest.yield_fixture
def server(config, resolver_dir):
    yield devns.server.DevNS(config)

@pytest.fixture
def Connection():
    class Connection(object):
        settimeout = MagicMock()
        bind = MagicMock()
        sendto = MagicMock()

        def __init__(self, responses, expected):
            self.responses = responses
            self.expected = expected

        def getsockname(self):
            return "0.0.0.0", 53535

        def recvfrom(self, length):
            response = self.responses.pop()
            if isinstance(response, tuple):
                return response
            raise response()

    return Connection
<commit_msg>Whoops, forgot py26 doesn't like 0 length field names<commit_after>import functools
import logging
import os
import pytest
import tempfile

import devns
import devns.cli

from mock import MagicMock

@pytest.fixture
def config():
    return devns.Config()

@pytest.yield_fixture
def resolver_dir(config):
    resolvers = []
    config.resolver_dir = os.path.join(
        tempfile.gettempdir(),
        "{0}-{1}".format(tempfile.gettempprefix(), "resolver")
    )
    resolvers.append(config.resolver_dir)
    yield config.resolver_dir
    resolvers.append(config.resolver_dir)
    for resolver in filter(None, set(resolvers)):
        if os.path.isdir(resolver):
            os.rmdir(resolver)

@pytest.fixture
def logger(request):
    return logging.getLogger(request.node.nodeid)

@pytest.fixture
def parse_args(config):
    return functools.partial(devns.cli.parse_args, config=config)

@pytest.yield_fixture
def server(config, resolver_dir):
    yield devns.server.DevNS(config)

@pytest.fixture
def Connection():
    class Connection(object):
        settimeout = MagicMock()
        bind = MagicMock()
        sendto = MagicMock()

        def __init__(self, responses, expected):
            self.responses = responses
            self.expected = expected

        def getsockname(self):
            return "0.0.0.0", 53535

        def recvfrom(self, length):
            response = self.responses.pop()
            if isinstance(response, tuple):
                return response
            raise response()

    return Connection
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U
#
# This file is part of Orion Context Broker.
#
# Orion Context Broker is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Orion Context Broker is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.
#
# For those usages not covered by this license please contact with
# iot_support at tid dot es

# Hint: use 'PYTHONIOENCODING=utf8 python check_metadata_id.py' if you are going to redirect the output of this
# script to a file

__author__ = 'fermin'

import sys
import json
import pymongo
from datetime import datetime

##############################################################
# BEGIN of the configuration part (don't touch above this line ;)

uri = 'mongodb://localhost:27017'

autofix = False

verbose = False

# END of the configuration part (don't touch below this line ;)
##############################################################

ATTRS = 'attrs'

def flatten(_id):
    """
    The way in which Python manage dictionaries doesn't make easy to be sure
    of field ordering, which is important for MongoDB in the case of using an
    embedded document for _id. This function helps.

    :param _id: JSON document containing id, type and servicePath
    :return: a "flatten" version of the _id
    """

    r = {'_id.id': _id['id']}

    if 'type' in _id:
       r['_id.type'] = _id['type']
    else:
       r['_id.type'] = {'$exists': False}

    if 'servicePath' in _id:
       r['_id.servicePath'] = _id['servicePath']
    else:
       r['_id.servicePath'] = {'$exists': False}

    return r

def update_ok(doc, check_attrs):
    """
    Check that entity document was updated correctly at DB.

    :param doc: the doc to check
    :param check_attrs: list of attributes which existente is checked
    """

    if not ATTRS in doc:
        #print "debug1: no attrs"
        return False

    for attr in check_attrs:

        if attr not in doc[ATTRS]:
            #print "debug2: %s" % attr
            return False

    return True

def fix_location(entity, geo_attr, coords):
    """
    Fix location entity, adding the missing field.

    :param entity: entity to fix
    :param geo_attr: name of the geo:point attribute
    :param coords: coordinates of the geo:point attribute
    :return: "OK" if update when ok, "FAIL xxxx" otherwise
    """

    # Location example for reference:
    # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } }

    # Get coordinates and revert it (GeoJSON uses that)
    try:
        # Sanity check
        if len(coords.split(',')) != 2:
            raise ValueError

        coordinates = [float(coords.split(',')[1]), float(coords.split(',')[0])]
    except ValueError:
        return 'FAIL coordinates parsing error <%s>' % coords

    location = {
        'attrName': geo_attr,
        'coords': {
            'type': 'Point',
            'coordinates': coordinates
        }
    }

    try:
        # Update document with the new attribute fields
        db[COL].update(flatten(entity['_id']), {'$set': {'location': location}})

        # Check update was ok (this is not an exhaustive checking that is better than nothing :)
        check_doc = db[COL].find_one(flatten(entity['_id']))

        if update_ok(check_doc, entity[ATTRS]):
            return 'OK'
        else:
            return 'FAIL'

    except pymongo.errors.WriteError as e:
        return 'FAIL mongo error %d' % e.code

def fix_empty_geopoint(entity, geo_attr):
    """
    Fix location with empty geo:point, seting it to 0,0

    :param entity: entity to fix
    :param geo_attr: name of the geo:point attribute
    :return: "OK" if update when ok, "FAIL xxxx" otherwise
    """

    # Location example for reference:
    # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } }

    entity[ATTRS][geo_attr]['value'] = '0,0'

    location = {
        'attrName': geo_attr,
        'coords': {
            'type': 'Point',
            'coordinates': [0, 0]
        }
    }

    try:
        # Update document with the new attribute fields
        db[COL].update(flatten(entity['_id']), {'$set': {ATTRS: entity[ATTRS], 'location': location}})

        # Check update was ok (this is not an exhaustive checking that is better than nothing :)
        check_doc = db[COL].find_one(flatten(entity['_id']))

        if update_ok(check_doc, entity[ATTRS]):
            return 'OK'
        else:
            return 'FAIL'

    except pymongo.errors.WriteError as e:
        return 'FAIL mongo error %d' % e.code

def extract_geo_attr(attrs):
    """
    Given a key-value of attributes, returns the attribute with geo: type or None
    if no geo: attribute is found.

    :param attrs: key-value of attribute to process
    :return: an attribute name or None
    """

    for attr in attrs.keys():
        if attrs[attr]['type'].startswith('geo:'):
            return attr

    return None

def check_ngsiv1_location(attrs, location):
    """
    Check if there is an attribute in the entity which match location field in the "NGSIv1 location way", i.e.
    attribute type doesn't means anything but the value coordinates format matching the location

    :param attrs: entity attributes object to look
    :param location: location field
    :return: TRue if check is ok, False otherwise
    """

    # Location example for reference:
    # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } }

    # Only Point is allowed in NGSIv1
    if location['coords']['type'] != 'Point':
        return False

    loc_attr = None
    for attr in attrs.keys():
        if attr == location['attrName']:
            loc_attr = attrs[attr]
            break

    if loc_attr is None:
        return False

    value = loc_attr['value']

    # Coordinates format check
    if len(value.split(',')) != 2:
        return False

    # Check coordinates are equal
    try:
        if float(value.split(',')[0]) != location['coords']['coordinates'][1]:
            return False
        if float(value.split(',')[1]) != location['coords']['coordinates'][0]:
            return False

    except ValueError:
        # Format error: some of the token is not a valid float
        return False

    return True

def date2string(time):
    """
    Convert date to string

    :param time: date time (as timestamp)
    :return: string representing the date time
    """

    return datetime.fromtimestamp(time).strftime("%Y-%m-%dT%H:%M:%SZ")

def safe_add(d, k):
    """
    Add or create a key to dictionary if it doesn't already exists.

    :param d:
    :param k:
    :return:
    """

    if not k in d.keys():
        d[k] = True

def msg(m):
    """
    Print a message if verbose is enabled
    :param m: message to print
    :return:
    """

    if verbose:
        print m

##########################
# Main program starts here

if len(sys.argv) != 2:
    print "invalid number of arguments, use: ./check_location_coherence.py <database_name>"
    sys.exit()

DB = sys.argv[1]
COL = 'entities'

# Warn user
if autofix:
    print "WARNING!!!! This script modifies your '%s' database. It is STRONGLY RECOMMENDED that you" % DB
    print "do a backup of your database before using it as described in https://fiware-orion.readthedocs.io/en/master/admin/database_admin/index.html#backup. Use this script at your own risk."
    print "If you are sure you want to continue type 'yes' and press Enter"

    confirm = raw_input()

    if (confirm != 'yes'):
        sys.exit()

client = pymongo.MongoClient(uri)
db = client[DB]

need_help = False
processed = 0

# At the end, processed = no id + id duplicate attrs + id single attrs
counter_analysis = {
    'ngeo-nloc': 0,
    'geo-loc': 0,
    'geo-nloc': 0,
    'ngeo-loc': 0,
    'legacy': 0,
    'unfixable': 0,
    'emptygeopoint': 0,
}
# At the end, processed = untouched + changed + error
counter_update = {
    'untouched': 0,
    'changed': 0,
    'error': 0,
}

erroneous_geo_types = {}

total = db[COL].count()

print "- processing entities collection (%d entities) looking for location field coherence, this may take a while... " % total

# The sort() is a way of ensuring that a modified document doesn't enters again at the end of the cursor (we have
# observed that this may happen with large collections, e.g ~50,000 entities). In addition, we have to use
# batch_size so the cursor doesn't expires at server (see http://stackoverflow.com/questions/10298354/mongodb-cursor-id-not-valid-error).
# The used batch_size value is an heuristic
for doc in db[COL].find().sort([('_id.id', 1), ('_id.type', -1), ('_id.servicePath', 1)]).batch_size(100):

    processed += 1

    # Progress meter
    #sys.stdout.write('- processing entity: %d/%d   \r' % (processed, total) )
    #sys.stdout.flush()

    location = None
    geo_attr = extract_geo_attr(doc[ATTRS])
    if 'location' in doc:
        location = doc['location']

    if location is None:
        if geo_attr is None:
            # Entity without geo: attribute and without location field. It's ok.
            counter_analysis['ngeo-nloc'] += 1
            counter_update['untouched'] += 1
        else:  # geo_attr is not None
            geo_type = doc[ATTRS][geo_attr]['type']
            geo_value = doc[ATTRS][geo_attr]['value']

            if geo_type == 'geo:point':

                if geo_value == '':
                    # Entity with empty geo:point is a degenerated case. Fixable, setting 0,0 as coordinates.
                    counter_analysis['emptygeopoint'] += 1
                    if autofix:
                        result = fix_empty_geopoint(doc, geo_attr)
                        msg('   - {0}: fixing empty geo:point {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']),
                                                                                date2string(doc['modDate']), result))
                        if result == 'OK':
                            counter_update['changed'] += 1
                        else:
                            counter_update['error'] += 1
                            need_help = True
                    else:
                        msg('   - {0}: empty geo:point {1} ({2})'.format(processed, json.dumps(doc['_id']),
                                                                              date2string(doc['modDate'])))

                        counter_update['untouched'] += 1
                        need_help = True
                else:
                    # Entity with geo:point attribute but without location field. Fixable
                    counter_analysis['geo-nloc'] += 1
                    if autofix:
                        result = fix_location(doc, geo_attr, geo_value)
                        msg('   - {0}: fixing entity {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']),
                                                                            date2string(doc['modDate']), result))
                        if result == 'OK':
                            counter_update['changed'] += 1
                        else:
                            counter_update['error'] += 1
                            need_help = True
                    else:
                        msg('   - {0}: entity w/ geo:point but wo/ location  {1} ({2})'.format(processed, json.dumps(doc['_id']),
                                                                                date2string(doc['modDate'])))
                        counter_update['untouched'] += 1
                        need_help = True
            else:
                # Entity with geo: attribute different than geo:point but without location field. Not fixable
                msg('   - {0}: unfixable {1} {2} ({3})'.format(processed, geo_type, json.dumps(doc['_id']),
                                                               date2string(doc['modDate'])))

                safe_add(erroneous_geo_types, geo_type)

                counter_analysis['unfixable'] += 1
                counter_update['untouched'] += 1
                need_help = True

    else:  # location is not None
        if geo_attr is None:
            # Entity without geo: attribute location but with location field. They may come from NGSIv1
            # or be a real problem.
            if check_ngsiv1_location(doc[ATTRS], location):
                counter_analysis['legacy'] += 1
                counter_update['untouched'] += 1
            else:
                counter_analysis['ngeo-loc'] += 1
                counter_update['untouched'] += 1
                need_help = True
        else:  # geo_attr is not None
            # Entity with geo: attribute and with location field. It's ok
            # FIXME: it may happen that the GeoJSON at location field doesn't correspond with the one calculated
            # from geo: attribute. However, we consider that possibility very rare so we are not checking it
            counter_analysis['geo-loc'] += 1
            counter_update['untouched'] += 1

print '- processing entity: %d/%d' % (processed, total)
print '- documents analyzed:                                                          %d' % processed
print '  * entities wo/ geo: and wo/ location field (ok):                             %d' % counter_analysis['ngeo-nloc']
print '  * entities w/  geo: and w/  location field (ok):                             %d' % counter_analysis['geo-loc']
print '  * entities wo/ geo: and w/  location field - coherent (legacy NGSIv1, ok):   %d' % counter_analysis['legacy']
print '  ! entities wo/ geo: and w/  location field - not coherent (unfixable):       %d' % counter_analysis['ngeo-loc']
print '  ! entities w/ geo:point and wo/ location field (fixable):                    %d' % counter_analysis['geo-nloc']
print '  ! entities w/ empty string in geo:point (fixable)                            %d' % counter_analysis['emptygeopoint']
print '  ! entities w/ other geo: and wo/ location field (unfixable)                  %d' % counter_analysis['unfixable']

if len(erroneous_geo_types.keys()) > 0:

    print '* geo: types found without associated location field (except geo:point):       %s' % ','.join(erroneous_geo_types.keys())

print '- documents processed:                                                         %d' % processed
print '  * untouched:                                                                 %d' % counter_update['untouched']
print '  * changed:                                                                   %d' % counter_update['changed']
print '  * attempt to change but error:                                               %d' % counter_update['error']

if need_help:
    print "------------------------------------------------------"
    print "WARNING: some problem was found during the process. Ask for help!"
<commit_msg>FIX check_location_coherence.py narrow down geo: type dectection<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U
#
# This file is part of Orion Context Broker.
#
# Orion Context Broker is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Orion Context Broker is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.
#
# For those usages not covered by this license please contact with
# iot_support at tid dot es

# Hint: use 'PYTHONIOENCODING=utf8 python check_metadata_id.py' if you are going to redirect the output of this
# script to a file

__author__ = 'fermin'

import sys
import json
import pymongo
from datetime import datetime

##############################################################
# BEGIN of the configuration part (don't touch above this line ;)

uri = 'mongodb://localhost:27017'

autofix = False

verbose = False

# END of the configuration part (don't touch below this line ;)
##############################################################

ATTRS = 'attrs'
GEO_TYPES = [ 'geo:point', 'geo:line', 'geo:box', 'geo:polygon', 'geo:json']

def flatten(_id):
    """
    The way in which Python manage dictionaries doesn't make easy to be sure
    of field ordering, which is important for MongoDB in the case of using an
    embedded document for _id. This function helps.

    :param _id: JSON document containing id, type and servicePath
    :return: a "flatten" version of the _id
    """

    r = {'_id.id': _id['id']}

    if 'type' in _id:
       r['_id.type'] = _id['type']
    else:
       r['_id.type'] = {'$exists': False}

    if 'servicePath' in _id:
       r['_id.servicePath'] = _id['servicePath']
    else:
       r['_id.servicePath'] = {'$exists': False}

    return r

def update_ok(doc, check_attrs):
    """
    Check that entity document was updated correctly at DB.

    :param doc: the doc to check
    :param check_attrs: list of attributes which existente is checked
    """

    if not ATTRS in doc:
        #print "debug1: no attrs"
        return False

    for attr in check_attrs:

        if attr not in doc[ATTRS]:
            #print "debug2: %s" % attr
            return False

    return True

def fix_location(entity, geo_attr, coords):
    """
    Fix location entity, adding the missing field.

    :param entity: entity to fix
    :param geo_attr: name of the geo:point attribute
    :param coords: coordinates of the geo:point attribute
    :return: "OK" if update when ok, "FAIL xxxx" otherwise
    """

    # Location example for reference:
    # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } }

    # Get coordinates and revert it (GeoJSON uses that)
    try:
        # Sanity check
        if len(coords.split(',')) != 2:
            raise ValueError

        coordinates = [float(coords.split(',')[1]), float(coords.split(',')[0])]
    except ValueError:
        return 'FAIL coordinates parsing error <%s>' % coords

    location = {
        'attrName': geo_attr,
        'coords': {
            'type': 'Point',
            'coordinates': coordinates
        }
    }

    try:
        # Update document with the new attribute fields
        db[COL].update(flatten(entity['_id']), {'$set': {'location': location}})

        # Check update was ok (this is not an exhaustive checking that is better than nothing :)
        check_doc = db[COL].find_one(flatten(entity['_id']))

        if update_ok(check_doc, entity[ATTRS]):
            return 'OK'
        else:
            return 'FAIL'

    except pymongo.errors.WriteError as e:
        return 'FAIL mongo error %d' % e.code

def fix_empty_geopoint(entity, geo_attr):
    """
    Fix location with empty geo:point, seting it to 0,0

    :param entity: entity to fix
    :param geo_attr: name of the geo:point attribute
    :return: "OK" if update when ok, "FAIL xxxx" otherwise
    """

    # Location example for reference:
    # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } }

    entity[ATTRS][geo_attr]['value'] = '0,0'

    location = {
        'attrName': geo_attr,
        'coords': {
            'type': 'Point',
            'coordinates': [0, 0]
        }
    }

    try:
        # Update document with the new attribute fields
        db[COL].update(flatten(entity['_id']), {'$set': {ATTRS: entity[ATTRS], 'location': location}})

        # Check update was ok (this is not an exhaustive checking that is better than nothing :)
        check_doc = db[COL].find_one(flatten(entity['_id']))

        if update_ok(check_doc, entity[ATTRS]):
            return 'OK'
        else:
            return 'FAIL'

    except pymongo.errors.WriteError as e:
        return 'FAIL mongo error %d' % e.code

def extract_geo_attr(attrs):
    """
    Given a key-value of attributes, returns the attribute with geo: type or None
    if no geo: attribute is found.

    :param attrs: key-value of attribute to process
    :return: an attribute name or None
    """

    for attr in attrs.keys():
        if attrs[attr]['type'] in GEO_TYPES:
            return attr

    return None

def check_ngsiv1_location(attrs, location):
    """
    Check if there is an attribute in the entity which match location field in the "NGSIv1 location way", i.e.
    attribute type doesn't means anything but the value coordinates format matching the location

    :param attrs: entity attributes object to look
    :param location: location field
    :return: TRue if check is ok, False otherwise
    """

    # Location example for reference:
    # { "attrName" : "position", "coords" : { "type" : "Point", "coordinates" : [ -1.52363, 42.9113 ] } }

    # Only Point is allowed in NGSIv1
    if location['coords']['type'] != 'Point':
        return False

    loc_attr = None
    for attr in attrs.keys():
        if attr == location['attrName']:
            loc_attr = attrs[attr]
            break

    if loc_attr is None:
        return False

    value = loc_attr['value']

    # Coordinates format check
    if len(value.split(',')) != 2:
        return False

    # Check coordinates are equal
    try:
        if float(value.split(',')[0]) != location['coords']['coordinates'][1]:
            return False
        if float(value.split(',')[1]) != location['coords']['coordinates'][0]:
            return False

    except ValueError:
        # Format error: some of the token is not a valid float
        return False

    return True

def date2string(time):
    """
    Convert date to string

    :param time: date time (as timestamp)
    :return: string representing the date time
    """

    return datetime.fromtimestamp(time).strftime("%Y-%m-%dT%H:%M:%SZ")

def safe_add(d, k):
    """
    Add or create a key to dictionary if it doesn't already exists.

    :param d:
    :param k:
    :return:
    """

    if not k in d.keys():
        d[k] = True

def msg(m):
    """
    Print a message if verbose is enabled
    :param m: message to print
    :return:
    """

    if verbose:
        print m

##########################
# Main program starts here

if len(sys.argv) != 2:
    print "invalid number of arguments, use: ./check_location_coherence.py <database_name>"
    sys.exit()

DB = sys.argv[1]
COL = 'entities'

# Warn user
if autofix:
    print "WARNING!!!! This script modifies your '%s' database. It is STRONGLY RECOMMENDED that you" % DB
    print "do a backup of your database before using it as described in https://fiware-orion.readthedocs.io/en/master/admin/database_admin/index.html#backup. Use this script at your own risk."
    print "If you are sure you want to continue type 'yes' and press Enter"

    confirm = raw_input()

    if (confirm != 'yes'):
        sys.exit()

client = pymongo.MongoClient(uri)
db = client[DB]

need_help = False
processed = 0

# At the end, processed = no id + id duplicate attrs + id single attrs
counter_analysis = {
    'ngeo-nloc': 0,
    'geo-loc': 0,
    'geo-nloc': 0,
    'ngeo-loc': 0,
    'legacy': 0,
    'unfixable': 0,
    'emptygeopoint': 0,
}
# At the end, processed = untouched + changed + error
counter_update = {
    'untouched': 0,
    'changed': 0,
    'error': 0,
}

erroneous_geo_types = {}

total = db[COL].count()

print "- processing entities collection (%d entities) looking for location field coherence, this may take a while... " % total

# The sort() is a way of ensuring that a modified document doesn't enters again at the end of the cursor (we have
# observed that this may happen with large collections, e.g ~50,000 entities). In addition, we have to use
# batch_size so the cursor doesn't expires at server (see http://stackoverflow.com/questions/10298354/mongodb-cursor-id-not-valid-error).
# The used batch_size value is an heuristic
for doc in db[COL].find().sort([('_id.id', 1), ('_id.type', -1), ('_id.servicePath', 1)]).batch_size(100):

    processed += 1

    # Progress meter
    #sys.stdout.write('- processing entity: %d/%d   \r' % (processed, total) )
    #sys.stdout.flush()

    location = None
    geo_attr = extract_geo_attr(doc[ATTRS])
    if 'location' in doc:
        location = doc['location']

    if location is None:
        if geo_attr is None:
            # Entity without geo: attribute and without location field. It's ok.
            counter_analysis['ngeo-nloc'] += 1
            counter_update['untouched'] += 1
        else:  # geo_attr is not None
            geo_type = doc[ATTRS][geo_attr]['type']
            geo_value = doc[ATTRS][geo_attr]['value']

            if geo_type == 'geo:point':

                if geo_value == '':
                    # Entity with empty geo:point is a degenerated case. Fixable, setting 0,0 as coordinates.
                    counter_analysis['emptygeopoint'] += 1
                    if autofix:
                        result = fix_empty_geopoint(doc, geo_attr)
                        msg('   - {0}: fixing empty geo:point {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']),
                                                                                date2string(doc['modDate']), result))
                        if result == 'OK':
                            counter_update['changed'] += 1
                        else:
                            counter_update['error'] += 1
                            need_help = True
                    else:
                        msg('   - {0}: empty geo:point {1} ({2})'.format(processed, json.dumps(doc['_id']),
                                                                              date2string(doc['modDate'])))

                        counter_update['untouched'] += 1
                        need_help = True
                else:
                    # Entity with geo:point attribute but without location field. Fixable
                    counter_analysis['geo-nloc'] += 1
                    if autofix:
                        result = fix_location(doc, geo_attr, geo_value)
                        msg('   - {0}: fixing entity {1} ({2}): {3}'.format(processed, json.dumps(doc['_id']),
                                                                            date2string(doc['modDate']), result))
                        if result == 'OK':
                            counter_update['changed'] += 1
                        else:
                            counter_update['error'] += 1
                            need_help = True
                    else:
                        msg('   - {0}: entity w/ geo:point but wo/ location  {1} ({2})'.format(processed, json.dumps(doc['_id']),
                                                                                date2string(doc['modDate'])))
                        counter_update['untouched'] += 1
                        need_help = True
            else:
                # Entity with geo: attribute different than geo:point but without location field. Not fixable
                msg('   - {0}: unfixable {1} {2} ({3})'.format(processed, geo_type, json.dumps(doc['_id']),
                                                               date2string(doc['modDate'])))

                safe_add(erroneous_geo_types, geo_type)

                counter_analysis['unfixable'] += 1
                counter_update['untouched'] += 1
                need_help = True

    else:  # location is not None
        if geo_attr is None:
            # Entity without geo: attribute location but with location field. They may come from NGSIv1
            # or be a real problem.
            if check_ngsiv1_location(doc[ATTRS], location):
                counter_analysis['legacy'] += 1
                counter_update['untouched'] += 1
            else:
                counter_analysis['ngeo-loc'] += 1
                counter_update['untouched'] += 1
                need_help = True
        else:  # geo_attr is not None
            # Entity with geo: attribute and with location field. It's ok
            # FIXME: it may happen that the GeoJSON at location field doesn't correspond with the one calculated
            # from geo: attribute. However, we consider that possibility very rare so we are not checking it
            counter_analysis['geo-loc'] += 1
            counter_update['untouched'] += 1

print '- processing entity: %d/%d' % (processed, total)
print '- documents analyzed:                                                          %d' % processed
print '  * entities wo/ geo: and wo/ location field (ok):                             %d' % counter_analysis['ngeo-nloc']
print '  * entities w/  geo: and w/  location field (ok):                             %d' % counter_analysis['geo-loc']
print '  * entities wo/ geo: and w/  location field - coherent (legacy NGSIv1, ok):   %d' % counter_analysis['legacy']
print '  ! entities wo/ geo: and w/  location field - not coherent (unfixable):       %d' % counter_analysis['ngeo-loc']
print '  ! entities w/ geo:point and wo/ location field (fixable):                    %d' % counter_analysis['geo-nloc']
print '  ! entities w/ empty string in geo:point (fixable)                            %d' % counter_analysis['emptygeopoint']
print '  ! entities w/ other geo: and wo/ location field (unfixable)                  %d' % counter_analysis['unfixable']

if len(erroneous_geo_types.keys()) > 0:

    print '* geo: types found without associated location field (except geo:point):       %s' % ','.join(erroneous_geo_types.keys())

print '- documents processed:                                                         %d' % processed
print '  * untouched:                                                                 %d' % counter_update['untouched']
print '  * changed:                                                                   %d' % counter_update['changed']
print '  * attempt to change but error:                                               %d' % counter_update['error']

if need_help:
    print "------------------------------------------------------"
    print "WARNING: some problem was found during the process. Ask for help!"
<|endoftext|>


<commit_before>"""Imported by pytest at the start of every test session.

# Fixture Goals

Fixtures herein are made available to every test collected by pytest. They are
designed with the following goals in mind:
- Running a test on a microvm is as easy as importing a microvm fixture.
- Adding a new microvm image (kernel, rootfs) for tests to run on is as easy as
  uploading that image to a key-value store, e.g., an s3 bucket.

# Solution

- Keep microvm test images in an s3 bucket, structured as follows:

``` tree
s3://<bucket-url>/microvm-images/
    <microvm_test_image_folder_n>/
        kernel/
            <optional_kernel_name.>vmlinux.bin
        fsfiles/
            <rootfs_name>rootfs.ext4
            <other_fsfile_n>
            ...
        <other_resource_n>
        ...
    ...
```

- Tag `<microvm_test_image_folder_n>` with the capabilities of that image:

``` json
TagSet = [{"key": "capability:<cap_name>", "value": ""}, ...]
```

- Make available fixtures that expose microvms based on any given capability.
  For example, a test function using the fixture `test_microvm_any` should run
  on all microvm images in the s3 bucket, while a test using the fixture
  `test_microvm_with_net` should only run on the microvm images tagged with
  `capability:net`. Note that a test function that uses a parameterized fixture
  will yield one test case for every possible parameter of that fixture. For
  example, using `test_microvm_any` in a test will create as many test cases
  as there are microvm images in the s3 bucket.

- Provide fixtures that simplify other common testing operations, like http
  over local unix domain sockets.

# Example

```
def test_with_any_microvm(test_microvm_any, uhttp):

    response = uhttp.put(
        test_microvm_any.microvm_cfg_url,
        json={'vcpu_count': 2}
    )
    assert(uhttp.is_good_response(response.status_code))

    # [...]

    response = uhttp.put(
        test_microvm_any.actions_url + '/1',
        json={'action_id': '1', 'action_type': 'InstanceStart'}
    )
    assert(uhttp.is_good_response(response.status_code))
```

The test above makes use of the "any" test microvm fixture, so this test will
be run on every microvm image in the bucket, each as a separate test case.

# Notes

- Reading up on pytest fixtures is probably needed when editing this file.

# TODO

- A fixture that wraps `subprocess.run('<command>, shell=True, check=True)`,
  and also controls output verbosity by appending `>/dev/null [&2>1]`.
- A fixture that allows per-test-function dependency installation.
- Monitor of socket file creation via inotify.
- Support generating fixtures with more than one capability. This is supported
  by the MicrovmImageFetcher, but not by the fixture template.
"""

import threading
import os
import shutil
import tempfile
import uuid

import pytest

from host_tools.network import UniqueIPv4Generator
from microvm_image import MicrovmImageS3Fetcher
from microvm import JailerContext, MicrovmSlot, Microvm

SPEC_S3_BUCKET = 'spec.firecracker'
"""The s3 bucket that holds global Firecracker specifications."""

DEFAULT_TEST_IMAGES_S3_BUCKET = 'spec.firecracker'
"""The default s3 bucket that holds Firecracker microvm test images."""

ENV_TEST_IMAGES_S3_BUCKET = 'TEST_MICROVM_IMAGES_S3_BUCKET'
"""Environment variable for configuring the test microvm s3 bucket.

If variable exists in `os.environ`, its value will be used as the s3 bucket
for microvm test images.
"""

ENV_TMPDIR_VAR = 'TR_TMPDIR'
"""Environment variable for configuring temporary directory.

If variable exists in `os.environ`, its value it will be used for the test
session root and temporary directories.
"""

DEFAULT_ROOT_TESTSESSION_PATH = '/tmp/firecracker_test_session/'
"""If ENV_TMPDIR_VAR is not set, this path will be used instead."""

IP4_GENERATOR_CREATE_LOCK = threading.Lock()

# Some tests create system-level resources; ensure we run as root.
if os.geteuid() != 0:
    raise PermissionError("Test session needs to be run as root.")

@pytest.fixture(autouse=True, scope='session')
def test_session_root_path():
    """Ensure and yield the testrun root directory.

    If created here, it is also destroyed during teardown. The root directory
    is created per test session.
    """
    created_test_session_root_path = False

    try:
        root_path = os.environ[ENV_TMPDIR_VAR]
    except Exception:
        root_path = DEFAULT_ROOT_TESTSESSION_PATH
        raise Exception

    if not os.path.exists(root_path):
        os.makedirs(root_path)
        created_test_session_root_path = True

    yield root_path

    if created_test_session_root_path:
        shutil.rmtree(root_path)

@pytest.fixture
def test_session_tmp_path(test_session_root_path):
    """Yield a random temporary directory. Destroyed on teardown."""
    # pylint: disable=redefined-outer-name
    # The fixture pattern causes a pylint false positive for that rule.

    tmp_path = tempfile.mkdtemp(prefix=test_session_root_path)
    yield tmp_path
    shutil.rmtree(tmp_path)

@pytest.fixture
def microvm_slot(test_session_root_path):
    """Yield a microvm slot with an UUID as the slot id."""
    # pylint: disable=redefined-outer-name
    # The fixture pattern causes a pylint false positive for that rule.

    microvm_slot_id = str(uuid.uuid4())
    slot = MicrovmSlot(
        jailer_context=JailerContext.default_with_id(microvm_slot_id),
        slot_id=microvm_slot_id,
        microvm_root_path=test_session_root_path
    )
    slot.setup()
    yield slot
    slot.teardown()

@pytest.fixture
def microvm(microvm_slot):
    """Yield a spawned microvm in a given microvm slot."""
    # pylint: disable=redefined-outer-name
    # The fixture pattern causes a pylint false positive for that rule.

    vm = Microvm(microvm_slot, microvm_id=str(uuid.uuid4()))
    vm.spawn()
    vm.wait_create()
    yield vm
    vm.kill()

@pytest.fixture
def network_config():
    """Yield a UniqueIPv4Generator."""
    with IP4_GENERATOR_CREATE_LOCK:
        ipv4_generator = UniqueIPv4Generator.get_instance()
    yield ipv4_generator

@pytest.fixture
def microvm_image_fetcher():
    """Return a borg object that knows about fetching microvm images.

    If `ENV_TEST_IMAGES_S3_BUCKET` is set in the environment, target the bucket
    specified therein, else use the default.
    """
    if ENV_TEST_IMAGES_S3_BUCKET in os.environ:
        test_images_s3_bucket = os.environ.get(ENV_TEST_IMAGES_S3_BUCKET)
    else:
        test_images_s3_bucket = DEFAULT_TEST_IMAGES_S3_BUCKET

    return MicrovmImageS3Fetcher(test_images_s3_bucket)

@pytest.fixture(
    params=microvm_image_fetcher().list_microvm_images(
        capability_filter=['*']
    )
)
def test_microvm_any(request, microvm, microvm_image_fetcher):
    """Yield a microvm that can have any image in the spec bucket.

    A test case using this fixture will run for every microvm_image.

    When using a pytest parameterized fixture, a test case is created for each
    parameter in the list. We generate the list dynamically based on the
    capability filter. This will result in
    `len(microvm_image_fetcher.list_microvm_images(capability_filter=['*']))`
    test cases for each test that depends on this fixture, each receiving a
    microvm slot with a different microvm image.
    """
    # pylint: disable=redefined-outer-name
    # The fixture pattern causes a pylint false positive for that rule.

    microvm_image_fetcher.get_microvm_image(request.param, microvm.slot)
    yield microvm

TEST_MICROVM_CAP_FIXTURE_TEMPLATE = (
    "@pytest.fixture("
    "    params=microvm_image_fetcher().list_microvm_images(\n"
    "        capability_filter=['CAP']\n"
    "    )\n"
    ")\n"
    "def test_microvm_with_CAP(request, microvm, microvm_image_fetcher):\n"
    "    microvm_image_fetcher.get_microvm_image(\n"
    "        request.param, microvm.slot\n"
    "    )\n"
    "    yield microvm"
)

# To make test writing easy, we want to dynamically create fixtures with all
# capabilities present in the test microvm images bucket. `pytest` doesn't
# provide a way to do that outright, but luckily all of python is just lists of
# of lists and a cursor, so exec() works fine here.
#
# TODO: Support generating fixtures with more than one capability. This is
# supported by the MicrovmImageFetcher, but not by the fixture template.
for capability in microvm_image_fetcher().enum_capabilities():
    test_microvm_cap_fixture = (
        TEST_MICROVM_CAP_FIXTURE_TEMPLATE.replace('CAP', capability)
    )
    # pylint: disable=exec-used
    # This is the most straightforward way to achieve this result.
    exec(test_microvm_cap_fixture)
<commit_msg>tests: add a check for Python version >= 3.6<commit_after>"""Imported by pytest at the start of every test session.

# Fixture Goals

Fixtures herein are made available to every test collected by pytest. They are
designed with the following goals in mind:
- Running a test on a microvm is as easy as importing a microvm fixture.
- Adding a new microvm image (kernel, rootfs) for tests to run on is as easy as
  uploading that image to a key-value store, e.g., an s3 bucket.

# Solution

- Keep microvm test images in an s3 bucket, structured as follows:

``` tree
s3://<bucket-url>/microvm-images/
    <microvm_test_image_folder_n>/
        kernel/
            <optional_kernel_name.>vmlinux.bin
        fsfiles/
            <rootfs_name>rootfs.ext4
            <other_fsfile_n>
            ...
        <other_resource_n>
        ...
    ...
```

- Tag `<microvm_test_image_folder_n>` with the capabilities of that image:

``` json
TagSet = [{"key": "capability:<cap_name>", "value": ""}, ...]
```

- Make available fixtures that expose microvms based on any given capability.
  For example, a test function using the fixture `test_microvm_any` should run
  on all microvm images in the s3 bucket, while a test using the fixture
  `test_microvm_with_net` should only run on the microvm images tagged with
  `capability:net`. Note that a test function that uses a parameterized fixture
  will yield one test case for every possible parameter of that fixture. For
  example, using `test_microvm_any` in a test will create as many test cases
  as there are microvm images in the s3 bucket.

- Provide fixtures that simplify other common testing operations, like http
  over local unix domain sockets.

# Example

```
def test_with_any_microvm(test_microvm_any, uhttp):

    response = uhttp.put(
        test_microvm_any.microvm_cfg_url,
        json={'vcpu_count': 2}
    )
    assert(uhttp.is_good_response(response.status_code))

    # [...]

    response = uhttp.put(
        test_microvm_any.actions_url + '/1',
        json={'action_id': '1', 'action_type': 'InstanceStart'}
    )
    assert(uhttp.is_good_response(response.status_code))
```

The test above makes use of the "any" test microvm fixture, so this test will
be run on every microvm image in the bucket, each as a separate test case.

# Notes

- Reading up on pytest fixtures is probably needed when editing this file.

# TODO

- A fixture that wraps `subprocess.run('<command>, shell=True, check=True)`,
  and also controls output verbosity by appending `>/dev/null [&2>1]`.
- A fixture that allows per-test-function dependency installation.
- Monitor of socket file creation via inotify.
- Support generating fixtures with more than one capability. This is supported
  by the MicrovmImageFetcher, but not by the fixture template.
"""

import threading
import os
import shutil
import sys
import tempfile
import uuid

import pytest

from host_tools.network import UniqueIPv4Generator
from microvm_image import MicrovmImageS3Fetcher
from microvm import JailerContext, MicrovmSlot, Microvm

SPEC_S3_BUCKET = 'spec.firecracker'
"""The s3 bucket that holds global Firecracker specifications."""

DEFAULT_TEST_IMAGES_S3_BUCKET = 'spec.firecracker'
"""The default s3 bucket that holds Firecracker microvm test images."""

ENV_TEST_IMAGES_S3_BUCKET = 'TEST_MICROVM_IMAGES_S3_BUCKET'
"""Environment variable for configuring the test microvm s3 bucket.

If variable exists in `os.environ`, its value will be used as the s3 bucket
for microvm test images.
"""

ENV_TMPDIR_VAR = 'TR_TMPDIR'
"""Environment variable for configuring temporary directory.

If variable exists in `os.environ`, its value it will be used for the test
session root and temporary directories.
"""

DEFAULT_ROOT_TESTSESSION_PATH = '/tmp/firecracker_test_session/'
"""If ENV_TMPDIR_VAR is not set, this path will be used instead."""

IP4_GENERATOR_CREATE_LOCK = threading.Lock()

# This codebase uses Python features available in Python 3.6 or above
if sys.version_info < (3, 6):
    raise SystemError("This codebase requires Python 3.6 or above.")

# Some tests create system-level resources; ensure we run as root.
if os.geteuid() != 0:
    raise PermissionError("Test session needs to be run as root.")

@pytest.fixture(autouse=True, scope='session')
def test_session_root_path():
    """Ensure and yield the testrun root directory.

    If created here, it is also destroyed during teardown. The root directory
    is created per test session.
    """
    created_test_session_root_path = False

    try:
        root_path = os.environ[ENV_TMPDIR_VAR]
    except Exception:
        root_path = DEFAULT_ROOT_TESTSESSION_PATH
        raise Exception

    if not os.path.exists(root_path):
        os.makedirs(root_path)
        created_test_session_root_path = True

    yield root_path

    if created_test_session_root_path:
        shutil.rmtree(root_path)

@pytest.fixture
def test_session_tmp_path(test_session_root_path):
    """Yield a random temporary directory. Destroyed on teardown."""
    # pylint: disable=redefined-outer-name
    # The fixture pattern causes a pylint false positive for that rule.

    tmp_path = tempfile.mkdtemp(prefix=test_session_root_path)
    yield tmp_path
    shutil.rmtree(tmp_path)

@pytest.fixture
def microvm_slot(test_session_root_path):
    """Yield a microvm slot with an UUID as the slot id."""
    # pylint: disable=redefined-outer-name
    # The fixture pattern causes a pylint false positive for that rule.

    microvm_slot_id = str(uuid.uuid4())
    slot = MicrovmSlot(
        jailer_context=JailerContext.default_with_id(microvm_slot_id),
        slot_id=microvm_slot_id,
        microvm_root_path=test_session_root_path
    )
    slot.setup()
    yield slot
    slot.teardown()

@pytest.fixture
def microvm(microvm_slot):
    """Yield a spawned microvm in a given microvm slot."""
    # pylint: disable=redefined-outer-name
    # The fixture pattern causes a pylint false positive for that rule.

    vm = Microvm(microvm_slot, microvm_id=str(uuid.uuid4()))
    vm.spawn()
    vm.wait_create()
    yield vm
    vm.kill()

@pytest.fixture
def network_config():
    """Yield a UniqueIPv4Generator."""
    with IP4_GENERATOR_CREATE_LOCK:
        ipv4_generator = UniqueIPv4Generator.get_instance()
    yield ipv4_generator

@pytest.fixture
def microvm_image_fetcher():
    """Return a borg object that knows about fetching microvm images.

    If `ENV_TEST_IMAGES_S3_BUCKET` is set in the environment, target the bucket
    specified therein, else use the default.
    """
    if ENV_TEST_IMAGES_S3_BUCKET in os.environ:
        test_images_s3_bucket = os.environ.get(ENV_TEST_IMAGES_S3_BUCKET)
    else:
        test_images_s3_bucket = DEFAULT_TEST_IMAGES_S3_BUCKET

    return MicrovmImageS3Fetcher(test_images_s3_bucket)

@pytest.fixture(
    params=microvm_image_fetcher().list_microvm_images(
        capability_filter=['*']
    )
)
def test_microvm_any(request, microvm, microvm_image_fetcher):
    """Yield a microvm that can have any image in the spec bucket.

    A test case using this fixture will run for every microvm_image.

    When using a pytest parameterized fixture, a test case is created for each
    parameter in the list. We generate the list dynamically based on the
    capability filter. This will result in
    `len(microvm_image_fetcher.list_microvm_images(capability_filter=['*']))`
    test cases for each test that depends on this fixture, each receiving a
    microvm slot with a different microvm image.
    """
    # pylint: disable=redefined-outer-name
    # The fixture pattern causes a pylint false positive for that rule.

    microvm_image_fetcher.get_microvm_image(request.param, microvm.slot)
    yield microvm

TEST_MICROVM_CAP_FIXTURE_TEMPLATE = (
    "@pytest.fixture("
    "    params=microvm_image_fetcher().list_microvm_images(\n"
    "        capability_filter=['CAP']\n"
    "    )\n"
    ")\n"
    "def test_microvm_with_CAP(request, microvm, microvm_image_fetcher):\n"
    "    microvm_image_fetcher.get_microvm_image(\n"
    "        request.param, microvm.slot\n"
    "    )\n"
    "    yield microvm"
)

# To make test writing easy, we want to dynamically create fixtures with all
# capabilities present in the test microvm images bucket. `pytest` doesn't
# provide a way to do that outright, but luckily all of python is just lists of
# of lists and a cursor, so exec() works fine here.
#
# TODO: Support generating fixtures with more than one capability. This is
# supported by the MicrovmImageFetcher, but not by the fixture template.
for capability in microvm_image_fetcher().enum_capabilities():
    test_microvm_cap_fixture = (
        TEST_MICROVM_CAP_FIXTURE_TEMPLATE.replace('CAP', capability)
    )
    # pylint: disable=exec-used
    # This is the most straightforward way to achieve this result.
    exec(test_microvm_cap_fixture)
<|endoftext|>


<commit_before>import os
import pytest
from thefuck import shells
from thefuck import conf, const
from thefuck.system import Path

shells.shell = shells.Generic()

def pytest_addoption(parser):
    """Adds `--enable-functional` argument."""
    group = parser.getgroup("thefuck")
    group.addoption('--enable-functional', action="store_true", default=False,
                    help="Enable functional tests")

@pytest.fixture
def no_memoize(monkeypatch):
    monkeypatch.setattr('thefuck.utils.memoize.disabled', True)

@pytest.fixture(autouse=True)
def settings(request):
    def _reset_settings():
        conf.settings.clear()
        conf.settings.update(const.DEFAULT_SETTINGS)

    request.addfinalizer(_reset_settings)
    conf.settings.user_dir = Path('~/.thefuck')
    return conf.settings

@pytest.fixture
def no_colors(settings):
    settings.no_colors = True

@pytest.fixture(autouse=True)
def no_cache(monkeypatch):
    monkeypatch.setattr('thefuck.utils.cache.disabled', True)

@pytest.fixture(autouse=True)
def functional(request):
    if request.node.get_marker('functional') \
            and not request.config.getoption('enable_functional'):
        pytest.skip('functional tests are disabled')

@pytest.fixture
def source_root():
    return Path(__file__).parent.parent.resolve()

@pytest.fixture
def set_shell(monkeypatch):
    def _set(cls):
        shell = cls()
        monkeypatch.setattr('thefuck.shells.shell', shell)
        return shell

    return _set

@pytest.fixture(autouse=True)
def os_environ(monkeypatch):
    env = {'PATH': os.environ['PATH']}
    monkeypatch.setattr('os.environ', env)
    return env
<commit_msg>#N/A: Fix tests with pytest 4<commit_after>import os
import pytest
from thefuck import shells
from thefuck import conf, const
from thefuck.system import Path

shells.shell = shells.Generic()

def pytest_addoption(parser):
    """Adds `--enable-functional` argument."""
    group = parser.getgroup("thefuck")
    group.addoption('--enable-functional', action="store_true", default=False,
                    help="Enable functional tests")

@pytest.fixture
def no_memoize(monkeypatch):
    monkeypatch.setattr('thefuck.utils.memoize.disabled', True)

@pytest.fixture(autouse=True)
def settings(request):
    def _reset_settings():
        conf.settings.clear()
        conf.settings.update(const.DEFAULT_SETTINGS)

    request.addfinalizer(_reset_settings)
    conf.settings.user_dir = Path('~/.thefuck')
    return conf.settings

@pytest.fixture
def no_colors(settings):
    settings.no_colors = True

@pytest.fixture(autouse=True)
def no_cache(monkeypatch):
    monkeypatch.setattr('thefuck.utils.cache.disabled', True)

@pytest.fixture(autouse=True)
def functional(request):
    if request.node.get_closest_marker('functional') \
            and not request.config.getoption('enable_functional'):
        pytest.skip('functional tests are disabled')

@pytest.fixture
def source_root():
    return Path(__file__).parent.parent.resolve()

@pytest.fixture
def set_shell(monkeypatch):
    def _set(cls):
        shell = cls()
        monkeypatch.setattr('thefuck.shells.shell', shell)
        return shell

    return _set

@pytest.fixture(autouse=True)
def os_environ(monkeypatch):
    env = {'PATH': os.environ['PATH']}
    monkeypatch.setattr('os.environ', env)
    return env
<|endoftext|>