<commit_before># cython: auto_cpdef=True

'''Python code for reading AVRO files'''

# This code is a modified version of the code at
# http://svn.apache.org/viewvc/avro/trunk/lang/py/src/avro/ which is under
# Apache 2.0 license (http://www.apache.org/licenses/LICENSE-2.0)

import json
from os import SEEK_CUR
from struct import unpack
from zlib import decompress

try:
    from ._six import MemoryIO, xrange, btou, utob
    from ._schema import acquaint_schema, extract_record_type
except ImportError:
    from .six import MemoryIO, xrange, btou, utob
    from .schema import acquaint_schema, extract_record_type

VERSION = 1
MAGIC = b'Obj' + utob(chr(VERSION))
SYNC_SIZE = 16
HEADER_SCHEMA = {
    'type': 'record',
    'name': 'org.apache.avro.file.Header',
    'fields': [
        {
            'name': 'magic',
            'type': {'type': 'fixed', 'name': 'magic', 'size': len(MAGIC)},
            },
        {
            'name': 'meta',
            'type': {'type': 'map', 'values': 'bytes'}
            },
        {
            'name': 'sync',
            'type': {'type': 'fixed', 'name': 'sync', 'size': SYNC_SIZE}
            },
    ]
}
MASK = 0xFF

def read_null(fo, schema):
    '''null is written as zero bytes.'''
    return None

def read_boolean(fo, schema):
    '''A boolean is written as a single byte whose value is either 0 (false) or
    1 (true).
    '''

    # technically 0x01 == true and 0x00 == false, but many languages will cast anything
    # other than 0 to True and only 0 to False
    return fo.read(1) != 0x00

def read_long(fo, schema):
    '''int and long values are written using variable-length, zig-zag
    coding.'''
    c = fo.read(1)

    # We do EOF checking only here, since most reader start here
    if not c:
        raise StopIteration

    b = ord(c)
    n = b & 0x7F
    shift = 7

    while (b & 0x80) != 0:
        b = ord(fo.read(1))
        n |= (b & 0x7F) << shift
        shift += 7

    return (n >> 1) ^ -(n & 1)

def read_float(fo, schema):
    '''A float is written as 4 bytes.

    The float is converted into a 32-bit integer using a method equivalent to
    Java's floatToIntBits and then encoded in little-endian format.
    '''

    return unpack('<f', fo.read(4))[0]

def read_double(fo, schema):
    '''A double is written as 8 bytes.

    The double is converted into a 64-bit integer using a method equivalent to
    Java's doubleToLongBits and then encoded in little-endian format.
    '''
    return unpack('<d', fo.read(8))[0]

def read_bytes(fo, schema):
    '''Bytes are encoded as a long followed by that many bytes of data.'''
    size = read_long(fo, schema)
    return fo.read(size)

def read_utf8(fo, schema):
    '''A string is encoded as a long followed by that many bytes of UTF-8
    encoded character data.
    '''
    return btou(read_bytes(fo, schema), 'utf-8')

def read_fixed(fo, schema):
    '''Fixed instances are encoded using the number of bytes declared in the
    schema.'''
    return fo.read(schema['size'])

def read_enum(fo, schema):
    '''An enum is encoded by a int, representing the zero-based position of the
    symbol in the schema.
    '''
    index = read_long(fo, schema)
    return schema['symbols'][index]

def read_array(fo, schema):
    '''Arrays are encoded as a series of blocks.

    Each block consists of a long count value, followed by that many array
    items.  A block with count zero indicates the end of the array.  Each item
    is encoded per the array's item schema.

    If a block's count is negative, then the count is followed immediately by a
    long block size, indicating the number of bytes in the block.  The actual
    count in this case is the absolute value of the count written.
    '''
    read_items = []

    block_count = read_long(fo, schema)

    while block_count != 0:
        if block_count < 0:
            block_count = -block_count
            # Read block size, unused
            read_long(fo, schema)

        for i in xrange(block_count):
            read_items.append(read_data(fo, schema['items']))
        block_count = read_long(fo, schema)

    return read_items

def read_map(fo, schema):
    '''Maps are encoded as a series of blocks.

    Each block consists of a long count value, followed by that many key/value
    pairs.  A block with count zero indicates the end of the map.  Each item is
    encoded per the map's value schema.

    If a block's count is negative, then the count is followed immediately by a
    long block size, indicating the number of bytes in the block.  The actual
    count in this case is the absolute value of the count written.
    '''
    read_items = {}
    block_count = read_long(fo, schema)
    while block_count != 0:
        if block_count < 0:
            block_count = -block_count
            # Read block size, unused
            read_long(fo, schema)

        for i in xrange(block_count):
            key = read_utf8(fo, schema)
            read_items[key] = read_data(fo, schema['values'])
        block_count = read_long(fo, schema)

    return read_items

def read_union(fo, schema):
    '''A union is encoded by first writing a long value indicating the
    zero-based position within the union of the schema of its value.

    The value is then encoded per the indicated schema within the union.
    '''
    # schema resolution
    index = read_long(fo, schema)
    return read_data(fo, schema[index])

def read_record(fo, schema):
    '''A record is encoded by encoding the values of its fields in the order
    that they are declared. In other words, a record is encoded as just the
    concatenation of the encodings of its fields.  Field values are encoded per
    their schema.

    Schema Resolution:
     * the ordering of fields may be different: fields are matched by name.
     * schemas for fields with the same name in both records are resolved
         recursively.
     * if the writer's record contains a field with a name not present in the
         reader's record, the writer's value for that field is ignored.
     * if the reader's record schema has a field that contains a default value,
         and writer's schema does not have a field with the same name, then the
         reader should use the default value from its field.
     * if the reader's record schema has a field with no default value, and
         writer's schema does not have a field with the same name, then the
         field's value is unset.
    '''
    record = {}
    for field in schema['fields']:
        record[field['name']] = read_data(fo, field['type'])

    return record

READERS = {
    'null': read_null,
    'boolean': read_boolean,
    'string': read_utf8,
    'int': read_long,
    'long': read_long,
    'float': read_float,
    'double': read_double,
    'bytes': read_bytes,
    'fixed': read_fixed,
    'enum': read_enum,
    'array': read_array,
    'map': read_map,
    'union': read_union,
    'error_union': read_union,
    'record': read_record,
    'error': read_record,
    'request': read_record,
}

def read_data(fo, schema):
    '''Read data from file object according to schema.'''

    return READERS[extract_record_type(schema)](fo, schema)

def skip_sync(fo, sync_marker):
    '''Skip sync marker, might raise StopIteration.'''
    mark = fo.read(SYNC_SIZE)

    if not mark:
        raise StopIteration

    if mark != sync_marker:
        fo.seek(-SYNC_SIZE, SEEK_CUR)

def null_read_block(fo):
    '''Read block in "null" codec.'''
    read_long(fo, None)
    return fo

def deflate_read_block(fo):
    '''Read block in "deflate" codec.'''
    data = read_bytes(fo, None)
    # -15 is the log of the window size; negative indicates "raw" (no
    # zlib headers) decompression.  See zlib.h.
    return MemoryIO(decompress(data, -15))

BLOCK_READERS = {
    'null': null_read_block,
    'deflate': deflate_read_block
}

try:
    import snappy

    def snappy_read_block(fo):
        length = read_long(fo, None)
        data = fo.read(length - 4)
        fo.read(4)  # CRC
        return MemoryIO(snappy.decompress(data))

    BLOCK_READERS['snappy'] = snappy_read_block
except ImportError:
    pass

def _iter_avro(fo, header, codec, schema):
    '''Return iterator over avro records.'''
    sync_marker = header['sync']
    # Value in schema is bytes

    read_block = BLOCK_READERS.get(codec)
    if not read_block:
        raise ValueError('Unrecognized codec: {0!r}'.format(codec))

    block_count = 0
    while True:
        skip_sync(fo, sync_marker)
        block_count = read_long(fo, None)
        block_fo = read_block(fo)

        for i in xrange(block_count):
            yield read_data(block_fo, schema)

class iter_avro:
    '''Custom iterator over avro file.

    Example:
        with open('some-file.avro', 'rb') as fo:
            avro = iter_avro(fo)
            schema = avro.schema

            for record in avro:
                process_record(record)
    '''
    def __init__(self, fo):
        self.fo = fo
        try:
            self._header = read_data(fo, HEADER_SCHEMA)
        except StopIteration:
            raise ValueError('cannot read header - is it an avro file?')

        # `meta` values are bytes. So, the actual decoding has to be external.
        self.schema = schema = \
            json.loads(btou(self._header['meta']['avro.schema']))
        self.codec = btou(self._header['meta'].get('avro.codec', b'null'))

        acquaint_schema(schema)
        self._records = _iter_avro(fo, self._header, self.codec, schema)

    def __iter__(self):
        return self._records

    def next(self):
        return next(self._records)
<commit_msg>Shorten line to please ci<commit_after># cython: auto_cpdef=True

'''Python code for reading AVRO files'''

# This code is a modified version of the code at
# http://svn.apache.org/viewvc/avro/trunk/lang/py/src/avro/ which is under
# Apache 2.0 license (http://www.apache.org/licenses/LICENSE-2.0)

import json
from os import SEEK_CUR
from struct import unpack
from zlib import decompress

try:
    from ._six import MemoryIO, xrange, btou, utob
    from ._schema import acquaint_schema, extract_record_type
except ImportError:
    from .six import MemoryIO, xrange, btou, utob
    from .schema import acquaint_schema, extract_record_type

VERSION = 1
MAGIC = b'Obj' + utob(chr(VERSION))
SYNC_SIZE = 16
HEADER_SCHEMA = {
    'type': 'record',
    'name': 'org.apache.avro.file.Header',
    'fields': [
        {
            'name': 'magic',
            'type': {'type': 'fixed', 'name': 'magic', 'size': len(MAGIC)},
            },
        {
            'name': 'meta',
            'type': {'type': 'map', 'values': 'bytes'}
            },
        {
            'name': 'sync',
            'type': {'type': 'fixed', 'name': 'sync', 'size': SYNC_SIZE}
            },
    ]
}
MASK = 0xFF

def read_null(fo, schema):
    '''null is written as zero bytes.'''
    return None

def read_boolean(fo, schema):
    '''A boolean is written as a single byte whose value is either 0 (false) or
    1 (true).
    '''

    # technically 0x01 == true and 0x00 == false, but many languages will cast
    # anything other than 0 to True and only 0 to False
    return fo.read(1) != 0x00

def read_long(fo, schema):
    '''int and long values are written using variable-length, zig-zag
    coding.'''
    c = fo.read(1)

    # We do EOF checking only here, since most reader start here
    if not c:
        raise StopIteration

    b = ord(c)
    n = b & 0x7F
    shift = 7

    while (b & 0x80) != 0:
        b = ord(fo.read(1))
        n |= (b & 0x7F) << shift
        shift += 7

    return (n >> 1) ^ -(n & 1)

def read_float(fo, schema):
    '''A float is written as 4 bytes.

    The float is converted into a 32-bit integer using a method equivalent to
    Java's floatToIntBits and then encoded in little-endian format.
    '''

    return unpack('<f', fo.read(4))[0]

def read_double(fo, schema):
    '''A double is written as 8 bytes.

    The double is converted into a 64-bit integer using a method equivalent to
    Java's doubleToLongBits and then encoded in little-endian format.
    '''
    return unpack('<d', fo.read(8))[0]

def read_bytes(fo, schema):
    '''Bytes are encoded as a long followed by that many bytes of data.'''
    size = read_long(fo, schema)
    return fo.read(size)

def read_utf8(fo, schema):
    '''A string is encoded as a long followed by that many bytes of UTF-8
    encoded character data.
    '''
    return btou(read_bytes(fo, schema), 'utf-8')

def read_fixed(fo, schema):
    '''Fixed instances are encoded using the number of bytes declared in the
    schema.'''
    return fo.read(schema['size'])

def read_enum(fo, schema):
    '''An enum is encoded by a int, representing the zero-based position of the
    symbol in the schema.
    '''
    index = read_long(fo, schema)
    return schema['symbols'][index]

def read_array(fo, schema):
    '''Arrays are encoded as a series of blocks.

    Each block consists of a long count value, followed by that many array
    items.  A block with count zero indicates the end of the array.  Each item
    is encoded per the array's item schema.

    If a block's count is negative, then the count is followed immediately by a
    long block size, indicating the number of bytes in the block.  The actual
    count in this case is the absolute value of the count written.
    '''
    read_items = []

    block_count = read_long(fo, schema)

    while block_count != 0:
        if block_count < 0:
            block_count = -block_count
            # Read block size, unused
            read_long(fo, schema)

        for i in xrange(block_count):
            read_items.append(read_data(fo, schema['items']))
        block_count = read_long(fo, schema)

    return read_items

def read_map(fo, schema):
    '''Maps are encoded as a series of blocks.

    Each block consists of a long count value, followed by that many key/value
    pairs.  A block with count zero indicates the end of the map.  Each item is
    encoded per the map's value schema.

    If a block's count is negative, then the count is followed immediately by a
    long block size, indicating the number of bytes in the block.  The actual
    count in this case is the absolute value of the count written.
    '''
    read_items = {}
    block_count = read_long(fo, schema)
    while block_count != 0:
        if block_count < 0:
            block_count = -block_count
            # Read block size, unused
            read_long(fo, schema)

        for i in xrange(block_count):
            key = read_utf8(fo, schema)
            read_items[key] = read_data(fo, schema['values'])
        block_count = read_long(fo, schema)

    return read_items

def read_union(fo, schema):
    '''A union is encoded by first writing a long value indicating the
    zero-based position within the union of the schema of its value.

    The value is then encoded per the indicated schema within the union.
    '''
    # schema resolution
    index = read_long(fo, schema)
    return read_data(fo, schema[index])

def read_record(fo, schema):
    '''A record is encoded by encoding the values of its fields in the order
    that they are declared. In other words, a record is encoded as just the
    concatenation of the encodings of its fields.  Field values are encoded per
    their schema.

    Schema Resolution:
     * the ordering of fields may be different: fields are matched by name.
     * schemas for fields with the same name in both records are resolved
         recursively.
     * if the writer's record contains a field with a name not present in the
         reader's record, the writer's value for that field is ignored.
     * if the reader's record schema has a field that contains a default value,
         and writer's schema does not have a field with the same name, then the
         reader should use the default value from its field.
     * if the reader's record schema has a field with no default value, and
         writer's schema does not have a field with the same name, then the
         field's value is unset.
    '''
    record = {}
    for field in schema['fields']:
        record[field['name']] = read_data(fo, field['type'])

    return record

READERS = {
    'null': read_null,
    'boolean': read_boolean,
    'string': read_utf8,
    'int': read_long,
    'long': read_long,
    'float': read_float,
    'double': read_double,
    'bytes': read_bytes,
    'fixed': read_fixed,
    'enum': read_enum,
    'array': read_array,
    'map': read_map,
    'union': read_union,
    'error_union': read_union,
    'record': read_record,
    'error': read_record,
    'request': read_record,
}

def read_data(fo, schema):
    '''Read data from file object according to schema.'''

    return READERS[extract_record_type(schema)](fo, schema)

def skip_sync(fo, sync_marker):
    '''Skip sync marker, might raise StopIteration.'''
    mark = fo.read(SYNC_SIZE)

    if not mark:
        raise StopIteration

    if mark != sync_marker:
        fo.seek(-SYNC_SIZE, SEEK_CUR)

def null_read_block(fo):
    '''Read block in "null" codec.'''
    read_long(fo, None)
    return fo

def deflate_read_block(fo):
    '''Read block in "deflate" codec.'''
    data = read_bytes(fo, None)
    # -15 is the log of the window size; negative indicates "raw" (no
    # zlib headers) decompression.  See zlib.h.
    return MemoryIO(decompress(data, -15))

BLOCK_READERS = {
    'null': null_read_block,
    'deflate': deflate_read_block
}

try:
    import snappy

    def snappy_read_block(fo):
        length = read_long(fo, None)
        data = fo.read(length - 4)
        fo.read(4)  # CRC
        return MemoryIO(snappy.decompress(data))

    BLOCK_READERS['snappy'] = snappy_read_block
except ImportError:
    pass

def _iter_avro(fo, header, codec, schema):
    '''Return iterator over avro records.'''
    sync_marker = header['sync']
    # Value in schema is bytes

    read_block = BLOCK_READERS.get(codec)
    if not read_block:
        raise ValueError('Unrecognized codec: {0!r}'.format(codec))

    block_count = 0
    while True:
        skip_sync(fo, sync_marker)
        block_count = read_long(fo, None)
        block_fo = read_block(fo)

        for i in xrange(block_count):
            yield read_data(block_fo, schema)

class iter_avro:
    '''Custom iterator over avro file.

    Example:
        with open('some-file.avro', 'rb') as fo:
            avro = iter_avro(fo)
            schema = avro.schema

            for record in avro:
                process_record(record)
    '''
    def __init__(self, fo):
        self.fo = fo
        try:
            self._header = read_data(fo, HEADER_SCHEMA)
        except StopIteration:
            raise ValueError('cannot read header - is it an avro file?')

        # `meta` values are bytes. So, the actual decoding has to be external.
        self.schema = schema = \
            json.loads(btou(self._header['meta']['avro.schema']))
        self.codec = btou(self._header['meta'].get('avro.codec', b'null'))

        acquaint_schema(schema)
        self._records = _iter_avro(fo, self._header, self.codec, schema)

    def __iter__(self):
        return self._records

    def next(self):
        return next(self._records)
<|endoftext|>


<commit_before>import unittest
from unittest.mock import patch
from dagda.api.service.monitor import start_monitor_by_container_id
from dagda.api.service.monitor import stop_monitor_by_container_id

# -- Test suite

class MonitorApiTestCase(unittest.TestCase):

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=False)
    def test_start_monitor_by_container_id_503(self,m):
        response, code = start_monitor_by_container_id('test_id')
        self.assertEqual(code, 503)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=False)
    def test_stop_monitor_by_container_id_503(self,m):
        response, code = stop_monitor_by_container_id('test_id')
        self.assertEqual(code, 503)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    def test_start_monitor_by_container_id_400(self, m):
        response, code = start_monitor_by_container_id(None)
        self.assertEqual(code, 400)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    def test_stop_monitor_by_container_id_400(self, m):
        response, code = stop_monitor_by_container_id(None)
        self.assertEqual(code, 400)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    def test_start_monitor_by_container_id_404(self, m):
        response, code = start_monitor_by_container_id('fake_id')
        self.assertEqual(code, 404)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    def test_stop_monitor_by_container_id_404(self, m):
        response, code = stop_monitor_by_container_id('fake_id')
        self.assertEqual(code, 404)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Added some tests<commit_after>import unittest
from unittest.mock import patch
from dagda.api.service.monitor import start_monitor_by_container_id
from dagda.api.service.monitor import stop_monitor_by_container_id

# -- Test suite

class MonitorApiTestCase(unittest.TestCase):

    # -- Mock internal classes

    class MockDockerDriver():
        def get_docker_image_name_from_container_id(self, id):
            return 'redis'

    class MockMongoDriverStartTrue():
        def is_there_a_started_monitoring(self, id):
            return True

        def update_runtime_monitoring_analysis(self, id):
            return

        def get_a_started_monitoring(self, id):
            return {'runtime_analysis': {'stop_timestamp': None}, 'status': None, '_id': 1234567890}

        def update_docker_image_scan_result_to_history(self, id, result):
            return

        def get_docker_image_history(self, image_name, id):
            return [{'runtime_analysis': {'stop_timestamp': None}, 'status': None, '_id': 1234567890}]

    class MockMongoDriverStartFalse():
        def is_there_a_started_monitoring(self, id):
            return False

        def insert_docker_image_scan_result_to_history(self, history):
            return 1234567890

    # -- Tests

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=False)
    def test_start_monitor_by_container_id_503(self,m):
        response, code = start_monitor_by_container_id('test_id')
        self.assertEqual(code, 503)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=False)
    def test_stop_monitor_by_container_id_503(self,m):
        response, code = stop_monitor_by_container_id('test_id')
        self.assertEqual(code, 503)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    def test_start_monitor_by_container_id_400(self, m):
        response, code = start_monitor_by_container_id(None)
        self.assertEqual(code, 400)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    def test_stop_monitor_by_container_id_400(self, m):
        response, code = stop_monitor_by_container_id(None)
        self.assertEqual(code, 400)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    def test_start_monitor_by_container_id_404(self, m):
        response, code = start_monitor_by_container_id('fake_id')
        self.assertEqual(code, 404)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    def test_stop_monitor_by_container_id_404(self, m):
        response, code = stop_monitor_by_container_id('fake_id')
        self.assertEqual(code, 404)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    @patch('api.internal.internal_server.InternalServer.get_docker_driver', return_value=MockDockerDriver())
    @patch('api.internal.internal_server.InternalServer.get_mongodb_driver', return_value=MockMongoDriverStartTrue())
    def test_start_monitor_by_container_id_400_started(self, m1, m2, m3):
        response, code = start_monitor_by_container_id('id')
        self.assertEqual(code, 400)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    @patch('api.internal.internal_server.InternalServer.get_docker_driver', return_value=MockDockerDriver())
    @patch('api.internal.internal_server.InternalServer.get_mongodb_driver', return_value=MockMongoDriverStartFalse())
    def test_stop_monitor_by_container_id_400_not_started(self, m1, m2, m3):
        response, code = stop_monitor_by_container_id('id')
        self.assertEqual(code, 400)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    @patch('api.internal.internal_server.InternalServer.get_docker_driver', return_value=MockDockerDriver())
    @patch('api.internal.internal_server.InternalServer.get_mongodb_driver', return_value=MockMongoDriverStartFalse())
    def test_start_monitor_by_container_id_OK(self, m1, m2, m3):
        response, code = start_monitor_by_container_id('id')
        self.assertEqual(code, 202)

    @patch('api.internal.internal_server.InternalServer.is_runtime_analysis_enabled', return_value=True)
    @patch('api.internal.internal_server.InternalServer.get_docker_driver', return_value=MockDockerDriver())
    @patch('api.internal.internal_server.InternalServer.get_mongodb_driver', return_value=MockMongoDriverStartTrue())
    def test_stop_monitor_by_container_id_OK(self, m1, m2, m3):
        response = stop_monitor_by_container_id('id')
        self.assertEqual(response, '{"_id": 1234567890, "runtime_analysis": {"stop_timestamp": null}, "status": null}')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Copyright 2019 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Wrapper around stratis CLI
"""
import datetime
import os
import time

from .utils import (exec_command, rs, umount_mdv, stratis_link,
                    size_representation)

# Some packaged systems might place this in /usr/sbin
STRATIS_CLI = os.getenv("STRATIS_CLI", "/usr/bin/stratis")

# Name prefix, so that we hopefully don't destroy any end user data by mistake!
TEST_PREF = os.getenv("STRATIS_UT_PREFIX", "STRATI$_DE$TROY_ME!_")

def p_n():
    """
    Return a random pool name
    :return: Random String
    """
    return TEST_PREF + "pool" + rs()

def fs_n():
    """
    Return a random FS name
    :return: Random String
    """
    return TEST_PREF + "fs" + rs()

class StratisCli:
    @staticmethod
    def cli_version():
        """
        Returns version of command line
        :return: String representation of version
        """
        stdout, _ = exec_command([STRATIS_CLI, "--version"])
        return stdout.strip()

    @staticmethod
    def daemon_version():
        """
        Returns the version of daemon
        :return: String representation of version
        """
        stdout, _ = exec_command([STRATIS_CLI, "daemon", "version"])
        return stdout.strip()

    @staticmethod
    def daemon_redundancy():
        """
        Returns what redundancy the daemon supports
        :return: (Desc, int)
        """
        stdout, _ = exec_command([STRATIS_CLI, "daemon", "redundancy"])
        parsed = stdout.strip().replace(" ", "").split(":")
        return parsed[0], int(parsed[1])

    @staticmethod
    def pool_list(specify_list=True):
        """
        Query the pools
        :param specify_list: Use pool list syntax
        :return: A dict,  Key being the pool name, the value being a dict with
                          keys [SIZE, USED]
        """
        cmd = [STRATIS_CLI, "pool"]
        if specify_list:
            cmd.append("list")
        std_out, _ = exec_command(cmd)
        lines = std_out.splitlines()

        header = lines.pop(0).strip().replace(" ", "")
        assert header == "NameTotalPhysicalSizeTotalPhysicalUsed"

        rc = {}
        for l in lines:
            name, size, size_units, used, used_units = l.split()

            if name.startswith(TEST_PREF):
                rc[name] = dict(
                    SIZE=size_representation(size, size_units),
                    USED=size_representation(used, used_units))
        return rc

    @staticmethod
    def fs_list(specify_list=True):
        """
        Query the file systems
        :param specify_list: Use fs list syntax
        :return: A dict,  Key being the fs name, the value being a dict with
                          keys [POOL_NAME, USED_SIZE, UUID, SYM_LINK, CREATED,
                                CREATED_TS]
        """
        cmd = [STRATIS_CLI, "fs"]
        if specify_list:
            cmd.append("list")

        std_out, _ = exec_command(cmd)
        lines = std_out.splitlines()

        header = lines.pop(0).strip().replace(" ", "")
        assert header == "PoolNameNameUsedCreatedDeviceUUID"

        rc = {}
        for l in lines:
            pool_name, name, used, used_units, month, day, year, hr_min, \
                sym_link, uuid = l.split()

            if not pool_name.startswith(TEST_PREF):
                continue

            created = "%s %s %s %s" % (month, day, year, hr_min)
            ts = time.mktime(
                datetime.datetime.strptime(created,
                                           "%b %d %Y %H:%M").timetuple())
            now = time.time()

            # Everything we have created should have occurred very recently, but
            # our CI systems can be slow, so give them some time, we might need
            # to remove this assert if it becomes problematic.
            assert (now - ts) < 300.0

            rc[name] = dict(
                POOL_NAME=pool_name,
                USED_SIZE=size_representation(used, used_units),
                UUID=uuid,
                SYM_LINK=sym_link,
                CREATED=created,
                CREATED_TS=ts)

        return rc

    @staticmethod
    def blockdev_list(specify_list=True):
        """
        Query the block devs
        :param specify_list: Use blockdev list syntax
        :return: A dict, Key is the device node, the value being a dict with
                         keys [POOL_NAME, SIZE, STATE, TIER]
        """
        cmd = [STRATIS_CLI, "blockdev"]
        if specify_list:
            cmd.append("list")

        std_out, _ = exec_command(cmd)
        lines = std_out.splitlines()

        header = lines.pop(0).strip().replace(" ", "")
        assert header == "PoolNameDeviceNodePhysicalSizeStateTier"

        rc = {}
        for l in lines:
            pool_name, device_node, size, size_units, state, tier = l.split()

            if pool_name.startswith(TEST_PREF):
                rc[device_node] = dict(
                    POOL_NAME=pool_name,
                    SIZE=size_representation(size, size_units),
                    STATE=state,
                    TIER=tier)

        return rc

    @staticmethod
    def pool_create(name, block_devices):
        """
        Creates a pool
        :param name: Name of pool
        :param block_devices: List of block devices to use
        :return: None
        """
        exec_command([STRATIS_CLI, "pool", "create", name] + block_devices)

        # Work around for
        # https://bugzilla.redhat.com/show_bug.cgi?id=1687002
        umount_mdv()

        full_path = stratis_link(name)
        assert os.path.exists(full_path)
        assert os.path.isdir(full_path)

    @staticmethod
    def pool_destroy(name):
        """
        Destroy a pool
        :param name: Name of pool to destroy
        :return: None
        """
        if name.startswith(TEST_PREF):
            exec_command([STRATIS_CLI, "pool", "destroy", name])

    @staticmethod
    def pool_rename(old_name, new_name):
        """
        Renames a pool
        :param old_name:    Current pool name
        :param new_name:    New pool name
        :return: None
        """
        if old_name.startswith(TEST_PREF) and new_name.startswith(TEST_PREF):
            exec_command([STRATIS_CLI, "pool", "rename", old_name, new_name])

            old_path = stratis_link(old_name)
            new_path = stratis_link(new_name)
            assert not os.path.exists(old_path)
            assert os.path.exists(new_path)
            assert os.path.isdir(new_path)

    @staticmethod
    def pool_add(pool_name, option, devices):
        """
        Adds one or more devices to a pool
        :param pool_name: Name of pool
        :param option: String, either ["add-cache", "add-data"]
        :param devices: List of devices to add
        :return: None
        """
        if pool_name.startswith(TEST_PREF):
            exec_command([STRATIS_CLI, "pool", option, pool_name] + devices)
            blockdevs = StratisCli.blockdev_list()
            for b in devices:
                assert b in blockdevs
                assert blockdevs[b]["POOL_NAME"] == pool_name

    @staticmethod
    def destroy_all():
        """
        Destroys all Stratis FS and pools!
        :return: None
        """
        umount_mdv()

        # Remove FS
        for name, fs in StratisCli.fs_list().items():
            StratisCli.fs_destroy(fs["POOL_NAME"], name)

        # Remove Pools
        for name in StratisCli.pool_list().keys():
            StratisCli.pool_destroy(name)

    @staticmethod
    def fs_create(pool_name, fs_name):
        """
        Create a FS
        :param pool_name: Pool to allocate FS from
        :param fs_name: FS name
        :return: None
        """
        if pool_name.startswith(TEST_PREF) and fs_name.startswith(TEST_PREF):
            exec_command([STRATIS_CLI, "fs", "create", pool_name, fs_name])

            # Check symlink
            full_path = stratis_link(pool_name, fs_name)
            assert os.path.exists(full_path)

    @staticmethod
    def fs_destroy(pool_name, fs_name):
        """
        Destroy a FS
        :param pool_name:  Pool which contains the FS
        :param fs_name: Name of FS to destroy
        :return: None
        """
        if pool_name.startswith(TEST_PREF):
            exec_command([STRATIS_CLI, "fs", "destroy", pool_name, fs_name])
            full_path = stratis_link(pool_name, fs_name)
            assert os.path.exists(full_path) is False

    @staticmethod
    def fs_rename(pool_name, old_name, new_name):
        """
        Rename a FS
        :param pool_name: Pool which contains the FS
        :param old_name: Existing FS name
        :param new_name: New FS name
        :return: None
        """
        if pool_name.startswith(TEST_PREF):
            exec_command(
                [STRATIS_CLI, "fs", "rename", pool_name, old_name, new_name])

            assert not os.path.exists(stratis_link(pool_name, old_name))
            assert os.path.exists(stratis_link(pool_name, new_name))

    @staticmethod
    def fs_ss_create(pool_name, fs_name, snapshot_name):
        """
        Create a Snap shot
        :param pool_name: Pool which contains FS to snapshot
        :param fs_name: File system to snap shot
        :param snapshot_name: Name of snap shot
        :return: None
        """
        if pool_name.startswith(TEST_PREF):
            exec_command([
                STRATIS_CLI, "fs", "snapshot", pool_name, fs_name,
                snapshot_name
            ])

            # Check symlink
            full_path = stratis_link(pool_name, snapshot_name)
            assert os.path.exists(full_path)
<commit_msg>Do not make assertions about elapsed time in fs_list<commit_after># Copyright 2019 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Wrapper around stratis CLI
"""
import datetime
import os
import time

from .utils import (exec_command, rs, umount_mdv, stratis_link,
                    size_representation)

# Some packaged systems might place this in /usr/sbin
STRATIS_CLI = os.getenv("STRATIS_CLI", "/usr/bin/stratis")

# Name prefix, so that we hopefully don't destroy any end user data by mistake!
TEST_PREF = os.getenv("STRATIS_UT_PREFIX", "STRATI$_DE$TROY_ME!_")

def p_n():
    """
    Return a random pool name
    :return: Random String
    """
    return TEST_PREF + "pool" + rs()

def fs_n():
    """
    Return a random FS name
    :return: Random String
    """
    return TEST_PREF + "fs" + rs()

class StratisCli:
    @staticmethod
    def cli_version():
        """
        Returns version of command line
        :return: String representation of version
        """
        stdout, _ = exec_command([STRATIS_CLI, "--version"])
        return stdout.strip()

    @staticmethod
    def daemon_version():
        """
        Returns the version of daemon
        :return: String representation of version
        """
        stdout, _ = exec_command([STRATIS_CLI, "daemon", "version"])
        return stdout.strip()

    @staticmethod
    def daemon_redundancy():
        """
        Returns what redundancy the daemon supports
        :return: (Desc, int)
        """
        stdout, _ = exec_command([STRATIS_CLI, "daemon", "redundancy"])
        parsed = stdout.strip().replace(" ", "").split(":")
        return parsed[0], int(parsed[1])

    @staticmethod
    def pool_list(specify_list=True):
        """
        Query the pools
        :param specify_list: Use pool list syntax
        :return: A dict,  Key being the pool name, the value being a dict with
                          keys [SIZE, USED]
        """
        cmd = [STRATIS_CLI, "pool"]
        if specify_list:
            cmd.append("list")
        std_out, _ = exec_command(cmd)
        lines = std_out.splitlines()

        header = lines.pop(0).strip().replace(" ", "")
        assert header == "NameTotalPhysicalSizeTotalPhysicalUsed"

        rc = {}
        for l in lines:
            name, size, size_units, used, used_units = l.split()

            if name.startswith(TEST_PREF):
                rc[name] = dict(
                    SIZE=size_representation(size, size_units),
                    USED=size_representation(used, used_units))
        return rc

    @staticmethod
    def fs_list(specify_list=True):
        """
        Query the file systems
        :param specify_list: Use fs list syntax
        :return: A dict,  Key being the fs name, the value being a dict with
                          keys [POOL_NAME, USED_SIZE, UUID, SYM_LINK, CREATED,
                                CREATED_TS]
        """
        cmd = [STRATIS_CLI, "fs"]
        if specify_list:
            cmd.append("list")

        std_out, _ = exec_command(cmd)
        lines = std_out.splitlines()

        header = lines.pop(0).strip().replace(" ", "")
        assert header == "PoolNameNameUsedCreatedDeviceUUID"

        rc = {}
        for l in lines:
            pool_name, name, used, used_units, month, day, year, hr_min, \
                sym_link, uuid = l.split()

            if not pool_name.startswith(TEST_PREF):
                continue

            created = "%s %s %s %s" % (month, day, year, hr_min)
            ts = time.mktime(
                datetime.datetime.strptime(created,
                                           "%b %d %Y %H:%M").timetuple())

            rc[name] = dict(
                POOL_NAME=pool_name,
                USED_SIZE=size_representation(used, used_units),
                UUID=uuid,
                SYM_LINK=sym_link,
                CREATED=created,
                CREATED_TS=ts)

        return rc

    @staticmethod
    def blockdev_list(specify_list=True):
        """
        Query the block devs
        :param specify_list: Use blockdev list syntax
        :return: A dict, Key is the device node, the value being a dict with
                         keys [POOL_NAME, SIZE, STATE, TIER]
        """
        cmd = [STRATIS_CLI, "blockdev"]
        if specify_list:
            cmd.append("list")

        std_out, _ = exec_command(cmd)
        lines = std_out.splitlines()

        header = lines.pop(0).strip().replace(" ", "")
        assert header == "PoolNameDeviceNodePhysicalSizeStateTier"

        rc = {}
        for l in lines:
            pool_name, device_node, size, size_units, state, tier = l.split()

            if pool_name.startswith(TEST_PREF):
                rc[device_node] = dict(
                    POOL_NAME=pool_name,
                    SIZE=size_representation(size, size_units),
                    STATE=state,
                    TIER=tier)

        return rc

    @staticmethod
    def pool_create(name, block_devices):
        """
        Creates a pool
        :param name: Name of pool
        :param block_devices: List of block devices to use
        :return: None
        """
        exec_command([STRATIS_CLI, "pool", "create", name] + block_devices)

        # Work around for
        # https://bugzilla.redhat.com/show_bug.cgi?id=1687002
        umount_mdv()

        full_path = stratis_link(name)
        assert os.path.exists(full_path)
        assert os.path.isdir(full_path)

    @staticmethod
    def pool_destroy(name):
        """
        Destroy a pool
        :param name: Name of pool to destroy
        :return: None
        """
        if name.startswith(TEST_PREF):
            exec_command([STRATIS_CLI, "pool", "destroy", name])

    @staticmethod
    def pool_rename(old_name, new_name):
        """
        Renames a pool
        :param old_name:    Current pool name
        :param new_name:    New pool name
        :return: None
        """
        if old_name.startswith(TEST_PREF) and new_name.startswith(TEST_PREF):
            exec_command([STRATIS_CLI, "pool", "rename", old_name, new_name])

            old_path = stratis_link(old_name)
            new_path = stratis_link(new_name)
            assert not os.path.exists(old_path)
            assert os.path.exists(new_path)
            assert os.path.isdir(new_path)

    @staticmethod
    def pool_add(pool_name, option, devices):
        """
        Adds one or more devices to a pool
        :param pool_name: Name of pool
        :param option: String, either ["add-cache", "add-data"]
        :param devices: List of devices to add
        :return: None
        """
        if pool_name.startswith(TEST_PREF):
            exec_command([STRATIS_CLI, "pool", option, pool_name] + devices)
            blockdevs = StratisCli.blockdev_list()
            for b in devices:
                assert b in blockdevs
                assert blockdevs[b]["POOL_NAME"] == pool_name

    @staticmethod
    def destroy_all():
        """
        Destroys all Stratis FS and pools!
        :return: None
        """
        umount_mdv()

        # Remove FS
        for name, fs in StratisCli.fs_list().items():
            StratisCli.fs_destroy(fs["POOL_NAME"], name)

        # Remove Pools
        for name in StratisCli.pool_list().keys():
            StratisCli.pool_destroy(name)

    @staticmethod
    def fs_create(pool_name, fs_name):
        """
        Create a FS
        :param pool_name: Pool to allocate FS from
        :param fs_name: FS name
        :return: None
        """
        if pool_name.startswith(TEST_PREF) and fs_name.startswith(TEST_PREF):
            exec_command([STRATIS_CLI, "fs", "create", pool_name, fs_name])

            # Check symlink
            full_path = stratis_link(pool_name, fs_name)
            assert os.path.exists(full_path)

    @staticmethod
    def fs_destroy(pool_name, fs_name):
        """
        Destroy a FS
        :param pool_name:  Pool which contains the FS
        :param fs_name: Name of FS to destroy
        :return: None
        """
        if pool_name.startswith(TEST_PREF):
            exec_command([STRATIS_CLI, "fs", "destroy", pool_name, fs_name])
            full_path = stratis_link(pool_name, fs_name)
            assert os.path.exists(full_path) is False

    @staticmethod
    def fs_rename(pool_name, old_name, new_name):
        """
        Rename a FS
        :param pool_name: Pool which contains the FS
        :param old_name: Existing FS name
        :param new_name: New FS name
        :return: None
        """
        if pool_name.startswith(TEST_PREF):
            exec_command(
                [STRATIS_CLI, "fs", "rename", pool_name, old_name, new_name])

            assert not os.path.exists(stratis_link(pool_name, old_name))
            assert os.path.exists(stratis_link(pool_name, new_name))

    @staticmethod
    def fs_ss_create(pool_name, fs_name, snapshot_name):
        """
        Create a Snap shot
        :param pool_name: Pool which contains FS to snapshot
        :param fs_name: File system to snap shot
        :param snapshot_name: Name of snap shot
        :return: None
        """
        if pool_name.startswith(TEST_PREF):
            exec_command([
                STRATIS_CLI, "fs", "snapshot", pool_name, fs_name,
                snapshot_name
            ])

            # Check symlink
            full_path = stratis_link(pool_name, snapshot_name)
            assert os.path.exists(full_path)
<|endoftext|>


<commit_before>from collections import UserList, UserDict
from functools import partial

def fnv32a(text):
    h = 0x811c9dc5
    for c in text:
        h = ((h ^ ord(c)) * 0x01000193) & 0xffffffff
    return h

def numbered_columns(array):
    # Can't use `not array` because numpy interprets it differently.
    if len(array) == 0:
        return []
    return [str(i) for i in range(len(array[0]))]

def iterate_items(obj):
    """Iterates over the object's key value pairs (key, value), where obj[key] == value."""
    if hasattr(obj, "items"):
        return obj.items()
    # If the object lacks the items() function, assume it's a list with keys [0, len(obj)-1].
    return enumerate(obj)

class Array(UserList):
    """Array stores the real-valued features.

    Args:
        columns: [str], the name of each column.
    """

    def __init__(self, columns=None):
        super().__init__()
        self._columns = list(columns) if columns is not None else []
        self.data = []

    def concat(self, other, prefix=""):
        """Adds the columns from `other` to `self`. `other` can be
        of any type that supports length, indexing and enumeration. Furthermore
        column names can be supplied in a `columns` attribute.

        Args:
            other: contains the new columns.
            prefix: str, optional prefix added to the new column names
                to avoid name clashes.
        """
        if len(self) == 0:
            self.data = [[] for i in range(len(other))]

        if len(self) != len(other):
            raise ValueError("array length does not match - have {} and {}".format(len(self), len(other)))

        columns = other.columns if hasattr(other, "columns") else numbered_columns(other)
        if prefix:
            columns = ["{}_{}".format(prefix, name) for name in columns]

        # Make sure the column names do not clash.
        for column in columns:
            if column in self._columns:
                raise ValueError("a column named '{}' already exists".format(column))

        self._columns += columns
        for i, row in enumerate(other):
            self.data[i].extend(row)

    @property
    def shape(self):
        """Returns the array shape."""
        return (len(self.data), len(self.data[0]))

    @property
    def columns(self):
        """Returns the column names."""
        if not self._columns:
            self._columns = numbered_columns(self)
        return self._columns

class BaseFeature(object):
    def set(self, *args, **kwargs):
        raise NotImplementedError()

    def push(self):
        raise NotImplementedError()

    def array(self):
        raise NotImplementedError()

class Pipe(object):
    """Pipe applies one ore more functions to a feature array.

    Args:
        feature: Feature|Group, of which the array shall be transformed.
        functions: callable, one ore more functions that take and return an array.
    """

    def __init__(self, feature, *functions):
        self.feature = feature
        self.functions = functions

    def array(self):
        """Returns the feature array with all functions of the pipe applied."""
        result = self.feature.array()
        for function in self.functions:
            result = function(result)
        return result

    def __getattr__(self, name):
        return getattr(self.feature, name)

class Group(BaseFeature):
    """Group produces real-valued feature arrays from one or more
    Feature/Group classes.

    Args:
        features: {str: Feature|Group}, instances of Feature/Group classes
            stored under their names.
    """

    def __init__(self, features):
        self.features = features

    def set(self, *args, **kwargs):
        # If there's only a single feature, allow omitting the feature name.
        # Otherwise name should be the first argument and the remaining arguments
        # get passed on to the feature's set() function.
        if len(self.features) == 1:
            name, = self.features.keys()
            if name == args[0]:
                args = args[1:]
        else:
            name = args[0]
            args = args[1:]
        self.features[name].set(*args, **kwargs)

    def push(self):
        for feature in self.features.values():
            feature.push()

    def array(self):
        result = Array()
        for name, feature in sorted(self.features.items()):
            result.concat(feature.array(), prefix=name)
        return result

    def __getattr__(self, name):
        function, *partial_applied_args = name.split("_")
        if function != "set" or not partial_applied_args:
            return self.__getattribute__(name)
        return partial(self.set, *partial_applied_args)

class Feature(BaseFeature):
    """Base class of all features.

    A feature produces one or more numerical values. These values
    are stored in so-called fields.

    Args:
        dimensions: int or [str] or None, either the number of dimensions this feature
            or a list of their names.

            If dimensions is an integer, the dimension names will be indexes from 0 to `dimensions-1`.
            If dimensions is None, the number of dimensions and their names will be
            determined dynamically.

    Attributes:
        dimensions: [int|str] or None, the names of the fields produced by this feature.
    """

    def __init__(self, dimensions=None):
        if hasattr(self, "Dimensions"):
            dimensions = self.Dimensions
        if type(dimensions) is int:
            dimensions = list(range(dimensions))
        self.dimensions = sorted(dimensions) if dimensions else None
        self.slot = {}
        self.rows = []

    def push(self):
        if self.dimensions:
            for dimension, value in iterate_items(self.slot):
                if dimension not in self.dimensions:
                    raise KeyError("unknown dimension '{}' (have dimensions {})".format(dimension, self.dimensions))
        self.rows.append(self.slot)
        self.slot = {}

    def array(self):
        # If dimensions is None, determine the dimensions by looking at the data.
        if self.dimensions is None:
            self.dimensions = sorted({dimension for row in self.rows for dimension, value in iterate_items(row)})
        result = Array(columns=self.dimensions)
        for i, row in enumerate(self.rows):
            values = [0.0] * len(self.dimensions)
            for dimension, value in iterate_items(row):
                values[self.dimensions.index(dimension)] = value
            result.data.append(values)
        return result

class Numerical(Feature):
    """Produces a single numerical value."""

    def set(self, *args):
        index = 0
        if len(args) == 1:
            value, = args
        else:
            index, value = args
        self.slot[index] = value

class Categorical(Feature):
    """Performs one hot encoding on categorical data.

    Args:
        values: list, the values to encode.
    """

    def __init__(self, values):
        values = sorted(set(values))
        super().__init__(dimensions=len(values))
        self.values = values

    def set(self, token, weight=1.0):
        if token in self.values:
            self.slot[self.values.index(token)] = weight

class Hashed(Feature):
    """Hashes arbitrary values into a fixed number of buckets.

    Args:
        hash: callable, the hash function to map features to buckets.
            Please note that Python's hash() function is randomized. Using it
            here maps values to different buckets on each program run.
        buckets: int, the number of buckets to use.
        replace: str or callable, defines a replacement strategy if a value
            gets assigned to a non-empty bucket. Supported strategies are
                - 'sum', which adds the value to bucket
                - 'max', which stores the maximum in the bucket
            Additionally, a callable function can be passed to implement
            a custom replacement strategy.
        random_sign: bool, if True, makes the sign of the weights depend on
            the hash values.
    """

    def __init__(self, hash=fnv32a, buckets=100, replace=None, random_sign=False):
        super().__init__(dimensions=buckets)
        self.buckets = buckets
        self.random_sign = random_sign
        self.replace = None
        if replace == "sum":
            self.func = lambda new, old: new + old
        if replace == "max":
            self.func = lambda new, old: max(new, old)
        if callable(replace):
            self.func = replace
        self.hash = hash

    def set(self, token, weight=1.0):
        key = self.hash(token)
        if self.random_sign:
            if key & 0x80000000 != 0:
                weight = -weight
        index = key % self.buckets
        if self.replace is not None and index in self.slot:
            weight = self.replace(weight, self.slot[index])
        self.slot[index] = weight
<commit_msg>move curried set function into its own class<commit_after>from collections import UserList, UserDict
from functools import partial

def fnv32a(text):
    h = 0x811c9dc5
    for c in text:
        h = ((h ^ ord(c)) * 0x01000193) & 0xffffffff
    return h

def numbered_columns(array):
    # Can't use `not array` because numpy interprets it differently.
    if len(array) == 0:
        return []
    return [str(i) for i in range(len(array[0]))]

def iterate_items(obj):
    """Iterates over the object's key value pairs (key, value), where obj[key] == value."""
    if hasattr(obj, "items"):
        return obj.items()
    # If the object lacks the items() function, assume it's a list with keys [0, len(obj)-1].
    return enumerate(obj)

class Array(UserList):
    """Array stores the real-valued features.

    Args:
        columns: [str], the name of each column.
    """

    def __init__(self, columns=None):
        super().__init__()
        self._columns = list(columns) if columns is not None else []
        self.data = []

    def concat(self, other, prefix=""):
        """Adds the columns from `other` to `self`. `other` can be
        of any type that supports length, indexing and enumeration. Furthermore
        column names can be supplied in a `columns` attribute.

        Args:
            other: contains the new columns.
            prefix: str, optional prefix added to the new column names
                to avoid name clashes.
        """
        if len(self) == 0:
            self.data = [[] for i in range(len(other))]

        if len(self) != len(other):
            raise ValueError("array length does not match - have {} and {}".format(len(self), len(other)))

        columns = other.columns if hasattr(other, "columns") else numbered_columns(other)
        if prefix:
            columns = ["{}_{}".format(prefix, name) for name in columns]

        # Make sure the column names do not clash.
        for column in columns:
            if column in self._columns:
                raise ValueError("a column named '{}' already exists".format(column))

        self._columns += columns
        for i, row in enumerate(other):
            self.data[i].extend(row)

    @property
    def shape(self):
        """Returns the array shape."""
        return (len(self.data), len(self.data[0]))

    @property
    def columns(self):
        """Returns the column names."""
        if not self._columns:
            self._columns = numbered_columns(self)
        return self._columns

class BaseFeature(object):
    def set(self, *args, **kwargs):
        raise NotImplementedError()

    def push(self):
        raise NotImplementedError()

    def array(self):
        raise NotImplementedError()

class CurriedSet(object):
    """Helper class that allows partially applying arguments to the object's set function
    by putting them into the function name. This allows more readable function calls:

    >>> obj.set("a", "b", "c", 1.0)
    >>> obj.set_a_b_c(1.0)
    """

    def __getattr__(self, name):
        function, *partial_applied_args = name.split("_")
        if function != "set" or not partial_applied_args:
            return self.__getattribute__(name)
        return partial(self.set, *partial_applied_args)

class Pipe(object):
    """Pipe applies one ore more functions to a feature array.

    Args:
        feature: Feature|Group, of which the array shall be transformed.
        functions: callable, one ore more functions that take and return an array.
    """

    def __init__(self, feature, *functions):
        self.feature = feature
        self.functions = functions

    def array(self):
        """Returns the feature array with all functions of the pipe applied."""
        result = self.feature.array()
        for function in self.functions:
            result = function(result)
        return result

    def __getattr__(self, name):
        return getattr(self.feature, name)

class Group(BaseFeature, CurriedSet):
    """Group produces real-valued feature arrays from one or more
    Feature/Group classes.

    Args:
        features: {str: Feature|Group}, instances of Feature/Group classes
            stored under their names.
    """

    def __init__(self, features):
        super().__init__()
        self.features = features

    def set(self, *args, **kwargs):
        # If there's only a single feature, allow omitting the feature name.
        # Otherwise name should be the first argument and the remaining arguments
        # get passed on to the feature's set() function.
        if len(self.features) == 1:
            name, = self.features.keys()
            if name == args[0]:
                args = args[1:]
        else:
            name = args[0]
            args = args[1:]
        self.features[name].set(*args, **kwargs)

    def push(self):
        for feature in self.features.values():
            feature.push()

    def array(self):
        result = Array()
        for name, feature in sorted(self.features.items()):
            result.concat(feature.array(), prefix=name)
        return result

class Feature(BaseFeature):
    """Base class of all features.

    A feature produces one or more numerical values. These values
    are stored in so-called fields.

    Args:
        dimensions: int or [str] or None, either the number of dimensions this feature
            or a list of their names.

            If dimensions is an integer, the dimension names will be indexes from 0 to `dimensions-1`.
            If dimensions is None, the number of dimensions and their names will be
            determined dynamically.

    Attributes:
        dimensions: [int|str] or None, the names of the fields produced by this feature.
    """

    def __init__(self, dimensions=None):
        if hasattr(self, "Dimensions"):
            dimensions = self.Dimensions
        if type(dimensions) is int:
            dimensions = list(range(dimensions))
        self.dimensions = sorted(dimensions) if dimensions else None
        self.slot = {}
        self.rows = []

    def push(self):
        if self.dimensions:
            for dimension, value in iterate_items(self.slot):
                if dimension not in self.dimensions:
                    raise KeyError("unknown dimension '{}' (have dimensions {})".format(dimension, self.dimensions))
        self.rows.append(self.slot)
        self.slot = {}

    def array(self):
        # If dimensions is None, determine the dimensions by looking at the data.
        if self.dimensions is None:
            self.dimensions = sorted({dimension for row in self.rows for dimension, value in iterate_items(row)})
        result = Array(columns=self.dimensions)
        for i, row in enumerate(self.rows):
            values = [0.0] * len(self.dimensions)
            for dimension, value in iterate_items(row):
                values[self.dimensions.index(dimension)] = value
            result.data.append(values)
        return result

class Numerical(Feature, CurriedSet):
    """Produces a single numerical value."""

    def set(self, *args):
        index = 0
        if len(args) == 1:
            value, = args
        else:
            index, value = args
        self.slot[index] = value

class Categorical(Feature, CurriedSet):
    """Performs one hot encoding on categorical data.

    Args:
        values: list, the values to encode.
    """

    def __init__(self, values):
        values = sorted(set(values))
        super().__init__(dimensions=len(values))
        self.values = values

    def set(self, token, weight=1.0):
        if token in self.values:
            self.slot[self.values.index(token)] = weight

class Hashed(Feature):
    """Hashes arbitrary values into a fixed number of buckets.

    Args:
        hash: callable, the hash function to map features to buckets.
            Please note that Python's hash() function is randomized. Using it
            here maps values to different buckets on each program run.
        buckets: int, the number of buckets to use.
        replace: str or callable, defines a replacement strategy if a value
            gets assigned to a non-empty bucket. Supported strategies are
                - 'sum', which adds the value to bucket
                - 'max', which stores the maximum in the bucket
            Additionally, a callable function can be passed to implement
            a custom replacement strategy.
        random_sign: bool, if True, makes the sign of the weights depend on
            the hash values.
    """

    def __init__(self, hash=fnv32a, buckets=100, replace=None, random_sign=False):
        super().__init__(dimensions=buckets)
        self.buckets = buckets
        self.random_sign = random_sign
        self.replace = None
        if replace == "sum":
            self.func = lambda new, old: new + old
        if replace == "max":
            self.func = lambda new, old: max(new, old)
        if callable(replace):
            self.func = replace
        self.hash = hash

    def set(self, token, weight=1.0):
        key = self.hash(token)
        if self.random_sign:
            if key & 0x80000000 != 0:
                weight = -weight
        index = key % self.buckets
        if self.replace is not None and index in self.slot:
            weight = self.replace(weight, self.slot[index])
        self.slot[index] = weight
<|endoftext|>


<commit_before>from unittest import TestCase
from dusty.compiler.nginx import (nginx_configuration_spec, _nginx_listen_string, _nginx_location_spec,
                                  _nginx_proxy_string, _nginx_server_spec)

def clense(string):
    return string.replace("\n", "").replace("\t", "").replace(" ", "")

class TestPortSpecCompiler(TestCase):
    def setUp(self):
        self.port_spec_dict_1 = {'nginx': [{'proxied_ip': '127.0.0.1',
                                            'proxied_port':'80',
                                            'host_address': 'local.gc.com',
                                            'host_port': '80'}]}
        self.port_spec_dict_2 = {'nginx': [{'proxied_ip': '127.0.0.0',
                                            'proxied_port':'8000',
                                            'host_address': 'local.gcapi.com',
                                            'host_port': '8001'}]}

    def test_nginx_configuration_spec(self):
        expected_output = clense("""http {
            server {
                listen local.gc.com:80;
                location / {
                    proxy_pass 127.0.0.1:80;
                }
            }
        }
        """)
        output = clense(nginx_configuration_spec(self.port_spec_dict_1))
        print output
        self.assertEqual(output, expected_output)
<commit_msg>passing first test<commit_after>from unittest import TestCase
from dusty.compiler.nginx import (nginx_configuration_spec, _nginx_listen_string, _nginx_location_spec,
                                  _nginx_proxy_string, _nginx_server_spec)

def clense(string):
    return string.replace("\n", "").replace("\t", "").replace(" ", "")

class TestPortSpecCompiler(TestCase):
    def setUp(self):
        self.port_spec_dict_1 = {'nginx': [{'proxied_ip': '127.0.0.1',
                                            'proxied_port':'80',
                                            'host_address': 'local.gc.com',
                                            'host_port': '80'}]}
        self.port_spec_dict_2 = {'nginx': [{'proxied_ip': '127.0.0.0',
                                            'proxied_port':'8000',
                                            'host_address': 'local.gcapi.com',
                                            'host_port': '8001'}]}

    def test_nginx_configuration_spec_1(self):
        expected_output = clense("""http {
            server {
                listen local.gc.com:80;
                location / {
                    proxy_pass 127.0.0.1:80;
                }
            }
        }
        """)
        output = clense(nginx_configuration_spec(self.port_spec_dict_1))
        print output
        self.assertEqual(output, expected_output)

    def test_nginx_configuration_spec_2(self):
        expected_output = clense("""http {
            server {
                listen local.gcapi.com:80001;
                location / {
                    proxy_pass 127.0.0.0:8000;
                }
            }
        }
        """)
        output = clense(nginx_configuration_spec(self.port_spec_dict_2))
        print output
        self.assertEqual(output, expected_output)
<|endoftext|>


<commit_before>import os
import unittest

from examples.sumo.bay_bridge import bay_bridge_example
from examples.sumo.bay_bridge_toll import bay_bridge_toll_example
from examples.sumo.bottlenecks import bottleneck_example
from examples.sumo.density_exp import run_bottleneck
from examples.sumo.figure_eight import figure_eight_example
from examples.sumo.grid import grid_example
from examples.sumo.highway import highway_example
from examples.sumo.loop_merge import loop_merge_example
from examples.sumo.merge import merge_example
from examples.sumo.minicity import minicity_example
from examples.sumo.sugiyama import sugiyama_example

from examples.rllib.cooperative_merge import setup_exps as coop_setup
from examples.rllib.figure_eight import setup_exps as figure_eight_setup
from examples.rllib.green_wave import setup_exps as green_wave_setup
from examples.rllib.stabilizing_highway import setup_exps as highway_setup
from examples.rllib.stabilizing_the_ring import setup_exps as ring_setup
from examples.rllib.velocity_bottleneck import setup_exps as bottleneck_setup
from examples.rllib.multiagent_exps.multiagent_figure_eight \
   import setup_exps as multi_figure_eight_setup
from examples.rllib.multiagent_exps.multiagent_stabilizing_the_ring \
    import setup_exps as multi_ring_setup

import ray
from ray.tune import run_experiments

os.environ['TEST_FLAG'] = 'True'

class TestSumoExamples(unittest.TestCase):
    """Tests the example scripts in examples/sumo.

    This is done by running the experiment function within each script for a
    few time steps. Note that, this does not test for any refactoring changes
    done to the functions within the experiment class.
    """

    def test_bottleneck(self):
        """Verifies that examples/sumo/bottlenecks.py is working."""
        # import the experiment variable from the example
        exp = bottleneck_example(20, 5, render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_figure_eight(self):
        """Verifies that examples/sumo/figure_eight.py is working."""
        # import the experiment variable from the example
        exp = figure_eight_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_grid(self):
        """Verifies that examples/sumo/grid.py is working."""
        # test the example in the absence of inflows
        exp = grid_example(render=False, use_inflows=False)
        exp.run(1, 5)

        # test the example in the presence of inflows
        exp = grid_example(render=False, use_inflows=True)
        exp.run(1, 5)

    def test_highway(self):
        """Verifies that examples/sumo/highway.py is working."""
        # import the experiment variable from the example
        exp = highway_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_merge(self):
        """Verifies that examples/sumo/merge.py is working."""
        # import the experiment variable from the example
        exp = merge_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_sugiyama(self):
        """Verifies that examples/sumo/sugiyama.py is working."""
        # import the experiment variable from the example
        exp = sugiyama_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_loop_merge(self):
        """Verify that examples/sumo/two_loops_merge_straight.py is working."""
        # import the experiment variable from the example
        exp = loop_merge_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_bay_bridge(self):
        """Verifies that examples/sumo/bay_bridge.py is working."""
        # import the experiment variable from the example
        exp = bay_bridge_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

        # import the experiment variable from the example with inflows
        exp = bay_bridge_example(render=False, use_inflows=True)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

        # import the experiment variable from the example with traffic lights
        exp = bay_bridge_example(render=False, use_traffic_lights=True)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_bay_bridge_toll(self):
        """Verifies that examples/sumo/bay_bridge_toll.py is working."""
        # import the experiment variable from the example
        exp = bay_bridge_toll_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_minicity(self):
        """Verifies that examples/sumo/minicity.py is working."""
        # import the experiment variable from the example
        exp = minicity_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_density_exp(self):
        """Verifies that examples/sumo/density_exp.py is working."""
        _ = run_bottleneck.remote(100, 1, 10, render=False)

class TestRllibExamples(unittest.TestCase):
    """Tests the example scripts in examples/sumo.

    This is done by running each experiment in that folder for five time-steps
    and confirming that it completes one rollout with two workers.
    # FIXME(ev) this test adds several minutes to the testing scheme
    """
    def setUp(self):
        if not ray.is_initialized():
            ray.init(num_cpus=1)

    def test_coop_merge(self):
        alg_run, env_name, config = coop_setup()
        self.run_exp(alg_run, env_name, config)

    def test_figure_eight(self):
        alg_run, env_name, config = figure_eight_setup()
        self.run_exp(alg_run, env_name, config)

    def test_green_wave(self):
        # test the example in the absence of inflows
        alg_run, env_name, config = green_wave_setup(use_inflows=False)
        self.run_exp(alg_run, env_name, config)

        # test the example in the presence of inflows
        alg_run, env_name, config = green_wave_setup(use_inflows=True)
        self.run_exp(alg_run, env_name, config)

    def test_stabilizing_highway(self):
        alg_run, env_name, config = highway_setup()
        self.run_exp(alg_run, env_name, config)

    def test_ring(self):
        alg_run, env_name, config = ring_setup()
        self.run_exp(alg_run, env_name, config)

    def test_bottleneck(self):
        alg_run, env_name, config = bottleneck_setup()
        self.run_exp(alg_run, env_name, config)

    def test_multi_figure_eight(self):
        alg_run, env_name, config = multi_figure_eight_setup()
        self.run_exp(alg_run, env_name, config)

    def test_multi_ring(self):
        alg_run, env_name, config = multi_ring_setup()
        self.run_exp(alg_run, env_name, config)

    @staticmethod
    def run_exp(alg_run, env_name, config):
        try:
            ray.init(num_cpus=1)
        except Exception:
            pass
        config['train_batch_size'] = 50
        config['horizon'] = 50
        config['sample_batch_size'] = 50
        config['num_workers'] = 0
        config['sgd_minibatch_size'] = 32

        run_experiments({
            'test': {
                'run': alg_run,
                'env': env_name,
                'config': {
                    **config
                },

                'checkpoint_freq': 1,
                'stop': {
                    'training_iteration': 1,
                },
            }
        })

if __name__ == '__main__':
    try:
        ray.init(num_cpus=1)
    except Exception:
        pass
    unittest.main()
    ray.shutdown()
<commit_msg>fix pr<commit_after>import os
import unittest

from examples.sumo.bay_bridge import bay_bridge_example
from examples.sumo.bay_bridge_toll import bay_bridge_toll_example
from examples.sumo.bottlenecks import bottleneck_example
from examples.sumo.density_exp import run_bottleneck
from examples.sumo.figure_eight import figure_eight_example
from examples.sumo.grid import grid_example
from examples.sumo.highway import highway_example
from examples.sumo.loop_merge import loop_merge_example
from examples.sumo.merge import merge_example
from examples.sumo.minicity import minicity_example
from examples.sumo.sugiyama import sugiyama_example

from examples.rllib.cooperative_merge import setup_exps as coop_setup
from examples.rllib.figure_eight import setup_exps as figure_eight_setup
from examples.rllib.green_wave import setup_exps as green_wave_setup
from examples.rllib.stabilizing_highway import setup_exps as highway_setup
from examples.rllib.stabilizing_the_ring import setup_exps as ring_setup
from examples.rllib.velocity_bottleneck import setup_exps as bottleneck_setup
from examples.rllib.multiagent_exps.multiagent_figure_eight \
   import setup_exps as multi_figure_eight_setup
from examples.rllib.multiagent_exps.multiagent_stabilizing_the_ring \
    import setup_exps as multi_ring_setup

import ray
from ray.tune import run_experiments

os.environ['TEST_FLAG'] = 'True'
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'

class TestSumoExamples(unittest.TestCase):
    """Tests the example scripts in examples/sumo.

    This is done by running the experiment function within each script for a
    few time steps. Note that, this does not test for any refactoring changes
    done to the functions within the experiment class.
    """

    def test_bottleneck(self):
        """Verifies that examples/sumo/bottlenecks.py is working."""
        # import the experiment variable from the example
        exp = bottleneck_example(20, 5, render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_figure_eight(self):
        """Verifies that examples/sumo/figure_eight.py is working."""
        # import the experiment variable from the example
        exp = figure_eight_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_grid(self):
        """Verifies that examples/sumo/grid.py is working."""
        # test the example in the absence of inflows
        exp = grid_example(render=False, use_inflows=False)
        exp.run(1, 5)

        # test the example in the presence of inflows
        exp = grid_example(render=False, use_inflows=True)
        exp.run(1, 5)

    def test_highway(self):
        """Verifies that examples/sumo/highway.py is working."""
        # import the experiment variable from the example
        exp = highway_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_merge(self):
        """Verifies that examples/sumo/merge.py is working."""
        # import the experiment variable from the example
        exp = merge_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_sugiyama(self):
        """Verifies that examples/sumo/sugiyama.py is working."""
        # import the experiment variable from the example
        exp = sugiyama_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_loop_merge(self):
        """Verify that examples/sumo/two_loops_merge_straight.py is working."""
        # import the experiment variable from the example
        exp = loop_merge_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_bay_bridge(self):
        """Verifies that examples/sumo/bay_bridge.py is working."""
        # import the experiment variable from the example
        exp = bay_bridge_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

        # import the experiment variable from the example with inflows
        exp = bay_bridge_example(render=False, use_inflows=True)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

        # import the experiment variable from the example with traffic lights
        exp = bay_bridge_example(render=False, use_traffic_lights=True)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_bay_bridge_toll(self):
        """Verifies that examples/sumo/bay_bridge_toll.py is working."""
        # import the experiment variable from the example
        exp = bay_bridge_toll_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_minicity(self):
        """Verifies that examples/sumo/minicity.py is working."""
        # import the experiment variable from the example
        exp = minicity_example(render=False)

        # run the experiment for a few time steps to ensure it doesn't fail
        exp.run(1, 5)

    def test_density_exp(self):
        """Verifies that examples/sumo/density_exp.py is working."""
        _ = run_bottleneck.remote(100, 1, 10, render=False)

class TestRllibExamples(unittest.TestCase):
    """Tests the example scripts in examples/sumo.

    This is done by running each experiment in that folder for five time-steps
    and confirming that it completes one rollout with two workers.
    # FIXME(ev) this test adds several minutes to the testing scheme
    """
    def setUp(self):
        if not ray.is_initialized():
            ray.init(num_cpus=1)

    def test_coop_merge(self):
        alg_run, env_name, config = coop_setup()
        self.run_exp(alg_run, env_name, config)

    def test_figure_eight(self):
        alg_run, env_name, config = figure_eight_setup()
        self.run_exp(alg_run, env_name, config)

    def test_green_wave(self):
        # test the example in the absence of inflows
        alg_run, env_name, config = green_wave_setup(use_inflows=False)
        self.run_exp(alg_run, env_name, config)

        # test the example in the presence of inflows
        alg_run, env_name, config = green_wave_setup(use_inflows=True)
        self.run_exp(alg_run, env_name, config)

    def test_stabilizing_highway(self):
        alg_run, env_name, config = highway_setup()
        self.run_exp(alg_run, env_name, config)

    def test_ring(self):
        alg_run, env_name, config = ring_setup()
        self.run_exp(alg_run, env_name, config)

    def test_bottleneck(self):
        alg_run, env_name, config = bottleneck_setup()
        self.run_exp(alg_run, env_name, config)

    def test_multi_figure_eight(self):
        alg_run, env_name, config = multi_figure_eight_setup()
        self.run_exp(alg_run, env_name, config)

    def test_multi_ring(self):
        alg_run, env_name, config = multi_ring_setup()
        self.run_exp(alg_run, env_name, config)

    @staticmethod
    def run_exp(alg_run, env_name, config):
        try:
            ray.init(num_cpus=1)
        except Exception:
            pass
        config['train_batch_size'] = 50
        config['horizon'] = 50
        config['sample_batch_size'] = 50
        config['num_workers'] = 0
        config['sgd_minibatch_size'] = 32

        run_experiments({
            'test': {
                'run': alg_run,
                'env': env_name,
                'config': {
                    **config
                },

                'checkpoint_freq': 1,
                'stop': {
                    'training_iteration': 1,
                },
            }
        })

if __name__ == '__main__':
    try:
        ray.init(num_cpus=1)
    except Exception:
        pass
    unittest.main()
    ray.shutdown()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Runs an analysis on your EC2 Security Group instance assignments.
"""

from boto.ec2 import connect_to_region
import re
import yaml
from time import time
from cachetools import ttl_cache
from flask import Flask, render_template

REGIONS = ['us-east-1','us-west-1','eu-west-1','ap-northeast-1','us-west-2']

DIGIT = re.compile("\d")
UNDERSCORE = re.compile("_")

@ttl_cache(ttl=5*60)
def get_all_reservations(ec2):
    return ec2.get_all_reservations()

def fetch_records(env):
  credentials = yaml.load(open('config/aws.yaml', 'r').read())[env]

  records = []
  for region in REGIONS:
    ec2 = connect_to_region(region, **credentials)
    for reservation in get_all_reservations(ec2):
      if len(reservation.instances) > 1:
        print "More than one instance in this reservation: %s" % reservation.id
        continue

      instance = reservation.instances[0]
      name = instance.tags.get('Name', instance.id)
      groups = [group.name for group in reservation.groups]

      m = DIGIT.search(name)
      role = name[0:m.start()] if m else "unknown"

      if "default" not in groups:
        print "%s missing default group" % name
      groups = [group for group in groups if group != "default"]

      has_correct_roles = len(groups) > 0
      for group in groups:
        if group.startswith("cassandra_"):
          m = UNDERSCORE.search(group)
          g = group[m.start()+1:len(group)] if m else group
        else:
          g = "graph" if group == "graphite" else group
        has_correct_roles = has_correct_roles and role == g
      records.append([region, role, name, groups, has_correct_roles])
  return records

app = Flask(__name__)

@app.route('/')
@app.route('/<env>')
def security_group_assignments(env='prod'):
  records = fetch_records(env)
  return render_template('assignments.html', records=records)

if __name__ == '__main__':
  app.run(debug=True)

<commit_msg>[adhoc:dalton] Refactor/simplify dalton analysis webpage<commit_after>#!/usr/bin/env python
"""
Runs an analysis on your EC2 Security Group instance assignments.
"""

from boto.ec2 import connect_to_region
import re
import yaml
from cachetools import ttl_cache
from flask import Flask, render_template

REGIONS = ['us-east-1','us-west-1','eu-west-1','ap-northeast-1','us-west-2']

DIGIT = re.compile("\d")
UNDERSCORE = re.compile("_")

@ttl_cache(ttl=5*60)
def get_all_reservations(ec2):
  return ec2.get_all_reservations()

def fetch_hosts_and_groups(env):
  credentials = yaml.load(open('config/aws.yaml', 'r').read())[env]

  records = []
  for region in REGIONS:
    ec2 = connect_to_region(region, **credentials)
    for reservation in get_all_reservations(ec2):
      if len(reservation.instances) > 1:
        print "Skipping reservation with more than one instance: %s" % reservation.id
        continue

      instance = reservation.instances[0]
      name = instance.tags.get('Name', instance.id)
      groups = [group.name for group in reservation.groups]

      m = DIGIT.search(name)
      role = name[0:m.start()] if m else "unknown"

      records.append([region, role, name, groups])
  return records

def analyze_security_group_assignments(records):
  analyzed = []
  for region, role, name, groups in records:
    if "default" not in groups:
      print "%s missing default group" % name
    groups = [group for group in groups if group != "default"]
    has_correct_groups = check_has_correct_security_groups(role, groups)
    analyzed.append([region, role, name, groups, has_correct_groups])
  return analyzed

def check_has_correct_security_groups(role, security_groups):
  has_correct_groups = len(security_groups) > 0
  for group in security_groups:
    if group.startswith("cassandra_"):
      m = UNDERSCORE.search(group)
      g = group[m.start()+1:len(group)] if m else group
    else:
      g = "graph" if group == "graphite" else group
    has_correct_groups = has_correct_groups and role == g
  return has_correct_groups

app = Flask(__name__)

@app.route('/')
@app.route('/<env>')
def security_group_assignments(env='prod'):
  records = analyze_security_group_assignments(fetch_hosts_and_groups(env))
  return render_template('assignments.html', records=records)

if __name__ == '__main__':
  app.run(debug=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Standalone utility functions for Mininet tests."""

import os
import socket

FAUCET_DIR = os.getenv('FAUCET_DIR', '../faucet')
RESERVED_FOR_TESTS_PORTS = (179, 5001, 5002, 6633, 6653, 9179)

def mininet_dpid(int_dpid):
    """Return stringified hex version, of int DPID for mininet."""
    return str('%x' % int(int_dpid))

def str_int_dpid(str_dpid):
    """Return stringified int version, of int or hex DPID from YAML."""
    str_dpid = str(str_dpid)
    if str_dpid.startswith('0x'):
        return str(int(str_dpid, 16))
    else:
        return str(int(str_dpid))

def find_free_port(ports_socket):
    """Retrieve a free TCP port from test server."""
    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.connect(ports_socket)
    buf = ''
    while not buf.find('\n') > -1:
        buf = buf + sock.recv(1024)
    return [int(x) for x in buf.strip().split()]

def serve_ports(ports_socket):
    """Implement a TCP server to dispense free TCP ports."""
    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.bind(ports_socket)
    sock.listen(1)
    ports_served = set()

    while True:
        connection, _ = sock.accept()
        while True:
            free_socket = socket.socket()
            free_socket.bind(('', 0))
            free_port = free_socket.getsockname()[1]
            free_socket.close()
            if free_port < 1024:
                continue
            if free_port in RESERVED_FOR_TESTS_PORTS:
                continue
            if free_port in ports_served:
                continue
            break
        ports_served.add(free_port)
        # pylint: disable=no-member
        connection.sendall('%u %u\n' % (free_port, len(ports_served)))
        connection.close()

def timeout_cmd(cmd, timeout):
    return 'timeout -sKILL %us stdbuf -o0 -e0 %s' % (timeout, cmd)

def timeout_soft_cmd(cmd, timeout):
    return 'timeout %us stdbuf -o0 -e0 %s' % (timeout, cmd)
<commit_msg>Don't need to reserve 9179 anymore.<commit_after>#!/usr/bin/env python

"""Standalone utility functions for Mininet tests."""

import os
import socket

FAUCET_DIR = os.getenv('FAUCET_DIR', '../faucet')
RESERVED_FOR_TESTS_PORTS = (179, 5001, 5002, 6633, 6653)

def mininet_dpid(int_dpid):
    """Return stringified hex version, of int DPID for mininet."""
    return str('%x' % int(int_dpid))

def str_int_dpid(str_dpid):
    """Return stringified int version, of int or hex DPID from YAML."""
    str_dpid = str(str_dpid)
    if str_dpid.startswith('0x'):
        return str(int(str_dpid, 16))
    else:
        return str(int(str_dpid))

def find_free_port(ports_socket):
    """Retrieve a free TCP port from test server."""
    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.connect(ports_socket)
    buf = ''
    while not buf.find('\n') > -1:
        buf = buf + sock.recv(1024)
    return [int(x) for x in buf.strip().split()]

def serve_ports(ports_socket):
    """Implement a TCP server to dispense free TCP ports."""
    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.bind(ports_socket)
    sock.listen(1)
    ports_served = set()

    while True:
        connection, _ = sock.accept()
        while True:
            free_socket = socket.socket()
            free_socket.bind(('', 0))
            free_port = free_socket.getsockname()[1]
            free_socket.close()
            if free_port < 1024:
                continue
            if free_port in RESERVED_FOR_TESTS_PORTS:
                continue
            if free_port in ports_served:
                continue
            break
        ports_served.add(free_port)
        # pylint: disable=no-member
        connection.sendall('%u %u\n' % (free_port, len(ports_served)))
        connection.close()

def timeout_cmd(cmd, timeout):
    return 'timeout -sKILL %us stdbuf -o0 -e0 %s' % (timeout, cmd)

def timeout_soft_cmd(cmd, timeout):
    return 'timeout %us stdbuf -o0 -e0 %s' % (timeout, cmd)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2017 EMBL - European Bioinformatics Institute
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import importlib
import pytest

from model_mommy import mommy

from django.core.urlresolvers import reverse

class TestDefaultAPI(object):

    @pytest.mark.parametrize(
        '_view',
        [
            'biomes',
            'experiments',
            'pipelines',
            'publications',
            'samples',
            'studies',
            pytest.mark.xfail('viewdoesnotexist'),
        ]
    )
    @pytest.mark.django_db
    def test_empty_list(self, client, _view):
        view_name = "%s-list" % _view
        url = reverse(view_name)
        response = client.get(url)
        assert response.status_code == 200
        # assert response.headers['Content-Type'] == \
        # 'application/vnd.api+json'
        rsp = response.json()
        assert rsp['meta']['pagination']['page'] == 1
        assert rsp['meta']['pagination']['pages'] == 1
        assert rsp['meta']['pagination']['count'] == 0

    @pytest.mark.parametrize(
        '_model, _view',
        [
            ('Biome', 'biomes'),
            ('ExperimentType', 'experiments'),
            ('Pipeline', 'pipelines'),
            ('Publication', 'publications'),
            ('Run', 'runs'),
            ('Sample', 'samples'),
            ('Study', 'studies'),
        ]
    )
    @pytest.mark.django_db
    def test_list(self, client, _model, _view):
        klass = getattr(importlib.import_module("emg_api.models"), _model)

        model_name = "emg_api.%s" % _model
        view_name = "%s-list" % _view

        # start from 1
        # https://code.djangoproject.com/ticket/17653
        for pk in range(1, 101):
            if _model in ('Sample', 'Study'):
                _biome = mommy.make('emg_api.Biome', pk=pk)
                mommy.make(model_name, pk=pk, biome=_biome, is_public=1)
            elif _model in ('Run'):
                mommy.make(model_name, pk=pk, run_status_id=4)
            else:
                mommy.make(model_name, pk=pk)

        url = reverse(view_name)
        response = client.get(url)
        assert response.status_code == 200
        rsp = response.json()

        # Meta
        assert rsp['meta']['pagination']['page'] == 1
        assert rsp['meta']['pagination']['pages'] == 5
        assert rsp['meta']['pagination']['count'] == 100

        # Links
        first_link = 'http://testserver/api/%s?page=1' % _view
        last_link = 'http://testserver/api/%s?page=5' % _view
        next_link = 'http://testserver/api/%s?page=2' % _view
        assert rsp['links']['first'] == first_link
        assert rsp['links']['last'] == last_link
        assert rsp['links']['next'] == next_link
        assert rsp['links']['prev'] is None

        # Data
        assert len(rsp['data']) == 20

        for d in rsp['data']:
            assert d['type'] == _model
            _attrs = [f.name for f in klass._meta.get_fields()]
            for a in d['attributes']:
                assert a in _attrs
<commit_msg>test default relationships<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2017 EMBL - European Bioinformatics Institute
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import importlib
import pytest

from model_mommy import mommy

from django.core.urlresolvers import reverse

class TestDefaultAPI(object):

    @pytest.mark.parametrize(
        '_view',
        [
            'biomes',
            'experiments',
            'pipelines',
            'publications',
            'samples',
            'studies',
            pytest.mark.xfail('viewdoesnotexist'),
        ]
    )
    @pytest.mark.django_db
    def test_empty_list(self, client, _view):
        view_name = "%s-list" % _view
        url = reverse(view_name)
        response = client.get(url)
        assert response.status_code == 200
        # assert response.headers['Content-Type'] == \
        # 'application/vnd.api+json'
        rsp = response.json()
        assert rsp['meta']['pagination']['page'] == 1
        assert rsp['meta']['pagination']['pages'] == 1
        assert rsp['meta']['pagination']['count'] == 0

    @pytest.mark.parametrize(
        '_model, _view, relations',
        [
            ('Biome', 'biomes', ['studies', 'samples']),
            ('ExperimentType', 'experiments', ['runs']),
            ('Pipeline', 'pipelines', ['runs']),
            ('Publication', 'publications', ['studies']),
            ('Run', 'runs', ['pipeline', 'sample']),
            ('Sample', 'samples', ['biome', 'study', 'runs']),
            ('Study', 'studies', ['biome', 'publications', 'samples']),
        ]
    )
    @pytest.mark.django_db
    def test_list(self, client, _model, _view, relations):
        klass = getattr(importlib.import_module("emg_api.models"), _model)

        model_name = "emg_api.%s" % _model
        view_name = "%s-list" % _view

        # start from 1
        # https://code.djangoproject.com/ticket/17653
        for pk in range(1, 101):
            if _model in ('Sample', 'Study'):
                _biome = mommy.make('emg_api.Biome', pk=pk)
                mommy.make(model_name, pk=pk, biome=_biome, is_public=1)
            elif _model in ('Run'):
                mommy.make(model_name, pk=pk, run_status_id=4)
            else:
                mommy.make(model_name, pk=pk)

        url = reverse(view_name)
        response = client.get(url)
        assert response.status_code == 200
        rsp = response.json()

        # Meta
        assert rsp['meta']['pagination']['page'] == 1
        assert rsp['meta']['pagination']['pages'] == 5
        assert rsp['meta']['pagination']['count'] == 100

        # Links
        first_link = 'http://testserver/api/%s?page=1' % _view
        last_link = 'http://testserver/api/%s?page=5' % _view
        next_link = 'http://testserver/api/%s?page=2' % _view
        assert rsp['links']['first'] == first_link
        assert rsp['links']['last'] == last_link
        assert rsp['links']['next'] == next_link
        assert rsp['links']['prev'] is None

        # Data
        assert len(rsp['data']) == 20

        for d in rsp['data']:
            assert d['type'] == _model
            _attrs = [f.name for f in klass._meta.get_fields()]
            for a in d['attributes']:
                assert a in _attrs
            assert relations == list(d['relationships'])
<|endoftext|>