<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# 'License'); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

#pylint: disable=wildcard-import,missing-docstring,too-many-public-methods

import unittest, json
from qpid_dispatch_internal.management.schema import Schema, BooleanType, EnumType, AttributeType, ValidationError, EnumValue, EntityType
import collections

def replace_od(thing):
    if isinstance(thing, collections.Mapping):
        return dict((k, replace_od(v)) for k,v in thing.iteritems())
    if isinstance(thing, list):
        return [replace_od(t) for t in thing]
    return thing

SCHEMA_1 = {
    "prefix":"org.example",
    "annotations": {
        "entityId": {
            "attributes": {
                "name": {"type":"String", "required": True, "unique":True},
                "type": {"type":"String", "required": True}
            }
        }
    },
    "entityTypes": {
        "container": {
            "singleton": True,
            "annotations" : ["entityId"],
            "attributes": {
                "workerThreads" : {"type":"Integer", "default": 1}
            }
        },
        "listener": {
            "annotations" : ["entityId"],
            "attributes": {
                "addr" : {"type":"String"}
            }
        },
        "connector": {
            "annotations" : ["entityId"],
            "attributes": {
                "addr" : {"type":"String"}
            }
        }
    }
}

class SchemaTest(unittest.TestCase):

    def test_bool(self):
        b = BooleanType()
        self.assertTrue(b.validate('on'))
        self.assertTrue(b.validate(True))
        self.assertFalse(b.validate(False))
        self.assertFalse(b.validate('no'))
        self.assertRaises(ValidationError, b.validate, 'x')

    def test_enum(self):
        e = EnumType(['a', 'b', 'c'])
        self.assertEqual(e.validate('a'), 'a')
        self.assertEqual(e.validate(1), 'b')
        self.assertEqual(e.validate('c'), 2)
        self.assertEqual(e.validate(2), 2)
        self.assertRaises(ValidationError, e.validate, 'foo')
        self.assertRaises(ValidationError, e.validate, 3)

        self.assertEqual('["x"]', json.dumps([EnumValue('x',3)]))

    def test_attribute_def(self):
        a = AttributeType('foo', 'String', default='FOO')
        self.assertEqual('FOO', a.missing_value())
        self.assertEqual(a.validate('x'), 'x')

        a = AttributeType('foo', 'String', default='FOO', required=True)
        self.assertEqual('FOO', a.missing_value())

        a = AttributeType('foo', 'String', required=True)
        self.assertRaises(ValidationError, a.missing_value) # Missing required value.

        a = AttributeType('foo', 'String', value='FOO') # Fixed value
        self.assertEqual('FOO', a.missing_value())
        self.assertEqual(a.validate('FOO'), 'FOO')
        self.assertRaises(ValidationError, a.validate, 'XXX') # Bad fixed value

        self.assertRaises(ValidationError, AttributeType, 'foo', 'String', value='FOO', default='BAR') # Illegal

        a = AttributeType('foo', 'Integer')
        self.assertEqual(3, a.validate(3))
        self.assertEqual(3, a.validate('3'))
        self.assertEqual(3, a.validate(3.0))
        self.assertRaises(ValidationError, a.validate, None)
        self.assertRaises(ValidationError, a.validate, "xxx")

    def test_entity_type(self):
        s = Schema(annotations={
            'i1':{'attributes': { 'foo1': {'type':'String', 'default':'FOO1'}}},
            'i2':{'attributes': { 'foo2': {'type':'String', 'default':'FOO2'}}}})

        e = EntityType('MyEntity', s, attributes={
            'foo': {'type':'String', 'default':'FOO'},
            'req': {'type':'Integer', 'required':True},
            'e': {'type':['x', 'y']}})
        self.assertRaises(ValidationError, e.validate, {}) # Missing required 'req'
        self.assertEqual(e.validate({'req':42}), {'foo': 'FOO', 'req': 42})
        # Try with an annotation
        e = EntityType('e2', s, attributes={'x':{'type':'Integer'}}, annotations=['i1', 'i2'])
        self.assertEqual(e.validate({'x':1}), {'x':1, 'foo1': 'FOO1', 'foo2': 'FOO2'})

    def test_entity_refs(self):
        e = EntityType('MyEntity', Schema(), attributes={
            'type': {'type': 'String', 'required': True, 'value': '$$entityType'},
            'name': {'type':'String', 'default':'$identity'},
            'identity': {'type':'String', 'default':'$name', "required": True}})

        self.assertEqual({'type': 'MyEntity', 'identity': 'x', 'name': 'x'},
                         e.validate({'identity':'x'}))
        self.assertEqual({'type': 'MyEntity', 'identity': 'x', 'name': 'x'},
                         e.validate({'name':'x'}))
        self.assertEqual({'type': 'MyEntity', 'identity': 'x', 'name': 'y'},
                         e.validate({'identity': 'x', 'name':'y'}))
        self.assertRaises(ValidationError, e.validate, {}) # Circular reference.

    def test_entity_annotation_refs(self):
        s = Schema(annotations={
            'i1': {'attributes': {
                'name': {'type':'String', 'default':'$identity'},
                'identity': {'type':'String', 'default':'$name', "required": True}}}})

        e = EntityType('MyEntity', s, attributes={}, annotations=['i1'])
        self.assertEqual({'identity': 'x', 'name': 'x'}, e.validate({'identity':'x'}))
        self.assertEqual({'identity': 'x', 'name': 'x'}, e.validate({'name':'x'}))
        self.assertEqual({'identity': 'x', 'name': 'y'}, e.validate({'identity': 'x', 'name':'y'}))
        self.assertRaises(ValidationError, e.validate, {})

    def test_schema_dump(self):
        s = Schema(**SCHEMA_1)
        self.maxDiff = None     # pylint: disable=invalid-name
        self.longMessage = True     # pylint: disable=invalid-name
        expect = {
            "prefix":"org.example",

            "annotations": {
                "entityId": {
                    "attributes": {
                        "name": {"required": True,
                                 "unique":True,
                                 "type": "String"}
                    }
                }
            },

            "entityTypes": {
                "container": {
                    "singleton": True,
                    "attributes": {
                        "name": {"type":"String", "unique":True, "required": True},
                        "workerThreads": {"type":"Integer", "default": 1}
                    }
                    },
                    "listener": {
                        "attributes": {
                            "name": {"type":"String", "unique":True, "required": True},
                            "addr" : {"type":"String"}
                        }
                    },
                "connector": {
                    "attributes": {
                        "name": {"type":"String", "unique":True, "required": True},
                        "addr" : {"type":"String"}
                    }
                }
            }
        }
        def jsontof(j,fname):
            with open(fname,'w') as f:
                json.dump(j, f, indent=4)
        self.assertDictEqual(expect, replace_od(s.dump()))

        s2 = Schema(**s.dump())
        self.assertEqual(s.dump(), s2.dump())

    def test_schema_validate(self):
        s = Schema(**SCHEMA_1)
        # Duplicate unique attribute 'name'
        m = [{'type': 'listener', 'name':'x'},
             {'type': 'listener', 'name':'x'}]
        self.assertRaises(ValidationError, s.validate_all, m)
        # Duplicate singleton entity 'container'
        m = [{'type': 'container', 'name':'x'},
             {'type': 'container', 'name':'y'}]
        self.assertRaises(ValidationError, s.validate_all, m)
        # Valid model
        m = [{'type': 'container', 'name':'x'},
             {'type': 'listener', 'name':'y'}]
        s.validate_all(m)

    def test_schema_entity(self):
        s = Schema(**SCHEMA_1)
        self.assertRaises(ValidationError, s.entity, {'type': 'nosuch'})
        self.assertRaises(ValidationError, s.entity, {'type': 'listener', 'nosuch': 'x'})
        e = s.entity({'type': 'listener', 'name':'x', 'addr':'foo'})
        self.assertEqual(e.attributes, {'type': 'org.example.listener', 'name':'x', 'addr':'foo'})
        self.assertEqual(e['addr'], 'foo')
        self.assertRaises(ValidationError, e.__setitem__, 'nosuch', 'x')
        try:
            e.nosuch = 'x'
            self.fail("Expected exception")
        except: pass

if __name__ == '__main__':
    unittest.main()
<commit_msg>NO-JIRA: Fix test failure in management.schema  unit tests.<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# 'License'); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

#pylint: disable=wildcard-import,missing-docstring,too-many-public-methods

import unittest, json
from qpid_dispatch_internal.management.schema import Schema, BooleanType, EnumType, AttributeType, ValidationError, EnumValue, EntityType
import collections

def replace_od(thing):
    if isinstance(thing, collections.Mapping):
        return dict((k, replace_od(v)) for k,v in thing.iteritems())
    if isinstance(thing, list):
        return [replace_od(t) for t in thing]
    return thing

SCHEMA_1 = {
    "prefix":"org.example",
    "annotations": {
        "entityId": {
            "attributes": {
                "name": {"type":"String", "required": True, "unique":True},
                "type": {"type":"String", "required": True}
            }
        }
    },
    "entityTypes": {
        "container": {
            "singleton": True,
            "annotations" : ["entityId"],
            "attributes": {
                "workerThreads" : {"type":"Integer", "default": 1}
            }
        },
        "listener": {
            "annotations" : ["entityId"],
            "attributes": {
                "addr" : {"type":"String"}
            }
        },
        "connector": {
            "annotations" : ["entityId"],
            "attributes": {
                "addr" : {"type":"String"}
            }
        }
    }
}

class SchemaTest(unittest.TestCase):

    def test_bool(self):
        b = BooleanType()
        self.assertTrue(b.validate('on'))
        self.assertTrue(b.validate(True))
        self.assertFalse(b.validate(False))
        self.assertFalse(b.validate('no'))
        self.assertRaises(ValidationError, b.validate, 'x')

    def test_enum(self):
        e = EnumType(['a', 'b', 'c'])
        self.assertEqual(e.validate('a'), 'a')
        self.assertEqual(e.validate(1), 'b')
        self.assertEqual(e.validate('c'), 2)
        self.assertEqual(e.validate(2), 2)
        self.assertRaises(ValidationError, e.validate, 'foo')
        self.assertRaises(ValidationError, e.validate, 3)

        self.assertEqual('["x"]', json.dumps([EnumValue('x',3)]))

    def test_attribute_def(self):
        a = AttributeType('foo', 'String', default='FOO')
        self.assertEqual('FOO', a.missing_value())
        self.assertEqual(a.validate('x'), 'x')

        a = AttributeType('foo', 'String', default='FOO', required=True)
        self.assertEqual('FOO', a.missing_value())

        a = AttributeType('foo', 'String', required=True)
        self.assertRaises(ValidationError, a.missing_value) # Missing required value.

        a = AttributeType('foo', 'String', value='FOO') # Fixed value
        self.assertEqual('FOO', a.missing_value())
        self.assertEqual(a.validate('FOO'), 'FOO')
        self.assertRaises(ValidationError, a.validate, 'XXX') # Bad fixed value

        self.assertRaises(ValidationError, AttributeType, 'foo', 'String', value='FOO', default='BAR') # Illegal

        a = AttributeType('foo', 'Integer')
        self.assertEqual(3, a.validate(3))
        self.assertEqual(3, a.validate('3'))
        self.assertEqual(3, a.validate(3.0))
        self.assertRaises(ValidationError, a.validate, None)
        self.assertRaises(ValidationError, a.validate, "xxx")

    def test_entity_type(self):
        s = Schema(annotations={
            'i1':{'attributes': { 'foo1': {'type':'String', 'default':'FOO1'}}},
            'i2':{'attributes': { 'foo2': {'type':'String', 'default':'FOO2'}}}})

        e = EntityType('MyEntity', s, attributes={
            'foo': {'type':'String', 'default':'FOO'},
            'req': {'type':'Integer', 'required':True},
            'e': {'type':['x', 'y']}})
        e.init()
        self.assertRaises(ValidationError, e.validate, {}) # Missing required 'req'
        self.assertEqual(e.validate({'req':42}), {'foo': 'FOO', 'req': 42})
        # Try with an annotation
        e = EntityType('e2', s, attributes={'x':{'type':'Integer'}}, annotations=['i1', 'i2'])
        e.init()
        self.assertEqual(e.validate({'x':1}), {'x':1, 'foo1': 'FOO1', 'foo2': 'FOO2'})

    def test_entity_refs(self):
        e = EntityType('MyEntity', Schema(), attributes={
            'type': {'type': 'String', 'required': True, 'value': '$$entityType'},
            'name': {'type':'String', 'default':'$identity'},
            'identity': {'type':'String', 'default':'$name', "required": True}})

        self.assertEqual({'type': 'MyEntity', 'identity': 'x', 'name': 'x'},
                         e.validate({'identity':'x'}))
        self.assertEqual({'type': 'MyEntity', 'identity': 'x', 'name': 'x'},
                         e.validate({'name':'x'}))
        self.assertEqual({'type': 'MyEntity', 'identity': 'x', 'name': 'y'},
                         e.validate({'identity': 'x', 'name':'y'}))
        self.assertRaises(ValidationError, e.validate, {}) # Circular reference.

    def test_entity_annotation_refs(self):
        s = Schema(annotations={
            'i1': {'attributes': {
                'name': {'type':'String', 'default':'$identity'},
                'identity': {'type':'String', 'default':'$name', "required": True}}}})

        e = EntityType('MyEntity', s, attributes={}, annotations=['i1'])
        e.init()
        self.assertEqual({'identity': 'x', 'name': 'x'}, e.validate({'identity':'x'}))
        self.assertEqual({'identity': 'x', 'name': 'x'}, e.validate({'name':'x'}))
        self.assertEqual({'identity': 'x', 'name': 'y'}, e.validate({'identity': 'x', 'name':'y'}))
        self.assertRaises(ValidationError, e.validate, {})

    def test_schema_dump(self):
        s = Schema(**SCHEMA_1)
        self.maxDiff = None     # pylint: disable=invalid-name
        self.longMessage = True     # pylint: disable=invalid-name
        expect = {
            "prefix":"org.example",

            "annotations": {
                "entityId": {
                    "attributes": {
                        "name": {"required": True,
                                 "unique":True,
                                 "type": "String"}
                    }
                }
            },

            "entityTypes": {
                "container": {
                    "singleton": True,
                    "attributes": {
                        "name": {"type":"String", "unique":True, "required": True},
                        "workerThreads": {"type":"Integer", "default": 1}
                    }
                    },
                    "listener": {
                        "attributes": {
                            "name": {"type":"String", "unique":True, "required": True},
                            "addr" : {"type":"String"}
                        }
                    },
                "connector": {
                    "attributes": {
                        "name": {"type":"String", "unique":True, "required": True},
                        "addr" : {"type":"String"}
                    }
                }
            }
        }
        def jsontof(j,fname):
            with open(fname,'w') as f:
                json.dump(j, f, indent=4)
        self.assertDictEqual(expect, replace_od(s.dump()))

        s2 = Schema(**s.dump())
        self.assertEqual(s.dump(), s2.dump())

    def test_schema_validate(self):
        s = Schema(**SCHEMA_1)
        # Duplicate unique attribute 'name'
        m = [{'type': 'listener', 'name':'x'},
             {'type': 'listener', 'name':'x'}]
        self.assertRaises(ValidationError, s.validate_all, m)
        # Duplicate singleton entity 'container'
        m = [{'type': 'container', 'name':'x'},
             {'type': 'container', 'name':'y'}]
        self.assertRaises(ValidationError, s.validate_all, m)
        # Valid model
        m = [{'type': 'container', 'name':'x'},
             {'type': 'listener', 'name':'y'}]
        s.validate_all(m)

    def test_schema_entity(self):
        s = Schema(**SCHEMA_1)
        self.assertRaises(ValidationError, s.entity, {'type': 'nosuch'})
        self.assertRaises(ValidationError, s.entity, {'type': 'listener', 'nosuch': 'x'})
        e = s.entity({'type': 'listener', 'name':'x', 'addr':'foo'})
        self.assertEqual(e.attributes, {'type': 'org.example.listener', 'name':'x', 'addr':'foo'})
        self.assertEqual(e['addr'], 'foo')
        self.assertRaises(ValidationError, e.__setitem__, 'nosuch', 'x')
        try:
            e.nosuch = 'x'
            self.fail("Expected exception")
        except: pass

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import os.path

from go.vumitools.contact import Contact

from vumi.utils import load_class

class ContactParserException(Exception):
    pass

class ContactFileParser(object):

    DEFAULT_HEADERS = {
        'name': 'Name',
        'surname': 'Surname',
        'bbm_pin': 'BBM Pin',
        'msisdn': 'Contact Number',
        'gtalk_id': 'GTalk (or XMPP) address',
        'dob': 'Date of Birth',
        'facebook_id': 'Facebook ID',
        'twitter_handle': 'Twitter handle',
        'email_address': 'Email address',
    }

    EXCLUDED_ATTRIBUTES = ['user_account', 'created_at', 'extra', 'groups']

    def is_header_row(self, columns):
        """
        Determines whether the given columns have something that might hint
        at the row being a row with column headers and not column values.
        """
        column_set = set([column.lower() for column in columns])
        hint_set = set(['phone', 'contact', 'msisdn', 'number'])
        return hint_set.intersection(column_set)

    @classmethod
    def get_file_extension(cls, file_name):
        name, extension = os.path.splitext(file_name)
        separator, suffix = extension.rsplit('.', 1)
        return suffix

    @classmethod
    def get_parser(cls, file_name):
        extension = cls.get_file_extension(file_name)

        parser = {
            'csv': 'go.contacts.parsers.csv_parser.CSVFileParser',
            'xls': 'go.contacts.parser.xls_parser.XLSFileParser',
            'xlsx': 'go.contacts.parser.xls_parser.XLSFileParser',
        }.get(extension)

        if parser:
            parser_class = load_class(*parser.rsplit('.', 1))
            return (extension, parser_class())
        else:
            raise ContactParserException('No parser available for type %s' % (
                extension,))

    def guess_headers_and_row(self, file_path):
        """
        Take a sample from the file path and determine if it has a header
        and provide a sample of the header if found along with existing
        values matched against the known headers.

        returns a Tuple:

            (header_found, known_headers, sample_data_row)
        """
        raise NotImplementedError('Subclasses should implement this.')

    def read_data_from_file(self, file_path, field_names):
        """
        Read the data from the file returning dictionaries of `field_names`
        versus the values found.
        """
        raise NotImplementedError('Subclasses should implement this.')

    def parse_file(self, file_path, field_names, has_header,
        excluded_attributes=None):
        """
        Parses the file and returns dictionaries ready to be fed
        the ContactStore.new_contact method.

        We need to know what we cannot set to avoid a file import overwriting
        things like account details. Excluded attributes is a list of contact
        attributes that are to be ignored. Defaults to EXCLUDED_ATTRIBUTES
        """
        excluded_attributes = excluded_attributes or self.EXCLUDED_ATTRIBUTES

        known_attributes = set([attribute
            for attribute in Contact.field_descriptors.keys()
            if attribute not in excluded_attributes])

        # We're expecting a generator so loop over it and save as contacts
        # in the contact_store, normalizing anything we need to
        data_dictionaries = self.read_data_from_file(file_path, field_names)
        for counter, data_dictionary in enumerate(data_dictionaries):

            # If we've determined that the first line of the file is
            # a header then skip it.
            if has_header and counter == 0:
                continue

            # Populate this with whatever we'll be sending to the
            # contact to be saved
            contact_dictionary = {}
            for key, value in data_dictionary.items():
                if key in known_attributes:
                    contact_dictionary[key] = value
                else:
                    extra = contact_dictionary.setdefault('extra', {})
                    extra[key] = value

            yield (counter if has_header else counter + 1, contact_dictionary)

<commit_msg>simpler extension splitting<commit_after>import os.path

from go.vumitools.contact import Contact

from vumi.utils import load_class

class ContactParserException(Exception):
    pass

class ContactFileParser(object):

    DEFAULT_HEADERS = {
        'name': 'Name',
        'surname': 'Surname',
        'bbm_pin': 'BBM Pin',
        'msisdn': 'Contact Number',
        'gtalk_id': 'GTalk (or XMPP) address',
        'dob': 'Date of Birth',
        'facebook_id': 'Facebook ID',
        'twitter_handle': 'Twitter handle',
        'email_address': 'Email address',
    }

    EXCLUDED_ATTRIBUTES = ['user_account', 'created_at', 'extra', 'groups']

    def is_header_row(self, columns):
        """
        Determines whether the given columns have something that might hint
        at the row being a row with column headers and not column values.
        """
        column_set = set([column.lower() for column in columns])
        hint_set = set(['phone', 'contact', 'msisdn', 'number'])
        return hint_set.intersection(column_set)

    @classmethod
    def get_file_extension(cls, file_name):
        name, extension = os.path.splitext(file_name)
        return extension[1:]

    @classmethod
    def get_parser(cls, file_name):
        extension = cls.get_file_extension(file_name)

        parser = {
            'csv': 'go.contacts.parsers.csv_parser.CSVFileParser',
            'xls': 'go.contacts.parser.xls_parser.XLSFileParser',
            'xlsx': 'go.contacts.parser.xls_parser.XLSFileParser',
        }.get(extension)

        if parser:
            parser_class = load_class(*parser.rsplit('.', 1))
            return (extension, parser_class())
        else:
            raise ContactParserException('No parser available for type %s' % (
                extension,))

    def guess_headers_and_row(self, file_path):
        """
        Take a sample from the file path and determine if it has a header
        and provide a sample of the header if found along with existing
        values matched against the known headers.

        returns a Tuple:

            (header_found, known_headers, sample_data_row)
        """
        raise NotImplementedError('Subclasses should implement this.')

    def read_data_from_file(self, file_path, field_names):
        """
        Read the data from the file returning dictionaries of `field_names`
        versus the values found.
        """
        raise NotImplementedError('Subclasses should implement this.')

    def parse_file(self, file_path, field_names, has_header,
        excluded_attributes=None):
        """
        Parses the file and returns dictionaries ready to be fed
        the ContactStore.new_contact method.

        We need to know what we cannot set to avoid a file import overwriting
        things like account details. Excluded attributes is a list of contact
        attributes that are to be ignored. Defaults to EXCLUDED_ATTRIBUTES
        """
        excluded_attributes = excluded_attributes or self.EXCLUDED_ATTRIBUTES

        known_attributes = set([attribute
            for attribute in Contact.field_descriptors.keys()
            if attribute not in excluded_attributes])

        # We're expecting a generator so loop over it and save as contacts
        # in the contact_store, normalizing anything we need to
        data_dictionaries = self.read_data_from_file(file_path, field_names)
        for counter, data_dictionary in enumerate(data_dictionaries):

            # If we've determined that the first line of the file is
            # a header then skip it.
            if has_header and counter == 0:
                continue

            # Populate this with whatever we'll be sending to the
            # contact to be saved
            contact_dictionary = {}
            for key, value in data_dictionary.items():
                if key in known_attributes:
                    contact_dictionary[key] = value
                else:
                    extra = contact_dictionary.setdefault('extra', {})
                    extra[key] = value

            yield (counter if has_header else counter + 1, contact_dictionary)

<|endoftext|>


<commit_before>#!/bin/env python3

import yaml
import time
import subprocess
import multiprocessing
import argparse
import shutil
import os
import json

parser = argparse.ArgumentParser(description='IO scheduler tester')
parser.add_argument('--directory', help='Directory to run on', default='/mnt')
parser.add_argument('--seastar-build-dir', help='Path to seastar build directory', default='./build/dev/', dest='bdir')
parser.add_argument('--duration', help='One run duration', type=int, default=60)
parser.add_argument('--read-reqsize', help='Size of a read request in kbytes', type=int, default=4)
parser.add_argument('--read-fibers', help='Number of reading fibers', type=int, default=5)
parser.add_argument('--read-shares', help='Shares for read workload', type=int, default=2500)
parser.add_argument('--write-reqsize', help='Size of a write request in kbytes', type=int, default=64)
parser.add_argument('--write-fibers', help='Number of writing fibers', type=int, default=2)
parser.add_argument('--write-shares', help='Shares for write workload', type=int, default=100)
parser.add_argument('--shards', help='Number of shards to use', type=int)
parser.add_argument('--sleep-type', help='The io_tester conf.options.sleep_type option', default='busyloop')
parser.add_argument('--pause-dist', help='The io_tester conf.option.pause_distribution option', default='uniform')
args = parser.parse_args()

class iotune:
    def __init__(self, args):
        self._iotune = args.bdir + '/apps/iotune/iotune'
        self._dir = args.directory

    def ensure_io_properties(self):
        if os.path.exists('io_properties.yaml'):
            print('Using existing io_properties file')
        else:
            print('Running iotune')
            subprocess.check_call([self._iotune, '--evaluation-directory', self._dir, '--properties-file', 'io_properties.yaml'])

class job:
    def __init__(self, typ, req_size_kb, shares = 100, prl = None, rps = None):
        self._typ = typ
        self._req_size = req_size_kb
        self._prl = prl
        self._rps = rps
        self._shares = shares

    def prl(self):
        return self._prl

    def rqsz(self):
        return self._req_size

    def to_conf_entry(self, name, options):
        ret = {
            'name': name,
            'shards': 'all',
            'type': self._typ,
            'shard_info': {
                'reqsize': f'{self._req_size}kB',
                'shares': self._shares,
            },
            'options': options,
        }
        if self._prl is not None:
            ret['shard_info']['parallelism'] = int(self._prl)
        if self._rps is not None:
            ret['shard_info']['rps'] = int(self._rps)
        return ret

class io_tester:
    def __init__(self, args):
        self._jobs = []
        self._io_tester = args.bdir + '/apps/io_tester/io_tester'
        self._dir = args.directory
        self._use_fraction = 0.8
        self._max_data_size_gb = 8
        self._job_options = {
            'sleep_type': args.sleep_type,
            'pause_distribution': args.pause_dist,
        }
        self._io_tester_args = [
            '--io-properties-file', 'io_properties.yaml',
            '--storage', self._dir,
            '--duration', f'{args.duration}',
        ]
        if args.shards is not None:
            self._io_tester_args += [ f'-c{args.shards}' ]

    def add_job(self, name, job):
        self._jobs.append(job.to_conf_entry(name, self._job_options))

    def _setup_data_sizes(self):
        du = shutil.disk_usage(self._dir)
        one_job_space_mb = int(du.free * self._use_fraction / len(self._jobs) / (100*1024*1024)) * 100 # round down to 100MB
        if one_job_space_mb > self._max_data_size_gb * 1024:
            one_job_space_mb = self._max_data_size_gb * 1024
        for j in self._jobs:
            j['data_size'] = f'{one_job_space_mb}MB'

    def names(self):
        return [ j.name for j in _jobs ]

    def run(self):
        if not self._jobs:
            raise 'Empty jobs'

        self._setup_data_sizes()
        yaml.Dumper.ignore_aliases = lambda *args : True
        yaml.dump(self._jobs, open('conf.yaml', 'w'))
        res = subprocess.check_output([self._io_tester, '--conf', 'conf.yaml', *self._io_tester_args], stderr=subprocess.PIPE)
        res = res.split(b'---\n')[1]
        res = yaml.safe_load(res)

        ret = {}
        for shard_res in res:
            for wl in shard_res:
                if wl == 'shard':
                    continue

                if wl not in ret:
                    ret[wl] = {
                            'throughput': 0.0,
                            'IOPS': 0.0,
                            'latencies': {
                                'p0.95': 0,
                            },
                            'stats': {
                                'total_requests': 0,
                                'io_queue_total_operations': 0,
                                'io_queue_total_exec_sec': 0,
                                'io_queue_total_delay_sec': 0,
                            },
                    }

                ret[wl]['throughput'] += shard_res[wl]['throughput']
                ret[wl]['IOPS'] += shard_res[wl]['IOPS']
                ret[wl]['latencies']['p0.95'] += shard_res[wl]['latencies']['p0.95']
                ret[wl]['stats']['total_requests'] += shard_res[wl]['stats']['total_requests']
                ret[wl]['stats']['io_queue_total_operations'] += shard_res[wl]['stats']['io_queue_total_operations']
                ret[wl]['stats']['io_queue_total_delay_sec'] += shard_res[wl]['stats']['io_queue_total_delay_sec']
                ret[wl]['stats']['io_queue_total_exec_sec'] += shard_res[wl]['stats']['io_queue_total_exec_sec']

        for wl in ret:
            ret[wl]['latencies']['p0.95'] /= len(res)

        return ret

def show_stat_header():
    print('-' * 20 + '8<' + '-' * 20)
    print('name           througput(kbs) iops lat95(us) queue-time(us) execution-time(us) K(bw) K(iops) K()')

def run_and_show_results(m, ioprop):
    def xtimes(st, nm):
        return (st[nm] * 1000000) / st["io_queue_total_operations"]

    res = m.run()
    for name in res:
        st = res[name]
        throughput = st['throughput']
        iops = st['IOPS']
        lats = st['latencies']
        stats = st['stats']

        if name.startswith('read'):
            k_bw = throughput / (ioprop['read_bandwidth']/1000)
            k_iops = iops / ioprop['read_iops']
        else:
            k_bw = throughput / (ioprop['write_bandwidth']/1000)
            k_iops = iops / ioprop['write_iops']

        print(f'{name:20} {int(throughput):7} {int(iops):5} {lats["p0.95"]:.1f} {xtimes(stats, "io_queue_total_delay_sec"):.1f} {xtimes(stats, "io_queue_total_exec_sec"):.1f} {k_bw:.3f} {k_iops:.3f} {k_bw + k_iops:.3f}')

iot = iotune(args)
iot.ensure_io_properties()

ioprop = yaml.safe_load(open('io_properties.yaml'))

for prop in ioprop['disks']:
    if prop['mountpoint'] == args.directory:
        ioprop = prop
        break
else:
    raise 'Cannot find required mountpoint in io-properties'

nr_cores = args.shards
if nr_cores is None:
    nr_cores = multiprocessing.cpu_count()

read_rps_per_shard = int(ioprop['read_iops'] / nr_cores * 0.5)
read_rps = read_rps_per_shard / args.read_fibers

print(f'Read RPS:{read_rps} fibers:{args.read_fibers}')

show_stat_header()

m = io_tester(args)
m.add_job(f'read_rated_{args.read_shares}', job('randread', args.read_reqsize, shares = args.read_shares, prl = args.read_fibers, rps = read_rps))
run_and_show_results(m, ioprop)

m = io_tester(args)
m.add_job(f'write_{args.write_shares}', job('seqwrite', args.write_reqsize, shares = args.write_shares, prl = args.write_fibers))
m.add_job(f'read_rated_{args.read_shares}', job('randread', args.read_reqsize, shares = args.read_shares, prl = args.read_fibers, rps = read_rps))
run_and_show_results(m, ioprop)
<commit_msg>test,iosched: Move current test into a function<commit_after>#!/bin/env python3

import yaml
import time
import subprocess
import multiprocessing
import argparse
import shutil
import os
import json

parser = argparse.ArgumentParser(description='IO scheduler tester')
parser.add_argument('--directory', help='Directory to run on', default='/mnt')
parser.add_argument('--seastar-build-dir', help='Path to seastar build directory', default='./build/dev/', dest='bdir')
parser.add_argument('--duration', help='One run duration', type=int, default=60)
parser.add_argument('--read-reqsize', help='Size of a read request in kbytes', type=int, default=4)
parser.add_argument('--read-fibers', help='Number of reading fibers', type=int, default=5)
parser.add_argument('--read-shares', help='Shares for read workload', type=int, default=2500)
parser.add_argument('--write-reqsize', help='Size of a write request in kbytes', type=int, default=64)
parser.add_argument('--write-fibers', help='Number of writing fibers', type=int, default=2)
parser.add_argument('--write-shares', help='Shares for write workload', type=int, default=100)
parser.add_argument('--shards', help='Number of shards to use', type=int)
parser.add_argument('--sleep-type', help='The io_tester conf.options.sleep_type option', default='busyloop')
parser.add_argument('--pause-dist', help='The io_tester conf.option.pause_distribution option', default='uniform')
args = parser.parse_args()

class iotune:
    def __init__(self, args):
        self._iotune = args.bdir + '/apps/iotune/iotune'
        self._dir = args.directory

    def ensure_io_properties(self):
        if os.path.exists('io_properties.yaml'):
            print('Using existing io_properties file')
        else:
            print('Running iotune')
            subprocess.check_call([self._iotune, '--evaluation-directory', self._dir, '--properties-file', 'io_properties.yaml'])

class job:
    def __init__(self, typ, req_size_kb, shares = 100, prl = None, rps = None):
        self._typ = typ
        self._req_size = req_size_kb
        self._prl = prl
        self._rps = rps
        self._shares = shares

    def prl(self):
        return self._prl

    def rqsz(self):
        return self._req_size

    def to_conf_entry(self, name, options):
        ret = {
            'name': name,
            'shards': 'all',
            'type': self._typ,
            'shard_info': {
                'reqsize': f'{self._req_size}kB',
                'shares': self._shares,
            },
            'options': options,
        }
        if self._prl is not None:
            ret['shard_info']['parallelism'] = int(self._prl)
        if self._rps is not None:
            ret['shard_info']['rps'] = int(self._rps)
        return ret

class io_tester:
    def __init__(self, args):
        self._jobs = []
        self._io_tester = args.bdir + '/apps/io_tester/io_tester'
        self._dir = args.directory
        self._use_fraction = 0.8
        self._max_data_size_gb = 8
        self._job_options = {
            'sleep_type': args.sleep_type,
            'pause_distribution': args.pause_dist,
        }
        self._io_tester_args = [
            '--io-properties-file', 'io_properties.yaml',
            '--storage', self._dir,
            '--duration', f'{args.duration}',
        ]
        if args.shards is not None:
            self._io_tester_args += [ f'-c{args.shards}' ]

    def add_job(self, name, job):
        self._jobs.append(job.to_conf_entry(name, self._job_options))

    def _setup_data_sizes(self):
        du = shutil.disk_usage(self._dir)
        one_job_space_mb = int(du.free * self._use_fraction / len(self._jobs) / (100*1024*1024)) * 100 # round down to 100MB
        if one_job_space_mb > self._max_data_size_gb * 1024:
            one_job_space_mb = self._max_data_size_gb * 1024
        for j in self._jobs:
            j['data_size'] = f'{one_job_space_mb}MB'

    def names(self):
        return [ j.name for j in _jobs ]

    def run(self):
        if not self._jobs:
            raise 'Empty jobs'

        self._setup_data_sizes()
        yaml.Dumper.ignore_aliases = lambda *args : True
        yaml.dump(self._jobs, open('conf.yaml', 'w'))
        res = subprocess.check_output([self._io_tester, '--conf', 'conf.yaml', *self._io_tester_args], stderr=subprocess.PIPE)
        res = res.split(b'---\n')[1]
        res = yaml.safe_load(res)

        ret = {}
        for shard_res in res:
            for wl in shard_res:
                if wl == 'shard':
                    continue

                if wl not in ret:
                    ret[wl] = {
                            'throughput': 0.0,
                            'IOPS': 0.0,
                            'latencies': {
                                'p0.95': 0,
                            },
                            'stats': {
                                'total_requests': 0,
                                'io_queue_total_operations': 0,
                                'io_queue_total_exec_sec': 0,
                                'io_queue_total_delay_sec': 0,
                            },
                    }

                ret[wl]['throughput'] += shard_res[wl]['throughput']
                ret[wl]['IOPS'] += shard_res[wl]['IOPS']
                ret[wl]['latencies']['p0.95'] += shard_res[wl]['latencies']['p0.95']
                ret[wl]['stats']['total_requests'] += shard_res[wl]['stats']['total_requests']
                ret[wl]['stats']['io_queue_total_operations'] += shard_res[wl]['stats']['io_queue_total_operations']
                ret[wl]['stats']['io_queue_total_delay_sec'] += shard_res[wl]['stats']['io_queue_total_delay_sec']
                ret[wl]['stats']['io_queue_total_exec_sec'] += shard_res[wl]['stats']['io_queue_total_exec_sec']

        for wl in ret:
            ret[wl]['latencies']['p0.95'] /= len(res)

        return ret

iot = iotune(args)
iot.ensure_io_properties()

ioprop = yaml.safe_load(open('io_properties.yaml'))

for prop in ioprop['disks']:
    if prop['mountpoint'] == args.directory:
        ioprop = prop
        break
else:
    raise 'Cannot find required mountpoint in io-properties'

def mixed_show_stat_header():
    print('-' * 20 + '8<' + '-' * 20)
    print('name           througput(kbs) iops lat95(us) queue-time(us) execution-time(us) K(bw) K(iops) K()')

def mixed_run_and_show_results(m, ioprop):
    def xtimes(st, nm):
        return (st[nm] * 1000000) / st["io_queue_total_operations"]

    res = m.run()
    for name in res:
        st = res[name]
        throughput = st['throughput']
        iops = st['IOPS']
        lats = st['latencies']
        stats = st['stats']

        if name.startswith('read'):
            k_bw = throughput / (ioprop['read_bandwidth']/1000)
            k_iops = iops / ioprop['read_iops']
        else:
            k_bw = throughput / (ioprop['write_bandwidth']/1000)
            k_iops = iops / ioprop['write_iops']

        print(f'{name:20} {int(throughput):7} {int(iops):5} {lats["p0.95"]:.1f} {xtimes(stats, "io_queue_total_delay_sec"):.1f} {xtimes(stats, "io_queue_total_exec_sec"):.1f} {k_bw:.3f} {k_iops:.3f} {k_bw + k_iops:.3f}')

def run_mixed_test(args, ioprop):
    nr_cores = args.shards
    if nr_cores is None:
        nr_cores = multiprocessing.cpu_count()

    read_rps_per_shard = int(ioprop['read_iops'] / nr_cores * 0.5)
    read_rps = read_rps_per_shard / args.read_fibers

    print(f'Read RPS:{read_rps} fibers:{args.read_fibers}')

    mixed_show_stat_header()

    m = io_tester(args)
    m.add_job(f'read_rated_{args.read_shares}', job('randread', args.read_reqsize, shares = args.read_shares, prl = args.read_fibers, rps = read_rps))
    mixed_run_and_show_results(m, ioprop)

    m = io_tester(args)
    m.add_job(f'write_{args.write_shares}', job('seqwrite', args.write_reqsize, shares = args.write_shares, prl = args.write_fibers))
    m.add_job(f'read_rated_{args.read_shares}', job('randread', args.read_reqsize, shares = args.read_shares, prl = args.read_fibers, rps = read_rps))
    mixed_run_and_show_results(m, ioprop)

run_mixed_test(args, ioprop)
<|endoftext|>


<commit_before>"""
Copyright (c) 2015 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

from __future__ import unicode_literals

import os
import logging

from atomic_reactor.core import DockerTasker
from atomic_reactor.inner import DockerBuildWorkflow
from atomic_reactor.plugin import PostBuildPluginsRunner
from atomic_reactor.util import ImageName
from tests.constants import INPUT_IMAGE, SOURCE, LOCALHOST_REGISTRY_HTTP
try:
    import dockpulp
    from atomic_reactor.plugins.post_push_to_pulp import PulpPushPlugin
except (ImportError, SyntaxError):
    dockpulp = None

import pytest
from flexmock import flexmock
from tests.docker_mock import mock_docker

class X(object):
    image_id = INPUT_IMAGE
    git_dockerfile_path = None
    git_path = None
    base_image = ImageName(repo="qwe", tag="asd")

@pytest.mark.skipif(dockpulp is None,
                    reason='dockpulp module not available')
def test_pulp(tmpdir):
    tasker = DockerTasker()
    workflow = DockerBuildWorkflow(SOURCE, "test-image")
    setattr(workflow, 'builder', X())
    setattr(workflow.builder, 'source', X())
    setattr(workflow.builder.source, 'dockerfile_path', None)
    setattr(workflow.builder.source, 'path', None)
    setattr(workflow, 'tag_conf', X())
    setattr(workflow.tag_conf, 'images', [ImageName(repo="image-name1"),
                                          ImageName(namespace="prefix",
                                                    repo="image-name2"),
                                          ImageName(repo="image-name3", tag="asd")])

    # Mock dockpulp and docker
    dockpulp.Pulp = flexmock(dockpulp.Pulp)
    dockpulp.Pulp.registry='registry.example.com'
    (flexmock(dockpulp.imgutils).should_receive('get_metadata')
     .with_args(object)
     .and_return([{'id': 'foo'}]))
    (flexmock(dockpulp.imgutils).should_receive('get_versions')
     .with_args(object)
     .and_return({'id': '1.6.0'}))
    flexmock(dockpulp.imgutils).should_receive('check_repo').and_return(0)
    (flexmock(dockpulp.Pulp)
     .should_receive('push_tar_to_pulp')
     .with_args(object, object))
    (flexmock(dockpulp.Pulp)
     .should_receive('crane')
     .with_args(repos=list))
    mock_docker()

    os.environ['SOURCE_SECRET_PATH'] = str(tmpdir)
    with open(os.path.join(str(tmpdir), "pulp.cer"), "wt") as cer:
        cer.write("pulp certificate\n")
    with open(os.path.join(str(tmpdir), "pulp.key"), "wt") as key:
        key.write("pulp key\n")

    runner = PostBuildPluginsRunner(tasker, workflow, [{
        'name': PulpPushPlugin.key,
        'args': {
            'pulp_registry_name': 'test'
        }}])
    runner.run()
    assert PulpPushPlugin.key is not None
    images = [i.to_str() for i in workflow.postbuild_results[PulpPushPlugin.key]]
    assert "registry.example.com/image-name1" in images
    assert "registry.example.com/prefix/image-name2" in images
    assert "registry.example.com/image-name3:asd" in images
<commit_msg>Increase test coverage for pulp_push plugin<commit_after>"""
Copyright (c) 2015 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

from __future__ import unicode_literals

import os
import logging

from atomic_reactor.core import DockerTasker
from atomic_reactor.inner import DockerBuildWorkflow
from atomic_reactor.plugin import PostBuildPluginsRunner
from atomic_reactor.util import ImageName
from tests.constants import INPUT_IMAGE, SOURCE, LOCALHOST_REGISTRY_HTTP
try:
    import dockpulp
    from atomic_reactor.plugins.post_push_to_pulp import PulpPushPlugin
except (ImportError, SyntaxError):
    dockpulp = None

import pytest
from flexmock import flexmock
from tests.docker_mock import mock_docker

class X(object):
    image_id = INPUT_IMAGE
    git_dockerfile_path = None
    git_path = None
    base_image = ImageName(repo="qwe", tag="asd")

@pytest.mark.skipif(dockpulp is None,
                    reason='dockpulp module not available')
def test_pulp(tmpdir):
    tasker = DockerTasker()
    workflow = DockerBuildWorkflow(SOURCE, "test-image")
    setattr(workflow, 'builder', X())
    setattr(workflow.builder, 'source', X())
    setattr(workflow.builder.source, 'dockerfile_path', None)
    setattr(workflow.builder.source, 'path', None)
    setattr(workflow, 'tag_conf', X())
    setattr(workflow.tag_conf, 'images', [ImageName(repo="image-name1"),
                                          ImageName(namespace="prefix",
                                                    repo="image-name2"),
                                          ImageName(repo="image-name3", tag="asd")])

    # Mock dockpulp and docker
    dockpulp.Pulp = flexmock(dockpulp.Pulp)
    dockpulp.Pulp.registry='registry.example.com'
    (flexmock(dockpulp.imgutils).should_receive('get_metadata')
     .with_args(object)
     .and_return([{'id': 'foo'}]))
    (flexmock(dockpulp.imgutils).should_receive('get_versions')
     .with_args(object)
     .and_return({'id': '1.6.0'}))
    (flexmock(dockpulp.imgutils).should_receive('check_repo')
     .and_return(3)
     .and_return(2)
     .and_return(1)
     .and_return(0))
    (flexmock(dockpulp.Pulp)
     .should_receive('push_tar_to_pulp')
     .with_args(object, object))
    (flexmock(dockpulp.Pulp)
     .should_receive('crane')
     .with_args(repos=list))
    mock_docker()

    os.environ['SOURCE_SECRET_PATH'] = str(tmpdir)
    with open(os.path.join(str(tmpdir), "pulp.cer"), "wt") as cer:
        cer.write("pulp certificate\n")
    with open(os.path.join(str(tmpdir), "pulp.key"), "wt") as key:
        key.write("pulp key\n")

    runner = PostBuildPluginsRunner(tasker, workflow, [{
        'name': PulpPushPlugin.key,
        'args': {
            'pulp_registry_name': 'test'
        }}])
    with pytest.raises(Exception) as rc3:
        runner.run()
    with pytest.raises(Exception) as rc2:
        runner.run()
    with pytest.raises(Exception) as rc1:
        runner.run()
    runner.run()
    assert PulpPushPlugin.key is not None
    images = [i.to_str() for i in workflow.postbuild_results[PulpPushPlugin.key]]
    assert "registry.example.com/image-name1" in images
    assert "registry.example.com/prefix/image-name2" in images
    assert "registry.example.com/image-name3:asd" in images
<|endoftext|>


<commit_before>
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

"""Bloodhound multiproduct utility APIs"""

from genshi.builder import tag

from trac.util.text import unquote_label
from trac.wiki.formatter import LinkFormatter
from trac.core import Component, ComponentMeta

#----------------------------
# Component replacement base
#----------------------------

class ReplacementComponentMeta(ComponentMeta):
    """Component replacement meta class"""
    def __new__(mcs, name, bases, d):
        if bases[0] != Component:
            bases = bases[1:]
            base_class = bases[0]

            # undo what has been done by ComponentMeta.__new___ for the
            # replacement component base class

            # remove implemented interfaces from registry for the base class
            for itf in base_class._implements:
                ComponentMeta._registry[itf] = filter(lambda c: c != base_class,
                                                      ComponentMeta._registry[itf])

            # remove base class from components list
            ComponentMeta._components = filter(lambda c: c != base_class,
                                               ComponentMeta._components)

            base_class._implements = []
            base_class.abstract = True

        return ComponentMeta.__new__(mcs, name, bases, d)

class ReplacementComponent(Component):
    """Base class for components that replace existing trac
    implementations"""
    __metaclass__ = ReplacementComponentMeta

#--------------------------
# Custom wiki formatters
#--------------------------

class EmbeddedLinkFormatter(LinkFormatter):
    """Format the inner TracLinks expression corresponding to resources
    in compound links e.g. product:PREFIX:ticket:1 , global:ticket:1
    """

    def __init__(self, env, context, parent_match=None):
        """Extend initializer signature to accept parent match

        @param parent_match: mapping object containing the following keys
                        - ns : namespace of parent resolver
                        - target : target supplied in to parent resolver
                        - label: label supplied in to parent resolver
                        - fullmatch : parent regex match (optional)
        """
        super(EmbeddedLinkFormatter, self).__init__(env, context)
        self.parent_match = parent_match
        self.auto_quote = False

    def match(self, wikitext):
        if self.auto_quote:
            parts = tuple(wikitext.split(':', 1))
            if len(parts) == 2:
                if parts[1]:
                    _wikitext = '%s:"%s"' % parts
                else:
                    _wikitext = '[%s:]' % parts[:1]
            else:
                _wikitext = wikitext
        return super(EmbeddedLinkFormatter, self).match(_wikitext)

    @staticmethod
    def enhance_link(link):
        return link

    def handle_match(self, fullmatch):
        if self.parent_match is None:
            return super(EmbeddedLinkFormatter, self).handle_match(fullmatch)

        for itype, match in fullmatch.groupdict().items():
            if match and not itype in self.wikiparser.helper_patterns:
                # Check for preceding escape character '!'
                if match[0] == '!':
                    # Erroneous expression. Nested link would be escaped
                    return tag.a(self.parent_match['label'], class_='missing')
                if itype in self.wikiparser.external_handlers:
                    #TODO: Important! Add product prefix in label (when needed?)
                    external_handler = self.wikiparser.external_handlers[itype]
                    link = external_handler(self, match, fullmatch)
                else:
                    internal_handler = getattr(self, '_%s_formatter' % itype)
                    link = internal_handler(match, fullmatch)
                return self.enhance_link(link)

    # Overridden formatter methods
    # TODO : Override more if necessary
    def _shref_formatter(self, match, fullmatch):
        if self.parent_match is None:
            return super(EmbeddedLinkFormatter, self)._shref_formatter(
                    match, fullmatch)
        ns = fullmatch.group('sns')
        target = unquote_label(fullmatch.group('stgt'))
        label = (self.parent_match['label']
                 if self.parent_match['label'] != self.parent_match['target']
                 else target)
        return self._make_link(ns, target, match, label, fullmatch)

    def _lhref_formatter(self, match, fullmatch):
        if self.parent_match is None:
            return super(EmbeddedLinkFormatter, self)._lhref_formatter(
                    match, fullmatch)
        rel = fullmatch.group('rel')
        ns = fullmatch.group('lns')
        target = unquote_label(fullmatch.group('ltgt'))
        label = (self.parent_match['label']
                 if self.parent_match['label'] != self.parent_match['target']
                 else fullmatch.group('label'))
        return self._make_lhref_link(match, fullmatch, rel, ns, target, label)

#----------------------
# Useful regex
#----------------------

IDENTIFIER = r'(?!\d)\w+'
<commit_msg>Set `svn:eol-style = native` on all source code files.<commit_after>
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

"""Bloodhound multiproduct utility APIs"""

from genshi.builder import tag

from trac.util.text import unquote_label
from trac.wiki.formatter import LinkFormatter
from trac.core import Component, ComponentMeta

#----------------------------
# Component replacement base
#----------------------------

class ReplacementComponentMeta(ComponentMeta):
    """Component replacement meta class"""
    def __new__(mcs, name, bases, d):
        if bases[0] != Component:
            bases = bases[1:]
            base_class = bases[0]

            # undo what has been done by ComponentMeta.__new___ for the
            # replacement component base class

            # remove implemented interfaces from registry for the base class
            for itf in base_class._implements:
                ComponentMeta._registry[itf] = filter(lambda c: c != base_class,
                                                      ComponentMeta._registry[itf])

            # remove base class from components list
            ComponentMeta._components = filter(lambda c: c != base_class,
                                               ComponentMeta._components)

            base_class._implements = []
            base_class.abstract = True

        return ComponentMeta.__new__(mcs, name, bases, d)

class ReplacementComponent(Component):
    """Base class for components that replace existing trac
    implementations"""
    __metaclass__ = ReplacementComponentMeta

#--------------------------
# Custom wiki formatters
#--------------------------

class EmbeddedLinkFormatter(LinkFormatter):
    """Format the inner TracLinks expression corresponding to resources
    in compound links e.g. product:PREFIX:ticket:1 , global:ticket:1
    """

    def __init__(self, env, context, parent_match=None):
        """Extend initializer signature to accept parent match

        @param parent_match: mapping object containing the following keys
                        - ns : namespace of parent resolver
                        - target : target supplied in to parent resolver
                        - label: label supplied in to parent resolver
                        - fullmatch : parent regex match (optional)
        """
        super(EmbeddedLinkFormatter, self).__init__(env, context)
        self.parent_match = parent_match
        self.auto_quote = False

    def match(self, wikitext):
        if self.auto_quote:
            parts = tuple(wikitext.split(':', 1))
            if len(parts) == 2:
                if parts[1]:
                    _wikitext = '%s:"%s"' % parts
                else:
                    _wikitext = '[%s:]' % parts[:1]
            else:
                _wikitext = wikitext
        return super(EmbeddedLinkFormatter, self).match(_wikitext)

    @staticmethod
    def enhance_link(link):
        return link

    def handle_match(self, fullmatch):
        if self.parent_match is None:
            return super(EmbeddedLinkFormatter, self).handle_match(fullmatch)

        for itype, match in fullmatch.groupdict().items():
            if match and not itype in self.wikiparser.helper_patterns:
                # Check for preceding escape character '!'
                if match[0] == '!':
                    # Erroneous expression. Nested link would be escaped
                    return tag.a(self.parent_match['label'], class_='missing')
                if itype in self.wikiparser.external_handlers:
                    #TODO: Important! Add product prefix in label (when needed?)
                    external_handler = self.wikiparser.external_handlers[itype]
                    link = external_handler(self, match, fullmatch)
                else:
                    internal_handler = getattr(self, '_%s_formatter' % itype)
                    link = internal_handler(match, fullmatch)
                return self.enhance_link(link)

    # Overridden formatter methods
    # TODO : Override more if necessary
    def _shref_formatter(self, match, fullmatch):
        if self.parent_match is None:
            return super(EmbeddedLinkFormatter, self)._shref_formatter(
                    match, fullmatch)
        ns = fullmatch.group('sns')
        target = unquote_label(fullmatch.group('stgt'))
        label = (self.parent_match['label']
                 if self.parent_match['label'] != self.parent_match['target']
                 else target)
        return self._make_link(ns, target, match, label, fullmatch)

    def _lhref_formatter(self, match, fullmatch):
        if self.parent_match is None:
            return super(EmbeddedLinkFormatter, self)._lhref_formatter(
                    match, fullmatch)
        rel = fullmatch.group('rel')
        ns = fullmatch.group('lns')
        target = unquote_label(fullmatch.group('ltgt'))
        label = (self.parent_match['label']
                 if self.parent_match['label'] != self.parent_match['target']
                 else fullmatch.group('label'))
        return self._make_lhref_link(match, fullmatch, rel, ns, target, label)

#----------------------
# Useful regex
#----------------------

IDENTIFIER = r'(?!\d)\w+'
<|endoftext|>


<commit_before>#!/usr/bin/env python

import collections
import re
import subprocess
import os, sys

DEFINED_SYMBOL_TYPE = 'S'
UNDEFINED_SYMBOL_TYPE = 'U'
UNKNOWN_SYMBOL_TYPE = '?'

def parse_symbol_string(symbol_string):
    """Takes a string like '                 U _OBJC_CLASS_$_NSView' and returns symbol type and name"""
    match = re.match(r".+ ([a-zA-Z]) (.*)$", symbol_string)
    if match:
        symbol_type = match.group(1)
        symbol_name = match.group(2)
        if symbol_type in [DEFINED_SYMBOL_TYPE, UNDEFINED_SYMBOL_TYPE]:
            return symbol_type, symbol_name
    return UNKNOWN_SYMBOL_TYPE, None

def global_symbols(filename):
    """Returns tuple of lists - (defined_symbols, undefined_symbols)."""
    defined_symbols = []
    undefined_symbols = []
    global_symbols_text = subprocess.check_output(["nm", "-g", filename])
    for line in global_symbols_text.splitlines():
        symbol_type, symbol_name = parse_symbol_string(line)
        if symbol_type == DEFINED_SYMBOL_TYPE:
            defined_symbols.append(symbol_name)
        elif symbol_type == UNDEFINED_SYMBOL_TYPE:
            undefined_symbols.append(symbol_name)
    return defined_symbols, undefined_symbols

class SymbolTable:
    """SymbolTable tracks in which file which symbol was defined."""
    def __init__(self):
        self._symbol_to_file_dict = dict()

    def add_defined_symbol_from_file(self, symbol_name, filename):
        conflicting_filename = self._symbol_to_file_dict.get(symbol_name)
        #assert conflicting_filename is None
        self._symbol_to_file_dict[symbol_name] = filename

    def file_for_symbol(self, symbol_name):
        return self._symbol_to_file_dict.get(symbol_name)

def short_filename(file_path):
    """Strips directory and extension from file path."""
    return os.path.splitext(os.path.split(file_path)[1])[0]

def readable_symbol_name(symbol_name):
    """Strips known unreadable prefixes."""
    prefixes = ('_OBJC_CLASS_$_', '_OBJC_METACLASS_$_')
    for prefix in prefixes:
        if symbol_name.startswith(prefix):
            symbol_name = symbol_name[len(prefix):]
            break  # Strip single prefix.
    return symbol_name

class DirectedGraph:
    def __init__(self):
        self._adjacency_matrix = collections.defaultdict(list)

    def add_edge_with_label(self, from_vertex, to_vertex, edge_label):
        self._adjacency_matrix[from_vertex].append((to_vertex, edge_label))

    def add_vertex(self, vertex):
        self._adjacency_matrix[vertex] = []

    def is_empty(self):
        return len(self._adjacency_matrix) == 0

    def _collect_same_destination_vertex(self):
        result = dict()
        for from_vertex, destinations in self._adjacency_matrix.iteritems():
            collected_destinations = collections.defaultdict(set)
            for to_vertex, edge_label in destinations:
                collected_destinations[to_vertex].add(edge_label)
            result[from_vertex] = collected_destinations
        return result

    def write_dot_file(self, filename, write_edge_labels=False):
        nice_adjacency_matrix = self._collect_same_destination_vertex()
        with open(filename, 'w') as f:
            f.write("digraph dependencies {\n")
            for from_vertex, destinations in nice_adjacency_matrix.iteritems():
                for to_vertex, edge_labels in destinations.iteritems():
                    f.write(" " * 4)
                    f.write("%s -> %s" % (from_vertex, to_vertex))
                    if write_edge_labels:
                        f.write(" [label='%s']" % ", ".join(sorted(edge_labels)))
                    f.write(";\n")
                # Write vertex without outgoing nodes
                if len(destinations) == 0:
                    f.write(" " * 4)
                    f.write("%s;\n" % from_vertex)
            f.write("}\n")

class Dependencies:
    def __init__(self, link_file_list_filename):
        with open(link_file_list_filename, "r") as f:
            files_to_process = f.read().splitlines()
        symbol_table = SymbolTable()
        undefined_symbols = []
        # Remember where each defined symbol is defined.
        for file in files_to_process:
            defined, undefined = global_symbols(file)
            for symbol in defined:
                symbol_table.add_defined_symbol_from_file(symbol, file)
            undefined_symbols.append((file, undefined))
        # Find dependencies for undefined symbols.
        dependency_graph = DirectedGraph()
        for file, undefined in undefined_symbols:
            short_file = short_filename(file)
            dependency_graph.add_vertex(short_file)
            for symbol in undefined:
                defined_file = symbol_table.file_for_symbol(symbol)
                if defined_file is not None:
                    dependency_graph.add_edge_with_label(short_file,
                        short_filename(defined_file), readable_symbol_name(symbol))
        self._dependency_graph = dependency_graph

    def dump(self, filename, write_edge_labels=False, include_marked_files=False):
        assert not self._dependency_graph.is_empty()
        self._dependency_graph.write_dot_file(filename, write_edge_labels)

def print_usage():
    print """You must provide LINK_FILE_LIST_FILE
Usage: dependency-viz LINK_FILE_LIST_FILE"""

def get_files_to_process():
    if len(sys.argv) != 2:
        print_usage()
        sys.exit(1)
    linkFileListFilename = sys.argv[1]
    with open(linkFileListFilename, "r") as f:
        files_to_process = f.read().splitlines()
    return files_to_process

def main():
    if len(sys.argv) != 2:
        print_usage()
        sys.exit(1)
    link_file_list_filename = sys.argv[1]
    dependencies = Dependencies(link_file_list_filename)
    dependencies.dump("dependency.dot")

if __name__ == "__main__":
    main()

# I have an idea to run dependency-viz from Python REPL, not as a standalone
# command line tool.  It will reduce UI overhead.
#
# How I want to use dependency-visualizer (wishful thinking):
# > dependencies = Dependencies(linkFileListFilename)
# > dependencies.mark_files(unitTestLinkFileListFilename)
#
# -- write a graph
# > dependencies.dump("dependency.dot", write_edge_labels, include_marked_files)
# > dependencies.dump_subgraph("dependency.dot", vertexes, write_edge_labels, include_marked_files)
#
# -- trivial getters
# > print dependencies.files()
# > print dependencies.marked_files()
#
# -- what [transitive] dependencies should be satisfied to add file to tests
# > print dependencies.required_dependencies(filename, cumulative|detailed)
#
# -- find which file is most needed
# > print dependencies.provided_dependencies(filename, cumulative|detailed)
# > dependencies.all_dependencies()  # returns filename with provided and required dependencies
#
# -- kinda cycle detection
# > dependencies.strong_connected_components()
<commit_msg>Mark files.<commit_after>#!/usr/bin/env python

import collections
import re
import subprocess
import os, sys

DEFINED_SYMBOL_TYPE = 'S'
UNDEFINED_SYMBOL_TYPE = 'U'
UNKNOWN_SYMBOL_TYPE = '?'

def parse_symbol_string(symbol_string):
    """Takes a string like '                 U _OBJC_CLASS_$_NSView' and returns symbol type and name"""
    match = re.match(r".+ ([a-zA-Z]) (.*)$", symbol_string)
    if match:
        symbol_type = match.group(1)
        symbol_name = match.group(2)
        if symbol_type in [DEFINED_SYMBOL_TYPE, UNDEFINED_SYMBOL_TYPE]:
            return symbol_type, symbol_name
    return UNKNOWN_SYMBOL_TYPE, None

def global_symbols(filename):
    """Returns tuple of lists - (defined_symbols, undefined_symbols)."""
    defined_symbols = []
    undefined_symbols = []
    global_symbols_text = subprocess.check_output(["nm", "-g", filename])
    for line in global_symbols_text.splitlines():
        symbol_type, symbol_name = parse_symbol_string(line)
        if symbol_type == DEFINED_SYMBOL_TYPE:
            defined_symbols.append(symbol_name)
        elif symbol_type == UNDEFINED_SYMBOL_TYPE:
            undefined_symbols.append(symbol_name)
    return defined_symbols, undefined_symbols

class SymbolTable:
    """SymbolTable tracks in which file which symbol was defined."""
    def __init__(self):
        self._symbol_to_file_dict = dict()

    def add_defined_symbol_from_file(self, symbol_name, filename):
        conflicting_filename = self._symbol_to_file_dict.get(symbol_name)
        #assert conflicting_filename is None
        self._symbol_to_file_dict[symbol_name] = filename

    def file_for_symbol(self, symbol_name):
        return self._symbol_to_file_dict.get(symbol_name)

def short_filename(file_path):
    """Strips directory and extension from file path."""
    return os.path.splitext(os.path.split(file_path)[1])[0]

def readable_symbol_name(symbol_name):
    """Strips known unreadable prefixes."""
    prefixes = ('_OBJC_CLASS_$_', '_OBJC_METACLASS_$_')
    for prefix in prefixes:
        if symbol_name.startswith(prefix):
            symbol_name = symbol_name[len(prefix):]
            break  # Strip single prefix.
    return symbol_name

class DirectedGraph:
    def __init__(self):
        self._adjacency_matrix = collections.defaultdict(list)

    def add_edge_with_label(self, from_vertex, to_vertex, edge_label):
        self._adjacency_matrix[from_vertex].append((to_vertex, edge_label))

    def add_vertex(self, vertex):
        self._adjacency_matrix[vertex] = []

    def is_empty(self):
        return len(self._adjacency_matrix) == 0

    def vertexes(self):
        """Returns all vertexes."""
        return frozenset(self._adjacency_matrix.keys())

    def _collect_same_destination_vertex(self):
        result = dict()
        for from_vertex, destinations in self._adjacency_matrix.iteritems():
            collected_destinations = collections.defaultdict(set)
            for to_vertex, edge_label in destinations:
                collected_destinations[to_vertex].add(edge_label)
            result[from_vertex] = collected_destinations
        return result

    def write_dot_file(self, filename, write_edge_labels=False):
        nice_adjacency_matrix = self._collect_same_destination_vertex()
        with open(filename, 'w') as f:
            f.write("digraph dependencies {\n")
            for from_vertex, destinations in nice_adjacency_matrix.iteritems():
                for to_vertex, edge_labels in destinations.iteritems():
                    f.write(" " * 4)
                    f.write("%s -> %s" % (from_vertex, to_vertex))
                    if write_edge_labels:
                        f.write(" [label='%s']" % ", ".join(sorted(edge_labels)))
                    f.write(";\n")
                # Write vertex without outgoing nodes
                if len(destinations) == 0:
                    f.write(" " * 4)
                    f.write("%s;\n" % from_vertex)
            f.write("}\n")

class Dependencies:
    def __init__(self, link_file_list_filename):
        with open(link_file_list_filename, "r") as f:
            files_to_process = f.read().splitlines()
        symbol_table = SymbolTable()
        undefined_symbols = []
        # Remember where each defined symbol is defined.
        for file in files_to_process:
            defined, undefined = global_symbols(file)
            for symbol in defined:
                symbol_table.add_defined_symbol_from_file(symbol, file)
            undefined_symbols.append((file, undefined))
        # Find dependencies for undefined symbols.
        dependency_graph = DirectedGraph()
        for file, undefined in undefined_symbols:
            short_file = short_filename(file)
            dependency_graph.add_vertex(short_file)
            for symbol in undefined:
                defined_file = symbol_table.file_for_symbol(symbol)
                if defined_file is not None:
                    dependency_graph.add_edge_with_label(short_file,
                        short_filename(defined_file), readable_symbol_name(symbol))
        self._dependency_graph = dependency_graph
        self._marked_files = frozenset()

    def mark_files(self, link_file_list_filename):
        with open(link_file_list_filename, "r") as f:
            files = f.read().splitlines()
        files = [short_filename(f) for f in files]
        self._marked_files = frozenset(files)

    def files(self):
        return self._dependency_graph.vertexes()

    def marked_files(self):
        return self._marked_files

    def dump(self, filename, write_edge_labels=False, include_marked_files=False):
        assert not self._dependency_graph.is_empty()
        self._dependency_graph.write_dot_file(filename, write_edge_labels)

def print_usage():
    print """You must provide LINK_FILE_LIST_FILE
Usage: dependency-viz LINK_FILE_LIST_FILE"""

def get_files_to_process():
    if len(sys.argv) != 2:
        print_usage()
        sys.exit(1)
    linkFileListFilename = sys.argv[1]
    with open(linkFileListFilename, "r") as f:
        files_to_process = f.read().splitlines()
    return files_to_process

def main():
    if len(sys.argv) != 2:
        print_usage()
        sys.exit(1)
    link_file_list_filename = sys.argv[1]
    dependencies = Dependencies(link_file_list_filename)
    dependencies.dump("dependency.dot")

if __name__ == "__main__":
    main()

# I have an idea to run dependency-viz from Python REPL, not as a standalone
# command line tool.  It will reduce UI overhead.
#
# How I want to use dependency-visualizer (wishful thinking):
# > dependencies = Dependencies(linkFileListFilename)
# > dependencies.mark_files(unitTestLinkFileListFilename)
#
# -- write a graph
# > dependencies.dump("dependency.dot", write_edge_labels, include_marked_files)
# > dependencies.dump_subgraph("dependency.dot", vertexes, write_edge_labels, include_marked_files)
#
# -- trivial getters
# > print dependencies.files()
# > print dependencies.marked_files()
#
# -- what [transitive] dependencies should be satisfied to add file to tests
# > print dependencies.required_dependencies(filename, cumulative|detailed)
#
# -- find which file is most needed
# > print dependencies.provided_dependencies(filename, cumulative|detailed)
# > dependencies.all_dependencies()  # returns filename with provided and required dependencies
#
# -- kinda cycle detection
# > dependencies.strong_connected_components()
<|endoftext|>


<commit_before>import random
import string

from fabric.contrib.files import exists, sed, sudo
from fabric.api import env, local, run

env.use_ssh_config = True

REPO_URL = 'https://github.com/nicorellius/fiblist.git'
SETTINGS_FOLDER = 'fiblist/conf/settings'
PROJECT = 'fiblist'
VIRTENV_FOLDER = '/home/{0}/dev/virtenvs/{1}'.format(env.user, PROJECT)

def deploy():

    site_folder = '/home/{0}/sites/{1}'.format(env.user, PROJECT)
    source_folder = '{0}/source'.format(site_folder)
    secret_key_file = '/etc/prv/{0}/secret_key.txt'.format(PROJECT)

    _create_directory_structure_if_necessary(site_folder)
    _get_latest_source(site_folder, source_folder)
    _update_settings(source_folder, env.host)
    _generate_secret_key(secret_key_file)
    _update_virtenv(source_folder)
    _update_static_files(source_folder)
    _update_database(source_folder)

def _create_directory_structure_if_necessary(site_folder):

    for subfolder in ('database', 'static', 'source'):
        run('mkdir -p {0}/{1}'.format(site_folder, subfolder))

def _get_latest_source(site_folder, source_folder):

    if exists('{0}/{1}'.format(site_folder, '.git')):
        run('cd {0} && git fetch'.format(source_folder))

    else:
        run('git clone {0} {1}'.format(REPO_URL, source_folder))

    current_commit = local('git log -n 1 --format=%H', capture=True)
    run('cd {0} && git reset --hard {1}'.format(source_folder, current_commit))

def _update_settings(source_folder, site_name):

    settings_file = '{0}/{1}/staging.py'.format(source_folder, SETTINGS_FOLDER)
    sed(settings_file, 'DEBUG = True', 'DEBUG = False')
    sed(
        settings_file,
        'ALLOWED_HOSTS = .+$',
        'ALLOWED_HOSTS = ["{0}"]'.format(site_name)
    )

def _generate_secret_key(secret_key_file):

    sudo('mkdir -p /etc/prv/{0}'.format(PROJECT))

    if not exists(secret_key_file):
        generated_key = ''.join(
            [random.SystemRandom().choice(''.format(
                string.ascii_letters,
                string.digits,
                string.punctuation)
            ) for _ in range(50)]
        )

        secret = open(secret_key_file, 'w')
        secret.write(generated_key)
        secret.close()

def _update_virtenv(source_folder):

    virtenv_folder = VIRTENV_FOLDER

    if not exists('{0}/bin/pip'.format(virtenv_folder)):
        run('mkvirtualenv --python=/usr/bin/python3 {0}'.format(virtenv_folder))

    run('{0}/bin/pip install -r {1}/requirements.txt'.format(
        virtenv_folder,
        source_folder
    ))

def _update_static_files(source_folder):
    run('cd {0} && bin/python3 manage.py collectstatic --noinput'.format(VIRTENV_FOLDER))

def _update_database(source_folder):
    run('cd {0} && bin/python3 manage.py migrate --noinput'.format(VIRTENV_FOLDER))
<commit_msg>fixed bug in random secret key generator.<commit_after>import random
import string

from fabric.contrib.files import exists, sed, sudo
from fabric.api import env, local, run

env.use_ssh_config = True

REPO_URL = 'https://github.com/nicorellius/fiblist.git'
SETTINGS_FOLDER = 'fiblist/conf/settings'
PROJECT = 'fiblist'
VIRTENV_FOLDER = '/home/{0}/dev/virtenvs/{1}'.format(env.user, PROJECT)

def deploy():

    site_folder = '/home/{0}/sites/{1}'.format(env.user, PROJECT)
    source_folder = '{0}/source'.format(site_folder)
    secret_key_file = '/etc/prv/{0}/secret_key.txt'.format(PROJECT)

    _create_directory_structure_if_necessary(site_folder)
    _get_latest_source(site_folder, source_folder)
    _update_settings(source_folder, env.host)
    _generate_secret_key(secret_key_file)
    _update_virtenv(source_folder)
    _update_static_files(source_folder)
    _update_database(source_folder)

def _create_directory_structure_if_necessary(site_folder):

    for subfolder in ('database', 'static', 'source'):
        run('mkdir -p {0}/{1}'.format(site_folder, subfolder))

def _get_latest_source(site_folder, source_folder):

    if exists('{0}/{1}'.format(site_folder, '.git')):
        run('cd {0} && git fetch'.format(source_folder))

    else:
        run('git clone {0} {1}'.format(REPO_URL, source_folder))

    current_commit = local('git log -n 1 --format=%H', capture=True)
    run('cd {0} && git reset --hard {1}'.format(source_folder, current_commit))

def _update_settings(source_folder, site_name):

    settings_file = '{0}/{1}/staging.py'.format(source_folder, SETTINGS_FOLDER)
    sed(settings_file, 'DEBUG = True', 'DEBUG = False')
    sed(
        settings_file,
        'ALLOWED_HOSTS = .+$',
        'ALLOWED_HOSTS = ["{0}"]'.format(site_name)
    )

def _generate_secret_key(secret_key_file):

    sudo('mkdir -p /etc/prv/{0}'.format(PROJECT))

    if not exists(secret_key_file):
        generated_key = ''.join(
            [random.SystemRandom().choice('{0}{1}{2}'.format(
                string.ascii_letters,
                string.digits,
                string.punctuation)
            ) for _ in range(50)]
        )

        secret = open(secret_key_file, 'w')
        secret.write(generated_key)
        secret.close()

def _update_virtenv(source_folder):

    virtenv_folder = VIRTENV_FOLDER

    if not exists('{0}/bin/pip'.format(virtenv_folder)):
        run('mkvirtualenv --python=/usr/bin/python3 {0}'.format(virtenv_folder))

    run('{0}/bin/pip install -r {1}/requirements.txt'.format(
        virtenv_folder,
        source_folder
    ))

def _update_static_files(source_folder):
    run('cd {0} && bin/python3 manage.py collectstatic --noinput'.format(VIRTENV_FOLDER))

def _update_database(source_folder):
    run('cd {0} && bin/python3 manage.py migrate --noinput'.format(VIRTENV_FOLDER))
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)),'../2d/numericalTanks/randomWaves'))

import pytest
from proteus.iproteus import *
from proteus import Comm
comm = Comm.get()
#import random_waves_so
#import random_waves as rw
import numpy as np
import collections as cll
import csv
from proteus.test_utils import TestTools
#from AnalysisTools import readProbeFile,signalFilter,zeroCrossing,reflStat

from proteus.defaults import (load_physics as load_p,
                              load_numerics as load_n,
                              load_system as load_so)

modulepath = os.path.join(os.path.dirname(os.path.abspath(__file__)),'../../2d/numericalTanks/randomWaves')
petsc_options = os.path.join(os.path.dirname(os.path.abspath(__file__)),"../../inputTemplates/petsc.options.asm")

class NumericResults:

    @staticmethod
    def _read_log(file_name):
        log_file = open(file_name,'r')
        return log_file

    @classmethod
    def create_log(cls,file_name):
        random_log = cls._read_log(file_name)
        return random_log

class TestRandomWavesTetgen(TestTools.AirWaterVVTest):

    @classmethod
    def setup_class(cls):
        pass

    @classmethod
    def teardown_class(cls):
        pass

    def setup_method(self,method):
        #pass
        Profiling.openLog("proteus.log",10)
        Profiling.logAllProcesses = True

    def teardown_method(self,method):
        Profiling.closeLog()

        """ Tear down function """
        FileList = ['random_waves.xmf',
                    'random_waves.h5']
        for file in FileList:
            if os.path.isfile(file):
                os.remove(file)
            else:
                pass

    def test_run(self):
        from petsc4py import PETSc
        pList = []
        nList = []
        so = load_so('random_waves_so',modulepath)
        for (p,n) in so.pnList:
            pList.append(load_p(p,modulepath))
            nList.append(load_n(n,modulepath))
            if pList[-1].name == None:
                pList[-1].name = p
        #so = random_waves_so
        so.name = "random_waves"
        if so.sList == []:
            for i in range(len(so.pnList)):
                s = default_s
                so.sList.append(s)
        Profiling.logLevel=7
        Profiling.verbose=True
        # PETSc solver configuration
        OptDB = PETSc.Options()
        with open(petsc_options) as f:
            all = f.read().split()
            i=0
            while i < len(all):
                if i < len(all)-1:
                    if all[i+1][0]!='-':
                        print "setting ", all[i].strip(), all[i+1]
                        OptDB.setValue(all[i].strip('-'),all[i+1])
                        i=i+2
                    else:
                        print "setting ", all[i].strip(), "True"
                        OptDB.setValue(all[i].strip('-'),True)
                        i=i+1
                else:
                    print "setting ", all[i].strip(), "True"
                    OptDB.setValue(all[i].strip('-'),True)
                    i=i+1
        so.tnList=[0.0,0.001,0.011]
        #so.tnList=[0.0,0.001]+[0.001 + i*0.01 for i in range(1,int(round(0.03/0.01))+1)]
        ns = NumericalSolution.NS_base(so,pList,nList,so.sList,opts)
        ns.calculateSolution('random_waves')

        random_log = NumericResults.create_log('proteus.log')

        text = random_log.read()

        #if text.find('CFL') != -1:
        if text.find('Step Failed,') != -1:

            a = "No convergence"
        else:
            a = "good"

        if a  == "No convergence":
            print ("Convergence issue")
            assert False
        else:
            assert True

        #def failed(filename,word):
        #    file = open(filename,"r")
        #    text = file.read()

        #    if text.find(word) != -1:
        #        a = "No convergence"
        #    else:
        #        a = "good"
        #    file.close()
        #    return a

        #b = failed('proteus.log','Step Failed,')

        #if b == "No convergence":
        #    print ("Convergence issue")
        #    assert False
        #else:
        #    assert True

        #assert(True)

#     def test_validate(self):
#         # Reading probes into the file
#         file_vof = 'column_gauges.csv'
#         # Exctracting probes
#         data_vof = readProbeFile(file_vof)
#         time = data_vof[2]
#         vof = data_vof[3]
#         # Eta for RandomWavesFast
#         tank_dim = rw.opts.tank_dim
#         waterLevel = rw.opts.water_level
#         eta_fast = np.array(tank_dim[1]-vof[:,0]-waterLevel)
#         # Eta for RandomWaves
#         Tp = rw.Tp
#         Hs = rw.Hs
#         mwl = rw.mwl
#         depth = rw.depth
#         waveDir = np.array(rw.waveDir)
#         g = np.array(rw.g)
#         N = rw.N
#         bandFactor = rw.bandFactor
#         spectName = rw.spectName
#         phi = rw.phi
#         wave_ref = rw.wt.RandomWaves(Tp,Hs,mwl,depth,waveDir,g,N,bandFactor,spectName,spectral_params=None,phi=phi,fast=True)
#         eta_ref = []
#         X = np.array([0., 0., 0.])
#         for i in range(0,len(time)):
#             eta_ref.append(wave_ref.eta(X,time[i]))
#         eta_ref = np.array(eta_ref)
#         # Validation of the results
#         S = 0.
#         c = 0.
#         for i in range(len(time)):
#             c += 1.
#             S += (eta_fast[i]-eta_ref[i])**2
#         err = np.sqrt(S/c)
#         err = 100*err/(rw.opts.Hs)
#         assert(err<10.)

if __name__ == '__main__':
    pass
<commit_msg>Fix random waves bug path<commit_after>#!/usr/bin/env python
import os
os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)),'../../2d/numericalTanks/randomWaves'))

import pytest
from proteus.iproteus import *
from proteus import Comm
comm = Comm.get()
#import random_waves_so
#import random_waves as rw
import numpy as np
import collections as cll
import csv
from proteus.test_utils import TestTools
#from AnalysisTools import readProbeFile,signalFilter,zeroCrossing,reflStat

from proteus.defaults import (load_physics as load_p,
                              load_numerics as load_n,
                              load_system as load_so)

modulepath = os.path.join(os.path.dirname(os.path.abspath(__file__)),'../../2d/numericalTanks/randomWaves')
petsc_options = os.path.join(os.path.dirname(os.path.abspath(__file__)),"../../inputTemplates/petsc.options.asm")

class NumericResults:

    @staticmethod
    def _read_log(file_name):
        log_file = open(file_name,'r')
        return log_file

    @classmethod
    def create_log(cls,file_name):
        random_log = cls._read_log(file_name)
        return random_log

class TestRandomWavesTetgen(TestTools.AirWaterVVTest):

    @classmethod
    def setup_class(cls):
        pass

    @classmethod
    def teardown_class(cls):
        pass

    def setup_method(self,method):
        #pass
        Profiling.openLog("proteus.log",10)
        Profiling.logAllProcesses = True

    def teardown_method(self,method):
        Profiling.closeLog()

        """ Tear down function """
        FileList = ['random_waves.xmf',
                    'random_waves.h5']
        for file in FileList:
            if os.path.isfile(file):
                os.remove(file)
            else:
                pass

    def test_run(self):
        from petsc4py import PETSc
        pList = []
        nList = []
        so = load_so('random_waves_so',modulepath)
        for (p,n) in so.pnList:
            pList.append(load_p(p,modulepath))
            nList.append(load_n(n,modulepath))
            if pList[-1].name == None:
                pList[-1].name = p
        #so = random_waves_so
        so.name = "random_waves"
        if so.sList == []:
            for i in range(len(so.pnList)):
                s = default_s
                so.sList.append(s)
        Profiling.logLevel=7
        Profiling.verbose=True
        # PETSc solver configuration
        OptDB = PETSc.Options()
        with open(petsc_options) as f:
            all = f.read().split()
            i=0
            while i < len(all):
                if i < len(all)-1:
                    if all[i+1][0]!='-':
                        print "setting ", all[i].strip(), all[i+1]
                        OptDB.setValue(all[i].strip('-'),all[i+1])
                        i=i+2
                    else:
                        print "setting ", all[i].strip(), "True"
                        OptDB.setValue(all[i].strip('-'),True)
                        i=i+1
                else:
                    print "setting ", all[i].strip(), "True"
                    OptDB.setValue(all[i].strip('-'),True)
                    i=i+1
        so.tnList=[0.0,0.001,0.011]
        #so.tnList=[0.0,0.001]+[0.001 + i*0.01 for i in range(1,int(round(0.03/0.01))+1)]
        ns = NumericalSolution.NS_base(so,pList,nList,so.sList,opts)
        ns.calculateSolution('random_waves')

        random_log = NumericResults.create_log('proteus.log')

        text = random_log.read()

        #if text.find('CFL') != -1:
        if text.find('Step Failed,') != -1:

            a = "No convergence"
        else:
            a = "good"

        if a  == "No convergence":
            print ("Convergence issue")
            assert False
        else:
            assert True

        #def failed(filename,word):
        #    file = open(filename,"r")
        #    text = file.read()

        #    if text.find(word) != -1:
        #        a = "No convergence"
        #    else:
        #        a = "good"
        #    file.close()
        #    return a

        #b = failed('proteus.log','Step Failed,')

        #if b == "No convergence":
        #    print ("Convergence issue")
        #    assert False
        #else:
        #    assert True

        #assert(True)

#     def test_validate(self):
#         # Reading probes into the file
#         file_vof = 'column_gauges.csv'
#         # Exctracting probes
#         data_vof = readProbeFile(file_vof)
#         time = data_vof[2]
#         vof = data_vof[3]
#         # Eta for RandomWavesFast
#         tank_dim = rw.opts.tank_dim
#         waterLevel = rw.opts.water_level
#         eta_fast = np.array(tank_dim[1]-vof[:,0]-waterLevel)
#         # Eta for RandomWaves
#         Tp = rw.Tp
#         Hs = rw.Hs
#         mwl = rw.mwl
#         depth = rw.depth
#         waveDir = np.array(rw.waveDir)
#         g = np.array(rw.g)
#         N = rw.N
#         bandFactor = rw.bandFactor
#         spectName = rw.spectName
#         phi = rw.phi
#         wave_ref = rw.wt.RandomWaves(Tp,Hs,mwl,depth,waveDir,g,N,bandFactor,spectName,spectral_params=None,phi=phi,fast=True)
#         eta_ref = []
#         X = np.array([0., 0., 0.])
#         for i in range(0,len(time)):
#             eta_ref.append(wave_ref.eta(X,time[i]))
#         eta_ref = np.array(eta_ref)
#         # Validation of the results
#         S = 0.
#         c = 0.
#         for i in range(len(time)):
#             c += 1.
#             S += (eta_fast[i]-eta_ref[i])**2
#         err = np.sqrt(S/c)
#         err = 100*err/(rw.opts.Hs)
#         assert(err<10.)

if __name__ == '__main__':
    pass
<|endoftext|>