<commit_before># Copyright 2015 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import re

from touchdown.core import argument, errors, resource, serializers

from . import provisioner

try:
    import fuselage
    from fuselage import argument as f_args, builder, bundle, resources
except ImportError:
    raise errors.Error('You need the fuselage package to use the fuselage_bundle resource')

def underscore(title):
    return re.sub(r'(?<=[a-z])(?=[A-Z])', u'_', title).lower()

arguments = {
    f_args.Boolean: lambda resource_type, klass, arg: argument.Boolean(field=arg),
    f_args.String: lambda resource_type, klass, arg: argument.String(field=arg),
    f_args.FullPath: lambda resource_type, klass, arg: argument.String(field=arg),
    f_args.File: lambda resource_type, klass, arg: argument.String(field=arg),
    f_args.Integer: lambda resource_type, klass, arg: argument.Integer(field=arg),
    f_args.Octal: lambda resource_type, klass, arg: argument.Integer(field=arg),
    f_args.Dict: lambda resource_type, klass, arg: argument.Dict(field=arg),
    f_args.List: lambda resource_type, klass, arg: argument.List(field=arg),
    f_args.SubscriptionArgument: lambda resource_type, klass, arg: argument.List(field=arg),
    f_args.PolicyArgument: lambda resource_type, klass, arg: argument.String(field=arg, choices=resource_type.policies.keys()),
}

class FuselageResource(resource.Resource):

    @classmethod
    def adapt(base_klass, resource_type):
        args = {
            'resource_name': underscore(resource_type.__resource_name__),
            'fuselage_class': resource_type,
            'root': argument.Resource(Bundle),
        }

        for arg, klass in resource_type.__args__.items():
            args[arg] = arguments[klass.__class__](resource_type, klass, arg)

        cls = type(
            resource_type.__resource_name__,
            (base_klass, ),
            args
        )

        def _(self, **kwargs):
            arguments = {'parent': self}
            arguments.update(kwargs)
            resource = cls(**arguments)
            if not self.resources:
                self.resources = []
            self.resources.append(resource)
            self.add_dependency(resource)
            return resource
        setattr(Bundle, 'add_%s' % cls.resource_name, _)

        return cls

class BundleSerializer(serializers.Serializer):

    def render(self, runner, value):
        b = bundle.ResourceBundle()
        for res in value:
            b.add(res.fuselage_class(
                **serializers.Resource().render(runner, res)
            ))
        return builder.build(b)

class Bundle(provisioner.Provisioner):

    resource_name = 'fuselage_bundle'

    always_apply = argument.Boolean()
    resources = argument.List(
        argument.Resource(FuselageResource),
        field='script',
        serializer=BundleSerializer(),
    )
    sudo = argument.Boolean(field='sudo', default=True)

class Describe(provisioner.Describe):

    name = 'describe'
    resource = Bundle

    def describe_object(self):
        if self.resource.always_apply:
            return {'Results': 'Pending'}

        if not self.resource.target:
            # If target is not set we are probably dealing with an AMI... YUCK
            # Bail out
            return {'Result': 'Pending'}

        serializer = serializers.Resource()
        if serializer.pending(self.runner, self.resource):
            return {'Result': 'Pending'}

        kwargs = serializer.render(self.runner, self.resource)

        try:
            client = self.runner.get_plan(self.resource.target).get_client()
        except errors.ServiceNotReady:
            return {'Result': 'Pending'}

        try:
            client.run_script(kwargs['script'], ['-s'])
        except errors.CommandFailed as e:
            if e.exit_code == 254:
                return {'Result': 'Success'}

        return {'Result': 'Pending'}

class Apply(provisioner.Apply):

    resource = Bundle

for attr, value in vars(resources).items():
    if type(value) == fuselage.resource.ResourceType:
        locals()[attr] = FuselageResource.adapt(value)
<commit_msg>Fuselage: Bundle is pending when its resources are pending<commit_after># Copyright 2015 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import re

from touchdown.core import argument, errors, resource, serializers

from . import provisioner

try:
    import fuselage
    from fuselage import argument as f_args, builder, bundle, resources
except ImportError:
    raise errors.Error('You need the fuselage package to use the fuselage_bundle resource')

def underscore(title):
    return re.sub(r'(?<=[a-z])(?=[A-Z])', u'_', title).lower()

arguments = {
    f_args.Boolean: lambda resource_type, klass, arg: argument.Boolean(field=arg),
    f_args.String: lambda resource_type, klass, arg: argument.String(field=arg),
    f_args.FullPath: lambda resource_type, klass, arg: argument.String(field=arg),
    f_args.File: lambda resource_type, klass, arg: argument.String(field=arg),
    f_args.Integer: lambda resource_type, klass, arg: argument.Integer(field=arg),
    f_args.Octal: lambda resource_type, klass, arg: argument.Integer(field=arg),
    f_args.Dict: lambda resource_type, klass, arg: argument.Dict(field=arg),
    f_args.List: lambda resource_type, klass, arg: argument.List(field=arg),
    f_args.SubscriptionArgument: lambda resource_type, klass, arg: argument.List(field=arg),
    f_args.PolicyArgument: lambda resource_type, klass, arg: argument.String(field=arg, choices=resource_type.policies.keys()),
}

class FuselageResource(resource.Resource):

    @classmethod
    def adapt(base_klass, resource_type):
        args = {
            'resource_name': underscore(resource_type.__resource_name__),
            'fuselage_class': resource_type,
            'root': argument.Resource(Bundle),
        }

        for arg, klass in resource_type.__args__.items():
            args[arg] = arguments[klass.__class__](resource_type, klass, arg)

        cls = type(
            resource_type.__resource_name__,
            (base_klass, ),
            args
        )

        def _(self, **kwargs):
            arguments = {'parent': self}
            arguments.update(kwargs)
            resource = cls(**arguments)
            if not self.resources:
                self.resources = []
            self.resources.append(resource)
            self.add_dependency(resource)
            return resource
        setattr(Bundle, 'add_%s' % cls.resource_name, _)

        return cls

class BundleSerializer(serializers.Serializer):

    def render(self, runner, value):
        b = bundle.ResourceBundle()
        for res in value:
            b.add(res.fuselage_class(
                **serializers.Resource().render(runner, res)
            ))
        return builder.build(b)

    def pending(self, runner, value):
        for res in value:
            if serializers.Resource().pending(runner, res):
                return True
        return False

class Bundle(provisioner.Provisioner):

    resource_name = 'fuselage_bundle'

    always_apply = argument.Boolean()
    resources = argument.List(
        argument.Resource(FuselageResource),
        field='script',
        serializer=BundleSerializer(),
    )
    sudo = argument.Boolean(field='sudo', default=True)

class Describe(provisioner.Describe):

    name = 'describe'
    resource = Bundle

    def describe_object(self):
        if self.resource.always_apply:
            return {'Results': 'Pending'}

        if not self.resource.target:
            # If target is not set we are probably dealing with an AMI... YUCK
            # Bail out
            return {'Result': 'Pending'}

        serializer = serializers.Resource()
        if serializer.pending(self.runner, self.resource):
            return {'Result': 'Pending'}

        kwargs = serializer.render(self.runner, self.resource)

        try:
            client = self.runner.get_plan(self.resource.target).get_client()
        except errors.ServiceNotReady:
            return {'Result': 'Pending'}

        try:
            client.run_script(kwargs['script'], ['-s'])
        except errors.CommandFailed as e:
            if e.exit_code == 254:
                return {'Result': 'Success'}

        return {'Result': 'Pending'}

class Apply(provisioner.Apply):

    resource = Bundle

for attr, value in vars(resources).items():
    if type(value) == fuselage.resource.ResourceType:
        locals()[attr] = FuselageResource.adapt(value)
<|endoftext|>


<commit_before>from trac.versioncontrol import diff

import unittest

class DiffTestCase(unittest.TestCase):

    def test_get_change_extent(self):
        self.assertEqual((3, 0), diff._get_change_extent('xxx', 'xxx'))
        self.assertEqual((0, 0), diff._get_change_extent('', 'xxx'))
        self.assertEqual((0, 0), diff._get_change_extent('xxx', ''))
        self.assertEqual((0, 0), diff._get_change_extent('xxx', 'yyy'))
        self.assertEqual((1, -1), diff._get_change_extent('xxx', 'xyx'))
        self.assertEqual((1, -1), diff._get_change_extent('xxx', 'xyyyx'))
        self.assertEqual((1, 0), diff._get_change_extent('xy', 'xzz'))
        self.assertEqual((1, -1), diff._get_change_extent('xyx', 'xzzx'))
        self.assertEqual((1, -1), diff._get_change_extent('xzzx', 'xyx'))

    def test_insert_blank_line(self):
        opcodes = diff._get_opcodes(['A', 'B'], ['A', 'B', ''],
                                     ignore_blank_lines=0)
        self.assertEqual(('equal', 0, 2, 0, 2), opcodes.next())
        self.assertEqual(('insert', 2, 2, 2, 3), opcodes.next())

        opcodes = diff._get_opcodes(['A', 'B'], ['A', 'B', ''],
                                     ignore_blank_lines=1)
        self.assertEqual(('equal', 0, 2, 0, 3), opcodes.next())

        opcodes = diff._get_opcodes(['A'], ['A', 'B', ''],
                                     ignore_blank_lines=1)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('insert', 1, 1, 1, 3), opcodes.next())

    def test_delete_blank_line(self):
        opcodes = diff._get_opcodes(['A', 'B', ''], ['A', 'B'],
                                     ignore_blank_lines=0)
        self.assertEqual(('equal', 0, 2, 0, 2), opcodes.next())
        self.assertEqual(('delete', 2, 3, 2, 2), opcodes.next())

        opcodes = diff._get_opcodes(['A', 'B', ''], ['A', 'B'],
                                     ignore_blank_lines=1)
        self.assertEqual(('equal', 0, 3, 0, 2), opcodes.next())

        opcodes = diff._get_opcodes(['A', 'B', ''], ['A'],
                                     ignore_blank_lines=1)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('delete', 1, 3, 1, 1), opcodes.next())

    def test_space_changes(self):
        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B  b'],
                                     ignore_space_changes=0)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('replace', 1, 2, 1, 2), opcodes.next())

        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B  b'],
                                     ignore_space_changes=1)
        self.assertEqual(('equal', 0, 2, 0, 2), opcodes.next())

    def test_case_changes(self):
        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B B'],
                                     ignore_case=0)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('replace', 1, 2, 1, 2), opcodes.next())

        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B B'],
                                     ignore_case=1)
        self.assertEqual(('equal', 0, 2, 0, 2), opcodes.next())

    def test_space_and_case_changes(self):
        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B  B'],
                                     ignore_case=0, ignore_space_changes=0)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('replace', 1, 2, 1, 2), opcodes.next())

        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B  B'],
                                     ignore_case=1, ignore_space_changes=1)
        self.assertEqual(('equal', 0, 2, 0, 2), opcodes.next())

    def test_grouped_opcodes_context1(self):
        opcodes = diff._get_opcodes(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'],
                                    ['A', 'B', 'C', 'd', 'e', 'f', 'G', 'H'])
        groups = diff._group_opcodes(opcodes, n=1)
        group = groups.next()
        self.assertEqual(('equal', 2, 3, 2, 3), group[0])
        self.assertEqual(('replace', 3, 6, 3, 6), group[1])
        self.assertEqual(('equal', 6, 7, 6, 7), group[2])

    def test_grouped_opcodes_insert_blank_line_at_top(self):
        """
        Regression test for #2090. Make sure that the equal block following an
        insert at the top of a file is correct.
        """
        opcodes = diff._get_opcodes(['B', 'C', 'D', 'E', 'F', 'G'],
                                    ['A', 'B', 'C', 'D', 'E', 'F', 'G'])
        groups = diff._group_opcodes(opcodes, n=3)
        self.assertEqual([('insert', 0, 0, 0, 1), ('equal', 0, 3, 1, 4)],
                         groups.next())

    def test_unified_diff_no_context(self):
        diff_lines = list(diff.unified_diff(['a'], ['b']))
        self.assertEqual(['@@ -1,1 +1,1 @@', '-a', '+b'], diff_lines)

    def test_quotes_not_marked_up(self):
        """Make sure that the escape calls leave quotes along, we don't need
        to escape them."""
        changes = diff.diff_blocks(['ab'], ['a"b'])
        self.assertEquals(len(changes), 1)
        blocks = changes[0]
        self.assertEquals(len(blocks), 1)
        block = blocks[0]
        self.assertEquals(block['type'], 'mod')
        self.assertEquals(str(block['base']['lines'][0]), 'ab')
        self.assertEquals(str(block['changed']['lines'][0]), 'a"b')

    def test_whitespace_marked_up1(self):
        """Regression test for #5795"""
        changes = diff.diff_blocks(['*a'], [' *a'])
        block = changes[0][0]
        self.assertEquals(block['type'], 'mod')
        self.assertEquals(str(block['base']['lines'][0]), '*a')
        self.assertEquals(str(block['changed']['lines'][0]), ' *a')

    def test_whitespace_marked_up2(self):
        """Related to #5795"""
        changes = diff.diff_blocks(['   a'], ['   b'])
        block = changes[0][0]
        self.assertEquals(block['type'], 'mod')
        self.assertEquals(str(block['base']['lines'][0]), '   a')
        self.assertEquals(str(block['changed']['lines'][0]), '   b')

    def test_whitespace_marked_up3(self):
        """Related to #5795"""
        changes = diff.diff_blocks(['a   '], ['b   '])
        block = changes[0][0]
        self.assertEquals(block['type'], 'mod')
        self.assertEquals(str(block['base']['lines'][0]), 'a   ')
        self.assertEquals(str(block['changed']['lines'][0]), 'b   ')

    def test_expandtabs_works_right(self):
        """Regression test for #4557"""
        changes = diff.diff_blocks(['aa\tb'], ['aaxb'])
        block = changes[0][0]
        self.assertEquals(block['type'], 'mod')
        self.assertEquals(str(block['base']['lines'][0]), 'aa      b')
        self.assertEquals(str(block['changed']['lines'][0]), 'aaxb')

def suite():
    return unittest.makeSuite(DiffTestCase, 'test')

if __name__ == '__main__':
    unittest.main()
<commit_msg>versioncontrol: make the diff tests a bit stricter, in preparation of diff refactoring<commit_after>from trac.versioncontrol import diff

import unittest

class DiffTestCase(unittest.TestCase):

    def test_get_change_extent(self):
        self.assertEqual((3, 0), diff._get_change_extent('xxx', 'xxx'))
        self.assertEqual((0, 0), diff._get_change_extent('', 'xxx'))
        self.assertEqual((0, 0), diff._get_change_extent('xxx', ''))
        self.assertEqual((0, 0), diff._get_change_extent('xxx', 'yyy'))
        self.assertEqual((1, -1), diff._get_change_extent('xxx', 'xyx'))
        self.assertEqual((1, -1), diff._get_change_extent('xxx', 'xyyyx'))
        self.assertEqual((1, 0), diff._get_change_extent('xy', 'xzz'))
        self.assertEqual((1, -1), diff._get_change_extent('xyx', 'xzzx'))
        self.assertEqual((1, -1), diff._get_change_extent('xzzx', 'xyx'))

    def test_insert_blank_line(self):
        opcodes = diff._get_opcodes(['A', 'B'], ['A', 'B', ''],
                                     ignore_blank_lines=0)
        self.assertEqual(('equal', 0, 2, 0, 2), opcodes.next())
        self.assertEqual(('insert', 2, 2, 2, 3), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

        opcodes = diff._get_opcodes(['A', 'B'], ['A', 'B', ''],
                                     ignore_blank_lines=1)
        self.assertEqual(('equal', 0, 2, 0, 3), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

        opcodes = diff._get_opcodes(['A'], ['A', 'B', ''],
                                     ignore_blank_lines=0)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('insert', 1, 1, 1, 3), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

        opcodes = diff._get_opcodes(['A'], ['A', 'B', ''],
                                     ignore_blank_lines=1)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('insert', 1, 1, 1, 3), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

    def test_delete_blank_line(self):
        opcodes = diff._get_opcodes(['A', 'B', ''], ['A', 'B'],
                                     ignore_blank_lines=0)
        self.assertEqual(('equal', 0, 2, 0, 2), opcodes.next())
        self.assertEqual(('delete', 2, 3, 2, 2), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

        opcodes = diff._get_opcodes(['A', 'B', ''], ['A', 'B'],
                                     ignore_blank_lines=1)
        self.assertEqual(('equal', 0, 3, 0, 2), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

        opcodes = diff._get_opcodes(['A', 'B', ''], ['A'],
                                     ignore_blank_lines=0)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('delete', 1, 3, 1, 1), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

        opcodes = diff._get_opcodes(['A', 'B', ''], ['A'],
                                     ignore_blank_lines=1)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('delete', 1, 3, 1, 1), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

    def test_space_changes(self):
        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B  b'],
                                     ignore_space_changes=0)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('replace', 1, 2, 1, 2), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B  b'],
                                     ignore_space_changes=1)
        self.assertEqual(('equal', 0, 2, 0, 2), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

    def test_case_changes(self):
        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B B'],
                                     ignore_case=0)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('replace', 1, 2, 1, 2), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B B'],
                                     ignore_case=1)
        self.assertEqual(('equal', 0, 2, 0, 2), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

    def test_space_and_case_changes(self):
        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B  B'],
                                     ignore_case=0, ignore_space_changes=0)
        self.assertEqual(('equal', 0, 1, 0, 1), opcodes.next())
        self.assertEqual(('replace', 1, 2, 1, 2), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

        opcodes = diff._get_opcodes(['A', 'B b'], ['A', 'B  B'],
                                     ignore_case=1, ignore_space_changes=1)
        self.assertEqual(('equal', 0, 2, 0, 2), opcodes.next())
        self.assertRaises(StopIteration, opcodes.next)

    def test_grouped_opcodes_context1(self):
        opcodes = diff._get_opcodes(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'],
                                    ['A', 'B', 'C', 'd', 'e', 'f', 'G', 'H'])
        groups = diff._group_opcodes(opcodes, n=1)
        group = groups.next()
        self.assertRaises(StopIteration, groups.next)
        self.assertEqual(('equal', 2, 3, 2, 3), group[0])
        self.assertEqual(('replace', 3, 6, 3, 6), group[1])
        self.assertEqual(('equal', 6, 7, 6, 7), group[2])

    def test_grouped_opcodes_insert_blank_line_at_top(self):
        """
        Regression test for #2090. Make sure that the equal block following an
        insert at the top of a file is correct.
        """
        opcodes = diff._get_opcodes(['B', 'C', 'D', 'E', 'F', 'G'],
                                    ['A', 'B', 'C', 'D', 'E', 'F', 'G'])
        groups = diff._group_opcodes(opcodes, n=3)
        self.assertEqual([('insert', 0, 0, 0, 1), ('equal', 0, 3, 1, 4)],
                         groups.next())
        self.assertRaises(StopIteration, groups.next)

    def test_unified_diff_no_context(self):
        diff_lines = list(diff.unified_diff(['a'], ['b']))
        self.assertEqual(['@@ -1,1 +1,1 @@', '-a', '+b'], diff_lines)

    def test_quotes_not_marked_up(self):
        """Make sure that the escape calls leave quotes along, we don't need
        to escape them."""
        changes = diff.diff_blocks(['ab'], ['a"b'])
        self.assertEquals(len(changes), 1)
        blocks = changes[0]
        self.assertEquals(len(blocks), 1)
        block = blocks[0]
        self.assertEquals(block['type'], 'mod')
        self.assertEquals(str(block['base']['lines'][0]), 'ab')
        self.assertEquals(str(block['changed']['lines'][0]), 'a"b')

    def test_whitespace_marked_up1(self):
        """Regression test for #5795"""
        changes = diff.diff_blocks(['*a'], [' *a'])
        block = changes[0][0]
        self.assertEquals(block['type'], 'mod')
        self.assertEquals(str(block['base']['lines'][0]), '*a')
        self.assertEquals(str(block['changed']['lines'][0]),
                          ' *a')

    def test_whitespace_marked_up2(self):
        """Related to #5795"""
        changes = diff.diff_blocks(['   a'], ['   b'])
        block = changes[0][0]
        self.assertEquals(block['type'], 'mod')
        self.assertEquals(str(block['base']['lines'][0]),
                          '   a')
        self.assertEquals(str(block['changed']['lines'][0]),
                          '   b')

    def test_whitespace_marked_up3(self):
        """Related to #5795"""
        changes = diff.diff_blocks(['a   '], ['b   '])
        block = changes[0][0]
        self.assertEquals(block['type'], 'mod')
        self.assertEquals(str(block['base']['lines'][0]),
                          'a   ')
        self.assertEquals(str(block['changed']['lines'][0]),
                          'b   ')

    def test_expandtabs_works_right(self):
        """Regression test for #4557"""
        changes = diff.diff_blocks(['aa\tb'], ['aaxb'])
        block = changes[0][0]
        self.assertEquals(block['type'], 'mod')
        self.assertEquals(str(block['base']['lines'][0]),
                          'aa      b')
        self.assertEquals(str(block['changed']['lines'][0]),
                          'aaxb')

def suite():
    return unittest.makeSuite(DiffTestCase, 'test')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.9.1 on 2016-01-30 20:17
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('database', '0002_contact_notes'),
    ]

    operations = [
#        migrations.AddField(
#            model_name='contact',
#            name='address',
#            field=models.TextField(blank=True, null=True),
#        ),

    ]
<commit_msg>Update<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.9.1 on 2016-01-30 20:17
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('database', '0002_contact_notes'),
    ]

    operations = [
        migrations.AddField(
            model_name='contact',
            name='address',
            field=models.TextField(blank=True, null=True),
        ),

    ]
<|endoftext|>


<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import division
from __future__ import unicode_literals

from collections import Mapping

from activedata_etl import etl2key, key2etl
from mo_dots import Data, listwrap, wrap, set_default
from mo_json import json2value
from mo_logs import Log, machine_metadata, strings
from mo_math import Math

from activedata_etl.transforms import TRY_AGAIN_LATER
from mo_hg.hg_mozilla_org import minimize_repo
from mo_times.dates import Date
from pyLibrary.env import elasticsearch
from pyLibrary.env.git import get_git_revision

DEBUG = True
DISABLE_LOG_PARSING = False
MAX_THREADS = 5

seen_tasks = {}
new_seen_tc_properties = set()

def process(source_key, source, destination, resources, please_stop=None):
    output = []
    lines = list(enumerate(source.read_lines()))
    for line_number, line in lines:
        if please_stop:
            Log.error("Shutdown detected. Stopping early")
        try:
            raw_treeherder = json2value(line)
            etl_source = consume(raw_treeherder, "etl")
            if etl_source.source.source.id <= 687:
                etl_source = wrap({"source": key2etl(source_key)})
            etl_source.type = "join"
            etl_source.source.type = "join"

            normalized = Data()
            normalized.etl = set_default({
                "id": line_number,
                "timestamp": Date.now(),
                "machine": machine_metadata,
                "revision": get_git_revision(),
                "type": "join"
            }, etl_source)
            normalize(source_key, resources, raw_treeherder, normalized)
            normalized = elasticsearch.scrub(normalized)
            output.append(normalized)
        except Exception as e:
            if TRY_AGAIN_LATER in e:
                raise e
            Log.warning("Treeherder line for key {{key}} not processed: {{line|quote}}", key=source_key, line=line, cause=e)

    keys = destination.extend({"id": etl2key(t.etl), "value": t} for t in output)
    return keys

def normalize(source_key, resources, raw_treeherder, new_treeherder):
    raw_job = raw_treeherder.job

    new_treeherder.job.type.name = consume(raw_job, "job_type.name")
    new_treeherder.job.type.description = consume(raw_job, "job_type.description")
    new_treeherder.job.type.symbol = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.symbol"),
        consume(raw_job, "signature.job_type_symbol")
    )
    new_treeherder.job.type.name = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.name"),
        consume(raw_job, "signature.job_type_name")
    )
    new_treeherder.job.type.group.symbol = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.job_group.symbol"),
        consume(raw_job, "signature.job_group_symbol"),
        consume(raw_job, "job_group.symbol")
    )

    new_treeherder.job.type.group.name = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.job_group.name"),
        consume(raw_job, "job_type.job_group"),
        consume(raw_job, "signature.job_group_name"),
        consume(raw_job, "job_group.name"),
        consume(raw_job, "job_group")
    )

    new_treeherder.job.guid = consume(raw_job, "guid")
    new_treeherder.job.id = consume(raw_job, "id")
    new_treeherder.job.coalesced_to_guid = consume(raw_job, "coalesced_to_guid")

    # BUILD
    new_treeherder.build.branch = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "repository"),
        consume(raw_job, "signature.repository")
    )
    new_treeherder.build.revision = consume(raw_job, "push.revision")
    new_treeherder.build.revision12 = new_treeherder.build.revision[0:12]
    new_treeherder.build.date = consume(raw_job, "push.time")

    new_treeherder.build.platform = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.platform"),
        consume(raw_job, "signature.build_platform")
    )
    new_treeherder.build.os = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.os_name"),
        consume(raw_job, "signature.build_os_name")
    )
    new_treeherder.build.architecture = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.architecture"),
        consume(raw_job, "signature.build_architecture")
    )
    new_treeherder.build.product = consume(raw_job, "product.name")

    new_treeherder.run.key = consume(raw_job, "signature.name")
    new_treeherder.run.reason = consume(raw_job, "reason")
    new_treeherder.run.tier = consume(raw_job, "tier")
    new_treeherder.run.result = consume(raw_job, "result")
    new_treeherder.run.state = consume(raw_job, "state")
    if raw_treeherder.job.signature.build_system_type in ["taskcluster", "buildbot", "fx-test-jenkins", "fx-test-jenkins-dev", "qa-master", "qa-preprod-master"]:
        consume(raw_job, "signature.build_system_type")
    else:
        Log.error("Know nothing about build_system_type=={{type}}", type=raw_treeherder.job.signature.build_system_type)

    # RUN MACHINE
    new_treeherder.run.machine.name = machine_name = consume(raw_job, "machine.name")
    split_name = machine_name.split("-")
    if Math.is_integer(split_name[-1]):
        new_treeherder.run.machine.pool = "-".join(split_name[:-1])
    new_treeherder.run.machine.os = consume(raw_job, "signature.machine_os_name")
    new_treeherder.run.machine.architecture = consume(raw_job, "signature.machine_architecture")
    new_treeherder.run.machine.platform = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "machine_platform"),
        consume(raw_job, "signature.machine_platform")
    )

    # ACTION
    new_treeherder.action.start_time = consume(raw_job, "start_time")
    new_treeherder.action.end_time = consume(raw_job, "end_time")
    new_treeherder.action.request_time = consume(raw_job, "submit_time")
    new_treeherder.action.duration = new_treeherder.action.end_time - new_treeherder.action.start_time
    new_treeherder.last_modified = consume(raw_job, "last_modified")

    new_treeherder.failure.auto_classification = consume(raw_job, "autoclassify_status")
    new_treeherder.failure.classification = consume(raw_job, "failure_classification")
    new_treeherder.failure.notes = consume(raw_job, "job_note")

    new_treeherder.repo = {
        "branch": {"name": new_treeherder.build.branch},
        "changeset": {"id": new_treeherder.build.revision}
    }
    try:
        if new_treeherder.build.branch not in NON_HG_BRANCHES:
            new_treeherder.repo = minimize_repo(resources.hg.get_revision(new_treeherder.repo))
    except Exception as e:
        Log.warning("Problem with getting info changeset {{changeset}}", changeset=new_treeherder.repo, cause=e)

    new_treeherder.bugs = consume(raw_job, "bug_job_map")

    pull_job_log(source_key, consume(raw_treeherder.job, "job_log"), new_treeherder)

    consume(raw_job, "push")
    consume(raw_job, "running_eta")
    consume(raw_job, "who")
    consume(raw_job, "signature.first_submission_timestamp")
    consume(raw_job, "signature.option_collection_hash")
    consume(raw_job, "signature.signature")
    pull_details(source_key, consume(raw_treeherder.job, "job_detail"), new_treeherder)

    new_treeherder.run.taskcluster.id = coalesce_w_conflict_detection(
        source_key,
        new_treeherder.run.taskcluster.id,
        consume(raw_job, "taskcluster_metadata.task_id")
    )
    new_treeherder.run.taskcluster.retry_id = consume(raw_job, "taskcluster_metadata.retry_id")

    pull_options(source_key, raw_treeherder, new_treeherder)

    remainder = raw_treeherder.leaves()
    if remainder:
        Log.error("Did not process {{paths}} for key={{key}}", key=source_key, paths=[k for k, _ in remainder])

def pull_options(source_key, raw_treeherder, new_treeherder):
    options = listwrap(consume(raw_treeherder, "job.option_collection.option"))
    for o in options:
        if o in _option_map:
            new_treeherder.build.type += _option_map[o]
        elif o in ["e10s"]:
            new_treeherder.run.type += [o]
        else:
            Log.warning("not known option {{option|quote}} while processing {{key}}", key=source_key, option=o)

_option_map = {
    "addon": ["addon"],
    "aarch64-debug": ["aarch64", "debug"],
    "arm-debug": ["arm", "debug"],
    "all": ["all"],
    "asan": ["asan"],
    "ccov": ["ccov"],
    "cc": ["ccov"],
    "debug": ["debug"],
    "fips": ["fips"],
    "fuzz": ["fuzz"],
    "gyp": ["gyp"],
    "gyp-asan": ["gyp", "asan"],
    "jsdcov": ["jsdcov"],
    "make": ["make"],
    "nostylo": ["stylo-disabled"],
    "opt": ["opt"],
    "pgo": ["pgo"]
}

def pull_job_log(source_key, job_log, new_treeherder):
    for d in listwrap(job_log):
        for f in listwrap(d.failure_line):
            f.message = None
            f.job_guid = None
            f.stackwalk_stdout = None
            f.stackwalk_stderr = None
            f.subtest = None
    new_treeherder.job_log = job_log

def pull_details(source_key, details, new_treeherder):
    for d in listwrap(details):
        if d.title == "Summary":
            new_treeherder.summary = d.value
        elif d.title == "buildbot_request_id":
            new_treeherder.run.buildbot.id = d.value
        elif d.title == "Inspect Task":
            if d.url.startswith("https://tools.taskcluster.net/task-inspector/#"):
                new_treeherder.run.taskcluster.id = strings.between(d.url, "https://tools.taskcluster.net/task-inspector/#", "/")
            else:
                Log.warning("Can not extract task for ket {{key}} from {{url}}", key=source_key, url=d.url)
        elif d.title == "CPU idle":
            new_treeherder.stats.cpu_idle = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU user":
            new_treeherder.stats.cpu_user = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU system":
            new_treeherder.stats.cpu_system = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU iowait":
            new_treeherder.stats.cpu_io_wait = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU usage":
            new_treeherder.stats.cpu_usage = float(d.value.strip("%").replace(",", ""))
        elif d.title == "I/O read bytes / time":
            new_treeherder.stats.io_read_bytes = float(d.value.split("/")[0].strip().replace(",", ""))
            new_treeherder.stats.io_read_time = float(d.value.split("/")[1].strip().replace(",", ""))
        elif d.title == "I/O write bytes / time":
            new_treeherder.stats.io_write_bytes = float(d.value.split("/")[0].strip().replace(",", ""))
            new_treeherder.stats.io_write_time = float(d.value.split("/")[1].strip().replace(",", ""))
        elif d.title == "Swap in / out":
            new_treeherder.stats.swap_in = float(d.value.split("/")[0].strip().replace(",", ""))
            new_treeherder.stats.swap_out = float(d.value.split("/")[1].strip().replace(",", ""))
        elif d.title in ["artifact uploaded", "One Click Loaner"]:
            pass
        elif d.title.find("-chunked"):
            pass
        elif d.title == None:
            if d.value in ["auto clobber", "purged clobber", "periodic clobber", "forced clobber"]:
                pass
            elif any(map(d.value.startswith, ["linker max vsize: ", "num_ctors: "])):
                new_treeherder.stats.linker_max_vsize = int(d.value.split(":")[1].strip())
            elif any(map(d.value.startswith, KNOWN_VALUES)):
                pass
            else:
                KNOWN_VALUES.append(d.value)
                Log.warning("value has no title {{value|quote}} while processing {{key}}", key=source_key, value=d.value)
        else:
            Log.warning("can not process detail with title of {{title}}", title=d.title)
    new_treeherder.job.details = details

def coalesce_w_conflict_detection(source_key, *args):
    output = None
    for a in args:
        if a == None:
            continue
        if isinstance(a, Mapping) and not a:
            continue
        if output == None:
            output = a
        elif a != output:
            Log.warning("tried to coalesce {{values_|json}} while processing {{key}}", key=source_key, values_=args)
        else:
            pass
    return output

def consume(props, key):
    output = props[key]
    try:
        props[key] = None
    except Exception:
        pass
    return output

KNOWN_VALUES = [
    "marionette: ",
    "--",
    "The following arguments ",
    "Tests will be run from the following files:",
    "gaia_revlink: ",
    "Unknown: ",
    "\t--this-chunk=1 --total-chunks=1 -- "
]

NON_HG_BRANCHES = [
    "bmo-master",
    "go-bouncer",
    "snippets-service",
    "snippets-service",
    "snippets-tests",
    "socorro",
    "stubattribution-tests"
]
<commit_msg>another branch<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import division
from __future__ import unicode_literals

from collections import Mapping

from activedata_etl import etl2key, key2etl
from mo_dots import Data, listwrap, wrap, set_default
from mo_json import json2value
from mo_logs import Log, machine_metadata, strings
from mo_math import Math

from activedata_etl.transforms import TRY_AGAIN_LATER
from mo_hg.hg_mozilla_org import minimize_repo
from mo_times.dates import Date
from pyLibrary.env import elasticsearch
from pyLibrary.env.git import get_git_revision

DEBUG = True
DISABLE_LOG_PARSING = False
MAX_THREADS = 5

seen_tasks = {}
new_seen_tc_properties = set()

def process(source_key, source, destination, resources, please_stop=None):
    output = []
    lines = list(enumerate(source.read_lines()))
    for line_number, line in lines:
        if please_stop:
            Log.error("Shutdown detected. Stopping early")
        try:
            raw_treeherder = json2value(line)
            etl_source = consume(raw_treeherder, "etl")
            if etl_source.source.source.id <= 687:
                etl_source = wrap({"source": key2etl(source_key)})
            etl_source.type = "join"
            etl_source.source.type = "join"

            normalized = Data()
            normalized.etl = set_default({
                "id": line_number,
                "timestamp": Date.now(),
                "machine": machine_metadata,
                "revision": get_git_revision(),
                "type": "join"
            }, etl_source)
            normalize(source_key, resources, raw_treeherder, normalized)
            normalized = elasticsearch.scrub(normalized)
            output.append(normalized)
        except Exception as e:
            if TRY_AGAIN_LATER in e:
                raise e
            Log.warning("Treeherder line for key {{key}} not processed: {{line|quote}}", key=source_key, line=line, cause=e)

    keys = destination.extend({"id": etl2key(t.etl), "value": t} for t in output)
    return keys

def normalize(source_key, resources, raw_treeherder, new_treeherder):
    raw_job = raw_treeherder.job

    new_treeherder.job.type.name = consume(raw_job, "job_type.name")
    new_treeherder.job.type.description = consume(raw_job, "job_type.description")
    new_treeherder.job.type.symbol = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.symbol"),
        consume(raw_job, "signature.job_type_symbol")
    )
    new_treeherder.job.type.name = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.name"),
        consume(raw_job, "signature.job_type_name")
    )
    new_treeherder.job.type.group.symbol = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.job_group.symbol"),
        consume(raw_job, "signature.job_group_symbol"),
        consume(raw_job, "job_group.symbol")
    )

    new_treeherder.job.type.group.name = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "job_type.job_group.name"),
        consume(raw_job, "job_type.job_group"),
        consume(raw_job, "signature.job_group_name"),
        consume(raw_job, "job_group.name"),
        consume(raw_job, "job_group")
    )

    new_treeherder.job.guid = consume(raw_job, "guid")
    new_treeherder.job.id = consume(raw_job, "id")
    new_treeherder.job.coalesced_to_guid = consume(raw_job, "coalesced_to_guid")

    # BUILD
    new_treeherder.build.branch = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "repository"),
        consume(raw_job, "signature.repository")
    )
    new_treeherder.build.revision = consume(raw_job, "push.revision")
    new_treeherder.build.revision12 = new_treeherder.build.revision[0:12]
    new_treeherder.build.date = consume(raw_job, "push.time")

    new_treeherder.build.platform = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.platform"),
        consume(raw_job, "signature.build_platform")
    )
    new_treeherder.build.os = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.os_name"),
        consume(raw_job, "signature.build_os_name")
    )
    new_treeherder.build.architecture = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "build_platform.architecture"),
        consume(raw_job, "signature.build_architecture")
    )
    new_treeherder.build.product = consume(raw_job, "product.name")

    new_treeherder.run.key = consume(raw_job, "signature.name")
    new_treeherder.run.reason = consume(raw_job, "reason")
    new_treeherder.run.tier = consume(raw_job, "tier")
    new_treeherder.run.result = consume(raw_job, "result")
    new_treeherder.run.state = consume(raw_job, "state")
    if raw_treeherder.job.signature.build_system_type in ["taskcluster", "buildbot", "fx-test-jenkins", "fx-test-jenkins-dev", "qa-master", "qa-preprod-master"]:
        consume(raw_job, "signature.build_system_type")
    else:
        Log.error("Know nothing about build_system_type=={{type}}", type=raw_treeherder.job.signature.build_system_type)

    # RUN MACHINE
    new_treeherder.run.machine.name = machine_name = consume(raw_job, "machine.name")
    split_name = machine_name.split("-")
    if Math.is_integer(split_name[-1]):
        new_treeherder.run.machine.pool = "-".join(split_name[:-1])
    new_treeherder.run.machine.os = consume(raw_job, "signature.machine_os_name")
    new_treeherder.run.machine.architecture = consume(raw_job, "signature.machine_architecture")
    new_treeherder.run.machine.platform = coalesce_w_conflict_detection(
        source_key,
        consume(raw_job, "machine_platform"),
        consume(raw_job, "signature.machine_platform")
    )

    # ACTION
    new_treeherder.action.start_time = consume(raw_job, "start_time")
    new_treeherder.action.end_time = consume(raw_job, "end_time")
    new_treeherder.action.request_time = consume(raw_job, "submit_time")
    new_treeherder.action.duration = new_treeherder.action.end_time - new_treeherder.action.start_time
    new_treeherder.last_modified = consume(raw_job, "last_modified")

    new_treeherder.failure.auto_classification = consume(raw_job, "autoclassify_status")
    new_treeherder.failure.classification = consume(raw_job, "failure_classification")
    new_treeherder.failure.notes = consume(raw_job, "job_note")

    new_treeherder.repo = {
        "branch": {"name": new_treeherder.build.branch},
        "changeset": {"id": new_treeherder.build.revision}
    }
    try:
        if new_treeherder.build.branch not in NON_HG_BRANCHES:
            new_treeherder.repo = minimize_repo(resources.hg.get_revision(new_treeherder.repo))
    except Exception as e:
        Log.warning("Problem with getting info changeset {{changeset}}", changeset=new_treeherder.repo, cause=e)

    new_treeherder.bugs = consume(raw_job, "bug_job_map")

    pull_job_log(source_key, consume(raw_treeherder.job, "job_log"), new_treeherder)

    consume(raw_job, "push")
    consume(raw_job, "running_eta")
    consume(raw_job, "who")
    consume(raw_job, "signature.first_submission_timestamp")
    consume(raw_job, "signature.option_collection_hash")
    consume(raw_job, "signature.signature")
    pull_details(source_key, consume(raw_treeherder.job, "job_detail"), new_treeherder)

    new_treeherder.run.taskcluster.id = coalesce_w_conflict_detection(
        source_key,
        new_treeherder.run.taskcluster.id,
        consume(raw_job, "taskcluster_metadata.task_id")
    )
    new_treeherder.run.taskcluster.retry_id = consume(raw_job, "taskcluster_metadata.retry_id")

    pull_options(source_key, raw_treeherder, new_treeherder)

    remainder = raw_treeherder.leaves()
    if remainder:
        Log.error("Did not process {{paths}} for key={{key}}", key=source_key, paths=[k for k, _ in remainder])

def pull_options(source_key, raw_treeherder, new_treeherder):
    options = listwrap(consume(raw_treeherder, "job.option_collection.option"))
    for o in options:
        if o in _option_map:
            new_treeherder.build.type += _option_map[o]
        elif o in ["e10s"]:
            new_treeherder.run.type += [o]
        else:
            Log.warning("not known option {{option|quote}} while processing {{key}}", key=source_key, option=o)

_option_map = {
    "addon": ["addon"],
    "aarch64-debug": ["aarch64", "debug"],
    "arm-debug": ["arm", "debug"],
    "all": ["all"],
    "asan": ["asan"],
    "ccov": ["ccov"],
    "cc": ["ccov"],
    "debug": ["debug"],
    "fips": ["fips"],
    "fuzz": ["fuzz"],
    "gyp": ["gyp"],
    "gyp-asan": ["gyp", "asan"],
    "jsdcov": ["jsdcov"],
    "make": ["make"],
    "nostylo": ["stylo-disabled"],
    "opt": ["opt"],
    "pgo": ["pgo"]
}

def pull_job_log(source_key, job_log, new_treeherder):
    for d in listwrap(job_log):
        for f in listwrap(d.failure_line):
            f.message = None
            f.job_guid = None
            f.stackwalk_stdout = None
            f.stackwalk_stderr = None
            f.subtest = None
    new_treeherder.job_log = job_log

def pull_details(source_key, details, new_treeherder):
    for d in listwrap(details):
        if d.title == "Summary":
            new_treeherder.summary = d.value
        elif d.title == "buildbot_request_id":
            new_treeherder.run.buildbot.id = d.value
        elif d.title == "Inspect Task":
            if d.url.startswith("https://tools.taskcluster.net/task-inspector/#"):
                new_treeherder.run.taskcluster.id = strings.between(d.url, "https://tools.taskcluster.net/task-inspector/#", "/")
            else:
                Log.warning("Can not extract task for ket {{key}} from {{url}}", key=source_key, url=d.url)
        elif d.title == "CPU idle":
            new_treeherder.stats.cpu_idle = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU user":
            new_treeherder.stats.cpu_user = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU system":
            new_treeherder.stats.cpu_system = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU iowait":
            new_treeherder.stats.cpu_io_wait = float(d.value.split("(")[0].replace(",", ""))
        elif d.title == "CPU usage":
            new_treeherder.stats.cpu_usage = float(d.value.strip("%").replace(",", ""))
        elif d.title == "I/O read bytes / time":
            new_treeherder.stats.io_read_bytes = float(d.value.split("/")[0].strip().replace(",", ""))
            new_treeherder.stats.io_read_time = float(d.value.split("/")[1].strip().replace(",", ""))
        elif d.title == "I/O write bytes / time":
            new_treeherder.stats.io_write_bytes = float(d.value.split("/")[0].strip().replace(",", ""))
            new_treeherder.stats.io_write_time = float(d.value.split("/")[1].strip().replace(",", ""))
        elif d.title == "Swap in / out":
            new_treeherder.stats.swap_in = float(d.value.split("/")[0].strip().replace(",", ""))
            new_treeherder.stats.swap_out = float(d.value.split("/")[1].strip().replace(",", ""))
        elif d.title in ["artifact uploaded", "One Click Loaner"]:
            pass
        elif d.title.find("-chunked"):
            pass
        elif d.title == None:
            if d.value in ["auto clobber", "purged clobber", "periodic clobber", "forced clobber"]:
                pass
            elif any(map(d.value.startswith, ["linker max vsize: ", "num_ctors: "])):
                new_treeherder.stats.linker_max_vsize = int(d.value.split(":")[1].strip())
            elif any(map(d.value.startswith, KNOWN_VALUES)):
                pass
            else:
                KNOWN_VALUES.append(d.value)
                Log.warning("value has no title {{value|quote}} while processing {{key}}", key=source_key, value=d.value)
        else:
            Log.warning("can not process detail with title of {{title}}", title=d.title)
    new_treeherder.job.details = details

def coalesce_w_conflict_detection(source_key, *args):
    output = None
    for a in args:
        if a == None:
            continue
        if isinstance(a, Mapping) and not a:
            continue
        if output == None:
            output = a
        elif a != output:
            Log.warning("tried to coalesce {{values_|json}} while processing {{key}}", key=source_key, values_=args)
        else:
            pass
    return output

def consume(props, key):
    output = props[key]
    try:
        props[key] = None
    except Exception:
        pass
    return output

KNOWN_VALUES = [
    "marionette: ",
    "--",
    "The following arguments ",
    "Tests will be run from the following files:",
    "gaia_revlink: ",
    "Unknown: ",
    "\t--this-chunk=1 --total-chunks=1 -- "
]

NON_HG_BRANCHES = [
    "bmo-master",
    "go-bouncer",
    "mozillians-tests",
    "snippets-service",
    "snippets-service",
    "snippets-tests",
    "socorro",
    "stubattribution-tests"
]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import django.middleware.csrf
from django.contrib.staticfiles.templatetags.staticfiles import static

def getConfig(request):
    config = {}
    config['host'] = request.get_host()
    config['hostName'] = request.get_host().decode('idna')
    config['csrf_token'] = django.middleware.csrf.get_token(request)
    config['static_url'] = static('')
    config['user'] = {}

    return config
<commit_msg>remove static from mock<commit_after># -*- coding: utf-8 -*-

import django.middleware.csrf

def getConfig(request):
    config = {}
    config['host'] = request.get_host()
    config['hostName'] = request.get_host().decode('idna')
    config['csrf_token'] = django.middleware.csrf.get_token(request)
    config['user'] = {}

    return config
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os
import subprocess
import sys

import yaml
from colorama import Fore, Style

changed = set(os.getenv('CHANGED').split(' '))
changed.discard('')

for appid in sorted(changed):
    metadata_file = 'metadata/%s.yml' % appid
    diff = subprocess.check_output(
        (
            'git diff --no-color --diff-filter=d FETCH_HEAD...HEAD -- ' + metadata_file
        ).split(' ')
    )

    with open(metadata_file) as fp:
        current = yaml.safe_load(fp)
    cmd = 'git apply --reverse'
    p = subprocess.run(cmd.split(' '), input=diff, stdout=subprocess.DEVNULL)
    if p.returncode:
        print(
            Fore.RED + ('ERROR: %s: %d' % (cmd, p.returncode)) + Style.RESET_ALL,
            file=sys.stderr,
        )
        sys.exit(p.returncode)

    to_build = []
    if os.path.exists(metadata_file):
        with open(metadata_file) as fp:
            previous = yaml.safe_load(fp)
        cmd = 'git apply'
        p = subprocess.run(cmd.split(' '), input=diff, stdout=subprocess.DEVNULL)
        if p.returncode:
            print(
                Fore.RED + ('ERROR: %s: %d' % (cmd, p.returncode)) + Style.RESET_ALL,
                file=sys.stderr,
            )
            sys.exit(p.returncode)

        previous_builds = dict()
        for build in previous['Builds']:
            previous_builds[build['versionCode']] = build

        for build in current['Builds']:
            vc = build['versionCode']
            if vc not in previous_builds:
                to_build.append(vc)
                continue
            if build != previous_builds[vc]:
                to_build.append(vc)
    else:
        # this is a brand new metadata file
        cmd = 'git checkout -- ' + metadata_file
        p = subprocess.run(cmd.split(' '), stdout=subprocess.DEVNULL)
        if p.returncode:
            print(
                Fore.RED + ('ERROR: %s: %d' % (cmd, p.returncode)) + Style.RESET_ALL,
                file=sys.stderr,
            )
            sys.exit(p.returncode)
        with open(metadata_file) as fp:
            data = yaml.safe_load(fp)
        for build in data['Builds']:
            to_build.append(build['versionCode'])

    signatures_dir = 'metadata/%s/signatures/' % appid
    diff = subprocess.check_output(
        (
            'git diff --name-only --no-color --diff-filter=d FETCH_HEAD...HEAD -- '
            + signatures_dir
        ).split(' ')
    )
    for f in diff.split():
        vc = int(os.path.basename(os.path.dirname(f)))
        if vc not in to_build:
            to_build.append(vc)

    for vc in to_build:
        print('%s:%d' % (appid, vc), end=' ')
<commit_msg>fix find-changed-builds.py<commit_after>#!/usr/bin/env python3

import os
import subprocess
import sys

import yaml
from colorama import Fore, Style

changed = set(os.getenv('CHANGED').split(' '))
changed.discard('')

for appid in sorted(changed):
    metadata_file = 'metadata/%s.yml' % appid
    diff = subprocess.check_output(
        (
            'git diff --no-color --diff-filter=d HEAD^...HEAD -- ' + metadata_file
        ).split(' ')
    )

    with open(metadata_file) as fp:
        current = yaml.safe_load(fp)
    cmd = 'git apply --reverse'
    p = subprocess.run(cmd.split(' '), input=diff, stdout=subprocess.DEVNULL)
    if p.returncode:
        print(
            Fore.RED + ('ERROR: %s: %d' % (cmd, p.returncode)) + Style.RESET_ALL,
            file=sys.stderr,
        )
        sys.exit(p.returncode)

    to_build = []
    if os.path.exists(metadata_file):
        with open(metadata_file) as fp:
            previous = yaml.safe_load(fp)
        cmd = 'git apply'
        p = subprocess.run(cmd.split(' '), input=diff, stdout=subprocess.DEVNULL)
        if p.returncode:
            print(
                Fore.RED + ('ERROR: %s: %d' % (cmd, p.returncode)) + Style.RESET_ALL,
                file=sys.stderr,
            )
            sys.exit(p.returncode)

        previous_builds = dict()
        for build in previous['Builds']:
            previous_builds[build['versionCode']] = build

        for build in current['Builds']:
            vc = build['versionCode']
            if vc not in previous_builds:
                to_build.append(vc)
                continue
            if build != previous_builds[vc]:
                to_build.append(vc)
    else:
        # this is a brand new metadata file
        cmd = 'git checkout -- ' + metadata_file
        p = subprocess.run(cmd.split(' '), stdout=subprocess.DEVNULL)
        if p.returncode:
            print(
                Fore.RED + ('ERROR: %s: %d' % (cmd, p.returncode)) + Style.RESET_ALL,
                file=sys.stderr,
            )
            sys.exit(p.returncode)
        with open(metadata_file) as fp:
            data = yaml.safe_load(fp)
        for build in data['Builds']:
            to_build.append(build['versionCode'])

    signatures_dir = 'metadata/%s/signatures/' % appid
    diff = subprocess.check_output(
        (
            'git diff --name-only --no-color --diff-filter=d FETCH_HEAD...HEAD -- '
            + signatures_dir
        ).split(' ')
    )
    for f in diff.split():
        vc = int(os.path.basename(os.path.dirname(f)))
        if vc not in to_build:
            to_build.append(vc)

    for vc in to_build:
        print('%s:%d' % (appid, vc), end=' ')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Thu Sep 08 16:16:41 2016

@author: SISQUAKE
"""
from optparse import OptionParser
import os
from Game import Game
import datetime
from listFile import listFilePath

# def main():

METADATA_DIRECTORY = '../../new_meta'
PRICE_DATA_DIRECTORY = '../../price_data'
TRAINING_OUTPUT_DIRECTORY = '../../traing_data'
TRAINING_RATE = 0.8

TARGET_COUNTRY = 'us'

parser = OptionParser()
parser.add_option("-m", dest="METADATA_DIRECTORY")
parser.add_option("-p", dest="PRICE_DATA_DIRECTORY")
parser.add_option("-t", dest="TRAINING_OUTPUT_DIRECTORY")
parser.add_option("-r", dest="TRAINING_RATE")
(options, args) = parser.parse_args()

if options.METADATA_DIRECTORY is not None:
    METADATA_DIRECTORY = options.METADATA_DIRECTORY
if options.PRICE_DATA_DIRECTORY is not None:
    PRICE_DATA_DIRECTORY = options.PRICE_DATA_DIRECTORY
if options.TRAINING_OUTPUT_DIRECTORY is not None:
    TRAINING_OUTPUT_DIRECTORY = options.TRAINING_OUTPUT_DIRECTORY
if options.TRAINING_RATE is not None:
    TRAINING_RATE = float(TRAINING_RATE)

if not os.path.exists(TRAINING_OUTPUT_DIRECTORY):
    print("Can't not find output directory, Create a new one")
    os.makedirs(TRAINING_OUTPUT_DIRECTORY)

# Store for Binary feature needs

developer_set = set()
publisher_set = set()
genres_set = set()
country_set = set()

# Store ID to Game Object

game_set = {}

# Create Game Object form metadata
list_meta = listFilePath(METADATA_DIRECTORY)

for meta in list_meta['file']:
    g = Game(meta['path'], TARGET_COUNTRY)
    game_set[g.id] = g

    developer_set.add(g.developer)
    publisher_set.add(g.publisher)
    genres_set = genres_set.union(set(g.geners))

# Add historical data from different conutries

list_c = listFilePath(PRICE_DATA_DIRECTORY)
for c in list_c['dir']:
    list_price = listFilePath(c['path'])
    for p in list_price['file']:
        country_set.add(c['name'])
        g_id = p['name'].split('_')[3].split('.')[0]
        game_set.get(g_id).addcountry(c['name'], p['path'])

# Data Cleaning on every country and get lables depend on the target country
for g in game_set.values():
    g.cleanDataAndEctractTime()
    g.getLabels()

# Get AND set of all games' countrysets

for g in game_set.values():
    country_set = country_set & set(g.country_set.keys())

# Output

train_f = open(TRAINING_OUTPUT_DIRECTORY + '/traing_matrix.csv', 'w')
train_o = []
test_f = open(TRAINING_OUTPUT_DIRECTORY + '/testing_matrix.csv', 'w')
test_o = []
data_f = open(TRAINING_OUTPUT_DIRECTORY + '/data_matrix.csv', 'w')

for g in game_set.values():
    g.setTestStartPoint(TRAINING_RATE)
    target_c = g.country_set[g.target_country]
    # filter some bias games
    if len(target_c.times) == 0 or target_c.mean_duration > 250:
        continue

    f = open(TRAINING_OUTPUT_DIRECTORY + '/app_training_' + g.id + '.csv', 'w')

    for i in range(len(g.country_set[g.target_country].times)):
        o = []
        # fliter preorder price data
        if g.days_from_release + i < 0:
            continue
        # Label
        if g.lables[i] > g.THRESHOLD:
            o.append('1')
        else:
            if g.lables[i] > 0:
                o.append('0')
            else:
                o.append('-1')
        # Developer
        for d in list(developer_set):
            if g.developer == d:
                o.append('1')
            else:
                o.append('0')
        # Publisher
        for p in list(publisher_set):
            if g.publisher == p:
                o.append('1')
            else:
                o.append('0')
        # Genres
        gens = set(g.geners)
        for gen in list(genres_set):
            if gen in gens:
                o.append('1')
            else:
                o.append('0')
        # Static
        # for c in g.country_set
        c = target_c
        o.extend([
            c.mean_org_price, c.var_org_price, c.mean_price,
            c.var_price, c.mean_duration, c.var_duration,
            c.mean_discount, c.var_discount
        ])

        # days_since_last_dis
        o.append(target_c.days_since_last_dis[i])
        for c in g.country_set.values():
            if c.name == target_c.name:
                continue
            o.append(c.days_since_last_dis[i])
        # days from release
        o.append(g.days_from_release + i)

        # date month, day, weekday
        t = datetime.datetime.strptime(target_c.times[i], '%Y-%m-%d')
        o.extend([t.month, t.day, t.weekday() + 1])

        o.append('\n')

        # str() to all data
        o = map(str, o)
        o = ','.join(o)
        f.write(o)
        if (i < g.test_begin - 1):
            train_o.append(o)
        else:
            test_o.append(o)

    f.close()

train_f.write(''.join(train_o))
test_f.write(''.join(test_o))
data_f.write(''.join(train_o+test_o))
# if __name__ == "__main__":
#    main();
<commit_msg>Add instance output<commit_after># -*- coding: utf-8 -*-
"""
Created on Thu Sep 08 16:16:41 2016

@author: SISQUAKE
"""
from optparse import OptionParser
import os
from Game import Game
import datetime
from listFile import listFilePath

# def main():

METADATA_DIRECTORY = '../../new_meta'
PRICE_DATA_DIRECTORY = '../../price_data'
TRAINING_OUTPUT_DIRECTORY = '../../traing_data'
TRAINING_RATE = 0.8

TARGET_COUNTRY = 'us'

parser = OptionParser()
parser.add_option("-m", dest="METADATA_DIRECTORY")
parser.add_option("-p", dest="PRICE_DATA_DIRECTORY")
parser.add_option("-t", dest="TRAINING_OUTPUT_DIRECTORY")
parser.add_option("-r", dest="TRAINING_RATE")
(options, args) = parser.parse_args()

if options.METADATA_DIRECTORY is not None:
    METADATA_DIRECTORY = options.METADATA_DIRECTORY
if options.PRICE_DATA_DIRECTORY is not None:
    PRICE_DATA_DIRECTORY = options.PRICE_DATA_DIRECTORY
if options.TRAINING_OUTPUT_DIRECTORY is not None:
    TRAINING_OUTPUT_DIRECTORY = options.TRAINING_OUTPUT_DIRECTORY
if options.TRAINING_RATE is not None:
    TRAINING_RATE = float(TRAINING_RATE)

if not os.path.exists(TRAINING_OUTPUT_DIRECTORY):
    print("Can't not find output directory, Create a new one")
    os.makedirs(TRAINING_OUTPUT_DIRECTORY)

# Store for Binary feature needs

developer_set = set()
publisher_set = set()
genres_set = set()
country_set = set()

# Store ID to Game Object

game_set = {}

# Create Game Object form metadata
list_meta = listFilePath(METADATA_DIRECTORY)

for meta in list_meta['file']:
    g = Game(meta['path'], TARGET_COUNTRY)
    game_set[g.id] = g

    developer_set.add(g.developer)
    publisher_set.add(g.publisher)
    genres_set = genres_set.union(set(g.geners))

# Add historical data from different conutries

list_c = listFilePath(PRICE_DATA_DIRECTORY)
for c in list_c['dir']:
    list_price = listFilePath(c['path'])
    for p in list_price['file']:
        country_set.add(c['name'])
        g_id = p['name'].split('_')[3].split('.')[0]
        game_set.get(g_id).addcountry(c['name'], p['path'])

# Data Cleaning on every country and get lables depend on the target country
for g in game_set.values():
    g.cleanDataAndEctractTime()
    g.getLabels()

# Get AND set of all games' countrysets

for g in game_set.values():
    country_set = country_set & set(g.country_set.keys())

# Output

train_f = open(TRAINING_OUTPUT_DIRECTORY + '/traing_matrix.csv', 'w')
train_o = []
test_f = open(TRAINING_OUTPUT_DIRECTORY + '/testing_matrix.csv', 'w')
test_o = []
data_f = open(TRAINING_OUTPUT_DIRECTORY + '/data_matrix.csv', 'w')
intstance_f = open(TRAINING_OUTPUT_DIRECTORY + '/instance.csv', 'w')

for g in game_set.values():
    g.setTestStartPoint(TRAINING_RATE)
    target_c = g.country_set[g.target_country]
    # filter some bias games
    if len(target_c.times) == 0 or target_c.mean_duration > 250:
        continue

    f = open(TRAINING_OUTPUT_DIRECTORY + '/app_training_' + g.id + '.csv', 'w')

    for i in range(len(target_c.times)):
        o = []
        # fliter preorder price data
        if g.days_from_release + i < 0:
            continue

        # Developer
        for d in list(developer_set):
            if g.developer == d:
                o.append('1')
            else:
                o.append('0')
        # Publisher
        for p in list(publisher_set):
            if g.publisher == p:
                o.append('1')
            else:
                o.append('0')
        # Genres
        gens = set(g.geners)
        for gen in list(genres_set):
            if gen in gens:
                o.append('1')
            else:
                o.append('0')
        # Static
        # for c in g.country_set
        c = target_c
        o.extend([
            c.mean_org_price, c.var_org_price, c.mean_price,
            c.var_price, c.mean_duration, c.var_duration,
            c.mean_discount, c.var_discount
        ])

        # days_since_last_dis
        o.append(target_c.days_since_last_dis[i])
        for c in g.country_set.values():
            if c.name == target_c.name:
                continue
            o.append(c.days_since_last_dis[i])
        # days from release
        o.append(g.days_from_release + i)

        # date month, day, weekday
        t = datetime.datetime.strptime(target_c.times[i], '%Y-%m-%d')
        o.extend([t.month, t.day, t.weekday() + 1])

        o.append('\n')
        # str() to all data
        o = map(str, o)
        o = ','.join(o)

        # output intstance with id append
        if (i == len(target_c.times)-1):
            intstance_f.write(g.id + ',' + o)

        # Label
        if g.lables[i] > g.THRESHOLD:
            o = '1,' + o
        else:
            if g.lables[i] > 0:
                o = '0,' + o
            else:
                o = '-1,' + o

        f.write(o)
        # separte testing and training data
        if (i < g.test_begin - 1):
            train_o.append(o)
        else:
            test_o.append(o)

    f.close()

intstance_f.close()

train_f.write(''.join(train_o))
test_f.write(''.join(test_o))
data_f.write(''.join(train_o+test_o))
train_f.close()
test_f.close()
data_f.close()
# if __name__ == "__main__":
#    main();
<|endoftext|>


<commit_before># some very basic analysis functions
def get_R(d, current_column='EI', voltage_column='EV'):
    """
    Takes in padnas.DataFrame and optional column labels, returns average Resistance
    """
    R = d[voltage_column]/d[current_column]
    # remove all infinite resistances
    R=R.replace([np.inf, -np.inf], np.nan).dropna()
    return R.abs().mean()

def reset_pulse_energy_estimate(peak,width,slope,lrs_res=2e3,hrs_res=15e3):
    lrs_energy=(0.5*slope+(0.5-slope))*width*peak*peak/lrs_res
    hrs_energy=(0.5*slope+(0.5-slope))*width*peak*peak/hrs_res
    return lrs_energy+hrs_energy

# base abstract procedures
def find_set_V(d):
    """
    Given a sweep datum, tries to find the voltage at which the jump occurs
    """
    jumpstep=d["EI"].diff().abs().idxmax()
    jumpV = d.get_value(jumpstep, "EV")
    return jumpV

def get_hist(d):
    """
    Given a Sweep datum with EI and EV, calculate the difference between the first and second pass through a EV value, return as a dataframe of index, EV and R
    """
    if not "R" in d.columns:
        d["R"]=d["EV"]/d["EI"]
    half=len(d)//2
    # get first half of EV; make sure we indice from 0 to half
    d1=d[:half]
    d1.index=range(half)

    # same for second half, but flip so that the EV align
    d2=d[half:]
    d2.index=range(half)
    d2=d2.iloc[::-1]
    d2.index=range(half)
    hist = d2-d1
    # dataframe now contains positive R where the second pass had a larger resistance than the first
    return pd.concat([d1["EV"],hist["R"]],axis=1)

def add_resistance(d):
    out = d
    out["R"]=d["EV"]/d["EI"]
    return out

def add_energy(datum):
    datum['cumulative_energy']=(datum['ET'].diff().fillna(0)*datum['EI']*datum['EV']).cumsum()
    return datum
<commit_msg>added max functions to analysis helpers<commit_after>import pandas as pd
import numpy as np
# some very basic analysis functions
def get_R(d, current_column='EI', voltage_column='EV'):
    """
    Takes in padnas.DataFrame and optional column labels, returns average Resistance
    """
    R = d[voltage_column]/d[current_column]
    # remove all infinite resistances
    R=R.replace([np.inf, -np.inf], np.nan).dropna()
    return R.abs().mean()

def reset_pulse_energy_estimate(peak,width,slope,lrs_res=2e3,hrs_res=15e3):
    lrs_energy=(0.5*slope+(0.5-slope))*width*peak*peak/lrs_res
    hrs_energy=(0.5*slope+(0.5-slope))*width*peak*peak/hrs_res
    return lrs_energy+hrs_energy

def get_maxCurrent(d, current_column='EI', voltage_column='EV'):
    return d[current_column].abs().max()

# base abstract procedures
def find_set_V(d):
    """
    Given a sweep datum, tries to find the voltage at which the jump occurs
    """
    jumpstep=d["EI"].diff().abs().idxmax()
    jumpV = d.get_value(jumpstep, "EV")
    return jumpV

def get_hist(d):
    """
    Given a Sweep datum with EI and EV, calculate the difference between the first and second pass through a EV value, return as a dataframe of index, EV and R
    """
    if not "R" in d.columns:
        d["R"]=d["EV"]/d["EI"]
    half=len(d)//2
    # get first half of EV; make sure we indice from 0 to half
    d1=d[:half]
    d1.index=range(half)

    # same for second half, but flip so that the EV align
    d2=d[half:]
    d2.index=range(half)
    d2=d2.iloc[::-1]
    d2.index=range(half)
    hist = d2-d1
    # dataframe now contains positive R where the second pass had a larger resistance than the first
    return pd.concat([d1["EV"],hist["R"]],axis=1)

def getMaxHist(d):
    eh = get_hist(d)
    return eh["R"].abs().max()

def add_resistance(d):
    out = d
    out["R"]=d["EV"]/d["EI"]
    return out

def add_energy(datum):
    datum['cumulative_energy']=(datum['ET'].diff().fillna(0)*datum['EI']*datum['EV']).cumsum()
    return datum
<|endoftext|>


<commit_before>#!/usr/bin/python
"""Run tests in parallel."""

import argparse
import glob
import itertools
import json
import multiprocessing
import os
import sys
import time

import jobset
import watch_dirs

# SimpleConfig: just compile with CONFIG=config, and run the binary to test
class SimpleConfig(object):

  def __init__(self, config):
    self.build_config = config
    self.maxjobs = 2 * multiprocessing.cpu_count()
    self.allow_hashing = (config != 'gcov')

  def run_command(self, binary):
    return [binary]

# ValgrindConfig: compile with some CONFIG=config, but use valgrind to run
class ValgrindConfig(object):

  def __init__(self, config, tool):
    self.build_config = config
    self.tool = tool
    self.maxjobs = 2 * multiprocessing.cpu_count()
    self.allow_hashing = False

  def run_command(self, binary):
    return ['valgrind', binary, '--tool=%s' % self.tool]

class CLanguage(object):

  def __init__(self, make_target, test_lang):
    self.allow_hashing = True
    self.make_target = make_target
    with open('tools/run_tests/tests.json') as f:
      js = json.load(f)
      self.binaries = [tgt['name']
                       for tgt in js
                       if tgt['language'] == test_lang]

  def test_binaries(self, config):
    return ['bins/%s/%s' % (config, binary) for binary in self.binaries]

  def make_targets(self):
    return ['buildtests_%s' % self.make_target]

  def build_steps(self):
    return []

class NodeLanguage(object):

  def __init__(self):
    self.allow_hashing = False

  def test_binaries(self, config):
    return ['tools/run_tests/run_node.sh']

  def make_targets(self):
    return ['static_c']

  def build_steps(self):
    return [['tools/run_tests/build_node.sh']]

class PhpLanguage(object):

  def __init__(self):
    self.allow_hashing = False

  def test_binaries(self, config):
    return ['src/php/bin/run_tests.sh']

  def make_targets(self):
    return ['static_c']

  def build_steps(self):
    return [['tools/run_tests/build_php.sh']]

class PythonLanguage(object):

  def __init__(self):
    self.allow_hashing = False

  def test_binaries(self, config):
    return ['tools/run_tests/run_python.sh']

  def make_targets(self):
    return[]

  def build_steps(self):
    return [['tools/run_tests/build_python.sh']]

# different configurations we can run under
_CONFIGS = {
    'dbg': SimpleConfig('dbg'),
    'opt': SimpleConfig('opt'),
    'tsan': SimpleConfig('tsan'),
    'msan': SimpleConfig('msan'),
    'asan': SimpleConfig('asan'),
    'gcov': SimpleConfig('gcov'),
    'memcheck': ValgrindConfig('valgrind', 'memcheck'),
    'helgrind': ValgrindConfig('dbg', 'helgrind')
    }

_DEFAULT = ['dbg', 'opt']
_LANGUAGES = {
    'c++': CLanguage('cxx', 'c++'),
    'c': CLanguage('c', 'c'),
    'node': NodeLanguage(),
    'php': PhpLanguage(),
    'python': PythonLanguage(),
}

# parse command line
argp = argparse.ArgumentParser(description='Run grpc tests.')
argp.add_argument('-c', '--config',
                  choices=['all'] + sorted(_CONFIGS.keys()),
                  nargs='+',
                  default=_DEFAULT)
argp.add_argument('-n', '--runs_per_test', default=1, type=int)
argp.add_argument('-f', '--forever',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('--newline_on_success',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('-l', '--language',
                  choices=sorted(_LANGUAGES.keys()),
                  nargs='+',
                  default=sorted(_LANGUAGES.keys()))
args = argp.parse_args()

# grab config
run_configs = set(_CONFIGS[cfg]
                  for cfg in itertools.chain.from_iterable(
                      _CONFIGS.iterkeys() if x == 'all' else [x]
                      for x in args.config))
build_configs = set(cfg.build_config for cfg in run_configs)

make_targets = []
languages = set(_LANGUAGES[l] for l in args.language)
build_steps = [['make',
                '-j', '%d' % (multiprocessing.cpu_count() + 1),
                'CONFIG=%s' % cfg] + list(set(
                    itertools.chain.from_iterable(l.make_targets()
                                                  for l in languages)))
               for cfg in build_configs] + list(
                   itertools.chain.from_iterable(l.build_steps()
                                                 for l in languages))

runs_per_test = args.runs_per_test
forever = args.forever

class TestCache(object):
  """Cache for running tests."""

  def __init__(self):
    self._last_successful_run = {}

  def should_run(self, cmdline, bin_hash):
    cmdline = ' '.join(cmdline)
    if cmdline not in self._last_successful_run:
      return True
    if self._last_successful_run[cmdline] != bin_hash:
      return True
    return False

  def finished(self, cmdline, bin_hash):
    self._last_successful_run[' '.join(cmdline)] = bin_hash

  def dump(self):
    return [{'cmdline': k, 'hash': v}
            for k, v in self._last_successful_run.iteritems()]

  def parse(self, exdump):
    self._last_successful_run = dict((o['cmdline'], o['hash']) for o in exdump)

  def save(self):
    with open('.run_tests_cache', 'w') as f:
      f.write(json.dumps(self.dump()))

  def maybe_load(self):
    if os.path.exists('.run_tests_cache'):
      with open('.run_tests_cache') as f:
        self.parse(json.loads(f.read()))

def _build_and_run(check_cancelled, newline_on_success, cache):
  """Do one pass of building & running tests."""
  # build latest sequentially
  if not jobset.run(build_steps, maxjobs=1):
    return 1

  # run all the tests
  one_run = dict(
      (' '.join(config.run_command(x)), config.run_command(x))
      for config in run_configs
      for language in args.language
      for x in _LANGUAGES[language].test_binaries(config.build_config)
      ).values()
  all_runs = itertools.chain.from_iterable(
      itertools.repeat(one_run, runs_per_test))
  if not jobset.run(all_runs, check_cancelled,
                    newline_on_success=newline_on_success,
                    maxjobs=min(c.maxjobs for c in run_configs),
                    cache=cache):
    return 2

  return 0

test_cache = (None
              if not all(x.allow_hashing
                         for x in itertools.chain(languages, run_configs))
              else TestCache())
if test_cache:
  test_cache.maybe_load()

if forever:
  success = True
  while True:
    dw = watch_dirs.DirWatcher(['src', 'include', 'test'])
    initial_time = dw.most_recent_change()
    have_files_changed = lambda: dw.most_recent_change() != initial_time
    previous_success = success
    success = _build_and_run(check_cancelled=have_files_changed,
                             newline_on_success=False,
                             cache=test_cache) == 0
    if not previous_success and success:
      jobset.message('SUCCESS',
                     'All tests are now passing properly',
                     do_newline=True)
    jobset.message('IDLE', 'No change detected')
    if test_cache: test_cache.save()
    while not have_files_changed():
      time.sleep(1)
else:
  result = _build_and_run(check_cancelled=lambda: False,
                          newline_on_success=args.newline_on_success,
                          cache=test_cache)
  if result == 0:
    jobset.message('SUCCESS', 'All tests passed', do_newline=True)
  else:
    jobset.message('FAILED', 'Some tests failed', do_newline=True)
  if test_cache: test_cache.save()
  sys.exit(result)
<commit_msg>Clean up formatting<commit_after>#!/usr/bin/python
"""Run tests in parallel."""

import argparse
import glob
import itertools
import json
import multiprocessing
import os
import sys
import time

import jobset
import watch_dirs

# SimpleConfig: just compile with CONFIG=config, and run the binary to test
class SimpleConfig(object):

  def __init__(self, config):
    self.build_config = config
    self.maxjobs = 2 * multiprocessing.cpu_count()
    self.allow_hashing = (config != 'gcov')

  def run_command(self, binary):
    return [binary]

# ValgrindConfig: compile with some CONFIG=config, but use valgrind to run
class ValgrindConfig(object):

  def __init__(self, config, tool):
    self.build_config = config
    self.tool = tool
    self.maxjobs = 2 * multiprocessing.cpu_count()
    self.allow_hashing = False

  def run_command(self, binary):
    return ['valgrind', binary, '--tool=%s' % self.tool]

class CLanguage(object):

  def __init__(self, make_target, test_lang):
    self.allow_hashing = True
    self.make_target = make_target
    with open('tools/run_tests/tests.json') as f:
      js = json.load(f)
      self.binaries = [tgt['name']
                       for tgt in js
                       if tgt['language'] == test_lang]

  def test_binaries(self, config):
    return ['bins/%s/%s' % (config, binary) for binary in self.binaries]

  def make_targets(self):
    return ['buildtests_%s' % self.make_target]

  def build_steps(self):
    return []

class NodeLanguage(object):

  def __init__(self):
    self.allow_hashing = False

  def test_binaries(self, config):
    return ['tools/run_tests/run_node.sh']

  def make_targets(self):
    return ['static_c']

  def build_steps(self):
    return [['tools/run_tests/build_node.sh']]

class PhpLanguage(object):

  def __init__(self):
    self.allow_hashing = False

  def test_binaries(self, config):
    return ['src/php/bin/run_tests.sh']

  def make_targets(self):
    return ['static_c']

  def build_steps(self):
    return [['tools/run_tests/build_php.sh']]

class PythonLanguage(object):

  def __init__(self):
    self.allow_hashing = False

  def test_binaries(self, config):
    return ['tools/run_tests/run_python.sh']

  def make_targets(self):
    return[]

  def build_steps(self):
    return [['tools/run_tests/build_python.sh']]

# different configurations we can run under
_CONFIGS = {
    'dbg': SimpleConfig('dbg'),
    'opt': SimpleConfig('opt'),
    'tsan': SimpleConfig('tsan'),
    'msan': SimpleConfig('msan'),
    'asan': SimpleConfig('asan'),
    'gcov': SimpleConfig('gcov'),
    'memcheck': ValgrindConfig('valgrind', 'memcheck'),
    'helgrind': ValgrindConfig('dbg', 'helgrind')
    }

_DEFAULT = ['dbg', 'opt']
_LANGUAGES = {
    'c++': CLanguage('cxx', 'c++'),
    'c': CLanguage('c', 'c'),
    'node': NodeLanguage(),
    'php': PhpLanguage(),
    'python': PythonLanguage(),
}

# parse command line
argp = argparse.ArgumentParser(description='Run grpc tests.')
argp.add_argument('-c', '--config',
                  choices=['all'] + sorted(_CONFIGS.keys()),
                  nargs='+',
                  default=_DEFAULT)
argp.add_argument('-n', '--runs_per_test', default=1, type=int)
argp.add_argument('-f', '--forever',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('--newline_on_success',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('-l', '--language',
                  choices=sorted(_LANGUAGES.keys()),
                  nargs='+',
                  default=sorted(_LANGUAGES.keys()))
args = argp.parse_args()

# grab config
run_configs = set(_CONFIGS[cfg]
                  for cfg in itertools.chain.from_iterable(
                      _CONFIGS.iterkeys() if x == 'all' else [x]
                      for x in args.config))
build_configs = set(cfg.build_config for cfg in run_configs)

make_targets = []
languages = set(_LANGUAGES[l] for l in args.language)
build_steps = [['make',
                '-j', '%d' % (multiprocessing.cpu_count() + 1),
                'CONFIG=%s' % cfg] + list(set(
                    itertools.chain.from_iterable(l.make_targets()
                                                  for l in languages)))
               for cfg in build_configs] + list(
                   itertools.chain.from_iterable(l.build_steps()
                                                 for l in languages))

runs_per_test = args.runs_per_test
forever = args.forever

class TestCache(object):
  """Cache for running tests."""

  def __init__(self):
    self._last_successful_run = {}

  def should_run(self, cmdline, bin_hash):
    cmdline = ' '.join(cmdline)
    if cmdline not in self._last_successful_run:
      return True
    if self._last_successful_run[cmdline] != bin_hash:
      return True
    return False

  def finished(self, cmdline, bin_hash):
    self._last_successful_run[' '.join(cmdline)] = bin_hash

  def dump(self):
    return [{'cmdline': k, 'hash': v}
            for k, v in self._last_successful_run.iteritems()]

  def parse(self, exdump):
    self._last_successful_run = dict((o['cmdline'], o['hash']) for o in exdump)

  def save(self):
    with open('.run_tests_cache', 'w') as f:
      f.write(json.dumps(self.dump()))

  def maybe_load(self):
    if os.path.exists('.run_tests_cache'):
      with open('.run_tests_cache') as f:
        self.parse(json.loads(f.read()))

def _build_and_run(check_cancelled, newline_on_success, cache):
  """Do one pass of building & running tests."""
  # build latest sequentially
  if not jobset.run(build_steps, maxjobs=1):
    return 1

  # run all the tests
  one_run = dict(
      (' '.join(config.run_command(x)), config.run_command(x))
      for config in run_configs
      for language in args.language
      for x in _LANGUAGES[language].test_binaries(config.build_config)
      ).values()
  all_runs = itertools.chain.from_iterable(
      itertools.repeat(one_run, runs_per_test))
  if not jobset.run(all_runs, check_cancelled,
                    newline_on_success=newline_on_success,
                    maxjobs=min(c.maxjobs for c in run_configs),
                    cache=cache):
    return 2

  return 0

test_cache = (None
              if not all(x.allow_hashing
                         for x in itertools.chain(languages, run_configs))
              else TestCache())
if test_cache:
  test_cache.maybe_load()

if forever:
  success = True
  while True:
    dw = watch_dirs.DirWatcher(['src', 'include', 'test'])
    initial_time = dw.most_recent_change()
    have_files_changed = lambda: dw.most_recent_change() != initial_time
    previous_success = success
    success = _build_and_run(check_cancelled=have_files_changed,
                             newline_on_success=False,
                             cache=test_cache) == 0
    if not previous_success and success:
      jobset.message('SUCCESS',
                     'All tests are now passing properly',
                     do_newline=True)
    jobset.message('IDLE', 'No change detected')
    if test_cache: test_cache.save()
    while not have_files_changed():
      time.sleep(1)
else:
  result = _build_and_run(check_cancelled=lambda: False,
                          newline_on_success=args.newline_on_success,
                          cache=test_cache)
  if result == 0:
    jobset.message('SUCCESS', 'All tests passed', do_newline=True)
  else:
    jobset.message('FAILED', 'Some tests failed', do_newline=True)
  if test_cache: test_cache.save()
  sys.exit(result)
<|endoftext|>