<commit_before>#vim: set encoding=utf-8
from unittest import TestCase
from regparser.notice import compiler
from regparser.tree.struct import Node, find

class CompilerTests(TestCase):
    def test_dict_to_node(self):
        dict_node = {
            'text': 'node text',
            'label': ['205', 'A'],
            'node_type': 'appendix'}

        node = compiler.dict_to_node(dict_node)

        self.assertEqual(
            node,
            Node('node text', [], ['205', 'A'], None, 'appendix'))

        dict_node['tagged_text'] = '<E> Tagged </E> text.'

        node = compiler.dict_to_node(dict_node)

        actual_node = Node('node text', [], ['205', 'A'], None, 'appendix')
        actual_node.tagged_text = '<E> Tagged </E> text.'

        created_node = compiler.dict_to_node(dict_node)

        self.assertEqual(actual_node, created_node)
        self.assertEqual(actual_node.tagged_text, created_node.tagged_text)

        dict_node = {
            'text': 'node text'
        }

        node = compiler.dict_to_node(dict_node)
        self.assertEqual(node, dict_node)

    def test_sort_labels(self):
        labels = [
            ['205', '2', 'a', 'i'], ['205', 'Subpart', 'A'], ['205', '2']]

        sorted_labels = compiler.sort_labels(labels)
        self.assertEqual(
            sorted_labels,
            [['205', 'Subpart', 'A'], ['205', '2'], ['205', '2', 'a', 'i']])

    def test_make_root_sortable(self):
        self.assertEqual(
            compiler.make_root_sortable(['205', 'B'], Node.APPENDIX),
            (1, 'B'))

        self.assertEqual(
            compiler.make_root_sortable(['205', 'Subpart', 'J'], Node.SUBPART),
            (0, 'J'))

        self.assertEqual(
            compiler.make_root_sortable(['205', 'Interp'], Node.INTERP),
            (2, ))

    def test_add_child(self):
        n1 = Node('n1', label=['205', '1'])
        n2 = Node('n2', label=['205', '2'])
        n4 = Node('n4', label=['205', '4'])

        children = [n1, n2, n4]

        reg_tree = compiler.RegulationTree(None)

        n3 = Node('n3', label=['205', '3'])
        reg_tree.add_child(children, n3)

        self.assertEqual(children, [n1, n2, n3, n4])
        for c in children:
            self.assertFalse(hasattr(c, 'sortable'))

    def tree_with_paragraphs(self):
        n1 = Node('n1', label=['205', '1'])
        n2 = Node('n2', label=['205', '2'])
        n4 = Node('n4', label=['205', '4'])

        n2a = Node('n2a', label=['205', '2', 'a'])
        n2b = Node('n2b', label=['205', '2', 'b'])
        n2.children = [n2a, n2b]

        root = Node('', label=['205'])
        root.children = [n1, n2, n4]
        return root

    def test_replace_node_and_subtree(self):
        n1 = Node('n1', label=['205', '1'])
        n2 = Node('n2', label=['205', '2'])
        n4 = Node('n4', label=['205', '4'])

        n2a = Node('n2a', label=['205', '2', 'a'])
        n2b = Node('n2b', label=['205', '2', 'b'])
        n2.children = [n2a, n2b]

        root = Node('', label=['205'])
        root.children = [n1, n2, n4]

        reg_tree = compiler.RegulationTree(root)

        a2 = Node('a2', label=['205', '2'])
        a2e = Node('a2e', label=['205', '2', 'e'])
        a2f = Node('a2f', label=['205', '2', 'f'])
        a2.children = [a2e, a2f]

        reg_tree.replace_node_and_subtree(a2)

        new_tree = Node('', label=[205])
        new_tree.children = [n1, a2, n4]

        self.assertEqual(new_tree, reg_tree.tree)
        self.assertEqual(None, find(reg_tree.tree, '205-2-a'))

    def test_add_node(self):
        root = self.tree_with_paragraphs()
        reg_tree = compiler.RegulationTree(root)

        n2ai = Node('n2ai', label=['205', '2', 'a', '1'])
        reg_tree.add_node(n2ai)
        self.assertNotEqual(reg_tree.tree, root)

        n2a = find(root, '205-2-a')
        n2a.children = [n2ai]
        self.assertEqual(reg_tree.tree, root)

    def test_move(self):
        root = self.tree_with_paragraphs()
        reg_tree = compiler.RegulationTree(root)
        reg_tree.move('205-2-a', ['205', '4', 'a'])

        moved = find(reg_tree.tree, '205-4-a')
        self.assertNotEqual(None, moved)
        self.assertEqual(moved.text, 'n2a')
        self.assertEqual(None, find(reg_tree.tree, '205-2-a'))

    def test_add_to_root(self):
        nsa = Node(
            'nsa',
            label=['205', 'Subpart', 'A'],
            node_type=Node.SUBPART)

        nappa = Node(
            'nappa',
            label=['205', 'Appendix', 'C'],
            node_type=Node.APPENDIX)

        root = Node('', label=['205'])
        root.children = [nsa, nappa]

        reg_tree = compiler.RegulationTree(root)

        nappb = Node(
            'nappb',
            label=['205', 'Appendix', 'B'], node_type=Node.APPENDIX)

        reg_tree.add_to_root(nappb)
        root.children = [nsa, nappb, nappa]

        self.assertEqual(reg_tree.tree, root)

    def test_add_section(self):
        nsa = Node(
            'nsa',
            label=['205', 'Subpart', 'A'],
            node_type=Node.SUBPART)

        nappa = Node(
            'nappa',
            label=['205', 'Appendix', 'C'],
            node_type=Node.APPENDIX)

        root = Node('', label=['205'])
        root.children = [nsa, nappa]

        n1 = Node('', label=['205', '1'])

        reg_tree = compiler.RegulationTree(root)
        reg_tree.add_section(n1, ['205', 'Subpart', 'A'])

        nsa.children = [n1]
        self.assertEqual(reg_tree.tree, root)

    def test_replace_node_text(self):
        root = self.tree_with_paragraphs()

        change = {'node': {'text': 'new text'}}
        reg_tree = compiler.RegulationTree(root)

        reg_tree.replace_node_text('205-2-a', change)
        changed_node = find(reg_tree.tree, '205-2-a')
        self.assertEqual(changed_node.text, 'new text')

    def test_replace_node_title(self):
        root = self.tree_with_paragraphs()

        change = {'node': {'title': 'new title'}}
        reg_tree = compiler.RegulationTree(root)

        reg_tree.replace_node_title('205-2-a', change)
        changed_node = find(reg_tree.tree, '205-2-a')
        self.assertEqual(changed_node.title, 'new title')

    def test_get_subparts(self):
        nsa = Node(
            'nsa',
            label=['205', 'Subpart', 'A'], node_type=Node.SUBPART)

        nsb = Node('nsb',
                   label=['205', 'Subpart', 'B'],
                   node_type=Node.SUBPART)

        nappa = Node('nappa',
                     label=['205', 'Appendix', 'C'],
                     node_type=Node.APPENDIX)

        root = Node('', label=['205'])
        root.children = [nsa, nsb, nappa]

        reg_tree = compiler.RegulationTree(root)
        subparts = reg_tree.get_subparts()
        labels = [s.label_id() for s in subparts]

        self.assertEqual(labels, ['205-Subpart-A', '205-Subpart-B'])

    def tree_with_subparts(self):
        nsa = Node('nsa',
                   label=['205', 'Subpart', 'A'],
                   node_type=Node.SUBPART)

        nsb = Node('nsb',
                   label=['205', 'Subpart', 'B'],
                   node_type=Node.SUBPART)

        nappa = Node('nappa',
                     label=['205', 'Appendix', 'C'],
                     node_type=Node.APPENDIX)

        root = Node('', label=['205'])
        root.children = [nsa, nsb, nappa]
        return root

    def test_create_new_subpart(self):
        root = self.tree_with_subparts()

        reg_tree = compiler.RegulationTree(root)
        reg_tree.create_new_subpart(['205', 'Subpart', 'C'])

        subparts = reg_tree.get_subparts()
        labels = [s.label_id() for s in subparts]

        self.assertEqual(
            labels, ['205-Subpart-A', '205-Subpart-B', '205-Subpart-C'])

    def test_get_subpart_for_node(self):
        root = self.tree_with_subparts()
        n1 = Node('n1', label=['205', '1'])
        nsb = find(root, '205-Subpart-B')
        nsb.children = [n1]

        reg_tree = compiler.RegulationTree(root)
        subpart = reg_tree.get_subpart_for_node('205-1')

        self.assertEqual(subpart.label_id(), '205-Subpart-B')

    def test_compile_reg_put_replace_whole_tree(self):
        root = self.tree_with_paragraphs()

        change2a = {
            'action': 'PUT',
            'node': {
                'text': 'new text',
                'label': ['205', '2', 'a'],
                'node_type': 'regtext'}}

        change2a1 = {
            'action': 'PUT',
            'node': {
                'text': '2a1 text',
                'label': ['205', '2', 'a', '1'],
                'node_type': 'regtext'}}

        notice_changes = {
            '205-2-a-1': [change2a1],
            '205-2-a': [change2a]
        }

        reg = compiler.compile_regulation(root, notice_changes)

        added_node = find(reg, '205-2-a')
        self.assertEqual(added_node.text, 'new text')

        deeper = find(reg, '205-2-a-1')
        self.assertEqual(deeper.text, '2a1 text')

    def test_compile_reg_put_text_only(self):
        root = self.tree_with_paragraphs()
        change2a = {
            'action': 'PUT',
            'field': '[text]',
            'node': {
                'text': 'new text',
                'label': ['205', '2', 'a'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2-a': [change2a]}
        reg = compiler.compile_regulation(root, notice_changes)

        changed_node = find(reg, '205-2-a')
        self.assertEqual(changed_node.text, 'new text')

    def test_compile_reg_post_no_subpart(self):
        root = self.tree_with_paragraphs()
        change2a1 = {
            'action': 'POST',
            'node': {
                'text': '2a1 text',
                'label': ['205', '2', 'a', '1'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2-a-1': [change2a1]}
        reg = compiler.compile_regulation(root, notice_changes)
        added_node = find(reg, '205-2-a-1')
        self.assertNotEqual(None, added_node)
        self.assertEqual(added_node.text, '2a1 text')

    def test_compile_add_to_subpart(self):
        root = self.tree_with_subparts()

        change = {
            'action': 'POST',
            'subpart': ['205', 'Subpart', 'B'],
            'node': {
                'text': '2 text',
                'label': ['205', '2'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2': [change]}
        reg = compiler.compile_regulation(root, notice_changes)
        added_node = find(reg, '205-2')
        self.assertNotEqual(None, added_node)
        self.assertEqual(added_node.text, '2 text')

    def test_compile_designate(self):
        root = self.tree_with_subparts()
        change = {
            'action': 'POST',
            'subpart': ['205', 'Subpart', 'B'],
            'node': {
                'text': '2 text',
                'label': ['205', '2'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2': [change]}
        reg = compiler.compile_regulation(root, notice_changes)

        subpart_b = find(reg, '205-Subpart-B')
        self.assertEqual(len(subpart_b.children), 1)

        subpart_a = find(reg, '205-Subpart-A')
        self.assertEqual(len(subpart_a.children), 0)

        change = {
            'action': 'DESIGNATE',
            'destination': ['205', 'Subpart', 'A']}

        notice_changes = {'205-2': [change]}

        new_reg = compiler.compile_regulation(reg, notice_changes)

        subpart_b = find(new_reg, '205-Subpart-B')
        self.assertEqual(len(subpart_b.children), 0)

        subpart_a = find(new_reg, '205-Subpart-A')
        self.assertEqual(len(subpart_a.children), 1)

    def test_delete(self):
        root = self.tree_with_paragraphs()
        reg_tree = compiler.RegulationTree(root)

        self.assertNotEqual(None, find(reg_tree.tree, '205-2-a'))
        reg_tree.delete('205-2-a')
        self.assertEqual(None, find(reg_tree.tree, '205-2-a'))

    def test_get_parent(self):
        root = self.tree_with_paragraphs()
        reg_tree = compiler.RegulationTree(root)

        node = find(reg_tree.tree, '205-2-a')
        parent = reg_tree.get_parent(node)
        self.assertEqual(parent.label, ['205', '2'])
<commit_msg>Empty node tests<commit_after>#vim: set encoding=utf-8
from unittest import TestCase
from regparser.notice import compiler
from regparser.tree.struct import Node, find

class CompilerTests(TestCase):
    def test_dict_to_node(self):
        dict_node = {
            'text': 'node text',
            'label': ['205', 'A'],
            'node_type': 'appendix'}

        node = compiler.dict_to_node(dict_node)

        self.assertEqual(
            node,
            Node('node text', [], ['205', 'A'], None, 'appendix'))

        dict_node['tagged_text'] = '<E> Tagged </E> text.'

        node = compiler.dict_to_node(dict_node)

        actual_node = Node('node text', [], ['205', 'A'], None, 'appendix')
        actual_node.tagged_text = '<E> Tagged </E> text.'

        created_node = compiler.dict_to_node(dict_node)

        self.assertEqual(actual_node, created_node)
        self.assertEqual(actual_node.tagged_text, created_node.tagged_text)

        dict_node = {
            'text': 'node text'
        }

        node = compiler.dict_to_node(dict_node)
        self.assertEqual(node, dict_node)

    def test_sort_labels(self):
        labels = [
            ['205', '2', 'a', 'i'], ['205', 'Subpart', 'A'], ['205', '2']]

        sorted_labels = compiler.sort_labels(labels)
        self.assertEqual(
            sorted_labels,
            [['205', 'Subpart', 'A'], ['205', '2'], ['205', '2', 'a', 'i']])

    def test_make_root_sortable(self):
        self.assertEqual(
            compiler.make_root_sortable(['205', 'B'], Node.APPENDIX),
            (1, 'B'))

        self.assertEqual(
            compiler.make_root_sortable(['205', 'Subpart', 'J'], Node.SUBPART),
            (0, 'J'))

        self.assertEqual(
            compiler.make_root_sortable(['205', 'Interp'], Node.INTERP),
            (2, ))

    def test_add_child(self):
        n1 = Node('n1', label=['205', '1'])
        n2 = Node('n2', label=['205', '2'])
        n4 = Node('n4', label=['205', '4'])

        children = [n1, n2, n4]

        reg_tree = compiler.RegulationTree(None)

        n3 = Node('n3', label=['205', '3'])
        reg_tree.add_child(children, n3)

        self.assertEqual(children, [n1, n2, n3, n4])
        for c in children:
            self.assertFalse(hasattr(c, 'sortable'))

    def tree_with_paragraphs(self):
        n1 = Node('n1', label=['205', '1'])
        n2 = Node('n2', label=['205', '2'])
        n4 = Node('n4', label=['205', '4'])

        n2a = Node('n2a', label=['205', '2', 'a'])
        n2b = Node('n2b', label=['205', '2', 'b'])
        n2.children = [n2a, n2b]

        root = Node('', label=['205'])
        root.children = [n1, n2, n4]
        return root

    def test_replace_node_and_subtree(self):
        n1 = Node('n1', label=['205', '1'])
        n2 = Node('n2', label=['205', '2'])
        n4 = Node('n4', label=['205', '4'])

        n2a = Node('n2a', label=['205', '2', 'a'])
        n2b = Node('n2b', label=['205', '2', 'b'])
        n2.children = [n2a, n2b]

        root = Node('', label=['205'])
        root.children = [n1, n2, n4]

        reg_tree = compiler.RegulationTree(root)

        a2 = Node('a2', label=['205', '2'])
        a2e = Node('a2e', label=['205', '2', 'e'])
        a2f = Node('a2f', label=['205', '2', 'f'])
        a2.children = [a2e, a2f]

        reg_tree.replace_node_and_subtree(a2)

        new_tree = Node('', label=[205])
        new_tree.children = [n1, a2, n4]

        self.assertEqual(new_tree, reg_tree.tree)
        self.assertEqual(None, find(reg_tree.tree, '205-2-a'))

    def test_add_node(self):
        root = self.tree_with_paragraphs()
        reg_tree = compiler.RegulationTree(root)

        n2ai = Node('n2ai', label=['205', '2', 'a', '1'])
        reg_tree.add_node(n2ai)
        self.assertNotEqual(reg_tree.tree, root)

        n2a = find(root, '205-2-a')
        n2a.children = [n2ai]
        self.assertEqual(reg_tree.tree, root)

    def test_move(self):
        root = self.tree_with_paragraphs()
        reg_tree = compiler.RegulationTree(root)
        reg_tree.move('205-2-a', ['205', '4', 'a'])

        moved = find(reg_tree.tree, '205-4-a')
        self.assertNotEqual(None, moved)
        self.assertEqual(moved.text, 'n2a')
        self.assertEqual(None, find(reg_tree.tree, '205-2-a'))

    def test_add_to_root(self):
        nsa = Node(
            'nsa',
            label=['205', 'Subpart', 'A'],
            node_type=Node.SUBPART)

        nappa = Node(
            'nappa',
            label=['205', 'Appendix', 'C'],
            node_type=Node.APPENDIX)

        root = Node('', label=['205'])
        root.children = [nsa, nappa]

        reg_tree = compiler.RegulationTree(root)

        nappb = Node(
            'nappb',
            label=['205', 'Appendix', 'B'], node_type=Node.APPENDIX)

        reg_tree.add_to_root(nappb)
        root.children = [nsa, nappb, nappa]

        self.assertEqual(reg_tree.tree, root)

    def test_add_section(self):
        nsa = Node(
            'nsa',
            label=['205', 'Subpart', 'A'],
            node_type=Node.SUBPART)

        nappa = Node(
            'nappa',
            label=['205', 'Appendix', 'C'],
            node_type=Node.APPENDIX)

        root = Node('', label=['205'])
        root.children = [nsa, nappa]

        n1 = Node('', label=['205', '1'])

        reg_tree = compiler.RegulationTree(root)
        reg_tree.add_section(n1, ['205', 'Subpart', 'A'])

        nsa.children = [n1]
        self.assertEqual(reg_tree.tree, root)

    def test_replace_node_text(self):
        root = self.tree_with_paragraphs()

        change = {'node': {'text': 'new text'}}
        reg_tree = compiler.RegulationTree(root)

        reg_tree.replace_node_text('205-2-a', change)
        changed_node = find(reg_tree.tree, '205-2-a')
        self.assertEqual(changed_node.text, 'new text')

    def test_replace_node_title(self):
        root = self.tree_with_paragraphs()

        change = {'node': {'title': 'new title'}}
        reg_tree = compiler.RegulationTree(root)

        reg_tree.replace_node_title('205-2-a', change)
        changed_node = find(reg_tree.tree, '205-2-a')
        self.assertEqual(changed_node.title, 'new title')

    def test_get_subparts(self):
        nsa = Node(
            'nsa',
            label=['205', 'Subpart', 'A'], node_type=Node.SUBPART)

        nsb = Node('nsb',
                   label=['205', 'Subpart', 'B'],
                   node_type=Node.SUBPART)

        nappa = Node('nappa',
                     label=['205', 'Appendix', 'C'],
                     node_type=Node.APPENDIX)

        root = Node('', label=['205'])
        root.children = [nsa, nsb, nappa]

        reg_tree = compiler.RegulationTree(root)
        subparts = reg_tree.get_subparts()
        labels = [s.label_id() for s in subparts]

        self.assertEqual(labels, ['205-Subpart-A', '205-Subpart-B'])

    def tree_with_subparts(self):
        nsa = Node('nsa',
                   label=['205', 'Subpart', 'A'],
                   node_type=Node.SUBPART)

        nsb = Node('nsb',
                   label=['205', 'Subpart', 'B'],
                   node_type=Node.SUBPART)

        nappa = Node('nappa',
                     label=['205', 'Appendix', 'C'],
                     node_type=Node.APPENDIX)

        root = Node('', label=['205'])
        root.children = [nsa, nsb, nappa]
        return root

    def test_create_new_subpart(self):
        root = self.tree_with_subparts()

        reg_tree = compiler.RegulationTree(root)
        reg_tree.create_new_subpart(['205', 'Subpart', 'C'])

        subparts = reg_tree.get_subparts()
        labels = [s.label_id() for s in subparts]

        self.assertEqual(
            labels, ['205-Subpart-A', '205-Subpart-B', '205-Subpart-C'])

    def test_get_subpart_for_node(self):
        root = self.tree_with_subparts()
        n1 = Node('n1', label=['205', '1'])
        nsb = find(root, '205-Subpart-B')
        nsb.children = [n1]

        reg_tree = compiler.RegulationTree(root)
        subpart = reg_tree.get_subpart_for_node('205-1')

        self.assertEqual(subpart.label_id(), '205-Subpart-B')

    def test_compile_reg_put_replace_whole_tree(self):
        root = self.tree_with_paragraphs()

        change2a = {
            'action': 'PUT',
            'node': {
                'text': 'new text',
                'label': ['205', '2', 'a'],
                'node_type': 'regtext'}}

        change2a1 = {
            'action': 'PUT',
            'node': {
                'text': '2a1 text',
                'label': ['205', '2', 'a', '1'],
                'node_type': 'regtext'}}

        notice_changes = {
            '205-2-a-1': [change2a1],
            '205-2-a': [change2a]
        }

        reg = compiler.compile_regulation(root, notice_changes)

        added_node = find(reg, '205-2-a')
        self.assertEqual(added_node.text, 'new text')

        deeper = find(reg, '205-2-a-1')
        self.assertEqual(deeper.text, '2a1 text')

    def test_compile_reg_put_text_only(self):
        root = self.tree_with_paragraphs()
        change2a = {
            'action': 'PUT',
            'field': '[text]',
            'node': {
                'text': 'new text',
                'label': ['205', '2', 'a'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2-a': [change2a]}
        reg = compiler.compile_regulation(root, notice_changes)

        changed_node = find(reg, '205-2-a')
        self.assertEqual(changed_node.text, 'new text')

    def test_compile_reg_post_no_subpart(self):
        root = self.tree_with_paragraphs()
        change2a1 = {
            'action': 'POST',
            'node': {
                'text': '2a1 text',
                'label': ['205', '2', 'a', '1'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2-a-1': [change2a1]}
        reg = compiler.compile_regulation(root, notice_changes)
        added_node = find(reg, '205-2-a-1')
        self.assertNotEqual(None, added_node)
        self.assertEqual(added_node.text, '2a1 text')

    def test_compile_add_to_subpart(self):
        root = self.tree_with_subparts()

        change = {
            'action': 'POST',
            'subpart': ['205', 'Subpart', 'B'],
            'node': {
                'text': '2 text',
                'label': ['205', '2'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2': [change]}
        reg = compiler.compile_regulation(root, notice_changes)
        added_node = find(reg, '205-2')
        self.assertNotEqual(None, added_node)
        self.assertEqual(added_node.text, '2 text')

    def test_compile_designate(self):
        root = self.tree_with_subparts()
        change = {
            'action': 'POST',
            'subpart': ['205', 'Subpart', 'B'],
            'node': {
                'text': '2 text',
                'label': ['205', '2'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2': [change]}
        reg = compiler.compile_regulation(root, notice_changes)

        subpart_b = find(reg, '205-Subpart-B')
        self.assertEqual(len(subpart_b.children), 1)

        subpart_a = find(reg, '205-Subpart-A')
        self.assertEqual(len(subpart_a.children), 0)

        change = {
            'action': 'DESIGNATE',
            'destination': ['205', 'Subpart', 'A']}

        notice_changes = {'205-2': [change]}

        new_reg = compiler.compile_regulation(reg, notice_changes)

        subpart_b = find(new_reg, '205-Subpart-B')
        self.assertEqual(len(subpart_b.children), 0)

        subpart_a = find(new_reg, '205-Subpart-A')
        self.assertEqual(len(subpart_a.children), 1)

    def test_delete(self):
        root = self.tree_with_paragraphs()
        reg_tree = compiler.RegulationTree(root)

        self.assertNotEqual(None, find(reg_tree.tree, '205-2-a'))
        reg_tree.delete('205-2-a')
        self.assertEqual(None, find(reg_tree.tree, '205-2-a'))

    def test_get_parent(self):
        root = self.tree_with_paragraphs()
        reg_tree = compiler.RegulationTree(root)

        node = find(reg_tree.tree, '205-2-a')
        parent = reg_tree.get_parent(node)
        self.assertEqual(parent.label, ['205', '2'])

    def test_create_empty(self):
        root = self.tree_with_paragraphs()
        reg_tree = compiler.RegulationTree(root)
        reg_tree.create_empty_node('205-4-a')

        node = find(reg_tree.tree, '205-4-a')
        self.assertNotEqual(None, node)
        self.assertEqual(node.label, ['205', '4', 'a'])

    def test_add_node_no_parent(self):
        root = self.tree_with_paragraphs()
        reg_tree = compiler.RegulationTree(root)

        node = Node('', label=['205', '3', 'a'], node_type=Node.REGTEXT)
        reg_tree.add_node(node)

        parent = find(reg_tree.tree, '205-3')
        self.assertNotEqual(None, parent)
        self.assertEqual(parent.text, '')
<|endoftext|>


<commit_before># -------------------------------------------------------------------------------
# Copyright IBM Corp. 2016
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------------
import os
import sys
import nbformat
import re
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert.preprocessors.execute import CellExecutionError
import logging
from ipykernel.kernelspec import KernelSpecManager, write_kernel_spec
from jupyter_client.manager import KernelManager
from jupyter_client.kernelspec import NoSuchKernel
import shutil
from difflib import SequenceMatcher
from optparse import OptionParser

__TEST_KERNEL_NAME__ = "PixiedustTravisTest"

if "debug" in os.environ:
    logging.basicConfig(level=logging.DEBUG)

if "compareratio" in os.environ:
    try:
        compareRatio = float(os.environ["compareratio"])
    except:
        compareRatio = 0.98
else:
    compareRatio = 0.98

def createKernelSpecIfNeeded(kernelName, useSpark):
    try:
        km = KernelManager(kernel_name=kernelName)
        km.kernel_spec
        return None
    except NoSuchKernel:
        print("Creating new Kernel {} target: {}".format(kernelName, useSpark) )
        if useSpark:
            sparkHome = os.environ["SPARK_HOME"]
            overrides={
                "argv": [
                    "python",
                    "-m",
                    "ipykernel",
                    "-f",
                    "{connection_file}"
                ],
                "env": {
                    "SCALA_HOME": "{0}".format(os.environ["SCALA_HOME"]),
                    "SPARK_HOME": "{0}".format(sparkHome),
                    "PYTHONPATH": "{0}/python/:{0}/python/lib/py4j-0.9-src.zip".format(sparkHome),
                    "PYTHONSTARTUP": "{0}/python/pyspark/shell.py".format(sparkHome),
                    "PYSPARK_SUBMIT_ARGS": "--driver-class-path {0}/data/libs/* --master local[10] pyspark-shell".format(os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~'))),
                    "SPARK_DRIVER_MEMORY":"10G",
                    "SPARK_LOCAL_IP":"127.0.0.1"
                }
            }
        else:
            overrides={
                "argv": [
                    "python",
                    "-m",
                    "ipykernel",
                    "-f",
                    "{connection_file}"
                ],
                "env": {
                    "PIXIEDUST_HOME": os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~'))
                }
            }

        path = write_kernel_spec(overrides=overrides)
        dest = KernelSpecManager().install_kernel_spec(path, kernel_name=kernelName, user=True)
        # cleanup afterward
        shutil.rmtree(path)
        return dest

class RestartKernelException(Exception):
    pass

class CompareOutputException(Exception):
    pass

class PixieDustTestExecutePreprocessor( ExecutePreprocessor ):
    def skipCell(self, cell):
        m = re.search("#TARGET=(.*)", cell.source, re.IGNORECASE)
        if m is not None:
            return (m.group(1).lower() == "spark") != self.useSpark or m.group(1) == "NO_RUN"
        return False

    def preprocess_cell(self, cell, resources, cell_index):
        if self.skipCell(cell):
            return cell, resources
        beforeOutputs = cell.outputs
        skipCompareOutput = "#SKIP_COMPARE_OUTPUT" in cell.source
        pixiedustDisplay = "display(" in cell.source
        try:
            logging.debug("Processing cell:\r\n{0}".format(cell.source))
            cell, resources = super(PixieDustTestExecutePreprocessor, self).preprocess_cell(cell, resources, cell_index)
            for output in cell.outputs:
                if "text" in output and "restart kernel" in output["text"].lower():
                    raise RestartKernelException()
            if not skipCompareOutput:
                if not pixiedustDisplay:
                    self.compareOutputs(beforeOutputs, cell.outputs, cell.source)
                else:
                    self.compareOutputs(beforeOutputs, cell.outputs, cell.source, True)
            return cell, resources
        except Exception as e:
            if isinstance(e, CellExecutionError):
                for output in cell.outputs:
                    logging.warn("Output Error is {}".format(output))
            cell.source="%pixiedustLog -l debug"
            cell, resources = super(PixieDustTestExecutePreprocessor, self).preprocess_cell(cell, resources, cell_index)
            logging.error("An error occurred executing the cell:\r\n{0}".format(cell.source))
            logging.info("Fetching pixiedust log...")
            if len(cell.outputs) > 0 and "text" in cell.outputs[0]:
                logging.warn(cell.outputs[0].text)
            else:
                logging.warn("Pixiedust Log is empty")
            raise

    def compareOutputs(self, beforeOutputs, afterOutputs, cellsource, useRatio=False):
        #return a measure of the sequences similarity as a float in the range [0, 1]
        seqmatcher = SequenceMatcher(None, '', '')

        #filter transient data from output
        def filterOutput(output):
            return "data" in output and "application/javascript" in output["data"]

        afterOutputs = [output for output in afterOutputs if not filterOutput(output)]

        if ( len(beforeOutputs) != len(afterOutputs)):
            logging.debug("Outputs do not match \r\nExpected:\r\n {0} \r\n\r\nActual:\r\n {1}".format(beforeOutputs, afterOutputs))
            raise CompareOutputException("Outputs do not match for cell:\r\n{0}".format(cellsource))

        for beforeOutput, afterOutput in list(zip(beforeOutputs,afterOutputs)):
            if "output_type" in beforeOutput and beforeOutput["output_type"] != "execute_result":
                if len(beforeOutput) != len(afterOutput):
                    logging.debug("output_type does not match \r\nExpected:\r\n {0} \r\n\r\nActual:\r\n {1}".format(beforeOutputs, afterOutputs))
                    raise CompareOutputException("output_type does not match for cell:\r\n{0}".format(cellsource))
                skip = ["execution_count"]
                expected = 0
                actual = 0
                longest = 0
                for key in beforeOutput:
                    if key not in skip:
                        before = str(beforeOutput[key])
                        after = str(afterOutput[key])
                        if useRatio:
                            seqmatcher.set_seqs(before, after)
                            expected = len(before)
                            actual = len(after)
                            ratio = seqmatcher.quick_ratio()
                            logging.info("expected_length: {0}, actual_length: {1}, sequence_ratio: {2}, compare_ratio: {3}".format(expected, actual, ratio, compareRatio))
                            if ratio < compareRatio:
                                logging.debug("output_type ({0}) below ratio ({1}) threshold \r\nExpected:\r\n {2} \r\n\r\nActual:\r\n {3}".format(key, ratio, before, after))
                                raise CompareOutputException("output_type ({0}) below ratio ({1}) threshold for cell:\r\n{2}".format(key, ratio, cellsource))
                        elif beforeOutput[key] != afterOutput[key]:
                            logging.debug("output_type ({0}) does not match \r\nExpected:\r\n {1} \r\n\r\nActual:\r\n {2}".format(key, before, after))
                            raise CompareOutputException("output_type ({0}) does not match for cell:\r\n{1}".format(key, cellsource))

def runNotebook(path, useSpark):
    ep = PixieDustTestExecutePreprocessor(timeout=3600, kernel_name= __TEST_KERNEL_NAME__)
    ep.useSpark = useSpark
    nb=nbformat.read(path, as_version=4)
    #set the kernel name to test
    nb.metadata.kernelspec.name=__TEST_KERNEL_NAME__
    if "pixiedust_test" in nb.metadata:
        testMeta = nb.metadata["pixiedust_test"]
        targetKernel = testMeta.get("target")
        if targetKernel is not None and (targetKernel.lower() == "spark") != useSpark:
            logging.warn("Skipping processing of notebook {0} based on pixiedust_test metadata".format(path))
            return
        skipVersions = testMeta.get("skipPython",[])
        for skipVersion in skipVersions:
            skipVersion = int(float(skipVersion))
            if skipVersion == sys.version_info.major:
                logging.warn("Skipping processing of notebook {0} based on pixiedust_test metadata".format(path))
                return
    try:
        ep.preprocess(nb, {'metadata':{'path': os.path.dirname(path)}})
    finally:
        dir = os.environ.get("PIXIEDUST_TEST_OUTPUT", os.path.expanduser('~') + "/pixiedust") + "/tests"
        if not os.path.exists(dir):
            os.makedirs( dir )
        nbformat.write(nb, dir + "/" + os.path.basename(path) + ".out")

if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option("-t", "--target", dest="target", default="spark", help="target kernel: spark or plain")

    (options, args) = parser.parse_args()
    useSpark = options.target == "spark"
    print("Starting Test Suite with target: {}".format(options.target) )
    if "PIXIEDUST_HOME" in os.environ:
        print("Using PIXIEDUST_HOME: ", os.environ["PIXIEDUST_HOME"])
    kernelPath = createKernelSpecIfNeeded(__TEST_KERNEL_NAME__, useSpark)
    try:
        inputDir = os.environ.get("PIXIEDUST_TEST_INPUT", './tests')
        for path in os.listdir( inputDir ):
            if path.endswith(".ipynb"):
                print("Processing notebook {0}".format(path))
                processed = False
                count = 0
                while (not processed and count < 10 ):
                    try:
                        processed = True
                        count += 1
                        runNotebook(inputDir + "/" + path, useSpark)
                    except RestartKernelException:
                        logging.warn("Restarting kernel...")
                        processed = False
                    except:
                        logging.warn("Fatal Error in Notebook {}.".format(path))
                        raise
                print("Finished processing notebook {0}".format(path))
    finally:
        if kernelPath:
            shutil.rmtree(kernelPath)

        #clean up PIXIEDUST_HOME if provided
        def rmTree(path):
            if os.path.exists(path):
                shutil.rmtree(path)
        def rmFile(path):
            if os.path.exists(path):
                os.remove(path)
        if "PIXIEDUST_HOME" in os.environ:
            rmTree(os.environ["PIXIEDUST_HOME"] + "/data/libs")
            rmTree(os.environ["PIXIEDUST_HOME"] + "/pixiedust")
            rmFile(os.environ["PIXIEDUST_HOME"] + "/pixiedust.db")<commit_msg>travis ci error<commit_after># -------------------------------------------------------------------------------
# Copyright IBM Corp. 2016
#
# Licensed under the Apache License, Version 2.0 (the 'License');
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------------
import os
import sys
import nbformat
import re
import glob
from nbconvert.preprocessors import ExecutePreprocessor
from nbconvert.preprocessors.execute import CellExecutionError
import logging
from ipykernel.kernelspec import KernelSpecManager, write_kernel_spec
from jupyter_client.manager import KernelManager
from jupyter_client.kernelspec import NoSuchKernel
import shutil
from difflib import SequenceMatcher
from optparse import OptionParser

__TEST_KERNEL_NAME__ = "PixiedustTravisTest"

if "debug" in os.environ:
    logging.basicConfig(level=logging.DEBUG)

if "compareratio" in os.environ:
    try:
        compareRatio = float(os.environ["compareratio"])
    except:
        compareRatio = 0.98
else:
    compareRatio = 0.98

def getPyJ(sparkHome):
    pyjs = glob.glob('{0}/python/lib/py4j-*.zip'.format(sparkHome))
    if not pyjs:
        return '{0}/python/lib/py4j-0.9-src.zip'.format(sparkHome)
    else:
        return pyjs[0]

def createKernelSpecIfNeeded(kernelName, useSpark):
    try:
        km = KernelManager(kernel_name=kernelName)
        km.kernel_spec
        return None
    except NoSuchKernel:
        print("Creating new Kernel {} target: {}".format(kernelName, useSpark) )
        if useSpark:
            sparkHome = os.environ["SPARK_HOME"]
            overrides={
                "argv": [
                    "python",
                    "-m",
                    "ipykernel",
                    "-f",
                    "{connection_file}"
                ],
                "env": {
                    "SCALA_HOME": "{0}".format(os.environ["SCALA_HOME"]),
                    "SPARK_HOME": "{0}".format(sparkHome),
                    "PYTHONPATH": "{0}/python/:{1}".format(sparkHome, getPyJ(sparkHome)),
                    "PYTHONSTARTUP": "{0}/python/pyspark/shell.py".format(sparkHome),
                    "PYSPARK_SUBMIT_ARGS": "--driver-class-path {0}/data/libs/* --master local[10] pyspark-shell".format(os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~'))),
                    "SPARK_DRIVER_MEMORY":"10G",
                    "SPARK_LOCAL_IP":"127.0.0.1"
                }
            }
        else:
            overrides={
                "argv": [
                    "python",
                    "-m",
                    "ipykernel",
                    "-f",
                    "{connection_file}"
                ],
                "env": {
                    "PIXIEDUST_HOME": os.environ.get("PIXIEDUST_HOME", os.path.expanduser('~'))
                }
            }

        path = write_kernel_spec(overrides=overrides)
        dest = KernelSpecManager().install_kernel_spec(path, kernel_name=kernelName, user=True)
        # cleanup afterward
        shutil.rmtree(path)
        return dest

class RestartKernelException(Exception):
    pass

class CompareOutputException(Exception):
    pass

class PixieDustTestExecutePreprocessor( ExecutePreprocessor ):
    def skipCell(self, cell):
        m = re.search("#TARGET=(.*)", cell.source, re.IGNORECASE)
        if m is not None:
            return (m.group(1).lower() == "spark") != self.useSpark or m.group(1) == "NO_RUN"
        return False

    def preprocess_cell(self, cell, resources, cell_index):
        if self.skipCell(cell):
            return cell, resources
        beforeOutputs = cell.outputs
        skipCompareOutput = "#SKIP_COMPARE_OUTPUT" in cell.source
        pixiedustDisplay = "display(" in cell.source
        try:
            logging.debug("Processing cell:\r\n{0}".format(cell.source))
            cell, resources = super(PixieDustTestExecutePreprocessor, self).preprocess_cell(cell, resources, cell_index)
            for output in cell.outputs:
                if "text" in output and "restart kernel" in output["text"].lower():
                    raise RestartKernelException()
            if not skipCompareOutput:
                if not pixiedustDisplay:
                    self.compareOutputs(beforeOutputs, cell.outputs, cell.source)
                else:
                    self.compareOutputs(beforeOutputs, cell.outputs, cell.source, True)
            return cell, resources
        except Exception as e:
            if isinstance(e, CellExecutionError):
                for output in cell.outputs:
                    logging.warn("Output Error is {}".format(output))
            cell.source="%pixiedustLog -l debug"
            cell, resources = super(PixieDustTestExecutePreprocessor, self).preprocess_cell(cell, resources, cell_index)
            logging.error("An error occurred executing the cell:\r\n{0}".format(cell.source))
            logging.info("Fetching pixiedust log...")
            if len(cell.outputs) > 0 and "text" in cell.outputs[0]:
                logging.warn(cell.outputs[0].text)
            else:
                logging.warn("Pixiedust Log is empty")
            raise

    def compareOutputs(self, beforeOutputs, afterOutputs, cellsource, useRatio=False):
        #return a measure of the sequences similarity as a float in the range [0, 1]
        seqmatcher = SequenceMatcher(None, '', '')

        #filter transient data from output
        def filterOutput(output):
            return "data" in output and "application/javascript" in output["data"]

        afterOutputs = [output for output in afterOutputs if not filterOutput(output)]

        if ( len(beforeOutputs) != len(afterOutputs)):
            logging.debug("Outputs do not match \r\nExpected:\r\n {0} \r\n\r\nActual:\r\n {1}".format(beforeOutputs, afterOutputs))
            raise CompareOutputException("Outputs do not match for cell:\r\n{0}".format(cellsource))

        for beforeOutput, afterOutput in list(zip(beforeOutputs,afterOutputs)):
            if "output_type" in beforeOutput and beforeOutput["output_type"] != "execute_result":
                if len(beforeOutput) != len(afterOutput):
                    logging.debug("output_type does not match \r\nExpected:\r\n {0} \r\n\r\nActual:\r\n {1}".format(beforeOutputs, afterOutputs))
                    raise CompareOutputException("output_type does not match for cell:\r\n{0}".format(cellsource))
                skip = ["execution_count"]
                expected = 0
                actual = 0
                longest = 0
                for key in beforeOutput:
                    if key not in skip:
                        before = str(beforeOutput[key])
                        after = str(afterOutput[key])
                        if useRatio:
                            seqmatcher.set_seqs(before, after)
                            expected = len(before)
                            actual = len(after)
                            ratio = seqmatcher.quick_ratio()
                            logging.info("expected_length: {0}, actual_length: {1}, sequence_ratio: {2}, compare_ratio: {3}".format(expected, actual, ratio, compareRatio))
                            if ratio < compareRatio:
                                logging.debug("output_type ({0}) below ratio ({1}) threshold \r\nExpected:\r\n {2} \r\n\r\nActual:\r\n {3}".format(key, ratio, before, after))
                                raise CompareOutputException("output_type ({0}) below ratio ({1}) threshold for cell:\r\n{2}".format(key, ratio, cellsource))
                        elif beforeOutput[key] != afterOutput[key]:
                            logging.debug("output_type ({0}) does not match \r\nExpected:\r\n {1} \r\n\r\nActual:\r\n {2}".format(key, before, after))
                            raise CompareOutputException("output_type ({0}) does not match for cell:\r\n{1}".format(key, cellsource))

def runNotebook(path, useSpark):
    ep = PixieDustTestExecutePreprocessor(timeout=3600, kernel_name= __TEST_KERNEL_NAME__)
    ep.useSpark = useSpark
    nb=nbformat.read(path, as_version=4)
    #set the kernel name to test
    nb.metadata.kernelspec.name=__TEST_KERNEL_NAME__
    if "pixiedust_test" in nb.metadata:
        testMeta = nb.metadata["pixiedust_test"]
        targetKernel = testMeta.get("target")
        if targetKernel is not None and (targetKernel.lower() == "spark") != useSpark:
            logging.warn("Skipping processing of notebook {0} based on pixiedust_test metadata".format(path))
            return
        skipVersions = testMeta.get("skipPython",[])
        for skipVersion in skipVersions:
            skipVersion = int(float(skipVersion))
            if skipVersion == sys.version_info.major:
                logging.warn("Skipping processing of notebook {0} based on pixiedust_test metadata".format(path))
                return
    try:
        ep.preprocess(nb, {'metadata':{'path': os.path.dirname(path)}})
    finally:
        dir = os.environ.get("PIXIEDUST_TEST_OUTPUT", os.path.expanduser('~') + "/pixiedust") + "/tests"
        if not os.path.exists(dir):
            os.makedirs( dir )
        nbformat.write(nb, dir + "/" + os.path.basename(path) + ".out")

if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option("-t", "--target", dest="target", default="spark", help="target kernel: spark or plain")

    (options, args) = parser.parse_args()
    useSpark = options.target == "spark"
    print("Starting Test Suite with target: {}".format(options.target) )
    if "PIXIEDUST_HOME" in os.environ:
        print("Using PIXIEDUST_HOME: ", os.environ["PIXIEDUST_HOME"])
    kernelPath = createKernelSpecIfNeeded(__TEST_KERNEL_NAME__, useSpark)
    try:
        inputDir = os.environ.get("PIXIEDUST_TEST_INPUT", './tests')
        for path in os.listdir( inputDir ):
            if path.endswith(".ipynb"):
                print("Processing notebook {0}".format(path))
                processed = False
                count = 0
                while (not processed and count < 10 ):
                    try:
                        processed = True
                        count += 1
                        runNotebook(inputDir + "/" + path, useSpark)
                    except RestartKernelException:
                        logging.warn("Restarting kernel...")
                        processed = False
                    except:
                        logging.warn("Fatal Error in Notebook {}.".format(path))
                        raise
                print("Finished processing notebook {0}".format(path))
    finally:
        if kernelPath:
            shutil.rmtree(kernelPath)

        #clean up PIXIEDUST_HOME if provided
        def rmTree(path):
            if os.path.exists(path):
                shutil.rmtree(path)
        def rmFile(path):
            if os.path.exists(path):
                os.remove(path)
        if "PIXIEDUST_HOME" in os.environ:
            rmTree(os.environ["PIXIEDUST_HOME"] + "/data/libs")
            rmTree(os.environ["PIXIEDUST_HOME"] + "/pixiedust")
            rmFile(os.environ["PIXIEDUST_HOME"] + "/pixiedust.db")<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Created on 09 Oct 2020
@author: Jade Page (jade.page@southcoastscience.com)

DESCRIPTION
The aws_identity script allows the user to change the identity of an already-configured greengrass install,
in our use case it is to change the greengrass identity of a device which was setup using a cloned
base image without having to reinstall the greengrass software

The script could also be used to setup a "blank" greengrass install, which does not already have an identity, but
does already have the greengrass software.

If no group name is provided, the host name will be read from the device to generate it.
If no core name is provided, the host name will be read from the device to generate it.
If the set flag is not provided, the current identity will be read from the persistent file.

SYNOPSIS
aws_identity.py [-s] [-g GROUP_NAME] [-c CORE_NAME] [-k] [-v]

EXAMPLES
./aws_identity.py -s -g scs-test-003-group -c scs-test-003-core -v

FILES
A persistent file is placed in the conf directory when the identity is set, so that it can be read again later.

SEE ALSO
scs_mfr/aws_deployment.py
scs_mfr/aws_group_setup.py

REFERENCES
Created with reference to amazon's own device setup script (URL may change if updated)
https://d1onfpft10uf5o.cloudfront.net/greengrass-device-setup/downloads/gg-device-setup-latest.sh
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/iot.html
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/greengrass.html

NOTES
ATS_ROOT_CA_RSA_2048_REMOTE_LOCATION in core.aws.greengrass aws_identity is a certificate provided by
amazon itself and may be subject to change e.g. via obsolescence - check here:
https://docs.aws.amazon.com/iot/latest/developerguide/server-authentication.html

root@scs-cube-116:/etc/systemd/system# cat ~scs/SCS/aws/plain_access_key.json | aws_identity -sk
Logger created
Core created
Cert created
Core attached
Traceback (most recent call last):
  File "/home/scs/SCS/scs_mfr/src/scs_mfr/aws_identity.py", line 98, in <module>
    aws_setup.setup_device()
  File "/home/scs/SCS/scs_core/src/scs_core/aws/greengrass/aws_identity.py", line 72, in setup_device
    self.create_thing()
  File "/home/scs/SCS/scs_core/src/scs_core/aws/greengrass/aws_identity.py", line 106, in create_thing
    policyDocument=json.dumps(policy_doc)
  File "/usr/lib/python3/dist-packages/botocore/client.py", line 357, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/lib/python3/dist-packages/botocore/client.py", line 661, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.errorfactory.ResourceAlreadyExistsException: An error occurred (ResourceAlreadyExistsException) when calling the CreatePolicy operation: Policy cannot be created - name already exists (name=scs-cube-116-core_basic_policy)
"""

import json
import os
import sys

from botocore.exceptions import NoCredentialsError

from scs_core.aws.client.access_key import AccessKey
from scs_core.aws.client.client import Client
from scs_core.aws.config.aws import AWS
from scs_core.aws.greengrass.aws_identity import AWSSetup

from scs_host.sys.host import Host

from scs_mfr.cmd.cmd_aws_identity import CmdAWSIdentity

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    key = None

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdAWSIdentity()

    if not cmd.is_valid():
        cmd.print_help(sys.stderr)
        exit(2)

    if cmd.verbose:
        print("aws_identity: %s" % cmd, file=sys.stderr)
        sys.stderr.flush()

    # ----------------------------------------------------------------------------------------------------------------
    # Check sudo

    if os.geteuid() != 0:
        print("aws_identity: you need to have root privileges to run this script.", file=sys.stderr)
        exit(1)

    # ----------------------------------------------------------------------------------------------------------------
    # resources

    try:
        key = AccessKey.from_stdin() if cmd.stdin else AccessKey.from_user()
    except ValueError:
        print("aws_identity: invalid key.", file=sys.stderr)
        exit(1)

    # ----------------------------------------------------------------------------------------------------------------
    # run...

    try:
        if cmd.setup:
            iot_client = Client.construct('iot', key)
            gg_client = Client.construct('greengrass', key)

            aws_setup = AWSSetup(iot_client, gg_client, AWS.core_name(), AWS.group_name())
            aws_setup.setup_device()
            aws_setup.save(Host)

        else:
            aws_setup = AWSSetup.load(Host)

            if aws_setup:
                json_file = aws_setup.as_json()
                print(json.dumps(json_file))
            else:
                print("aws_identity: no identity found.", file=sys.stderr)

    except KeyboardInterrupt:
        print(file=sys.stderr)

    except (EOFError, NoCredentialsError):
        print("aws_identity: credentials error.", file=sys.stderr)
        exit(1)
<commit_msg>Cleaning<commit_after>#!/usr/bin/env python3

"""
Created on 09 Oct 2020
@author: Jade Page (jade.page@southcoastscience.com)

DESCRIPTION
The aws_identity script allows the user to change the identity of an already-configured greengrass install,
in our use case it is to change the greengrass identity of a device which was setup using a cloned
base image without having to reinstall the greengrass software

The script could also be used to setup a "blank" greengrass install, which does not already have an identity, but
does already have the greengrass software.

If no group name is provided, the host name will be read from the device to generate it.
If no core name is provided, the host name will be read from the device to generate it.
If the set flag is not provided, the current identity will be read from the persistent file.

SYNOPSIS
aws_identity.py [-s] [-g GROUP_NAME] [-c CORE_NAME] [-k] [-v]

EXAMPLES
./aws_identity.py -s -g scs-test-003-group -c scs-test-003-core -v

FILES
A persistent file is placed in the conf directory when the identity is set, so that it can be read again later.

SEE ALSO
scs_mfr/aws_deployment.py
scs_mfr/aws_group_setup.py

REFERENCES
Created with reference to amazon's own device setup script (URL may change if updated)
https://d1onfpft10uf5o.cloudfront.net/greengrass-device-setup/downloads/gg-device-setup-latest.sh
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/iot.html
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/greengrass.html

NOTES
ATS_ROOT_CA_RSA_2048_REMOTE_LOCATION in core.aws.greengrass aws_identity is a certificate provided by
amazon itself and may be subject to change e.g. via obsolescence - check here:
https://docs.aws.amazon.com/iot/latest/developerguide/server-authentication.html
"""

import json
import os
import sys

from botocore.exceptions import NoCredentialsError

from scs_core.aws.client.access_key import AccessKey
from scs_core.aws.client.client import Client
from scs_core.aws.config.aws import AWS
from scs_core.aws.greengrass.aws_identity import AWSSetup

from scs_host.sys.host import Host

from scs_mfr.cmd.cmd_aws_identity import CmdAWSIdentity

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    key = None

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdAWSIdentity()

    if not cmd.is_valid():
        cmd.print_help(sys.stderr)
        exit(2)

    if cmd.verbose:
        print("aws_identity: %s" % cmd, file=sys.stderr)
        sys.stderr.flush()

    # ----------------------------------------------------------------------------------------------------------------
    # Check sudo

    if os.geteuid() != 0:
        print("aws_identity: you need to have root privileges to run this script.", file=sys.stderr)
        exit(1)

    # ----------------------------------------------------------------------------------------------------------------
    # resources

    try:
        key = AccessKey.from_stdin() if cmd.stdin else AccessKey.from_user()
    except ValueError:
        print("aws_identity: invalid key.", file=sys.stderr)
        exit(1)

    # ----------------------------------------------------------------------------------------------------------------
    # run...

    try:
        if cmd.setup:
            iot_client = Client.construct('iot', key)
            gg_client = Client.construct('greengrass', key)

            aws_setup = AWSSetup(iot_client, gg_client, AWS.core_name(), AWS.group_name())
            aws_setup.setup_device()
            aws_setup.save(Host)

        else:
            aws_setup = AWSSetup.load(Host)

            if aws_setup:
                json_file = aws_setup.as_json()
                print(json.dumps(json_file))
            else:
                print("aws_identity: no identity found.", file=sys.stderr)

    except KeyboardInterrupt:
        print(file=sys.stderr)

    except (EOFError, NoCredentialsError):
        print("aws_identity: credentials error.", file=sys.stderr)
        exit(1)
<|endoftext|>


<commit_before>import unittest
from lxml import etree
from saml import schema
from saml.schema import saml, samlp
from datetime import datetime, timedelta

class AssertionTest(unittest.TestCase):
    def setUp(self):
        self.a = saml.Assertion()
        self.a.subject = saml.Subject()
        self.a.subject.id = saml.NameID("leckey.ryan@gmail.com")
        self.a.subject.id.format = saml.NameID.Format.EMAIL
        self.a.subject.confirm = saml.SubjectConfirmation()
        self.a.subject.confirm.data = saml.SubjectConfirmationData()
        self.a.subject.confirm.data.address = "127.0.0.1"
        self.a.subject.confirm.data.in_response_to = "saml://crm"
        self.a.subject.confirm.data.not_before = datetime.utcnow()
        self.a.subject.confirm.data.not_on_or_after = datetime.utcnow()
        self.a.issuer = saml.Issuer("saml://concordus")
        self.a.statements = []
        self.a.statements.append(saml.AuthenticationStatement())
        self.a.statements[0].session_not_on_or_after = self.a.statements[0].instant + timedelta(minutes=2)
        self.a.statements[0].context = saml.AuthenticationContext()
        self.a.statements[0].context.reference = saml.AuthenticationContext.Reference.PREVIOUS_SESSION
        self.a.statements.append(saml.AuthenticationStatement())
        self.a.statements[1].session_not_on_or_after = self.a.statements[0].instant + timedelta(minutes=2)
        self.a.statements[1].context = saml.AuthenticationContext()
        self.a.statements[1].context.reference = saml.AuthenticationContext.Reference.PREVIOUS_SESSION
        self.a.statements.append(saml.AuthenticationStatement())
        self.a.statements[2].session_not_on_or_after = self.a.statements[0].instant + timedelta(minutes=2)
        self.a.statemeArtifactResponseTestnts[2].context = saml.AuthenticationContext()
        self.a.statements[2].context.reference = saml.AuthenticationContext.Reference.PREVIOUS_SESSION

    def runTest(self):
        #print self.a

        self.x = saml.Assertion.serialize(self.a)
        self.s = etree.tostring(self.x, pretty_print=True)
        print(self.s)
        self.e = etree.XML(self.s)
        self.c = saml.Assertion.deserialize(self.e)
        print('--------------------------')
        self.x2 = saml.Assertion.serialize(self.c)
        self.s2 = etree.tostring(self.x2, pretty_print=True)
        self.assertEquals(self.s, self.s2, "XML doesn't match")

        print(self.s2)

<commit_msg>Small fix.<commit_after>import unittest
from lxml import etree
from saml import schema
from saml.schema import saml, samlp
from datetime import datetime, timedelta

class AssertionTest(unittest.TestCase):
    def setUp(self):
        self.a = saml.Assertion()
        self.a.subject = saml.Subject()
        self.a.subject.id = saml.NameID("leckey.ryan@gmail.com")
        self.a.subject.id.format = saml.NameID.Format.EMAIL
        self.a.subject.confirm = saml.SubjectConfirmation()
        self.a.subject.confirm.data = saml.SubjectConfirmationData()
        self.a.subject.confirm.data.address = "127.0.0.1"
        self.a.subject.confirm.data.in_response_to = "saml://crm"
        self.a.subject.confirm.data.not_before = datetime.utcnow()
        self.a.subject.confirm.data.not_on_or_after = datetime.utcnow()
        self.a.issuer = saml.Issuer("saml://concordus")
        self.a.statements = []
        self.a.statements.append(saml.AuthenticationStatement())
        self.a.statements[0].session_not_on_or_after = self.a.statements[0].instant + timedelta(minutes=2)
        self.a.statements[0].context = saml.AuthenticationContext()
        self.a.statements[0].context.reference = saml.AuthenticationContext.Reference.PREVIOUS_SESSION
        self.a.statements.append(saml.AuthenticationStatement())
        self.a.statements[1].session_not_on_or_after = self.a.statements[0].instant + timedelta(minutes=2)
        self.a.statements[1].context = saml.AuthenticationContext()
        self.a.statements[1].context.reference = saml.AuthenticationContext.Reference.PREVIOUS_SESSION
        self.a.statements.append(saml.AuthenticationStatement())
        self.a.statements[2].session_not_on_or_after = self.a.statements[0].instant + timedelta(minutes=2)
        self.a.statements[2].context = saml.AuthenticationContext()
        self.a.statements[2].context.reference = saml.AuthenticationContext.Reference.PREVIOUS_SESSION

    def runTest(self):
        #print self.a

        self.x = saml.Assertion.serialize(self.a)
        self.s = etree.tostring(self.x, pretty_print=True)
        print(self.s)
        self.e = etree.XML(self.s)
        self.c = saml.Assertion.deserialize(self.e)
        print('--------------------------')
        self.x2 = saml.Assertion.serialize(self.c)
        self.s2 = etree.tostring(self.x2, pretty_print=True)
        self.assertEquals(self.s, self.s2, "XML doesn't match")

        print(self.s2)

<|endoftext|>


<commit_before>"""
sentry.models.tagkey
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from django.core.urlresolvers import reverse
from django.db import models

from sentry.constants import MAX_TAG_KEY_LENGTH, TAG_LABELS
from sentry.db.models import Model, BoundedPositiveIntegerField, sane_repr
from sentry.manager import TagKeyManager
from sentry.utils.http import absolute_uri

class TagKey(Model):
    """
    Stores references to available filters keys.
    """
    project = models.ForeignKey('sentry.Project')
    key = models.CharField(max_length=MAX_TAG_KEY_LENGTH)
    values_seen = BoundedPositiveIntegerField(default=0)
    label = models.CharField(max_length=64, null=True)

    objects = TagKeyManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_filterkey'
        unique_together = (('project', 'key'),)

    __repr__ = sane_repr('project_id', 'key')

    def get_label(self):
        return self.label \
            or TAG_LABELS.get(self.key) \
            or self.key.replace('_', ' ').title()

    def get_absolute_url(self):
        # HACK(dcramer): quick and dirty way to support code/users
        if self.key == 'sentry:user':
            url_name = 'sentry-users'
        elif self.key == 'sentry:filename':
            url_name = 'sentry-explore-code'
        elif self.key == 'sentry:function':
            url_name = 'sentry-explore-code-by-function'
        else:
            url_name = 'sentry-explore-tag'
            return absolute_uri(reverse(url_name, args=[
                self.project.team.slug, self.project.slug, self.key]))

        return absolute_uri(reverse(url_name, args=[
            self.project.team.slug, self.project.slug]))
<commit_msg>Move absolute_url lookups to a dict<commit_after>"""
sentry.models.tagkey
~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from django.core.urlresolvers import reverse
from django.db import models

from sentry.constants import MAX_TAG_KEY_LENGTH, TAG_LABELS
from sentry.db.models import Model, BoundedPositiveIntegerField, sane_repr
from sentry.manager import TagKeyManager
from sentry.utils.http import absolute_uri

class TagKey(Model):
    """
    Stores references to available filters keys.
    """
    DEFAULT_URL_NAME = 'sentry-explore-tag'
    URL_NAMES = {
        'sentry:user': 'sentry-users',
        'sentry:filenames': 'sentry-explore-code',
        'sentry:function': 'sentry-explore-code-by-function',
    }

    project = models.ForeignKey('sentry.Project')
    key = models.CharField(max_length=MAX_TAG_KEY_LENGTH)
    values_seen = BoundedPositiveIntegerField(default=0)
    label = models.CharField(max_length=64, null=True)

    objects = TagKeyManager()

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_filterkey'
        unique_together = (('project', 'key'),)

    __repr__ = sane_repr('project_id', 'key')

    def get_label(self):
        return self.label \
            or TAG_LABELS.get(self.key) \
            or self.key.replace('_', ' ').title()

    def get_absolute_url(self):
        # HACK(dcramer): quick and dirty way to support code/users
        try:
            url_name = self.URL_NAMES[self.key]
        except KeyError:
            url_name = self.DEFAULT_URL_NAME
            return absolute_uri(reverse(url_name, args=[
                self.project.team.slug, self.project.slug, self.key]))

        return absolute_uri(reverse(url_name, args=[
            self.project.team.slug, self.project.slug]))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#   Copyright [2012] [Patrick Ancillotti]
#   Copyright [2012] [Jason Kölker]
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

from jsonklog.formatters import JSONFormatter

class RequireJSONFormatter(object):

    """Mixin class to require a Handler be configured with a JSONFormmater """

    def setFormatter(self, fmt):

        if not isinstance(fmt, JSONFormatter):
            raise TypeError("%s requires a JSONFormatter" %
                            self.__class__.__name__)
        self.formatter = fmt
<commit_msg>Some formatting changes<commit_after># -*- coding: utf-8 -*-
#
#   Copyright [2012] [Patrick Ancillotti]
#   Copyright [2012] [Jason Kölker]
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

from jsonklog.formatters import JSONFormatter

class RequireJSONFormatter(object):

    """ Mixin class to require a Handler be configured with a JSONFormmater """

    def setFormatter(self, fmt):

        if not isinstance(fmt, JSONFormatter):
            raise TypeError("%s requires a JSONFormatter" %
                            self.__class__.__name__)

        self.formatter = fmt
<|endoftext|>


<commit_before>import os

from django.http import HttpRequest
from django.core.urlresolvers import reverse
from django.conf import settings

from .base import BoardAppTest
from board.views import new_post, post_list, edit_post
from board.models import Post, Board, Comment, EditedPostHistory
from board.forms import PostForm

class CreatePostPageTest(BoardAppTest):
    def test_new_post_page_returns_correct_html(self):
        response = self.client.get(reverse('board:new_post', args=[self.default_board.slug]))

        response_decoded = self.remove_csrf(response.content.decode())
        self.assertIn('settings_id_fields', response_decoded)

    def test_new_post_can_save_a_POST_request(self):
        response = self.client.post(reverse('board:new_post', args=[self.default_board.slug]), {
            'post_title_text': 'NEW POST TITLE',
            'fields': 'NEW POST CONTENT',
        })

        self.assertEqual(Post.objects.count(), 1)
        first_new_post = Post.objects.first()
        self.assertEqual(first_new_post.title, 'NEW POST TITLE')

    def test_new_post_page_redirects_after_POST(self):
        response = self.client.post(reverse('board:new_post', args=[self.default_board.slug]), {
            'post_title_text': 'NEW POST TITLE',
            'fields': 'NEW POST CONTENT',
        })

        self.assertRedirects(response, reverse('board:post_list', args=[self.default_board.slug]))

    def test_create_post_page_only_saves_items_when_necessary(self):
        request = HttpRequest()
        new_post(request, self.default_board.slug)
        self.assertEqual(Post.objects.count(), 0)

class PostListTest(BoardAppTest):
    def get_response_from_post_list_search_request(self, search_flag, query):
        response = self.client.get(reverse('board:post_list', args=[self.default_board.slug]), data={
            'search_flag': search_flag,
            'query': query
        })
        return response

    def test_post_list_page_displays_all_list_titles(self):
        Post.objects.create(board=self.default_board, title='turtle1', content='slow')
        Post.objects.create(board=self.default_board, title='turtle2', content='slowslow')

        request = HttpRequest()
        response = post_list(request, self.default_board.slug)

        self.assertIn('turtle1', response.content.decode())
        self.assertIn('turtle2', response.content.decode())

    def test_post_list_page_displays_searched_post(self):
        repeat = 3
        for i in range(repeat):
            Post.objects.create(
                board=self.default_board,
                title='Hi, ' + str(i),
                content='content ' + str(i)
            )

        response = self.get_response_from_post_list_search_request('TITLE', 'Hi, 1')
        self.assertContains(response, 'Hi, 1')
        self.assertNotContains(response, 'Hi, 0')

        response = self.get_response_from_post_list_search_request('CONTENT', 'content 1')
        self.assertContains(response, 'Hi, 1')
        self.assertNotContains(response, 'Hi, 2')

        response = self.get_response_from_post_list_search_request('BOTH', 'Hi, 1')
        self.assertContains(response, 'Hi, 1')

        response = self.get_response_from_post_list_search_request('BOTH', 'content 2')
        self.assertContains(response, 'Hi, 2')

    def test_post_list_page_displays_all_posts_when_search_flag_is_invalid(self):
        repeat = 3
        for i in range(repeat):
            Post.objects.create(
                board=self.default_board,
                title='Hi, ' + str(i),
                content='content ' + str(i)
            )

        response = self.get_response_from_post_list_search_request('INVALID_FLAG', 'any query')
        self.assertContains(response, repeat)

class PostListViewTest(BoardAppTest):
    def test_use_post_list_template(self):
        response = self.client.get(reverse('board:post_list', args=[self.default_board.slug]))
        self.assertTemplateUsed(response, 'post_list.html')

    def test_use_pagination_template(self):
        response = self.client.get(reverse('board:post_list', args=[self.default_board.slug]))
        self.assertTemplateUsed(response, 'pagination.html')

    def test_view_default_page_list_when_board_open(self):
        response = self.client.get(reverse('board:post_list', args=[self.default_board.slug]))
        self.assertContains(response, '
'+str(1)+'
')

class DeletePostTest(BoardAppTest):
    def test_delete_only_post_selected_to_delete(self):
        delete_post = Post.objects.create(board=self.default_board, title='delete post', content='content')
        other_post = Post.objects.create(board=self.default_board, title='other post', content='content')

        self.assertEqual(delete_post.is_deleted, False)
        self.assertEqual(other_post.is_deleted, False)

        self.client.post(reverse('board:delete_post', args=[delete_post.id]))

        delete_post.refresh_from_db()
        other_post.refresh_from_db()

        self.assertEqual(delete_post.is_deleted, True)
        self.assertEqual(other_post.is_deleted, False)

    def test_redirect_to_post_list_after_delete_post(self):
        delete_post = Post.objects.create(board=self.default_board, title='delete post', content='content')
        response = self.client.post(reverse('board:delete_post', args=[delete_post.id]))

        self.assertRedirects(response, reverse('board:post_list', args=[self.default_board.slug]))

    def test_does_not_view_but_remain_in_DB_after_delete(self):
        delete_post = Post.objects.create(board=self.default_board, title='delete post', content='content')

        viewed_list = Post.objects.filter(is_deleted=False)
        self.assertIn(delete_post, viewed_list)

        self.client.post(reverse('board:delete_post', args=[delete_post.id]))

        viewed_list = Post.objects.filter(is_deleted=False)
        self.assertNotIn(delete_post, viewed_list)

        all_list = Post.objects.all()
        self.assertIn(delete_post, all_list)

    def test_can_not_access_with_GET(self):
        delete_post = Post.objects.create(board=self.default_board, title='delete post', content='content')
        response = self.client.get(reverse('board:delete_post', args=[delete_post.id]))

        self.assertEqual(response.status_code, 405)

class PostViewTest(BoardAppTest):
    def get_response_for_some_post_view(self):
        post_ = Post.objects.create(board=self.default_board, title='post of title', content='post of content')
        return self.client.get(reverse('board:view_post', args=[post_.id]))

    def test_uses_list_template(self):
        response = PostViewTest.get_response_for_some_post_view(self)
        self.assertTemplateUsed(response, 'view_post.html')

    def test_use_pagination_template(self):
        response = PostViewTest.get_response_for_some_post_view(self)
        self.assertTemplateUsed(response, 'pagination.html')

    def test_view_default_page_list_when_post_open(self):
        response = PostViewTest.get_response_for_some_post_view(self)
        self.assertContains(response, '
'+str(1)+'
')

    def test_passes_correct_post_to_template(self):
        other_post = Post.objects.create(
            board=self.default_board,
            title='other post of title',
            content='other post of content'
        )
        correct_post = Post.objects.create(
            board=self.default_board,
            title='correct post of title',
            content='correct post of content'
        )

        response = self.client.get(reverse('board:view_post', args=[correct_post.id]))

        self.assertEqual(response.context['post'], correct_post)

    def test_view_post_page_displays_correct_title_and_content(self):
        other_post = Post.objects.create(
            board=self.default_board,
            title='other post of title',
            content='other post of content'
        )
        correct_post = Post.objects.create(
            board=self.default_board,
            title='correct post of title',
            content='correct post of content'
        )

        response = self.client.get(reverse('board:view_post', args=[correct_post.id]))

        self.assertContains(response, 'correct post of title')
        self.assertContains(response, 'correct post of content')
        self.assertNotContains(response, 'other post of title')
        self.assertNotContains(response, 'other post of content')

class EditPostTest(BoardAppTest):
    @classmethod
    def setUpTestData(cls):
        super().setUpTestData()
        cls.default_post = Post.objects.create(
            board=cls.default_board,
            title='some post title',
            content='some post content'
        )

    def test_use_modify_template(self):
        response = self.client.get(reverse('board:edit_post', args=[self.default_post.id]))
        self.assertTemplateUsed(response, 'edit_post.html')

    def test_uses_post_form(self):
        response = self.client.get(reverse('board:edit_post', args=[self.default_post.id]))
        self.assertIsInstance(response.context['form'], PostForm)

    def test_POST_redirects_to_post_list(self):
        response = self.client.post(reverse('board:edit_post', args=[self.default_post.id]), {
            'post_title_text': 'Edited title',
            'fields': 'Edited content',
        })

        self.assertRedirects(response, reverse('board:post_list', args=[self.default_post.board.slug]))

    def test_record_edited_post_history_when_post_edited(self):
        saved_edited_post_history = EditedPostHistory.objects.all()
        self.assertEqual(saved_edited_post_history.count(), 0)

        response = self.client.post(reverse('board:edit_post', args=[self.default_post.id]), {
            'post_title_text': 'Edited title',
            'fields': 'Edited content',
            })

        saved_edited_post_history = EditedPostHistory.objects.all()
        self.assertEqual(saved_edited_post_history.count(), 1)
        self.assertEqual(saved_edited_post_history[0].title, 'some post title')
        self.assertEqual(saved_edited_post_history[0].content, 'some post content')

    def test_edited_post_history_is_related_to_post(self):
        response = self.client.post(reverse('board:edit_post', args=[self.default_post.id]), {
            'post_title_text': 'Edited title',
            'fields': 'Edited content',
        })

        edited_post_history = EditedPostHistory.objects.first()
        self.assertEqual(edited_post_history.post, self.default_post)

class NewCommentTest(BoardAppTest):
    @classmethod
    def setUpTestData(cls):
        super().setUpTestData()
        cls.default_post = Post.objects.create(
            board=cls.default_board,
            title='some post title',
            content='some post content'
        )

    def test_can_create_comment(self):
        self.client.post(
            reverse('board:new_comment', args=[self.default_post.id]),
            data={'comment_content': 'This is a comment'}
        )

        self.assertEqual(Comment.objects.count(), 1)

    def test_can_not_access_with_GET_on_new_comment(self):
        response = self.client.get(
            reverse('board:new_comment', args=[self.default_post.id]),
            data={'comment_content': 'This is a comment'}
        )

        self.assertEqual(response.status_code, 405)

class DeleteCommentTest(BoardAppTest):
    @classmethod
    def setUpTestData(cls):
        super().setUpTestData()
        cls.default_post = Post.objects.create(
            board=cls.default_board,
            title='some post title',
            content='some post content'
        )

    def test_can_delete_comment(self):
        comment = Comment.objects.create(
            post=self.default_post,
            content='This is a comment'
        )
        response = self.client.post(
            reverse('board:delete_comment', args=[self.default_post.id, comment.id])
        )

        self.assertEqual(Comment.objects.count(), 1)

    def test_can_not_access_with_GET_on_delete_comment(self):
        comment = Comment.objects.create(
            post=self.default_post,
            content='This is a comment'
        )
        response = self.client.get(
            reverse('board:delete_comment', args=[self.default_post.id, comment.id])
        )

        self.assertEqual(response.status_code, 405)

class FileUploadTest(BoardAppTest):
    def test_save_uploaded_file(self):
        uploaded_file_name = settings.BASE_DIR + '/test_file/test.txt'
        saved_file_name = settings.BASE_DIR + '/file/test.txt'

        if os.path.isfile(saved_file_name):
            os.remove(saved_file_name)

        uploaded_file = open(uploaded_file_name)
        response = self.client.post(reverse('board:new_post', args=[self.default_board.slug]), {
            'post_title_text': 'NEW POST TITLE',
            'fields': 'NEW POST CONTENT',
            'file': uploaded_file,
        })

        uploaded_file.seek(0)
        saved_file = open(saved_file_name)

        self.assertEqual(uploaded_file.read(os.path.getsize(uploaded_file_name)),
                         saved_file.read(os.path.getsize(saved_file_name)))
<commit_msg>Fix test file naming<commit_after>import os

from django.http import HttpRequest
from django.core.urlresolvers import reverse
from django.conf import settings

from .base import BoardAppTest
from board.views import new_post, post_list, edit_post
from board.models import Post, Board, Comment, EditedPostHistory
from board.forms import PostForm

class CreatePostPageTest(BoardAppTest):
    def test_new_post_page_returns_correct_html(self):
        response = self.client.get(reverse('board:new_post', args=[self.default_board.slug]))

        response_decoded = self.remove_csrf(response.content.decode())
        self.assertIn('settings_id_fields', response_decoded)

    def test_new_post_can_save_a_POST_request(self):
        response = self.client.post(reverse('board:new_post', args=[self.default_board.slug]), {
            'post_title_text': 'NEW POST TITLE',
            'fields': 'NEW POST CONTENT',
        })

        self.assertEqual(Post.objects.count(), 1)
        first_new_post = Post.objects.first()
        self.assertEqual(first_new_post.title, 'NEW POST TITLE')

    def test_new_post_page_redirects_after_POST(self):
        response = self.client.post(reverse('board:new_post', args=[self.default_board.slug]), {
            'post_title_text': 'NEW POST TITLE',
            'fields': 'NEW POST CONTENT',
        })

        self.assertRedirects(response, reverse('board:post_list', args=[self.default_board.slug]))

    def test_create_post_page_only_saves_items_when_necessary(self):
        request = HttpRequest()
        new_post(request, self.default_board.slug)
        self.assertEqual(Post.objects.count(), 0)

class PostListTest(BoardAppTest):
    def get_response_from_post_list_search_request(self, search_flag, query):
        response = self.client.get(reverse('board:post_list', args=[self.default_board.slug]), data={
            'search_flag': search_flag,
            'query': query
        })
        return response

    def test_post_list_page_displays_all_list_titles(self):
        Post.objects.create(board=self.default_board, title='turtle1', content='slow')
        Post.objects.create(board=self.default_board, title='turtle2', content='slowslow')

        request = HttpRequest()
        response = post_list(request, self.default_board.slug)

        self.assertIn('turtle1', response.content.decode())
        self.assertIn('turtle2', response.content.decode())

    def test_post_list_page_displays_searched_post(self):
        repeat = 3
        for i in range(repeat):
            Post.objects.create(
                board=self.default_board,
                title='Hi, ' + str(i),
                content='content ' + str(i)
            )

        response = self.get_response_from_post_list_search_request('TITLE', 'Hi, 1')
        self.assertContains(response, 'Hi, 1')
        self.assertNotContains(response, 'Hi, 0')

        response = self.get_response_from_post_list_search_request('CONTENT', 'content 1')
        self.assertContains(response, 'Hi, 1')
        self.assertNotContains(response, 'Hi, 2')

        response = self.get_response_from_post_list_search_request('BOTH', 'Hi, 1')
        self.assertContains(response, 'Hi, 1')

        response = self.get_response_from_post_list_search_request('BOTH', 'content 2')
        self.assertContains(response, 'Hi, 2')

    def test_post_list_page_displays_all_posts_when_search_flag_is_invalid(self):
        repeat = 3
        for i in range(repeat):
            Post.objects.create(
                board=self.default_board,
                title='Hi, ' + str(i),
                content='content ' + str(i)
            )

        response = self.get_response_from_post_list_search_request('INVALID_FLAG', 'any query')
        self.assertContains(response, repeat)

class PostListViewTest(BoardAppTest):
    def test_use_post_list_template(self):
        response = self.client.get(reverse('board:post_list', args=[self.default_board.slug]))
        self.assertTemplateUsed(response, 'post_list.html')

    def test_use_pagination_template(self):
        response = self.client.get(reverse('board:post_list', args=[self.default_board.slug]))
        self.assertTemplateUsed(response, 'pagination.html')

    def test_view_default_page_list_when_board_open(self):
        response = self.client.get(reverse('board:post_list', args=[self.default_board.slug]))
        self.assertContains(response, '
'+str(1)+'
')

class DeletePostTest(BoardAppTest):
    def test_delete_only_post_selected_to_delete(self):
        delete_post = Post.objects.create(board=self.default_board, title='delete post', content='content')
        other_post = Post.objects.create(board=self.default_board, title='other post', content='content')

        self.assertEqual(delete_post.is_deleted, False)
        self.assertEqual(other_post.is_deleted, False)

        self.client.post(reverse('board:delete_post', args=[delete_post.id]))

        delete_post.refresh_from_db()
        other_post.refresh_from_db()

        self.assertEqual(delete_post.is_deleted, True)
        self.assertEqual(other_post.is_deleted, False)

    def test_redirect_to_post_list_after_delete_post(self):
        delete_post = Post.objects.create(board=self.default_board, title='delete post', content='content')
        response = self.client.post(reverse('board:delete_post', args=[delete_post.id]))

        self.assertRedirects(response, reverse('board:post_list', args=[self.default_board.slug]))

    def test_does_not_view_but_remain_in_DB_after_delete(self):
        delete_post = Post.objects.create(board=self.default_board, title='delete post', content='content')

        viewed_list = Post.objects.filter(is_deleted=False)
        self.assertIn(delete_post, viewed_list)

        self.client.post(reverse('board:delete_post', args=[delete_post.id]))

        viewed_list = Post.objects.filter(is_deleted=False)
        self.assertNotIn(delete_post, viewed_list)

        all_list = Post.objects.all()
        self.assertIn(delete_post, all_list)

    def test_can_not_access_with_GET(self):
        delete_post = Post.objects.create(board=self.default_board, title='delete post', content='content')
        response = self.client.get(reverse('board:delete_post', args=[delete_post.id]))

        self.assertEqual(response.status_code, 405)

class PostViewTest(BoardAppTest):
    def get_response_for_some_post_view(self):
        post_ = Post.objects.create(board=self.default_board, title='post of title', content='post of content')
        return self.client.get(reverse('board:view_post', args=[post_.id]))

    def test_uses_list_template(self):
        response = PostViewTest.get_response_for_some_post_view(self)
        self.assertTemplateUsed(response, 'view_post.html')

    def test_use_pagination_template(self):
        response = PostViewTest.get_response_for_some_post_view(self)
        self.assertTemplateUsed(response, 'pagination.html')

    def test_view_default_page_list_when_post_open(self):
        response = PostViewTest.get_response_for_some_post_view(self)
        self.assertContains(response, '
'+str(1)+'
')

    def test_passes_correct_post_to_template(self):
        other_post = Post.objects.create(
            board=self.default_board,
            title='other post of title',
            content='other post of content'
        )
        correct_post = Post.objects.create(
            board=self.default_board,
            title='correct post of title',
            content='correct post of content'
        )

        response = self.client.get(reverse('board:view_post', args=[correct_post.id]))

        self.assertEqual(response.context['post'], correct_post)

    def test_view_post_page_displays_correct_title_and_content(self):
        other_post = Post.objects.create(
            board=self.default_board,
            title='other post of title',
            content='other post of content'
        )
        correct_post = Post.objects.create(
            board=self.default_board,
            title='correct post of title',
            content='correct post of content'
        )

        response = self.client.get(reverse('board:view_post', args=[correct_post.id]))

        self.assertContains(response, 'correct post of title')
        self.assertContains(response, 'correct post of content')
        self.assertNotContains(response, 'other post of title')
        self.assertNotContains(response, 'other post of content')

class EditPostTest(BoardAppTest):
    @classmethod
    def setUpTestData(cls):
        super().setUpTestData()
        cls.default_post = Post.objects.create(
            board=cls.default_board,
            title='some post title',
            content='some post content'
        )

    def test_use_modify_template(self):
        response = self.client.get(reverse('board:edit_post', args=[self.default_post.id]))
        self.assertTemplateUsed(response, 'edit_post.html')

    def test_uses_post_form(self):
        response = self.client.get(reverse('board:edit_post', args=[self.default_post.id]))
        self.assertIsInstance(response.context['form'], PostForm)

    def test_POST_redirects_to_post_list(self):
        response = self.client.post(reverse('board:edit_post', args=[self.default_post.id]), {
            'post_title_text': 'Edited title',
            'fields': 'Edited content',
        })

        self.assertRedirects(response, reverse('board:post_list', args=[self.default_post.board.slug]))

    def test_record_edited_post_history_when_post_edited(self):
        saved_edited_post_history = EditedPostHistory.objects.all()
        self.assertEqual(saved_edited_post_history.count(), 0)

        response = self.client.post(reverse('board:edit_post', args=[self.default_post.id]), {
            'post_title_text': 'Edited title',
            'fields': 'Edited content',
            })

        saved_edited_post_history = EditedPostHistory.objects.all()
        self.assertEqual(saved_edited_post_history.count(), 1)
        self.assertEqual(saved_edited_post_history[0].title, 'some post title')
        self.assertEqual(saved_edited_post_history[0].content, 'some post content')

    def test_edited_post_history_is_related_to_post(self):
        response = self.client.post(reverse('board:edit_post', args=[self.default_post.id]), {
            'post_title_text': 'Edited title',
            'fields': 'Edited content',
        })

        edited_post_history = EditedPostHistory.objects.first()
        self.assertEqual(edited_post_history.post, self.default_post)

class NewCommentTest(BoardAppTest):
    @classmethod
    def setUpTestData(cls):
        super().setUpTestData()
        cls.default_post = Post.objects.create(
            board=cls.default_board,
            title='some post title',
            content='some post content'
        )

    def test_can_create_comment(self):
        self.client.post(
            reverse('board:new_comment', args=[self.default_post.id]),
            data={'comment_content': 'This is a comment'}
        )

        self.assertEqual(Comment.objects.count(), 1)

    def test_can_not_access_with_GET_on_new_comment(self):
        response = self.client.get(
            reverse('board:new_comment', args=[self.default_post.id]),
            data={'comment_content': 'This is a comment'}
        )

        self.assertEqual(response.status_code, 405)

class DeleteCommentTest(BoardAppTest):
    @classmethod
    def setUpTestData(cls):
        super().setUpTestData()
        cls.default_post = Post.objects.create(
            board=cls.default_board,
            title='some post title',
            content='some post content'
        )

    def test_can_delete_comment(self):
        comment = Comment.objects.create(
            post=self.default_post,
            content='This is a comment'
        )
        response = self.client.post(
            reverse('board:delete_comment', args=[self.default_post.id, comment.id])
        )

        self.assertEqual(Comment.objects.count(), 1)

    def test_can_not_access_with_GET_on_delete_comment(self):
        comment = Comment.objects.create(
            post=self.default_post,
            content='This is a comment'
        )
        response = self.client.get(
            reverse('board:delete_comment', args=[self.default_post.id, comment.id])
        )

        self.assertEqual(response.status_code, 405)

class FileUploadTest(BoardAppTest):
    def test_save_upload_file(self):
        upload_test_file_name = settings.BASE_DIR + '/test_file/test.txt'
        saved_test_file_name = settings.BASE_DIR + '/file/test.txt'

        if os.path.isfile(saved_test_file_name):
            os.remove(saved_test_file_name)

        upload_file = open(upload_test_file_name)
        self.client.post(reverse('board:new_post', args=[self.default_board.slug]), {
            'post_title_text': 'NEW POST TITLE',
            'fields': 'NEW POST CONTENT',
            'file': upload_file,
        })

        upload_file.seek(0)
        saved_file = open(saved_test_file_name)

        self.assertEqual(upload_file.read(os.path.getsize(upload_test_file_name)),
                         saved_file.read(os.path.getsize(saved_test_file_name)))
<|endoftext|>


<commit_before>
from itertools import product
from collections import defaultdict

from scipy.spatial.distance import euclidean
import pulp

def tokens_to_fracdict(tokens):
    cntdict = defaultdict(lambda : 0)
    for token in tokens:
        cntdict[token] += 1
    totalcnt = sum(cntdict.values())
    return {token: float(cnt)/totalcnt for token, cnt in cntdict.items()}

# use PuLP
def word_mover_distance_probspec(first_sent_tokens, second_sent_tokens, wvmodel, lpFile=None):
    all_tokens = list(set(first_sent_tokens+second_sent_tokens))
    wordvecs = {token: wvmodel[token] for token in all_tokens}

    first_sent_buckets = tokens_to_fracdict(first_sent_tokens)
    second_sent_buckets = tokens_to_fracdict(second_sent_tokens)

    T = pulp.LpVariable.dicts('T_matrix', list(product(all_tokens, all_tokens)), lowBound=0)

    prob = pulp.LpProblem('WMD', sense=pulp.LpMinimize)
    prob += pulp.lpSum([T[token1, token2]*euclidean(wordvecs[token1], wordvecs[token2])
                        for token1, token2 in product(all_tokens, all_tokens)])
    for token2 in second_sent_buckets:
        prob += pulp.lpSum([T[token1, token2] for token1 in first_sent_buckets])==second_sent_buckets[token2]
    for token1 in first_sent_buckets:
        prob += pulp.lpSum([T[token1, token2] for token2 in second_sent_buckets])==first_sent_buckets[token1]

    if lpFile!=None:
        prob.writeLP(lpFile)

    prob.solve()

    return prob

def word_mover_distance(first_sent_tokens, second_sent_tokens, wvmodel, lpFile=None):
    prob = word_mover_distance_probspec(first_sent_tokens, second_sent_tokens, wvmodel, lpFile=lpFile)
    return pulp.value(prob.objective)<commit_msg>allow flexible distance metric<commit_after>
from itertools import product
from collections import defaultdict

from scipy.spatial.distance import euclidean
import pulp

def tokens_to_fracdict(tokens):
    cntdict = defaultdict(lambda : 0)
    for token in tokens:
        cntdict[token] += 1
    totalcnt = sum(cntdict.values())
    return {token: float(cnt)/totalcnt for token, cnt in cntdict.items()}

# use PuLP
def word_mover_distance_probspec(first_sent_tokens, second_sent_tokens, wvmodel, distancefunc=euclidean, lpFile=None):
    all_tokens = list(set(first_sent_tokens+second_sent_tokens))
    wordvecs = {token: wvmodel[token] for token in all_tokens}

    first_sent_buckets = tokens_to_fracdict(first_sent_tokens)
    second_sent_buckets = tokens_to_fracdict(second_sent_tokens)

    T = pulp.LpVariable.dicts('T_matrix', list(product(all_tokens, all_tokens)), lowBound=0)

    prob = pulp.LpProblem('WMD', sense=pulp.LpMinimize)
    prob += pulp.lpSum([T[token1, token2]*distancefunc(wordvecs[token1], wordvecs[token2])
                        for token1, token2 in product(all_tokens, all_tokens)])
    for token2 in second_sent_buckets:
        prob += pulp.lpSum([T[token1, token2] for token1 in first_sent_buckets])==second_sent_buckets[token2]
    for token1 in first_sent_buckets:
        prob += pulp.lpSum([T[token1, token2] for token2 in second_sent_buckets])==first_sent_buckets[token1]

    if lpFile!=None:
        prob.writeLP(lpFile)

    prob.solve()

    return prob

def word_mover_distance(first_sent_tokens, second_sent_tokens, wvmodel, distancefunc=euclidean, lpFile=None):
    prob = word_mover_distance_probspec(first_sent_tokens, second_sent_tokens, wvmodel,
                                        distancefunc=distancefunc, lpFile=lpFile)
    return pulp.value(prob.objective)<|endoftext|>