<commit_before>import os
import random
import re
import subprocess

from ccmlib import common
from dtest import Tester, debug
from tools import since

class TestOfflineTools(Tester):

    # In 2.0, we will get this error log message due to jamm not being
    # in the classpath
    ignore_log_patterns = ["Unable to initialize MemoryMeter"]

    def sstablelevelreset_test(self):
        """
        Insert data and call sstablelevelreset on a series of
        tables. Confirm level is reset to 0 using its output.
        Test a variety of possible errors and ensure response is resonable.
        @since 2.1.5
        @jira_ticket CASSANDRA-7614
        """
        cluster = self.cluster
        cluster.populate(1).start(wait_for_binary_proto=True)
        node1 = cluster.nodelist()[0]

        # test by trying to run on nonexistent keyspace
        cluster.stop(gently=False)
        (output, error, rc) = node1.run_sstablelevelreset("keyspace1", "standard1", output=True)
        self.assertIn("ColumnFamily not found: keyspace1/standard1", error)
        # this should return exit code 1
        self.assertEqual(rc, 1, msg=str(rc))

        # now test by generating keyspace but not flushing sstables
        cluster.start(wait_for_binary_proto=True)
        node1.stress(['write', 'n=100', '-schema', 'replication(factor=1)'])
        cluster.stop(gently=False)

        (output, error, rc) = node1.run_sstablelevelreset("keyspace1", "standard1", output=True)
        self.assertTrue(len(error) == 0 or "Max sstable size of" in error, error)
        self.assertIn("Found no sstables, did you give the correct keyspace", output)
        self.assertEqual(rc, 0, msg=str(rc))

        # test by writing small amount of data and flushing (all sstables should be level 0)
        cluster.start(wait_for_binary_proto=True)
        session = self.patient_cql_connection(node1)
        session.execute("ALTER TABLE keyspace1.standard1 with compaction={'class': 'LeveledCompactionStrategy', 'sstable_size_in_mb':1};")
        node1.stress(['write', 'n=1K', '-schema', 'replication(factor=1)'])
        node1.flush()
        cluster.stop(gently=False)

        (output, error, rc) = node1.run_sstablelevelreset("keyspace1", "standard1", output=True)
        self.assertTrue(len(error) == 0 or "Max sstable size of" in error, error)
        self.assertIn("since it is already on level 0", output)
        self.assertEqual(rc, 0, msg=str(rc))

        # test by loading large amount data so we have multiple levels and checking all levels are 0 at end
        cluster.start(wait_for_binary_proto=True)
        node1.stress(['write', 'n=50K', '-rate', 'threads=20', '-schema', 'replication(factor=1)'])
        cluster.flush()
        self.wait_for_compactions(node1)
        cluster.stop()

        initial_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))
        (output, error, rc) = node1.run_sstablelevelreset("keyspace1", "standard1", output=True)
        final_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))

        self.assertTrue(len(error) == 0 or "Max sstable size of" in error, error)
        self.assertEqual(rc, 0, msg=str(rc))

        debug(initial_levels)
        debug(final_levels)

        # let's make sure there was at least L1 beforing resetting levels
        self.assertTrue(max(initial_levels) > 0)

        # let's check all sstables are on L0 after sstablelevelreset
        self.assertTrue(max(final_levels) == 0)

    def get_levels(self, data):
        levels = []
        for sstable in data:
            (metadata, error, rc) = sstable
            level = int(re.findall("SSTable Level: [0-9]", metadata)[0][-1])
            levels.append(level)
        return levels

    def wait_for_compactions(self, node):
        pattern = re.compile("pending tasks: 0")
        while True:
            output, err = node.nodetool("compactionstats", capture_output=True)
            if pattern.search(output):
                break

    def sstableofflinerelevel_test(self):
        """
        Generate sstables of varying levels.
        Reset sstables to L0 with sstablelevelreset
        Run sstableofflinerelevel and ensure tables are promoted correctly
        Also test a variety of bad inputs including nonexistent keyspace and sstables
        @since 2.1.5
        @jira_ticket CASSANRDA-8031
        """
        cluster = self.cluster
        cluster.populate(1).start(wait_for_binary_proto=True)
        node1 = cluster.nodelist()[0]

        # NOTE - As of now this does not return when it encounters Exception and causes test to hang, temporarily commented out
        # test by trying to run on nonexistent keyspace
        # cluster.stop(gently=False)
        # (output, error, rc) = node1.run_sstableofflinerelevel("keyspace1", "standard1", output=True)
        # self.assertTrue("java.lang.IllegalArgumentException: Unknown keyspace/columnFamily keyspace1.standard1" in error)
        # # this should return exit code 1
        # self.assertEqual(rc, 1, msg=str(rc))
        # cluster.start()

        # now test by generating keyspace but not flushing sstables

        node1.stress(['write', 'n=1', '-schema', 'replication(factor=1)'])
        cluster.stop(gently=False)

        (output, error, rc) = node1.run_sstableofflinerelevel("keyspace1", "standard1", output=True)

        self.assertIn("No sstables to relevel for keyspace1.standard1", output)
        self.assertEqual(rc, 1, msg=str(rc))

        # test by flushing (sstable should be level 0)
        cluster.start(wait_for_binary_proto=True)
        session = self.patient_cql_connection(node1)
        session.execute("ALTER TABLE keyspace1.standard1 with compaction={'class': 'LeveledCompactionStrategy', 'sstable_size_in_mb':1};")

        node1.stress(['write', 'n=1K', '-schema', 'replication(factor=1)'])

        node1.flush()
        cluster.stop()

        (output, error, rc) = node1.run_sstableofflinerelevel("keyspace1", "standard1", output=True)
        self.assertIn("L0=1", output)
        self.assertEqual(rc, 0, msg=str(rc))

        # test by loading large amount data so we have multiple sstables
        cluster.start(wait_for_binary_proto=True)
        node1.stress(['write', 'n=100K', '-schema', 'replication(factor=1)'])
        node1.flush()
        self.wait_for_compactions(node1)
        cluster.stop()

        # Let's reset all sstables to L0
        initial_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))
        (output, error, rc) = node1.run_sstablelevelreset("keyspace1", "standard1", output=True)
        final_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))

        # let's make sure there was at least 3 levels (L0, L1 and L2)
        self.assertTrue(max(initial_levels) > 1)
        # let's check all sstables are on L0 after sstablelevelreset
        self.assertTrue(max(final_levels) == 0)

        # time to relevel sstables
        initial_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))
        (output, error, rc) = node1.run_sstableofflinerelevel("keyspace1", "standard1", output=True)
        final_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))

        debug(initial_levels)
        debug(final_levels)

        # let's check sstables were promoted after releveling
        self.assertTrue(max(final_levels) > 1)

    @since('2.2')
    def sstableverify_test(self):
        """
        Generate sstables and test offline verification works correctly
        Test on bad input: nonexistent keyspace and sstables
        Test on potential situations: deleted sstables, corrupted sstables
        """

        cluster = self.cluster
        cluster.populate(3).start(wait_for_binary_proto=True)
        node1, node2, node3 = cluster.nodelist()

        # test on nonexistent keyspace
        (out, err, rc) = node1.run_sstableverify("keyspace1", "standard1", output=True)
        self.assertIn("Unknown keyspace/table keyspace1.standard1", err)
        self.assertEqual(rc, 1, msg=str(rc))

        # test on nonexistent sstables:
        node1.stress(['write', 'n=100', '-schema', 'replication(factor=3)'])
        (out, err, rc) = node1.run_sstableverify("keyspace1", "standard1", output=True)
        self.assertEqual(rc, 0, msg=str(rc))

        # Generate multiple sstables and test works properly in the simple case
        node1.stress(['write', 'n=100K', '-schema', 'replication(factor=3)'])
        node1.flush()
        node1.stress(['write', 'n=100K', '-schema', 'replication(factor=3)'])
        node1.flush()
        cluster.stop()

        (out, error, rc) = node1.run_sstableverify("keyspace1", "standard1", output=True)

        self.assertEqual(rc, 0, msg=str(rc))

        # STDOUT of the sstableverify command consists of multiple lines which may contain
        # Java-normalized paths. To later compare these with Python-normalized paths, we
        # map over each line of out and replace Java-normalized paths with Python equivalents.
        outlines = map(lambda line: re.sub("(?<=path=').*(?=')",
                                           lambda match: os.path.normcase(match.group(0)),
                                           line),
                       out.splitlines())

        # check output is correct for each sstable
        sstables = self._get_final_sstables(node1, "keyspace1", "standard1")

        for sstable in sstables:
            verified = False
            hashcomputed = False
            for line in outlines:
                if sstable in line:
                    if "Verifying BigTableReader" in line:
                        verified = True
                    elif "Checking computed hash of BigTableReader" in line:
                        hashcomputed = True
                    else:
                        debug(line)

            debug(verified)
            debug(hashcomputed)
            debug(sstable)
            self.assertTrue(verified and hashcomputed)

        # now try intentionally corrupting an sstable to see if hash computed is different and error recognized
        sstable1 = sstables[1]
        with open(sstable1, 'r') as f:
            sstabledata = bytearray(f.read())
        with open(sstable1, 'w') as out:
            position = random.randrange(0, len(sstabledata))
            sstabledata[position] = (sstabledata[position] + 1) % 256
            out.write(sstabledata)

        # use verbose to get some coverage on it
        (out, error, rc) = node1.run_sstableverify("keyspace1", "standard1", options=['-v'], output=True)

        # Process sstableverify output to normalize paths in string to Python casing as above
        error = re.sub("(?<=Corrupted: ).*", lambda match: os.path.normcase(match.group(0)), error)

        self.assertIn("Corrupted: " + sstable1, error)
        self.assertEqual(rc, 1, msg=str(rc))

    def sstableexpiredblockers_test(self):
        cluster = self.cluster
        cluster.populate(1).start(wait_for_binary_proto=True)
        [node1] = cluster.nodelist()
        session = self.patient_cql_connection(node1)
        self.create_ks(session, 'ks', 1)
        session.execute("create table ks.cf (key int PRIMARY KEY, val int) with gc_grace_seconds=0")
        # create a blocker:
        session.execute("insert into ks.cf (key, val) values (1,1)")
        node1.flush()
        session.execute("delete from ks.cf where key = 2")
        node1.flush()
        session.execute("delete from ks.cf where key = 3")
        node1.flush()
        [(out, error, rc)] = node1.run_sstableexpiredblockers(keyspace="ks", column_family="cf")
        self.assertIn("blocks 2 expired sstables from getting dropped", out)

    def _get_final_sstables(self, node, ks, table):
        """
        Return the node final sstable data files, excluding the temporary tables.
        If sstableutil exists (>= 3.0) then we rely on this tool since the table
        file names no longer contain tmp in their names (CASSANDRA-7066).
        """
        # Get all sstable data files
        allsstables = map(os.path.normcase, node.get_sstables(ks, table))

        # Remove any temporary files
        tool_bin = node.get_tool('sstableutil')
        if os.path.isfile(tool_bin):
            args = [tool_bin, '--type', 'tmp', ks, table]
            env = common.make_cassandra_env(node.get_install_cassandra_root(), node.get_node_cassandra_root())
            p = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (stdout, stderr) = p.communicate()
            tmpsstables = map(os.path.normcase, stdout.splitlines())

            ret = list(set(allsstables) - set(tmpsstables))
        else:
            ret = [sstable for sstable in allsstables if "tmp" not in sstable[50:]]

        return ret
<commit_msg>ignore disk space warning in sstablelevelreset_test<commit_after>import os
import random
import re
import subprocess

from ccmlib import common
from dtest import Tester, debug
from tools import since

class TestOfflineTools(Tester):

    # In 2.0, we will get this error log message due to jamm not being
    # in the classpath
    ignore_log_patterns = ["Unable to initialize MemoryMeter"]

    def sstablelevelreset_test(self):
        """
        Insert data and call sstablelevelreset on a series of
        tables. Confirm level is reset to 0 using its output.
        Test a variety of possible errors and ensure response is resonable.
        @since 2.1.5
        @jira_ticket CASSANDRA-7614
        """
        cluster = self.cluster
        cluster.populate(1).start(wait_for_binary_proto=True)
        node1 = cluster.nodelist()[0]

        # test by trying to run on nonexistent keyspace
        cluster.stop(gently=False)
        (output, error, rc) = node1.run_sstablelevelreset("keyspace1", "standard1", output=True)
        self.assertIn("ColumnFamily not found: keyspace1/standard1", error)
        # this should return exit code 1
        self.assertEqual(rc, 1, msg=str(rc))

        # now test by generating keyspace but not flushing sstables
        cluster.start(wait_for_binary_proto=True)
        node1.stress(['write', 'n=100', '-schema', 'replication(factor=1)'])
        cluster.stop(gently=False)

        (output, error, rc) = node1.run_sstablelevelreset("keyspace1", "standard1", output=True)
        self._check_stderr_error(error)
        self.assertIn("Found no sstables, did you give the correct keyspace", output)
        self.assertEqual(rc, 0, msg=str(rc))

        # test by writing small amount of data and flushing (all sstables should be level 0)
        cluster.start(wait_for_binary_proto=True)
        session = self.patient_cql_connection(node1)
        session.execute("ALTER TABLE keyspace1.standard1 with compaction={'class': 'LeveledCompactionStrategy', 'sstable_size_in_mb':1};")
        node1.stress(['write', 'n=1K', '-schema', 'replication(factor=1)'])
        node1.flush()
        cluster.stop(gently=False)

        (output, error, rc) = node1.run_sstablelevelreset("keyspace1", "standard1", output=True)
        self._check_stderr_error(error)
        self.assertIn("since it is already on level 0", output)
        self.assertEqual(rc, 0, msg=str(rc))

        # test by loading large amount data so we have multiple levels and checking all levels are 0 at end
        cluster.start(wait_for_binary_proto=True)
        node1.stress(['write', 'n=50K', '-rate', 'threads=20', '-schema', 'replication(factor=1)'])
        cluster.flush()
        self.wait_for_compactions(node1)
        cluster.stop()

        initial_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))
        (output, error, rc) = node1.run_sstablelevelreset("keyspace1", "standard1", output=True)
        final_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))
        self._check_stderr_error(error)
        self.assertEqual(rc, 0, msg=str(rc))

        debug(initial_levels)
        debug(final_levels)

        # let's make sure there was at least L1 beforing resetting levels
        self.assertTrue(max(initial_levels) > 0)

        # let's check all sstables are on L0 after sstablelevelreset
        self.assertTrue(max(final_levels) == 0)

    def get_levels(self, data):
        levels = []
        for sstable in data:
            (metadata, error, rc) = sstable
            level = int(re.findall("SSTable Level: [0-9]", metadata)[0][-1])
            levels.append(level)
        return levels

    def wait_for_compactions(self, node):
        pattern = re.compile("pending tasks: 0")
        while True:
            output, err = node.nodetool("compactionstats", capture_output=True)
            if pattern.search(output):
                break

    def sstableofflinerelevel_test(self):
        """
        Generate sstables of varying levels.
        Reset sstables to L0 with sstablelevelreset
        Run sstableofflinerelevel and ensure tables are promoted correctly
        Also test a variety of bad inputs including nonexistent keyspace and sstables
        @since 2.1.5
        @jira_ticket CASSANRDA-8031
        """
        cluster = self.cluster
        cluster.populate(1).start(wait_for_binary_proto=True)
        node1 = cluster.nodelist()[0]

        # NOTE - As of now this does not return when it encounters Exception and causes test to hang, temporarily commented out
        # test by trying to run on nonexistent keyspace
        # cluster.stop(gently=False)
        # (output, error, rc) = node1.run_sstableofflinerelevel("keyspace1", "standard1", output=True)
        # self.assertTrue("java.lang.IllegalArgumentException: Unknown keyspace/columnFamily keyspace1.standard1" in error)
        # # this should return exit code 1
        # self.assertEqual(rc, 1, msg=str(rc))
        # cluster.start()

        # now test by generating keyspace but not flushing sstables

        node1.stress(['write', 'n=1', '-schema', 'replication(factor=1)'])
        cluster.stop(gently=False)

        (output, error, rc) = node1.run_sstableofflinerelevel("keyspace1", "standard1", output=True)

        self.assertIn("No sstables to relevel for keyspace1.standard1", output)
        self.assertEqual(rc, 1, msg=str(rc))

        # test by flushing (sstable should be level 0)
        cluster.start(wait_for_binary_proto=True)
        session = self.patient_cql_connection(node1)
        session.execute("ALTER TABLE keyspace1.standard1 with compaction={'class': 'LeveledCompactionStrategy', 'sstable_size_in_mb':1};")

        node1.stress(['write', 'n=1K', '-schema', 'replication(factor=1)'])

        node1.flush()
        cluster.stop()

        (output, error, rc) = node1.run_sstableofflinerelevel("keyspace1", "standard1", output=True)
        self.assertIn("L0=1", output)
        self.assertEqual(rc, 0, msg=str(rc))

        # test by loading large amount data so we have multiple sstables
        cluster.start(wait_for_binary_proto=True)
        node1.stress(['write', 'n=100K', '-schema', 'replication(factor=1)'])
        node1.flush()
        self.wait_for_compactions(node1)
        cluster.stop()

        # Let's reset all sstables to L0
        initial_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))
        (output, error, rc) = node1.run_sstablelevelreset("keyspace1", "standard1", output=True)
        final_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))

        # let's make sure there was at least 3 levels (L0, L1 and L2)
        self.assertTrue(max(initial_levels) > 1)
        # let's check all sstables are on L0 after sstablelevelreset
        self.assertTrue(max(final_levels) == 0)

        # time to relevel sstables
        initial_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))
        (output, error, rc) = node1.run_sstableofflinerelevel("keyspace1", "standard1", output=True)
        final_levels = self.get_levels(node1.run_sstablemetadata(keyspace="keyspace1", column_families=["standard1"]))

        debug(initial_levels)
        debug(final_levels)

        # let's check sstables were promoted after releveling
        self.assertTrue(max(final_levels) > 1)

    @since('2.2')
    def sstableverify_test(self):
        """
        Generate sstables and test offline verification works correctly
        Test on bad input: nonexistent keyspace and sstables
        Test on potential situations: deleted sstables, corrupted sstables
        """

        cluster = self.cluster
        cluster.populate(3).start(wait_for_binary_proto=True)
        node1, node2, node3 = cluster.nodelist()

        # test on nonexistent keyspace
        (out, err, rc) = node1.run_sstableverify("keyspace1", "standard1", output=True)
        self.assertIn("Unknown keyspace/table keyspace1.standard1", err)
        self.assertEqual(rc, 1, msg=str(rc))

        # test on nonexistent sstables:
        node1.stress(['write', 'n=100', '-schema', 'replication(factor=3)'])
        (out, err, rc) = node1.run_sstableverify("keyspace1", "standard1", output=True)
        self.assertEqual(rc, 0, msg=str(rc))

        # Generate multiple sstables and test works properly in the simple case
        node1.stress(['write', 'n=100K', '-schema', 'replication(factor=3)'])
        node1.flush()
        node1.stress(['write', 'n=100K', '-schema', 'replication(factor=3)'])
        node1.flush()
        cluster.stop()

        (out, error, rc) = node1.run_sstableverify("keyspace1", "standard1", output=True)

        self.assertEqual(rc, 0, msg=str(rc))

        # STDOUT of the sstableverify command consists of multiple lines which may contain
        # Java-normalized paths. To later compare these with Python-normalized paths, we
        # map over each line of out and replace Java-normalized paths with Python equivalents.
        outlines = map(lambda line: re.sub("(?<=path=').*(?=')",
                                           lambda match: os.path.normcase(match.group(0)),
                                           line),
                       out.splitlines())

        # check output is correct for each sstable
        sstables = self._get_final_sstables(node1, "keyspace1", "standard1")

        for sstable in sstables:
            verified = False
            hashcomputed = False
            for line in outlines:
                if sstable in line:
                    if "Verifying BigTableReader" in line:
                        verified = True
                    elif "Checking computed hash of BigTableReader" in line:
                        hashcomputed = True
                    else:
                        debug(line)

            debug(verified)
            debug(hashcomputed)
            debug(sstable)
            self.assertTrue(verified and hashcomputed)

        # now try intentionally corrupting an sstable to see if hash computed is different and error recognized
        sstable1 = sstables[1]
        with open(sstable1, 'r') as f:
            sstabledata = bytearray(f.read())
        with open(sstable1, 'w') as out:
            position = random.randrange(0, len(sstabledata))
            sstabledata[position] = (sstabledata[position] + 1) % 256
            out.write(sstabledata)

        # use verbose to get some coverage on it
        (out, error, rc) = node1.run_sstableverify("keyspace1", "standard1", options=['-v'], output=True)

        # Process sstableverify output to normalize paths in string to Python casing as above
        error = re.sub("(?<=Corrupted: ).*", lambda match: os.path.normcase(match.group(0)), error)

        self.assertIn("Corrupted: " + sstable1, error)
        self.assertEqual(rc, 1, msg=str(rc))

    def sstableexpiredblockers_test(self):
        cluster = self.cluster
        cluster.populate(1).start(wait_for_binary_proto=True)
        [node1] = cluster.nodelist()
        session = self.patient_cql_connection(node1)
        self.create_ks(session, 'ks', 1)
        session.execute("create table ks.cf (key int PRIMARY KEY, val int) with gc_grace_seconds=0")
        # create a blocker:
        session.execute("insert into ks.cf (key, val) values (1,1)")
        node1.flush()
        session.execute("delete from ks.cf where key = 2")
        node1.flush()
        session.execute("delete from ks.cf where key = 3")
        node1.flush()
        [(out, error, rc)] = node1.run_sstableexpiredblockers(keyspace="ks", column_family="cf")
        self.assertIn("blocks 2 expired sstables from getting dropped", out)

    def _check_stderr_error(self, error):
        if len(error) > 0:
            for line in error.splitlines():
                self.assertTrue("Max sstable size of" in line or "Consider adding more capacity" in line,
                                'Found line \n\n"{line}"\n\n in error\n\n{error}'.format(line=line, error=error))

    def _get_final_sstables(self, node, ks, table):
        """
        Return the node final sstable data files, excluding the temporary tables.
        If sstableutil exists (>= 3.0) then we rely on this tool since the table
        file names no longer contain tmp in their names (CASSANDRA-7066).
        """
        # Get all sstable data files
        allsstables = map(os.path.normcase, node.get_sstables(ks, table))

        # Remove any temporary files
        tool_bin = node.get_tool('sstableutil')
        if os.path.isfile(tool_bin):
            args = [tool_bin, '--type', 'tmp', ks, table]
            env = common.make_cassandra_env(node.get_install_cassandra_root(), node.get_node_cassandra_root())
            p = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (stdout, stderr) = p.communicate()
            tmpsstables = map(os.path.normcase, stdout.splitlines())

            ret = list(set(allsstables) - set(tmpsstables))
        else:
            ret = [sstable for sstable in allsstables if "tmp" not in sstable[50:]]

        return ret
<|endoftext|>


<commit_before># vim: ts=4 noet filetype=python
# This file is part of RCubic
#
# Copyright (c) 2012 Wireless Generation, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import os
import logging
import re
import fnmatch
import subprocess

from RCubic import exectree
from RCubic.RCubicUtilities import ConfigurationError

class RCubicScript(object):
    def __init__(self, filepath, version, override, phase, logdir, whitelist, blacklist, regexval, group):
        self.path = filepath
        self.name = filepath.split("/")[-1]
        self.version = version
        self.override = override
        self.logfile = "{0}/{1}.log".format(logdir, self.name)

        with open(self.path) as fd:
            script = fd.read()
        self.hdep = self._param_split(self._get_param(script, "HDEP"))
        self.sdep = self._param_split(self._get_param(script, "SDEP"))
        self.cdep = self._param_split(self._get_param(script, "CDEP"))
        self.idep = self._get_param(script, "IDEP", None)
        self.iterator = self._param_split(self._get_param(script, "ITER"))
        self.resources = self._param_split(self._get_param(script, "RESOURCES"))
        self.resources.append("default")
        self.products = self._param_split(self._get_param(script, "PRODUCT"))
        sphase = self._param_split(self._get_param(script, "PHASE"))
        self.group = group
        if regexval is None:
            self.regexval = True
        else:
            r = regexval.search(script)
            self.regexval = r is not None
        self.href = ""

        if len(sphase) >= 1:
            phase = RCubicScriptParser.PHASES[sphase[0]]
        self.phase = phase

        if len(blacklist) > 0 and self.name in blacklist:
            self.path = "-"
        elif len(whitelist) > 0 and self.name not in whitelist:
            self.path = "-"

    def _get_param(self, script, field, default=None):
        fieldre = re.compile(r"^[\s]*#%s:.*$" % (field), re.MULTILINE)
        begin = re.compile(r"^#[A-Z0-9]+:[\s]*")
        line = fieldre.search(script)
        return begin.sub("", line.group(0), 1) if line else default

    def _param_split(self, param):
        # Warning: code change not covered in tests
        # Split output by delimiters, eliminate empty strings
        separator = re.compile(r"[,;\s]+")
        return filter(None, separator.split(param)) if param else []

    def _parseHeaderLine(self, line):
        # Warning: code change not covered in tests
        # Split output by delimiters, eliminate empty strings
        begin = re.compile(r"^#[A-Z0-9]+:[\s]*")
        separator = re.compile(r"[,;\s]+")
        return filter(None, separator.split(begin.sub("", line, 1)))

class RCubicGroup(object):

    def __init__(self, element):
        try:
            self.version = element.attrib["version"]
            self.name = element.attrib["group"]
        except KeyError:
            raise ConfigurationError(
                    "Element on line %i of %s is missing version or group attributes."
                    % (element.sourceline, element.base)
            )

        try:
            self.phase = RCubicScriptParser.PHASES[
                    element.attrib.get("phase", "DEFAULT").upper()
            ]
        except:
            raise ConfigurationError(
                    "Attribute phase on line %i of %s has unrecognized value: '%s'."
                    % (element.sourceline, element.base, self.phase)
            )

        def booler(element, attrib, default):
            value = element.attrib.get(attrib, default).lower()
            if value not in ["true", "false"]:
                raise ConfigurationError(
                        "Attribute {0} is not (true|false) on line {1} of {2}."
                        .format(attrib, element.sourceline, element.base)
                )
            return value == "true"

        self.autoselect = booler(element, "autoSelect", "true")
        self.fulloverride = booler(element, "fullOverride", "false")
        self.forceselect = False
        self.scripts = []

    def __str__(self):
        return self.name

    def is_success(self):
        return all(script.job.is_success() for script in self.scripts)

    def is_done(self):
        return all(script.job.is_done() for script in self.scripts)

    def add_script(self, rs, override=False):
        if override:
            logging.debug("{0} is being overriden by {1}".format(rs.name, rs.path))
            for script in self.scripts:
                if script.name == rs.name:
                    self.scripts.remove(script)
                    break
        self.scripts.append(rs)

class RCubicScriptParser(object):
    PHASES = {"DEFAULT": 0, "EARLY": -1, "LATE": 1}

    def __init__(self, groups, logdir, workdir, whitelist, blacklist, regexval, resources):
        self.groups = groups
        self.logdir = logdir
        self.workdir = workdir
        if blacklist and whitelist:
            logging.warning("Conflicting whitelist/blacklist option. Ignoring blacklist.")
            blacklist = []
        elif blacklist is None:
            blacklist = []
        elif whitelist is None:
            whitelist = []
        self.blacklist = blacklist
        self.whitelist = whitelist
        if regexval is not None:
            regexval = re.compile(regexval, re.MULTILINE)
        self.regexval = regexval
        self.resources = resources
        self.unusedresources = []
        self.tree = None
        self.subtrees = {}

    def scripts(self):
        return [script for group in self.groups for script in group.scripts]

    def read_dirs(self, directory, override=False):
        failed_groups = []
        for group in self.groups:
            groupdir = "{0}/{1}".format(directory, group)
            logging.debug("processing group {0} {1} {2}.".format(group.name, groupdir, override))
            if not override:
                if not os.path.exists(groupdir):
                    failed_groups.append(group)
                    continue
                if group.fulloverride:
                    continue
            else:
                if not os.path.exists(groupdir):
                    continue
            for filename in os.listdir(groupdir):
                filepath = "{0}/{1}".format(groupdir, filename)
                if filename.startswith("{0}_".format(group)):
                    rs = RCubicScript(
                            filepath,
                            group.version,
                            override,
                            group.phase,
                            self.logdir,
                            self.whitelist,
                            self.blacklist,
                            self.regexval,
                            group,
                    )
                    group.add_script(rs, override)
                else:
                    logging.debug(
                                    "Skipping {0}/{1}, does not start with {2}_."
                                    .format(filepath, group)
                            )

    def _glob_expand(self, deps):
        rval = []
        for dep in deps:
            matched = False
            for script in self.scripts():
                if fnmatch.fnmatchcase(script.name, dep):
                    # we return script names instead of job instances to let
                    # ExecTree handle dangling deps
                    rval.append(script.name)
                    matched = True
            if not matched:
                rval.append(dep)
        return rval

    def eval_args(self, script):
        logging.debug("iterator: {0}, cwd: {1}".format(script.iterator, self.workdir))
        with open("/dev/null", "w") as devnull:
            if hasattr(subprocess, "check_output"):
                output = subprocess.check_output(script.iterator, stderr=devnull, cwd=self.workdir)
            else:
                p = subprocess.Popen(script.iterator, stdout=subprocess.PIPE, stderr=devnull, cwd=self.workdir)
                output = p.communicate()[0]

        # Split output by delimiters, eliminate empty strings
        separator = re.compile(r"[,;\s]+")
        args = filter(None, separator.split(output))

        logging.debug("Arguments {0}".format(args))
        return args

    def set_href(self, gerrit, project, githash, repopath):
        logging.debug("set hrefs")
        for script in self.scripts():
            script.href = "{0}/gitweb?p={1};a=blob;f={2};hb={3}".format(
                    gerrit, project, script.path[len(repopath) + 1:], githash
            )

    def init_tree(self):
        self.tree = exectree.ExecTree()
        self.tree.cwd = self.workdir
        self.tree.name = "rcubic"

        # Initialize all sub trees
        for script in self.scripts():
            if len(script.iterator) > 0:
                tree = exectree.ExecTree()
                tree.cwd = self.workdir
                tree.name = script.name
                tree.iterator = exectree.ExecIter(
                        "{0}_iter".format(script.name),
                        self.eval_args(script)
                )
                self.subtrees[script.name] = tree

        # Initialize Resources
        for resource, limit in self.resources.items():
            exectree.ExecResource(self.tree, resource, limit)

        # Initialize jobs and add to trees
        for script in self.scripts():
            script.job = exectree.ExecJob(
                    script.name,
                    script.path,
                    logfile=script.logfile,
                    arguments=[script.version],
                    href=script.href
            )
            if script.name in self.subtrees:
                script.job.jobpath = None
                script.job.subtree = self.subtrees[script.name]
            if script.override:
                script.job.tcolor = "deepskyblue"
            for resource in script.resources:
                r = self.tree.find_resource(resource)
                if r is None:
                    if resource not in self.unusedresources:
                        self.unusedresources.append(resource)
                else:
                    script.job.resources.append(r)
            if script.idep is None:
                self.tree.add_job(script.job)
            else:
                # todo handle exception nicely
                self.subtrees[script.idep].add_job(script.job)
        # Check for undefined resources
        if len(self.unusedresources) > 0:
            logging.warning(
                    "Resources referenced but not defined: {0}."
                    .format(", ".join(self.unusedresources))
            )

        # Initialize and set up dependencies
        for script in self.scripts():
            logging.debug("proccessing script: {0}".format(script.name))
            if script.idep is None:
                tree = self.tree
            else:
                tree = self.subtrees[script.idep]

            for dep in self._glob_expand(script.hdep):
                try:
                    d = tree.add_dep(dep, script.job)
                except exectree.JobUndefinedError:
                    if script.job.is_defined():
                        raise
                    dep = exectree.ExecJob(dep, "-", mustcomplete=False)
                    tree.add_job(dep)
                    d = tree.add_dep(dep, script.job)
                d.color = {"defined": "deepskyblue", "undefined": "red"}
            for dep in self._glob_expand(script.sdep):
                try:
                    d = tree.add_dep(dep, script.job)
                except exectree.JobUndefinedError:
                    dep = exectree.ExecJob(dep, "-", mustcomplete=False)
                    tree.add_job(dep)
                    d = tree.add_dep(dep, script.job)
                d.color = {"defined": "lawngreen", "undefined": "palegreen"}
            for cdep in self._glob_expand(script.cdep):
                # logging.debug("adding dep to ")
                try:
                    d = tree.add_dep(script.job, cdep)
                except exectree.JobUndefinedError:
                    cdep = exectree.ExecJob(cdep, "-", mustcomplete=False)
                    tree.add_job(cdep)
                    d = tree.add_dep(script.job, cdep)
                d.color = {"defined": "lawngreen", "undefined": "palegreen"}
            # stems = self.tree.stems()
            for pdep in self.scripts():
                # if pdep.phase < script.phase and pdep.job in stems:
                if pdep.phase < script.phase and pdep.idep is None and script.idep is None:
                    d = tree.add_dep(pdep.job, script.job)
                    if d is None:
                        continue
                    d.color = {"defined": "gold2", "undefined": "gold2"}
        # logging.debug("tree:\n{0}".format(etree.tostring(self.tree.xml(), pretty_print=True)))
        return self.tree
<commit_msg>RCubicScript.py: use variable initialization with default<commit_after># vim: ts=4 noet filetype=python
# This file is part of RCubic
#
# Copyright (c) 2012 Wireless Generation, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import os
import logging
import re
import fnmatch
import subprocess

from RCubic import exectree
from RCubic.RCubicUtilities import ConfigurationError

class RCubicScript(object):
    def __init__(self, filepath, version, override, phase, logdir, whitelist, blacklist, regexval, group):
        self.path = filepath
        self.name = filepath.split("/")[-1]
        self.version = version
        self.override = override
        self.logfile = "{0}/{1}.log".format(logdir, self.name)

        with open(self.path) as fd:
            script = fd.read()
        self.hdep = self._param_split(self._get_param(script, "HDEP"))
        self.sdep = self._param_split(self._get_param(script, "SDEP"))
        self.cdep = self._param_split(self._get_param(script, "CDEP"))
        self.idep = self._get_param(script, "IDEP", None)
        self.iterator = self._param_split(self._get_param(script, "ITER"))
        self.resources = self._param_split(self._get_param(script, "RESOURCES"))
        self.resources.append("default")
        self.products = self._param_split(self._get_param(script, "PRODUCT"))
        sphase = self._param_split(self._get_param(script, "PHASE"))
        self.group = group
        if regexval is None:
            self.regexval = True
        else:
            r = regexval.search(script)
            self.regexval = r is not None
        self.href = ""

        if len(sphase) >= 1:
            phase = RCubicScriptParser.PHASES[sphase[0]]
        self.phase = phase

        if len(blacklist) > 0 and self.name in blacklist:
            self.path = "-"
        elif len(whitelist) > 0 and self.name not in whitelist:
            self.path = "-"

    def _get_param(self, script, field, default=None):
        fieldre = re.compile(r"^[\s]*#%s:.*$" % (field), re.MULTILINE)
        begin = re.compile(r"^#[A-Z0-9]+:[\s]*")
        line = fieldre.search(script)
        return begin.sub("", line.group(0), 1) if line else default

    def _param_split(self, param):
        # Warning: code change not covered in tests
        # Split output by delimiters, eliminate empty strings
        separator = re.compile(r"[,;\s]+")
        return filter(None, separator.split(param)) if param else []

    def _parseHeaderLine(self, line):
        # Warning: code change not covered in tests
        # Split output by delimiters, eliminate empty strings
        begin = re.compile(r"^#[A-Z0-9]+:[\s]*")
        separator = re.compile(r"[,;\s]+")
        return filter(None, separator.split(begin.sub("", line, 1)))

class RCubicGroup(object):

    def __init__(self, element):
        try:
            self.version = element.attrib["version"]
            self.name = element.attrib["group"]
        except KeyError:
            raise ConfigurationError(
                    "Element on line %i of %s is missing version or group attributes."
                    % (element.sourceline, element.base)
            )

        try:
            self.phase = RCubicScriptParser.PHASES[
                    element.attrib.get("phase", "DEFAULT").upper()
            ]
        except:
            raise ConfigurationError(
                    "Attribute phase on line %i of %s has unrecognized value: '%s'."
                    % (element.sourceline, element.base, self.phase)
            )

        def booler(element, attrib, default):
            value = element.attrib.get(attrib, default).lower()
            if value not in ["true", "false"]:
                raise ConfigurationError(
                        "Attribute {0} is not (true|false) on line {1} of {2}."
                        .format(attrib, element.sourceline, element.base)
                )
            return value == "true"

        self.autoselect = booler(element, "autoSelect", "true")
        self.fulloverride = booler(element, "fullOverride", "false")
        self.forceselect = False
        self.scripts = []

    def __str__(self):
        return self.name

    def is_success(self):
        return all(script.job.is_success() for script in self.scripts)

    def is_done(self):
        return all(script.job.is_done() for script in self.scripts)

    def add_script(self, rs, override=False):
        if override:
            logging.debug("{0} is being overriden by {1}".format(rs.name, rs.path))
            for script in self.scripts:
                if script.name == rs.name:
                    self.scripts.remove(script)
                    break
        self.scripts.append(rs)

class RCubicScriptParser(object):
    PHASES = {"DEFAULT": 0, "EARLY": -1, "LATE": 1}

    def __init__(self, groups, logdir, workdir, whitelist, blacklist, regexval, resources):
        self.groups = groups
        self.logdir = logdir
        self.workdir = workdir
        if blacklist and whitelist:
            logging.warning("Conflicting whitelist/blacklist option. Ignoring blacklist.")
            blacklist = []
        self.blacklist = blacklist or []
        self.whitelist = whitelist or []
        self.regexval = re.compile(regexval, re.MULTILINE) if regexval else None
        self.resources = resources
        self.unusedresources = []
        self.tree = None
        self.subtrees = {}

    def scripts(self):
        return [script for group in self.groups for script in group.scripts]

    def read_dirs(self, directory, override=False):
        failed_groups = []
        for group in self.groups:
            groupdir = "{0}/{1}".format(directory, group)
            logging.debug("processing group {0} {1} {2}.".format(group.name, groupdir, override))
            if not override:
                if not os.path.exists(groupdir):
                    failed_groups.append(group)
                    continue
                if group.fulloverride:
                    continue
            else:
                if not os.path.exists(groupdir):
                    continue
            for filename in os.listdir(groupdir):
                filepath = "{0}/{1}".format(groupdir, filename)
                if filename.startswith("{0}_".format(group)):
                    rs = RCubicScript(
                            filepath,
                            group.version,
                            override,
                            group.phase,
                            self.logdir,
                            self.whitelist,
                            self.blacklist,
                            self.regexval,
                            group,
                    )
                    group.add_script(rs, override)
                else:
                    logging.debug(
                                    "Skipping {0}/{1}, does not start with {2}_."
                                    .format(filepath, group)
                            )

    def _glob_expand(self, deps):
        rval = []
        for dep in deps:
            matched = False
            for script in self.scripts():
                if fnmatch.fnmatchcase(script.name, dep):
                    # we return script names instead of job instances to let
                    # ExecTree handle dangling deps
                    rval.append(script.name)
                    matched = True
            if not matched:
                rval.append(dep)
        return rval

    def eval_args(self, script):
        logging.debug("iterator: {0}, cwd: {1}".format(script.iterator, self.workdir))
        with open("/dev/null", "w") as devnull:
            if hasattr(subprocess, "check_output"):
                output = subprocess.check_output(script.iterator, stderr=devnull, cwd=self.workdir)
            else:
                p = subprocess.Popen(script.iterator, stdout=subprocess.PIPE, stderr=devnull, cwd=self.workdir)
                output = p.communicate()[0]

        # Split output by delimiters, eliminate empty strings
        separator = re.compile(r"[,;\s]+")
        args = filter(None, separator.split(output))

        logging.debug("Arguments {0}".format(args))
        return args

    def set_href(self, gerrit, project, githash, repopath):
        logging.debug("set hrefs")
        for script in self.scripts():
            script.href = "{0}/gitweb?p={1};a=blob;f={2};hb={3}".format(
                    gerrit, project, script.path[len(repopath) + 1:], githash
            )

    def init_tree(self):
        self.tree = exectree.ExecTree()
        self.tree.cwd = self.workdir
        self.tree.name = "rcubic"

        # Initialize all sub trees
        for script in self.scripts():
            if len(script.iterator) > 0:
                tree = exectree.ExecTree()
                tree.cwd = self.workdir
                tree.name = script.name
                tree.iterator = exectree.ExecIter(
                        "{0}_iter".format(script.name),
                        self.eval_args(script)
                )
                self.subtrees[script.name] = tree

        # Initialize Resources
        for resource, limit in self.resources.items():
            exectree.ExecResource(self.tree, resource, limit)

        # Initialize jobs and add to trees
        for script in self.scripts():
            script.job = exectree.ExecJob(
                    script.name,
                    script.path,
                    logfile=script.logfile,
                    arguments=[script.version],
                    href=script.href
            )
            if script.name in self.subtrees:
                script.job.jobpath = None
                script.job.subtree = self.subtrees[script.name]
            if script.override:
                script.job.tcolor = "deepskyblue"
            for resource in script.resources:
                r = self.tree.find_resource(resource)
                if r is None:
                    if resource not in self.unusedresources:
                        self.unusedresources.append(resource)
                else:
                    script.job.resources.append(r)
            if script.idep is None:
                self.tree.add_job(script.job)
            else:
                # todo handle exception nicely
                self.subtrees[script.idep].add_job(script.job)
        # Check for undefined resources
        if len(self.unusedresources) > 0:
            logging.warning(
                    "Resources referenced but not defined: {0}."
                    .format(", ".join(self.unusedresources))
            )

        # Initialize and set up dependencies
        for script in self.scripts():
            logging.debug("proccessing script: {0}".format(script.name))
            if script.idep is None:
                tree = self.tree
            else:
                tree = self.subtrees[script.idep]

            for dep in self._glob_expand(script.hdep):
                try:
                    d = tree.add_dep(dep, script.job)
                except exectree.JobUndefinedError:
                    if script.job.is_defined():
                        raise
                    dep = exectree.ExecJob(dep, "-", mustcomplete=False)
                    tree.add_job(dep)
                    d = tree.add_dep(dep, script.job)
                d.color = {"defined": "deepskyblue", "undefined": "red"}
            for dep in self._glob_expand(script.sdep):
                try:
                    d = tree.add_dep(dep, script.job)
                except exectree.JobUndefinedError:
                    dep = exectree.ExecJob(dep, "-", mustcomplete=False)
                    tree.add_job(dep)
                    d = tree.add_dep(dep, script.job)
                d.color = {"defined": "lawngreen", "undefined": "palegreen"}
            for cdep in self._glob_expand(script.cdep):
                # logging.debug("adding dep to ")
                try:
                    d = tree.add_dep(script.job, cdep)
                except exectree.JobUndefinedError:
                    cdep = exectree.ExecJob(cdep, "-", mustcomplete=False)
                    tree.add_job(cdep)
                    d = tree.add_dep(script.job, cdep)
                d.color = {"defined": "lawngreen", "undefined": "palegreen"}
            # stems = self.tree.stems()
            for pdep in self.scripts():
                # if pdep.phase < script.phase and pdep.job in stems:
                if pdep.phase < script.phase and pdep.idep is None and script.idep is None:
                    d = tree.add_dep(pdep.job, script.job)
                    if d is None:
                        continue
                    d.color = {"defined": "gold2", "undefined": "gold2"}
        # logging.debug("tree:\n{0}".format(etree.tostring(self.tree.xml(), pretty_print=True)))
        return self.tree
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

# Site information
AUTHOR = 'Scott Berrevoets'
SITENAME = 'Scott Berrevoets'
SITEURL = 'http://scottberrevoets.com'

# Show line numbers in code snippets
MD_EXTENSIONS = ['codehilite(css_class=highlight, linenums = True)', 'extra']

# Path specifications
RELATIVE_URLS = True
PATH = 'content'
THEME = './theme'

ARTICLE_URL = '{date:%Y}/{date:%m}/{date:%d}/{slug}/'
ARTICLE_SAVE_AS = '{date:%Y}/{date:%m}/{date:%d}/{slug}/index.html'

PAGE_URL = '{slug}.html'
PAGE_SAVE_AS = '{slug}.html'

# Pagination
DEFAULT_PAGINATION = 5
PAGINATION_PATTERNS = (
    (1, '{base_name}/', '{base_name}/index.html'),
    (2, '{base_name}/page/{number}/', '{base_name}/page/{number}/index.html'),
)

# Create and link to index and archive
DIRECT_TEMPLATES = ('index', 'archive')
MENUITEMS = [('Blog', '/'), ('Archive', '/archive.html')]

# Disable authors, categories, and tags
AUTHOR_SAVE_AS = False
CATEGORY_SAVE_AS = False
TAG_SAVE_AS = False

# Locale settings
DEFAULT_LANG = 'en'
TIMEZONE = 'America/Los_Angeles'
DATE_FORMATS = { 'en': '%d %B %Y' }

# Social media
DISPLAY_SOCIAL_MEDIA = True
SOCIALMEDIAITEMS = [('twitter.svg', 'https://twitter.com/ScottBerrevoets'),
                    ('github.svg', 'https://github.com/sberrevoets'),
                    ('stackoverflow.svg', 'http://stackoverflow.com/users/751268/scott-berrevoets'),
                    ('youtube.svg', 'https://www.youtube.com/watch?v=dQw4w9WgXcQ')]

# No feeds in development mode
FEED_DOMAIN = SITEURL

FEED_ATOM = None
FEED_RSS = None
FEED_ALL_ATOM = None
FEED_ALL_RSS = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

<commit_msg>Change MD_EXTENSIONS to MARKDOWN<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

# Site information
AUTHOR = 'Scott Berrevoets'
SITENAME = 'Scott Berrevoets'
SITEURL = 'http://scottberrevoets.com'

# Show line numbers in code snippets
MARKDOWN = {
  'extension_configs': {
    'markdown.extensions.codehilite': {'css_class': 'highlight', 'linenums': True},
    'markdown.extensions.extra': {},
    'markdown.extensions.meta': {},
  },
  'output_format': 'html5',
}

# Path specifications
RELATIVE_URLS = True
PATH = 'content'
THEME = './theme'

ARTICLE_URL = '{date:%Y}/{date:%m}/{date:%d}/{slug}/'
ARTICLE_SAVE_AS = '{date:%Y}/{date:%m}/{date:%d}/{slug}/index.html'

PAGE_URL = '{slug}.html'
PAGE_SAVE_AS = '{slug}.html'

# Pagination
DEFAULT_PAGINATION = 5
PAGINATION_PATTERNS = (
    (1, '{base_name}/', '{base_name}/index.html'),
    (2, '{base_name}/page/{number}/', '{base_name}/page/{number}/index.html'),
)

# Create and link to index and archive
DIRECT_TEMPLATES = ('index', 'archive')
MENUITEMS = [('Blog', '/'), ('Archive', '/archive.html')]

# Disable authors, categories, and tags
AUTHOR_SAVE_AS = False
CATEGORY_SAVE_AS = False
TAG_SAVE_AS = False

# Locale settings
DEFAULT_LANG = 'en'
TIMEZONE = 'America/Los_Angeles'
DATE_FORMATS = { 'en': '%d %B %Y' }

# Social media
DISPLAY_SOCIAL_MEDIA = True
SOCIALMEDIAITEMS = [('twitter.svg', 'https://twitter.com/ScottBerrevoets'),
                    ('github.svg', 'https://github.com/sberrevoets'),
                    ('stackoverflow.svg', 'http://stackoverflow.com/users/751268/scott-berrevoets'),
                    ('youtube.svg', 'https://www.youtube.com/watch?v=dQw4w9WgXcQ')]

# No feeds in development mode
FEED_DOMAIN = SITEURL

FEED_ATOM = None
FEED_RSS = None
FEED_ALL_ATOM = None
FEED_ALL_RSS = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

<|endoftext|>


<commit_before>RECORD_HTTP_TRANSACTIONS = False

RAW_PROCESSING = ['cassandra']
NORMALIZED_PROCESSING = ['elasticsearch', 'cassandra']
<commit_msg>Update local-dist.py<commit_after>RAW_PROCESSING = ['cassandra']
NORMALIZED_PROCESSING = ['elasticsearch', 'cassandra']
<|endoftext|>


<commit_before>#room base
class Room():
    '''
    models abstract data type Room, not intended to be instanciated
    '''
    #keep track of all rooms created
    number_of_rooms = 0

    def __init__(self, max_occupants, name):
        '''
        return Room with no occupats
        '''
        self.__name = name
        self.__max_occupants = max_occupants
        self.__occupants = []
        self.__id = Room.number_of_rooms
        Room.number_of_rooms += 1

    @property
    def id(self):
        return __id

    @property
    def name(self):
        '''
        property name, return name for room
        '''
        return self.__name

    @name.setter
    def name(self, new_name):
        '''
        setter: set's name for room
        '''
        self.__name = new_name

    @property
    def max_occupants(self):
        return self.__max_occupants

    @property
    def current_population(self):
        '''
        return number of ppl in room
        '''
        return len(self.__occupants)

    def is_full(self):
        '''
        return True if room is full, else false
        '''
        return len(self.__occupants) == Room.max_occupants

    def is_in_room(self, person):
        '''
        returns true if person in room_name
        '''
        return person in self.__occupants

    def add_occupant(self, person):
        '''
        adds person to room_name
        '''
        if self.current_population == self.max_occupants:
            pass
            #raise full error
        if not self.is_in_room(person):
            self.__occupants.append(person)

        #raise some error

    def remove_occupant(self, person):
        '''
        removes person from room
        '''
        if self.is_in_room(person):
            del self.__occupants[self.__occupants.index(person)]
        #raise some error
    def get_occupants(self):
        '''
        returns a generator with all occupants
        '''
        occupants = self.__occupants[:]
        def occupants():
            for person in occupants:
                yield person
        return occupants()

#create a Office
class Office(Room):
    #number of offices a
    number_of_offices = 0

    max_occupants = 6
    def __init__(self, name):
        Room.__init__(self, Office.max_occupants, name)
        Office.number_of_offices += 1

#create a LivingSpace
class LivingSpace(Room):
    #number of LivingSpaces
    number_of_livingspace = 0

    max_occupants = 4
    def __init__(self, name):
        Room.__init__(self, LivingSpace.max_occupants, name)
        LivingSpace.number_of_livingspace += 1
<commit_msg>Feat: add properties to Felllow and Staff<commit_after>#room base
class Room():
    '''
    models abstract data type Room, not intended to be instanciated
    '''
    #keep track of all rooms created
    number_of_rooms = 0

    def __init__(self, max_occupants, name):
        '''
        return Room with no occupats
        '''
        Room.number_of_rooms += 1
        self.__name = name
        self.__max_occupants = max_occupants
        self.__occupants = []
        self.__id = Room.number_of_rooms

    @property
    def id(self):
        '''
        return id, unique identifier for room
        '''
        return self.__id

    @property
    def name(self):
        '''
        property name, return name for room
        '''
        return self.__name

    @name.setter
    def name(self, new_name):
        '''
        setter: set's name for room
        '''
        self.__name = new_name

    @property
    def max_occupants(self):
        return self.__max_occupants

    @property
    def current_population(self):
        '''
        return number of ppl in room
        '''
        return len(self.__occupants)

    def is_full(self):
        '''
        return True if room is full, else false
        '''
        return len(self.__occupants) == Room.max_occupants

    def is_in_room(self, person):
        '''
        returns true if person in room_name
        '''
        return person in self.__occupants

    def add_occupant(self, person):
        '''
        adds person to room_name
        '''
        if self.current_population == self.max_occupants:
            pass
            #raise full error
        if not self.is_in_room(person):
            self.__occupants.append(person)

        #raise some error

    def remove_occupant(self, person):
        '''
        removes person from room
        '''
        if self.is_in_room(person):
            del self.__occupants[self.__occupants.index(person)]
        #raise some error
    def get_occupants(self):
        '''
        returns a generator with all occupants
        '''
        occupants = self.__occupants[:]
        def occupants():
            for person in occupants:
                yield person
        return occupants()

#create a Office
class Office(Room):
    #number of offices a
    number_of_offices = 0

    max_occupants = 6
    def __init__(self, name):
        Office.number_of_offices += 1
        Room.__init__(self, Office.max_occupants, name)

#create a LivingSpace
class LivingSpace(Room):
    #number of LivingSpaces
    number_of_livingspace = 0

    max_occupants = 4
    def __init__(self, name):
        LivingSpace.number_of_livingspace += 1
        Room.__init__(self, LivingSpace.max_occupants, name)

################################################################################
class Person():
    #number of Person
    number_of_person = 0

    def __init__(self, name):
        Person.number_of_person += 1
        self.__id = Person.number_of_person
        self.__name = name
        self.__office = None

    @property
    def name(self):
        return self.__name
    @name.setter
    def name(self, new_name):
        self.__name = new_name

    @property
    def id(self):
        return self.__id

    @property
    def office(self):
        return self.__office

    @office.setter
    def office(self, office):
        self.__office = office

    def remove_office(self):
        self.office = None

    def is_allocated_office(self):
        return not (not self.office)

class Fellow(Person):
    #number of fellows
    number_of_fellows = 0

    def __init__(self, name, wants_living = False):
        Fellow.number_of_fellows += 1
        Person.__init__(self, name)
        self.__wants_living = wants_living
        self.__livingspace = None

    def is_allocated_living(self):
        return not not self.__livingspace

    @property
    def wants_living(self):
        return self.__wants_living
    @property
    def livingspace(self):
        return self.__livingspace
    @livingspace.setter
    def livingspace(self, space):
        if self.__wants_living:
            self.__livingspace = space
        #throw value error
    def remove_livingspace(self):
        self.__livingspace = None

class Staff(Person):
    #number of Staff
    number_of_staff = 0
    def __init__(self, name):
        Staff.number_of_staff += 1
        Person.__init__(self, name)
<|endoftext|>


<commit_before>"""
This file is part of Arakoon, a distributed key-value store. Copyright
(C) 2010 Incubaid BVBA

Licensees holding a valid Incubaid license may use this file in
accordance with Incubaid's Arakoon commercial license agreement. For
more information on how to enter into this agreement, please contact
Incubaid (contact details can be found on www.arakoon.org/licensing).

Alternatively, this file may be redistributed and/or modified under
the terms of the GNU Affero General Public License version 3, as
published by the Free Software Foundation. Under this license, this
file is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.

See the GNU Affero General Public License for more details.
You should have received a copy of the
GNU Affero General Public License along with this program (file "COPYING").
If not, see <http://www.gnu.org/licenses/>.
"""

from pymonkey import q

class ArakoonConfig:
    """
    Class for automated setup of Arakoon nodes
    """
    def addNode(self, name, ip = "127.0.0.1", client_port = 7080, messaging_port = 10000,
                log_level = "info",
                log_dir = None,
                home = None,
                tlog_dir = None):
        """
        Add a node to the configuration

        The function also creates

        @param name the name of the node, should be unique across the environment
        @param ip the ip this node shoulc be contacted on
        @param client_port the port the clients should use to contact this node
        @param messaging_port the port the other nodes should use to contact this node
        @param log_level the loglevel (debug info notice warning error fatal)
        @param log_dir the directory used for logging
        @param home the directory used for the nodes data
        @param tlog_dir the directory used for tlogs (if none, home will be used)
        """
        self.__validateName(name)
        self.__validateLogLevel(log_level)

        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            config.addSection("global")
            config.addParam("global","nodes", "")

        nodes = self.__getNodes(config)

        if name in nodes:
            raise Exception("There is already a node with name %s configured" % name)

        nodes.append(name)
        config.addSection(name)
        config.addParam(name, "name", name)
        config.addParam(name, "ip", ip)
        config.addParam(name, "client_port", client_port)
        config.addParam(name, "messaging_port", messaging_port)
        config.addParam(name, "log_level", log_level)

        if log_dir is None:
            log_dir = q.system.fs.joinPaths(q.dirs.logDir, "arakoon", name)
        config.addParam(name, "log_dir", log_dir)

        if home is None:
            home = q.system.fs.joinPaths(q.dirs.varDir, "db", "arakoon", name)
        config.addParam(name, "home", home)

        if tlog_dir:
            confid.addParam(name,"tlog_dir", tlog_dir)

        config.setParam("global","nodes", ",".join(nodes))

        config.write()

    def removeNode(self, name):
        """
        Remove a node from the configuration

        @param name the name of the node as configured in the config file
        """
        self.__validateName(name)

        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            raise Exception("No node with name %s configured" % name)

        nodes = self.__getNodes(config)

        if name in nodes:
            self.removeLocalNode(name)

            config.removeSection(name)

            nodes.remove(name)
            config.setParam("global","nodes", ",".join(nodes))

            config.write()
            return

        raise Exception("No node with name %s configured" % name)

    def setMasterLease(self, duration=None):
        """
        Set the master lease duration

        @param duration The duration of the master lease in seconds
        """
        section = "global"
        key = "lease_expiry"

        config = q.config.getInifile("arakoon")

        if not config.checkSection( section ):
            raise Exception("Section '%s' not found in arakoon config" % section )

        if duration is not None:
            if not isinstance( duration, int ) :
                raise AttributeError( "Invalid value for lease duration (expected int, got: '%s')" % duration)
            if config.checkParam(section, key):
                config.setParam(section, key, duration)
            else:
                config.addParam(section, key, duration)
        else:
            config.removeParam(section, key)

        config.write()

    def forceMaster(self, name=None):
        """
        Force a master

        @param name the name of the master to force. If None there is no longer a forced master
        """
        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            raise Exception("No node with name %s configured" % name)

        if name:
            nodes = self.__getNodes(config)

            self.__validateName(name)
            if not name in nodes:
                raise Exception("No node with name %s configured" % name)

            if config.checkParam("global", "master"):
                config.setParam("global", "master", name)
            else:
                config.addParam("global", "master", name)
        else:
            config.removeParam("global", "master")

        config.write()

    def setQuorum(self, quorum=None):
        """
        Set the quorom

        The quorom dictates on how many nodes need to acknowlegde the new value beforce it becomes accepted.
        The default is (nodes/2)+1

        @param quorm the forced quorom. If None, the default is used
        """
        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            config.addSection("global")
            config.addParam("global","nodes", "")

        if quorum:
            try :
                if ( int(quorum) != quorum or quorum < 0 or quorum > len( self.listNodes())) :
                    raise Exception ( "Illegal value for quorum" )

            except:
                raise Exception( "Illegal value for quorum" )

            if config.checkParam("global", "quorum"):
                config.setParam("global", "quorum", int(quorum))
            else:
                config.addParam("global", "quorum", int(quorum))
        else:
            config.removeParam("global", "quorum")

        config.write()

    def getClientConfig(self):
        """
        Get an object that contains all node information

        @return dict the dict can be used as param for the ArakoonConfig object
        """
        config = q.config.getInifile("arakoon")

        clientconfig = dict()

        if config.checkSection("global"):
            nodes = self.__getNodes(config)

            for name in nodes:
                clientconfig[name] = (config.getValue(name, "ip"), int(config.getValue(name, "client_port")))

        return clientconfig

    def listNodes(self):
        """
        Get a list of all node names

        @return list of strings containing the node names
        """
        config = q.config.getInifile("arakoon")

        return self.__getNodes(config)

    def getNodeConfig(self, name):
        """
        Get the parameters of a node section

        @param name the name of the node

        @return dict keys and values of the nodes parameters
        """
        self.__validateName(name)

        config = q.config.getInifile("arakoon")

        nodes = self.__getNodes(config)

        if name in nodes:
            return config.getSectionAsDict(name)
        else:
            raise Exception("No node with name %s configured" % name)

    def createDirs(self, name):
        """
        Create the Directories for a local arakoon node

        @param name the name of the node as configured in the config file
        """
        self.__validateName(name)

        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            raise Exception("No node with name %s configured" % name)

        nodes = self.__getNodes(config)

        if name in nodes:
            home = config.getValue(name, "home")
            q.system.fs.createDir(home)

            if config.checkParam(name, "tlog_dir"):
                tlog_dir = config.getValue(name, "tlog_dir")
                q.system.fs.createDir(tlog_dir)

            log_dir = config.getValue(name, "log_dir")
            q.system.fs.createDir(log_dir)

            return

        raise Exception("No node with name %s configured" % name)

    def removeDirs(self, name):
        """
        Remove the Directories for a local arakoon node

        @param name the name of the node as configured in the config file
        """
        self.__validateName(name)

        config = q.config.getInifile('arakoon')

        if not config.checkSection('global'):
            raise Exception("No node with name %s configured" % name)

        nodes = self.__getNodes(config)

        if name in nodes:
            home = config.getValue(name, "home")
            q.system.fs.removeDirTree(home)

            if config.checkParam(name, "tlog_dir"):
                tlog_dir = config.getValue(name, "tlog_dir")
                q.system.fs.removeDirTree(tlog_dir)

            log_dir = config.getValue(name, "log_dir")
            q.system.fs.removeDirTree(log_dir)
            return

        raise Exception("No node with name %s configured" % name)

    def addLocalNode(self, name):
        """
        Add a node to the list of nodes that have to be started locally

        @param name the name of the node as configured in the config file
        """
        self.__validateName(name)

        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            raise Exception("No node with name %s configured" % name)

        nodes = self.__getNodes(config)

        if name in nodes:
            nodesconfig = q.config.getInifile("arakoonservernodes")

            if not nodesconfig.checkSection("global"):
                nodesconfig.addSection("global")
                nodesconfig.addParam("global","nodes", "")

            nodes = self.__getNodes(nodesconfig)
            if name in nodes:
                raise Exception("There is already a node with name %s configured" % name)
            nodes.append(name)
            nodesconfig.setParam("global","nodes", ",".join(nodes))

            nodesconfig.write()

            return

        raise Exception("No node with name %s configured" % name)

    def removeLocalNode(self, name):
        """
        Remove a node from the list of nodes that have to be started locally

        @param name the name of the node as configured in the config file
        """
        self.__validateName(name)

        config = q.config.getInifile("arakoonservernodes")

        if not config.checkSection("global"):
            return

        nodes = self.__getNodes(config)

        if name in nodes:
            nodes.remove(name)
            config.setParam("global","nodes", ",".join(nodes))
            config.write()

    def listLocalNodes(self):
        """
        Get a list of the local nodes

        @return list of strings containing the node names
        """
        config = q.config.getInifile("arakoonservernodes")

        return self.__getNodes(config)

    def setUp(self, number_of_nodes):
        """
        Sets up a local environment

        @param number_of_nodes the number of nodes in the environment

        @return the dict that can be used as a param for the ArakoonConfig object
        """
        for i in range(0, number_of_nodes):
            nodeName = "arakoon_%s" %i
            self.addNode(name=nodeName, client_port=7080+i,messaging_port=10000+i)
            self.addLocalNode(nodeName)
            self.createDirs(nodeName)

        if number_of_nodes > 0:
            self.forceMaster("arakoon_0")

    def tearDown(self, removeDirs=True):
        """
        Tears down a local environment

        @param removeDirs remove the log and home dir
        """
        config = q.config.getInifile('arakoon')

        if not config.checkSection('global'):
            return

        nodes = self.__getNodes(config)

        for node in nodes:
            if removeDirs:
                self.removeDirs(node)

            self.removeNode(node)

        if self.__getForcedMaster(config):
            self.forceMaster()

    def __getForcedMaster(self, config):
        if not config.checkSection("global"):
            return []

        if config.checkParam("global", "master"):
            return config.getValue("global", "master").strip()
        else:
            return []

    def __getNodes(self, config):
        if not config.checkSection("global"):
            return []

        nodes = config.getValue("global", "nodes").strip()
        # "".split(",") -> ['']
        if nodes == "":
            return []
        nodes = nodes.split(",")
        nodes = map(lambda x: x.strip(), nodes)

        return nodes

    def __validateName(self, name):
        if name is None:
            raise Exception("A name should be passed. None is not an option")

        if not type(name) == type(str()):
            raise Exception("Name should be of type string")

        for char in [' ', ',', '#']:
            if char in name:
                raise Exception("name should not contain %s" % char)

    def __validateLogLevel(self, name):
        if not name in ["info", "debug", "notice", "warning", "error", "fatal"]:
            raise Exception("%s is not a valid log level" % name)
<commit_msg>fixed several typos<commit_after>"""
This file is part of Arakoon, a distributed key-value store. Copyright
(C) 2010 Incubaid BVBA

Licensees holding a valid Incubaid license may use this file in
accordance with Incubaid's Arakoon commercial license agreement. For
more information on how to enter into this agreement, please contact
Incubaid (contact details can be found on www.arakoon.org/licensing).

Alternatively, this file may be redistributed and/or modified under
the terms of the GNU Affero General Public License version 3, as
published by the Free Software Foundation. Under this license, this
file is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.

See the GNU Affero General Public License for more details.
You should have received a copy of the
GNU Affero General Public License along with this program (file "COPYING").
If not, see <http://www.gnu.org/licenses/>.
"""

from pymonkey import q

class ArakoonConfig:
    """
    Class for automated setup of Arakoon nodes
    """
    def addNode(self, name, ip = "127.0.0.1", client_port = 7080, messaging_port = 10000,
                log_level = "info",
                log_dir = None,
                home = None,
                tlog_dir = None):
        """
        Add a node to the configuration

        The function also creates

        @param name the name of the node, should be unique across the environment
        @param ip the ip this node shoulc be contacted on
        @param client_port the port the clients should use to contact this node
        @param messaging_port the port the other nodes should use to contact this node
        @param log_level the loglevel (debug info notice warning error fatal)
        @param log_dir the directory used for logging
        @param home the directory used for the nodes data
        @param tlog_dir the directory used for tlogs (if none, home will be used)
        """
        self.__validateName(name)
        self.__validateLogLevel(log_level)

        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            config.addSection("global")
            config.addParam("global","nodes", "")

        nodes = self.__getNodes(config)

        if name in nodes:
            raise Exception("There is already a node with name %s configured" % name)

        nodes.append(name)
        config.addSection(name)
        config.addParam(name, "name", name)
        config.addParam(name, "ip", ip)
        config.addParam(name, "client_port", client_port)
        config.addParam(name, "messaging_port", messaging_port)
        config.addParam(name, "log_level", log_level)

        if log_dir is None:
            log_dir = q.system.fs.joinPaths(q.dirs.logDir, "arakoon", name)
        config.addParam(name, "log_dir", log_dir)

        if home is None:
            home = q.system.fs.joinPaths(q.dirs.varDir, "db", "arakoon", name)
        config.addParam(name, "home", home)

        if tlog_dir:
            config.addParam(name,"tlog_dir", tlog_dir)

        config.setParam("global","nodes", ",".join(nodes))

        config.write()

    def removeNode(self, name):
        """
        Remove a node from the configuration

        @param name the name of the node as configured in the config file
        """
        self.__validateName(name)

        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            raise Exception("No node with name %s configured" % name)

        nodes = self.__getNodes(config)

        if name in nodes:
            self.removeLocalNode(name)

            config.removeSection(name)

            nodes.remove(name)
            config.setParam("global","nodes", ",".join(nodes))

            config.write()
            return

        raise Exception("No node with name %s configured" % name)

    def setMasterLease(self, duration=None):
        """
        Set the master lease duration

        @param duration The duration of the master lease in seconds
        """
        section = "global"
        key = "lease_expiry"

        config = q.config.getInifile("arakoon")

        if not config.checkSection( section ):
            raise Exception("Section '%s' not found in arakoon config" % section )

        if duration is not None:
            if not isinstance( duration, int ) :
                raise AttributeError( "Invalid value for lease duration (expected int, got: '%s')" % duration)
            if config.checkParam(section, key):
                config.setParam(section, key, duration)
            else:
                config.addParam(section, key, duration)
        else:
            config.removeParam(section, key)

        config.write()

    def forceMaster(self, name=None):
        """
        Force a master

        @param name the name of the master to force. If None there is no longer a forced master
        """
        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            raise Exception("No node with name %s configured" % name)

        if name:
            nodes = self.__getNodes(config)

            self.__validateName(name)
            if not name in nodes:
                raise Exception("No node with name %s configured" % name)

            if config.checkParam("global", "master"):
                config.setParam("global", "master", name)
            else:
                config.addParam("global", "master", name)
        else:
            config.removeParam("global", "master")

        config.write()

    def setQuorum(self, quorum=None):
        """
        Set the quorom

        The quorom dictates on how many nodes need to acknowlegde the new value before it becomes accepted.
        The default is (nodes/2)+1

        @param quorum the forced quorom. If None, the default is used
        """
        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            config.addSection("global")
            config.addParam("global","nodes", "")

        if quorum:
            try :
                if ( int(quorum) != quorum or quorum < 0 or quorum > len( self.listNodes())) :
                    raise Exception ( "Illegal value for quorum" )

            except:
                raise Exception( "Illegal value for quorum" )

            if config.checkParam("global", "quorum"):
                config.setParam("global", "quorum", int(quorum))
            else:
                config.addParam("global", "quorum", int(quorum))
        else:
            config.removeParam("global", "quorum")

        config.write()

    def getClientConfig(self):
        """
        Get an object that contains all node information

        @return dict the dict can be used as param for the ArakoonConfig object
        """
        config = q.config.getInifile("arakoon")

        clientconfig = dict()

        if config.checkSection("global"):
            nodes = self.__getNodes(config)

            for name in nodes:
                clientconfig[name] = (config.getValue(name, "ip"), int(config.getValue(name, "client_port")))

        return clientconfig

    def listNodes(self):
        """
        Get a list of all node names

        @return list of strings containing the node names
        """
        config = q.config.getInifile("arakoon")

        return self.__getNodes(config)

    def getNodeConfig(self, name):
        """
        Get the parameters of a node section

        @param name the name of the node

        @return dict keys and values of the nodes parameters
        """
        self.__validateName(name)

        config = q.config.getInifile("arakoon")

        nodes = self.__getNodes(config)

        if name in nodes:
            return config.getSectionAsDict(name)
        else:
            raise Exception("No node with name %s configured" % name)

    def createDirs(self, name):
        """
        Create the Directories for a local arakoon node

        @param name the name of the node as configured in the config file
        """
        self.__validateName(name)

        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            raise Exception("No node with name %s configured" % name)

        nodes = self.__getNodes(config)

        if name in nodes:
            home = config.getValue(name, "home")
            q.system.fs.createDir(home)

            if config.checkParam(name, "tlog_dir"):
                tlog_dir = config.getValue(name, "tlog_dir")
                q.system.fs.createDir(tlog_dir)

            log_dir = config.getValue(name, "log_dir")
            q.system.fs.createDir(log_dir)

            return

        raise Exception("No node with name %s configured" % name)

    def removeDirs(self, name):
        """
        Remove the Directories for a local arakoon node

        @param name the name of the node as configured in the config file
        """
        self.__validateName(name)

        config = q.config.getInifile('arakoon')

        if not config.checkSection('global'):
            raise Exception("No node with name %s configured" % name)

        nodes = self.__getNodes(config)

        if name in nodes:
            home = config.getValue(name, "home")
            q.system.fs.removeDirTree(home)

            if config.checkParam(name, "tlog_dir"):
                tlog_dir = config.getValue(name, "tlog_dir")
                q.system.fs.removeDirTree(tlog_dir)

            log_dir = config.getValue(name, "log_dir")
            q.system.fs.removeDirTree(log_dir)
            return

        raise Exception("No node with name %s configured" % name)

    def addLocalNode(self, name):
        """
        Add a node to the list of nodes that have to be started locally

        @param name the name of the node as configured in the config file
        """
        self.__validateName(name)

        config = q.config.getInifile("arakoon")

        if not config.checkSection("global"):
            raise Exception("No node with name %s configured" % name)

        nodes = self.__getNodes(config)

        if name in nodes:
            nodesconfig = q.config.getInifile("arakoonservernodes")

            if not nodesconfig.checkSection("global"):
                nodesconfig.addSection("global")
                nodesconfig.addParam("global","nodes", "")

            nodes = self.__getNodes(nodesconfig)
            if name in nodes:
                raise Exception("There is already a node with name %s configured" % name)
            nodes.append(name)
            nodesconfig.setParam("global","nodes", ",".join(nodes))

            nodesconfig.write()

            return

        raise Exception("No node with name %s configured" % name)

    def removeLocalNode(self, name):
        """
        Remove a node from the list of nodes that have to be started locally

        @param name the name of the node as configured in the config file
        """
        self.__validateName(name)

        config = q.config.getInifile("arakoonservernodes")

        if not config.checkSection("global"):
            return

        nodes = self.__getNodes(config)

        if name in nodes:
            nodes.remove(name)
            config.setParam("global","nodes", ",".join(nodes))
            config.write()

    def listLocalNodes(self):
        """
        Get a list of the local nodes

        @return list of strings containing the node names
        """
        config = q.config.getInifile("arakoonservernodes")

        return self.__getNodes(config)

    def setUp(self, number_of_nodes):
        """
        Sets up a local environment

        @param number_of_nodes the number of nodes in the environment

        @return the dict that can be used as a param for the ArakoonConfig object
        """
        for i in range(0, number_of_nodes):
            nodeName = "arakoon_%s" %i
            self.addNode(name=nodeName, client_port=7080+i,messaging_port=10000+i)
            self.addLocalNode(nodeName)
            self.createDirs(nodeName)

        if number_of_nodes > 0:
            self.forceMaster("arakoon_0")

    def tearDown(self, removeDirs=True):
        """
        Tears down a local environment

        @param removeDirs remove the log and home dir
        """
        config = q.config.getInifile('arakoon')

        if not config.checkSection('global'):
            return

        nodes = self.__getNodes(config)

        for node in nodes:
            if removeDirs:
                self.removeDirs(node)

            self.removeNode(node)

        if self.__getForcedMaster(config):
            self.forceMaster()

    def __getForcedMaster(self, config):
        if not config.checkSection("global"):
            return []

        if config.checkParam("global", "master"):
            return config.getValue("global", "master").strip()
        else:
            return []

    def __getNodes(self, config):
        if not config.checkSection("global"):
            return []

        nodes = config.getValue("global", "nodes").strip()
        # "".split(",") -> ['']
        if nodes == "":
            return []
        nodes = nodes.split(",")
        nodes = map(lambda x: x.strip(), nodes)

        return nodes

    def __validateName(self, name):
        if name is None:
            raise Exception("A name should be passed. None is not an option")

        if not type(name) == type(str()):
            raise Exception("Name should be of type string")

        for char in [' ', ',', '#']:
            if char in name:
                raise Exception("name should not contain %s" % char)

    def __validateLogLevel(self, name):
        if not name in ["info", "debug", "notice", "warning", "error", "fatal"]:
            raise Exception("%s is not a valid log level" % name)
<|endoftext|>


<commit_before>#!/usr/bin/python env
#Write the install shell script

import string

def writeShell(s,config,src):
    """
     Write the part for installation of a script
    """
    script=string.Template("""#!/bin/bash
tmpdir=$$(pwd)
sed -n -e '1,/^exit 0$$/!p' $$0> "$${tmpdir}/${name}.tar.gz" 2>/dev/null
tar zxvf ${name}.tar.gz
rm ${name}.tar.gz
exit 0
""")
    src_script=string.Template("""#!/bin/bash
tmpdir=$$(pwd)/tmp_dir
mkdir $$tmpdir
sed -n -e '1,/^exit 0$$/!p' $$0> "$${tmpdir}/${name}.tar.gz" 2>/dev/null
cd $$tmpdir
tar zxvf ${name}.tar.gz
rm ${name}.tar.gz
./configure $config
make
make install
make clean
echo "${tmpdir} can be removed"
exit 0
""")
    filename=s
    scriptname=''.join([filename,'.sh'])
    f=open(scriptname,'w')
    r={'name':filename,'config':config}
    if src:
        f.write(src_script.substitute(r))
    else:
        f.write(script.substitute(name=r['name']))
    f.close()
    return scriptname
<commit_msg>Fixed a bug<commit_after>#!/usr/bin/python env
#Write the install shell script

import string

def writeShell(s,config,src):
    """
     Write the part for installation of a script
    """
    script=string.Template("""#!/bin/bash
tmpdir=$$(pwd)
sed -n -e '1,/^exit 0$$/!p' $$0> "$${tmpdir}/${name}.tar.gz" 2>/dev/null
tar zxvf ${name}.tar.gz
rm ${name}.tar.gz
exit 0
""")
    src_script=string.Template("""#!/bin/bash
tmpdir=$$(pwd)/tmp_dir
mkdir $$tmpdir
sed -n -e '1,/^exit 0$$/!p' $$0> "$${tmpdir}/${name}.tar.gz" 2>/dev/null
cd $$tmpdir
tar zxvf ${name}.tar.gz
rm ${name}.tar.gz
./configure $config
make
make install
make clean
echo "$${tmpdir} can be removed"
exit 0
""")
    filename=s
    scriptname=''.join([filename,'.sh'])
    f=open(scriptname,'w')
    r={'name':filename,'config':config}
    if src:
        f.write(src_script.substitute(r))
    else:
        f.write(script.substitute(name=r['name']))
    f.close()
    return scriptname
<|endoftext|>