<commit_before>import configparser
import unittest
import uuid
from pathlib import Path

from TM1py import Element, Hierarchy, Dimension
from TM1py.Objects import Cube
from TM1py.Objects import Rules
from TM1py.Services import TM1Service

from .TestUtils import skip_if_insufficient_version

PREFIX = "TM1py_Tests_Cube_"

class TestCubeMethods(unittest.TestCase):
    tm1 = None
    cube_name = PREFIX + "some_name"
    dimension_names = [
        PREFIX + "dimension1",
        PREFIX + "dimension2",
        PREFIX + "dimension3"]

    @classmethod
    def setUp(cls):

        # Connection to TM1
        cls.config = configparser.ConfigParser()
        cls.config.read(Path(__file__).parent.joinpath('config.ini'))
        cls.tm1 = TM1Service(**cls.config['tm1srv01'])

        for dimension_name in cls.dimension_names:
            elements = [Element('Element {}'.format(str(j)), 'Numeric') for j in range(1, 1001)]
            hierarchy = Hierarchy(dimension_name=dimension_name,
                                  name=dimension_name,
                                  elements=elements)
            dimension = Dimension(dimension_name, [hierarchy])
            if not cls.tm1.dimensions.exists(dimension.name):
                cls.tm1.dimensions.create(dimension)

        # Build Cube
        cube = Cube(cls.cube_name, cls.dimension_names)
        if not cls.tm1.cubes.exists(cls.cube_name):
            cls.tm1.cubes.create(cube)
        c = Cube(cls.cube_name, dimensions=cls.dimension_names, rules=Rules(''))
        if cls.tm1.cubes.exists(c.name):
            cls.tm1.cubes.delete(c.name)
        cls.tm1.cubes.create(c)

    def test_get_cube(self):
        c = self.tm1.cubes.get(self.cube_name)
        self.assertIsInstance(c, Cube)
        self.assertEqual(c.dimensions, self.dimension_names)

        cubes = self.tm1.cubes.get_all()
        control_cubes = self.tm1.cubes.get_control_cubes()
        model_cubes = self.tm1.cubes.get_model_cubes()
        self.assertEqual(len(cubes), len(control_cubes + model_cubes))

    def test_update_cube(self):
        c = self.tm1.cubes.get(self.cube_name)
        c.rules = Rules("SKIPCHECK;\nFEEDERS;")
        self.tm1.cubes.update(c)
        # test if rule was actually updated
        c = self.tm1.cubes.get(self.cube_name)
        self.assertEqual(c.rules.text, "SKIPCHECK;\nFEEDERS;")
        self.assertTrue(c.skipcheck)

    def test_get_control_cubes(self):
        control_cubes = self.tm1.cubes.get_control_cubes()
        self.assertGreater(len(control_cubes), 0)
        for cube in control_cubes:
            self.assertTrue(cube.name.startswith("}"))

    def test_get_model_cubes(self):
        model_cubes = self.tm1.cubes.get_model_cubes()
        self.assertGreater(len(model_cubes), 0)
        for cube in model_cubes:
            self.assertFalse(cube.name.startswith("}"))

    def test_get_dimension_names(self):
        dimension_names = self.tm1.cubes.get_dimension_names(self.cube_name)
        self.assertEqual(dimension_names, self.dimension_names)

    def test_get_random_intersection(self):
        intersection1 = self.tm1.cubes.get_random_intersection(cube_name=self.cube_name, unique_names=False)
        intersection2 = self.tm1.cubes.get_random_intersection(cube_name=self.cube_name, unique_names=False)
        self.assertNotEqual(intersection1, intersection2)
        intersection1 = self.tm1.cubes.get_random_intersection(cube_name=self.cube_name, unique_names=True)
        intersection2 = self.tm1.cubes.get_random_intersection(cube_name=self.cube_name, unique_names=True)
        self.assertNotEqual(intersection1, intersection2)

    def test_exists(self):
        self.assertTrue(self.tm1.cubes.exists(self.cube_name))
        self.assertFalse(self.tm1.cubes.exists(uuid.uuid4()))

    def test_create_delete_cube(self):
        cube_name = PREFIX + "Some_Other_Name"
        # element with index 0 is Sandboxes
        dimension_names = self.tm1.dimensions.get_all_names()[1:3]
        cube = Cube(cube_name, dimension_names)

        all_cubes_before = self.tm1.cubes.get_all_names()
        self.tm1.cubes.create(cube)
        all_cubes_after = self.tm1.cubes.get_all_names()
        self.assertEqual(
            len(all_cubes_before) + 1,
            len(all_cubes_after))
        self.assertEqual(
            self.tm1.cubes.get_dimension_names(cube_name),
            dimension_names)

        all_cubes_before = self.tm1.cubes.get_all_names()
        self.tm1.cubes.delete(cube_name)
        all_cubes_after = self.tm1.cubes.get_all_names()
        self.assertEqual(len(all_cubes_before) - 1, len(all_cubes_after))

    @skip_if_insufficient_version(version="11.4")
    def test_get_storage_dimension_order(self):
        dimensions = self.tm1.cubes.get_storage_dimension_order(cube_name=self.cube_name)
        self.assertEqual(dimensions, self.dimension_names)

    def test_get_number_of_cubes(self):
        number_of_cubes = self.tm1.cubes.get_number_of_cubes()
        self.assertIsInstance(number_of_cubes, int)

    @skip_if_insufficient_version(version="11.4")
    def test_update_storage_dimension_order(self):
        self.tm1.cubes.update_storage_dimension_order(
            cube_name=self.cube_name,
            dimension_names=reversed(self.dimension_names))
        dimensions = self.tm1.cubes.get_storage_dimension_order(self.cube_name)
        self.assertEqual(
            list(reversed(dimensions)),
            self.dimension_names)

    @skip_if_insufficient_version(version="11.6")
    def test_load(self):
        response = self.tm1.cubes.load(cube_name=self.cube_name)
        self.assertTrue(response.ok)

    @skip_if_insufficient_version(version="11.6")
    def test_unload(self):
        response = self.tm1.cubes.unload(cube_name=self.cube_name)
        self.assertTrue(response.ok)

    def test_lock(self):
        response = self.tm1.cubes.lock(cube_name=self.cube_name)
        self.assertTrue(response.ok)

    def test_unlock(self):
        self.tm1.cubes.lock(cube_name=self.cube_name)
        response = self.tm1.cubes.unlock(cube_name=self.cube_name)
        self.assertTrue(response.ok)

    def test_check_rules_without_errors(self):
        errors = self.tm1.cubes.check_rules(cube_name=self.cube_name)
        self.assertEqual(0, len(errors))

    def test_check_rules_with_errors(self):
        cube = self.tm1.cubes.get(cube_name=self.cube_name)
        cube.rules = "SKIPCHECK"
        self.tm1.cubes.update(cube)

        errors = self.tm1.cubes.check_rules(cube_name=self.cube_name)
        self.assertEqual(1, len(errors))

    @classmethod
    def tearDown(cls):
        cls.tm1.cubes.delete(cls.cube_name)
        for dimension in cls.dimension_names:
            cls.tm1.dimensions.delete(dimension)
        cls.tm1.logout()

if __name__ == '__main__':
    unittest.main()
<commit_msg>move globals to class properties<commit_after>import configparser
import unittest
import uuid
from pathlib import Path

from TM1py import Element, Hierarchy, Dimension
from TM1py.Objects import Cube
from TM1py.Objects import Rules
from TM1py.Services import TM1Service

from .TestUtils import skip_if_insufficient_version

class TestCubeMethods(unittest.TestCase):

    @classmethod
    def setUp(cls):

        # Connection to TM1
        cls.config = configparser.ConfigParser()
        cls.config.read(Path(__file__).parent.joinpath('config.ini'))
        cls.tm1 = TM1Service(**cls.config['tm1srv01'])

        cls.prefix = "TM1py_Tests_Cube_"

        cls.cube_name = cls.prefix + "some_name"
        cls.dimension_names = [
            cls.prefix + "dimension1",
            cls.prefix + "dimension2",
            cls.prefix + "dimension3"]

        for dimension_name in cls.dimension_names:
            elements = [Element('Element {}'.format(str(j)), 'Numeric') for j in range(1, 1001)]
            hierarchy = Hierarchy(dimension_name=dimension_name,
                                  name=dimension_name,
                                  elements=elements)
            dimension = Dimension(dimension_name, [hierarchy])
            if not cls.tm1.dimensions.exists(dimension.name):
                cls.tm1.dimensions.create(dimension)

        # Build Cube
        cube = Cube(cls.cube_name, cls.dimension_names)
        if not cls.tm1.cubes.exists(cls.cube_name):
            cls.tm1.cubes.create(cube)
        c = Cube(cls.cube_name, dimensions=cls.dimension_names, rules=Rules(''))
        if cls.tm1.cubes.exists(c.name):
            cls.tm1.cubes.delete(c.name)
        cls.tm1.cubes.create(c)

    def test_get_cube(self):
        c = self.tm1.cubes.get(self.cube_name)
        self.assertIsInstance(c, Cube)
        self.assertEqual(c.dimensions, self.dimension_names)

        cubes = self.tm1.cubes.get_all()
        control_cubes = self.tm1.cubes.get_control_cubes()
        model_cubes = self.tm1.cubes.get_model_cubes()
        self.assertEqual(len(cubes), len(control_cubes + model_cubes))

    def test_update_cube(self):
        c = self.tm1.cubes.get(self.cube_name)
        c.rules = Rules("SKIPCHECK;\nFEEDERS;")
        self.tm1.cubes.update(c)
        # test if rule was actually updated
        c = self.tm1.cubes.get(self.cube_name)
        self.assertEqual(c.rules.text, "SKIPCHECK;\nFEEDERS;")
        self.assertTrue(c.skipcheck)

    def test_get_control_cubes(self):
        control_cubes = self.tm1.cubes.get_control_cubes()
        self.assertGreater(len(control_cubes), 0)
        for cube in control_cubes:
            self.assertTrue(cube.name.startswith("}"))

    def test_get_model_cubes(self):
        model_cubes = self.tm1.cubes.get_model_cubes()
        self.assertGreater(len(model_cubes), 0)
        for cube in model_cubes:
            self.assertFalse(cube.name.startswith("}"))

    def test_get_dimension_names(self):
        dimension_names = self.tm1.cubes.get_dimension_names(self.cube_name)
        self.assertEqual(dimension_names, self.dimension_names)

    def test_get_random_intersection(self):
        intersection1 = self.tm1.cubes.get_random_intersection(cube_name=self.cube_name, unique_names=False)
        intersection2 = self.tm1.cubes.get_random_intersection(cube_name=self.cube_name, unique_names=False)
        self.assertNotEqual(intersection1, intersection2)
        intersection1 = self.tm1.cubes.get_random_intersection(cube_name=self.cube_name, unique_names=True)
        intersection2 = self.tm1.cubes.get_random_intersection(cube_name=self.cube_name, unique_names=True)
        self.assertNotEqual(intersection1, intersection2)

    def test_exists(self):
        self.assertTrue(self.tm1.cubes.exists(self.cube_name))
        self.assertFalse(self.tm1.cubes.exists(uuid.uuid4()))

    def test_create_delete_cube(self):
        cube_name = self.prefix + "Some_Other_Name"
        # element with index 0 is Sandboxes
        dimension_names = self.tm1.dimensions.get_all_names()[1:3]
        cube = Cube(cube_name, dimension_names)

        all_cubes_before = self.tm1.cubes.get_all_names()
        self.tm1.cubes.create(cube)
        all_cubes_after = self.tm1.cubes.get_all_names()
        self.assertEqual(
            len(all_cubes_before) + 1,
            len(all_cubes_after))
        self.assertEqual(
            self.tm1.cubes.get_dimension_names(cube_name),
            dimension_names)

        all_cubes_before = self.tm1.cubes.get_all_names()
        self.tm1.cubes.delete(cube_name)
        all_cubes_after = self.tm1.cubes.get_all_names()
        self.assertEqual(len(all_cubes_before) - 1, len(all_cubes_after))

    @skip_if_insufficient_version(version="11.4")
    def test_get_storage_dimension_order(self):
        dimensions = self.tm1.cubes.get_storage_dimension_order(cube_name=self.cube_name)
        self.assertEqual(dimensions, self.dimension_names)

    def test_get_number_of_cubes(self):
        number_of_cubes = self.tm1.cubes.get_number_of_cubes()
        self.assertIsInstance(number_of_cubes, int)

    @skip_if_insufficient_version(version="11.4")
    def test_update_storage_dimension_order(self):
        self.tm1.cubes.update_storage_dimension_order(
            cube_name=self.cube_name,
            dimension_names=reversed(self.dimension_names))
        dimensions = self.tm1.cubes.get_storage_dimension_order(self.cube_name)
        self.assertEqual(
            list(reversed(dimensions)),
            self.dimension_names)

    @skip_if_insufficient_version(version="11.6")
    def test_load(self):
        response = self.tm1.cubes.load(cube_name=self.cube_name)
        self.assertTrue(response.ok)

    @skip_if_insufficient_version(version="11.6")
    def test_unload(self):
        response = self.tm1.cubes.unload(cube_name=self.cube_name)
        self.assertTrue(response.ok)

    def test_lock(self):
        response = self.tm1.cubes.lock(cube_name=self.cube_name)
        self.assertTrue(response.ok)

    def test_unlock(self):
        self.tm1.cubes.lock(cube_name=self.cube_name)
        response = self.tm1.cubes.unlock(cube_name=self.cube_name)
        self.assertTrue(response.ok)

    def test_check_rules_without_errors(self):
        errors = self.tm1.cubes.check_rules(cube_name=self.cube_name)
        self.assertEqual(0, len(errors))

    def test_check_rules_with_errors(self):
        cube = self.tm1.cubes.get(cube_name=self.cube_name)
        cube.rules = "SKIPCHECK"
        self.tm1.cubes.update(cube)

        errors = self.tm1.cubes.check_rules(cube_name=self.cube_name)
        self.assertEqual(1, len(errors))

    @classmethod
    def tearDown(cls):
        cls.tm1.cubes.delete(cls.cube_name)
        for dimension in cls.dimension_names:
            cls.tm1.dimensions.delete(dimension)
        cls.tm1.logout()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-----------------------------------------------------------------------
#
#       tune.py -- hill climbing of the evaluation vector
#
#-----------------------------------------------------------------------

import floyd as engine
import json
import math
import sys

#-----------------------------------------------------------------------
#       options
#-----------------------------------------------------------------------

cpu = 0

depth = 0

nrSteps = 6  # 1.....2.....3..X..4.....5.....6
nrSteps = 4  # 1.........2....X....3.........4
#nrSteps = 2 # 1..............X..............2

maxActive = float('+Inf') # "dirty mode"

quit = False

#-----------------------------------------------------------------------
#       readTestsFromEpd
#-----------------------------------------------------------------------

def readTestsFromEpd(input):
        """Read test postions from EPD
        Each line must contain a 4-field FEN followed by a 'result' opcode and operand,
        optionally followed by 'white' and 'black' opcodes for the players' ratings."""

        tests = []
        target = { '1-0' : 1.0, '0-1' : 0.0, '1/2-1/2' : 0.5 }

        for line in input:
                line = line.strip().split()
                pos = ' '.join(line[0:4])
                result = line[5].rstrip(';')
                if line[6] == 'white' and line[8] == 'black':
                        whiteElo = int(line[7].rstrip(';'))
                        blackElo = int(line[9].rstrip(';'))
                        pos = '%s %d' % (pos, whiteElo - blackElo)
                tests.append((pos, target[result] if line[1] == 'w' else 1.0 - target[result]))

        return tests

#-----------------------------------------------------------------------
#       getVector
#-----------------------------------------------------------------------

def getVector():
        vector, names = [], []
        try:
                coef = 0
                while True:
                        value, id = engine.setCoefficient(coef, 0)
                        engine.setCoefficient(coef, value) # TODO: HACK
                        vector.append(value)
                        names.append(id)
                        coef += 1
        except IndexError:
                pass
        return vector, names

#-----------------------------------------------------------------------
#       evaluateVector
#-----------------------------------------------------------------------

def evaluateVector(tests, passive, useCache):
        """
        tests    : list of (fen, result) tuples
        passive  : (in/out) dict of positions whose score has stayed constant
        useCache : flag to indicate when cached values can be used
        """

        sumSquaredErrors = 0.0
        scores = []
        for pos, target in tests:
                if not useCache or pos not in passive:
                        score, move = engine.search(pos, depth) # slow
                        if pos in passive and score != passive[pos]:
                                del passive[pos]
                else:
                        score = passive[pos] # fast
                scores.append(score)
                p = scoreToP(score)
                sumSquaredErrors += (p - target) * (p - target)

        return math.sqrt(sumSquaredErrors / len(tests)), scores

#-----------------------------------------------------------------------
#       scoreToP
#-----------------------------------------------------------------------

def scoreToP(score):
        """Convert a score in pawns to a winning probability (0..1)"""
        return 1 / (1 + 10 ** (-score/4.0))

#-----------------------------------------------------------------------
#       tuneSingle
#-----------------------------------------------------------------------

def tuneSingle(coef, tests, initialValue, initialResidual, initialScores):
        """Tune a single coefficient using robust a form of hill-climbing"""

        print 'evaluate id %s residual %.9f value %d' % (names[coef], initialResidual, initialValue)

        cache = { initialValue: initialResidual } # value -> residual

        bestValue, bestResidual, bestScores = initialValue, initialResidual, initialScores

        # Initial window scales with the magnitude of the initial value
        window = calcWindow(initialValue)

        # For switching to quick mode
        positions = [item[0] for item in tests]
        passive = dict(zip(positions, bestScores))
        lastActive, streak = None, 0
        quick, dirty = False, False

        # Loop until an improvement is found or the search is exhausted
        exhausted = False
        while not exhausted:

                # Center the window around the current best value
                center = bestValue
                minValue, maxValue = center - window/2, center + window/2
                stepSize = window / (nrSteps - 1)

                # Walk through the range in equal steps. Always complete all steps
                exhausted = True
                for step in range(nrSteps):
                        nextValue = int(round(minValue + step * stepSize))
                        if nextValue in cache:
                                continue
                        exhausted = False

                        engine.setCoefficient(coef, nextValue)
                        nextResidual, nextScores = evaluateVector(tests, passive, quick or dirty)
                        cache[nextValue] = nextResidual

                        active = len(positions) - len(passive)
                        print 'evaluate id %s residual %.9f' % (names[coef], nextResidual),
                        if not quick and not dirty:
                                print 'active %d' % active,
                        print 'value %d' % nextValue,

                        # Track changes in number of active positions for this parameter
                        if active != lastActive:
                                streak = 0
                        lastActive = active
                        streak += 1

                        # Determine if the result is an improvement
                        if (nextResidual, abs(nextValue)) < (bestResidual, abs(bestValue)):
                                bestValue, bestResidual, bestScores = nextValue, nextResidual, nextScores
                                print 'best'
                        else:
                                print # newline

                # Shrink the window if the best value is near the center
                if abs(bestValue - center) < window / 4:
                        if bestValue != initialValue:
                                break # Early termination (go to next parameter)
                        window /= 2
                        quick = streak >= nrSteps
                else:
                        if min(bestValue - minValue, maxValue - bestValue) < window / 8:
                                window *= 1.25 # Slight increase when at the edge

                dirty = lastActive >= maxActive

        # Update vector
        engine.setCoefficient(coef, bestValue)

        return bestValue, bestResidual, bestScores, active

#-----------------------------------------------------------------------
#       calcWindow
#-----------------------------------------------------------------------

def calcWindow(value):
        sigmoid = 1.0 / (1.0 + math.exp(-value * 1e-3))
        slope = sigmoid * (1.0 - sigmoid)
        window = 0.02 / max(slope, 0.01) * 1e3 # clip when below 1% (queens, rooks)
        return window

#-----------------------------------------------------------------------
#       writeVector
#-----------------------------------------------------------------------

def writeVector(vector, filename):
        with open(filename, 'w') as fp:
                json.dump(dict(zip(names, vector)), fp, indent=1, separators=(',', ': '), sort_keys=True)
                fp.write('\n')

#-----------------------------------------------------------------------
#       main
#-----------------------------------------------------------------------

if __name__ == '__main__':

        # -- Step 0: Get vector definition from module

        vector, names = getVector()

        # -- Step 1: Parse command line arguments

        if len(sys.argv) == 1:
                print 'Usage: python tune.py [options] <vector> [ <parameter> ... ]'
                print 'Options:'
                print '    vector         - JSON file to tune (input/output)'
                print '    parameter ...  - names of parameter(s) to tune (empty means all)'
                print 'Arguments:'
                print '    -n <cpu>       - (not implemented)'
                print '    -d <depth>     - search depth per position (default 0)'
                print '    -s <steps>     - probe steps before window shrink (default 2)'
                print '    -m <count>     - active positions before short-cut evaluation (default +Inf)'
                print '    -q             - print the current residual and quit'
                sys.exit(0)

        argi = 1

        while len(sys.argv) >= argi and sys.argv[argi][0] == '-':

                if sys.argv[argi] == '-n':
                        cpu = int(sys.argv[argi+1])
                        argi += 2
                        continue

                if sys.argv[argi] == '-d':
                        depth = int(sys.argv[argi+1])
                        argi += 2
                        continue

                if sys.argv[argi] == '-s':
                        nrSteps = int(sys.argv[argi+1])
                        argi += 2
                        continue

                if sys.argv[argi] == '-m':
                        maxActive = int(sys.argv[argi+1])
                        argi += 2
                        continue

                if sys.argv[argi] == '-q':
                        quit = True
                        argi += 1
                        continue

                print 'No such option:', sys.argv[argi]
                sys.exit(1)

        # Read vector from file, if any
        filename = sys.argv[argi]
        argi += 1
        try:
                with open(filename, 'r') as fp:
                        values = dict(zip(names, vector))
                        values.update(json.load(fp))
                        vector = [values[name] for name in names]
                        for coef in range(len(vector)):
                                engine.setCoefficient(coef, vector[coef])
        except IOError as err:
                print err
                print 'continue'

        coefList = range(len(vector))
        if len(sys.argv) > argi:
                coefList = [coef for coef in coefList if names[coef] in sys.argv[argi:]]

        # -- Step 2: Read positions from stdin

        tests = readTestsFromEpd(sys.stdin)

        # -- Step 3: Prepare. Calculate initial scores and residual

        bestResidual, bestScores = evaluateVector(tests, {}, False)

        print 'vector filename %s residual %.9f positions %d depth %d' % (repr(filename), bestResidual, len(tests), depth)
        print

        if quit:
                sys.exit(0)

        # -- Step 4: Tune all, half the set and repeat tuning until no more halving

        nrRounds = 0
        exitValue = 1
        exhausted = False
        while len(coefList) > 0 and not exhausted:
                nrRounds += 1
                print 'round %d count %d' % (nrRounds, len(coefList))
                print

                deltas = {}
                exhausted = True
                for coef in coefList:
                        oldValue = vector[coef]
                        newValue, newResidual, newScores, active = tuneSingle(coef, tests, oldValue, bestResidual, bestScores)

                        deltaResidual = newResidual - bestResidual
                        deltas[coef] = deltaResidual

                        if newValue != oldValue:
                                print 'update id %s residual %.9f delta %.3e active %d oldValue %d newValue %d' % (
                                        names[coef], newResidual, deltaResidual, active, oldValue, newValue)
                                vector[coef] = newValue
                                writeVector(vector, filename)
                                exitValue = 0
                                bestResidual, bestScores = newResidual, newScores
                                exhausted = False
                        print

                # Keep the most volatile half for the next round
                coefList = sorted(coefList, key=lambda x:deltas[x])
                newLen = len(coefList) // 2
                coefList = coefList[:newLen]

        # -- Step 5: Report result and exit

        print 'vector filename %s residual %.9f positions %d depth %d' % (repr(filename), bestResidual, len(tests), depth)
        print

        sys.exit(exitValue)

#-----------------------------------------------------------------------
#
#-----------------------------------------------------------------------

<commit_msg>tune.py help text<commit_after>#!/usr/bin/env python
#-----------------------------------------------------------------------
#
#       tune.py -- hill climbing of the evaluation vector
#
#-----------------------------------------------------------------------

import floyd as engine
import json
import math
import sys

#-----------------------------------------------------------------------
#       options
#-----------------------------------------------------------------------

cpu = 0

depth = 0

nrSteps = 6  # 1.....2.....3..X..4.....5.....6
nrSteps = 4  # 1.........2....X....3.........4
#nrSteps = 2 # 1..............X..............2

maxActive = float('+Inf') # "dirty mode"

quit = False

#-----------------------------------------------------------------------
#       readTestsFromEpd
#-----------------------------------------------------------------------

def readTestsFromEpd(input):
        """Read test postions from EPD
        Each line must contain a 4-field FEN followed by a 'result' opcode and operand,
        optionally followed by 'white' and 'black' opcodes for the players' ratings."""

        tests = []
        target = { '1-0' : 1.0, '0-1' : 0.0, '1/2-1/2' : 0.5 }

        for line in input:
                line = line.strip().split()
                pos = ' '.join(line[0:4])
                result = line[5].rstrip(';')
                if line[6] == 'white' and line[8] == 'black':
                        whiteElo = int(line[7].rstrip(';'))
                        blackElo = int(line[9].rstrip(';'))
                        pos = '%s %d' % (pos, whiteElo - blackElo)
                tests.append((pos, target[result] if line[1] == 'w' else 1.0 - target[result]))

        return tests

#-----------------------------------------------------------------------
#       getVector
#-----------------------------------------------------------------------

def getVector():
        vector, names = [], []
        try:
                coef = 0
                while True:
                        value, id = engine.setCoefficient(coef, 0)
                        engine.setCoefficient(coef, value) # TODO: HACK
                        vector.append(value)
                        names.append(id)
                        coef += 1
        except IndexError:
                pass
        return vector, names

#-----------------------------------------------------------------------
#       evaluateVector
#-----------------------------------------------------------------------

def evaluateVector(tests, passive, useCache):
        """
        tests    : list of (fen, result) tuples
        passive  : (in/out) dict of positions whose score has stayed constant
        useCache : flag to indicate when cached values can be used
        """

        sumSquaredErrors = 0.0
        scores = []
        for pos, target in tests:
                if not useCache or pos not in passive:
                        score, move = engine.search(pos, depth) # slow
                        if pos in passive and score != passive[pos]:
                                del passive[pos]
                else:
                        score = passive[pos] # fast
                scores.append(score)
                p = scoreToP(score)
                sumSquaredErrors += (p - target) * (p - target)

        return math.sqrt(sumSquaredErrors / len(tests)), scores

#-----------------------------------------------------------------------
#       scoreToP
#-----------------------------------------------------------------------

def scoreToP(score):
        """Convert a score in pawns to a winning probability (0..1)"""
        return 1 / (1 + 10 ** (-score/4.0))

#-----------------------------------------------------------------------
#       tuneSingle
#-----------------------------------------------------------------------

def tuneSingle(coef, tests, initialValue, initialResidual, initialScores):
        """Tune a single coefficient using robust a form of hill-climbing"""

        print 'evaluate id %s residual %.9f value %d' % (names[coef], initialResidual, initialValue)

        cache = { initialValue: initialResidual } # value -> residual

        bestValue, bestResidual, bestScores = initialValue, initialResidual, initialScores

        # Initial window scales with the magnitude of the initial value
        window = calcWindow(initialValue)

        # For switching to quick mode
        positions = [item[0] for item in tests]
        passive = dict(zip(positions, bestScores))
        lastActive, streak = None, 0
        quick, dirty = False, False

        # Loop until an improvement is found or the search is exhausted
        exhausted = False
        while not exhausted:

                # Center the window around the current best value
                center = bestValue
                minValue, maxValue = center - window/2, center + window/2
                stepSize = window / (nrSteps - 1)

                # Walk through the range in equal steps. Always complete all steps
                exhausted = True
                for step in range(nrSteps):
                        nextValue = int(round(minValue + step * stepSize))
                        if nextValue in cache:
                                continue
                        exhausted = False

                        engine.setCoefficient(coef, nextValue)
                        nextResidual, nextScores = evaluateVector(tests, passive, quick or dirty)
                        cache[nextValue] = nextResidual

                        active = len(positions) - len(passive)
                        print 'evaluate id %s residual %.9f' % (names[coef], nextResidual),
                        if not quick and not dirty:
                                print 'active %d' % active,
                        print 'value %d' % nextValue,

                        # Track changes in number of active positions for this parameter
                        if active != lastActive:
                                streak = 0
                        lastActive = active
                        streak += 1

                        # Determine if the result is an improvement
                        if (nextResidual, abs(nextValue)) < (bestResidual, abs(bestValue)):
                                bestValue, bestResidual, bestScores = nextValue, nextResidual, nextScores
                                print 'best'
                        else:
                                print # newline

                # Shrink the window if the best value is near the center
                if abs(bestValue - center) < window / 4:
                        if bestValue != initialValue:
                                break # Early termination (go to next parameter)
                        window /= 2
                        quick = streak >= nrSteps
                else:
                        if min(bestValue - minValue, maxValue - bestValue) < window / 8:
                                window *= 1.25 # Slight increase when at the edge

                dirty = lastActive >= maxActive

        # Update vector
        engine.setCoefficient(coef, bestValue)

        return bestValue, bestResidual, bestScores, active

#-----------------------------------------------------------------------
#       calcWindow
#-----------------------------------------------------------------------

def calcWindow(value):
        sigmoid = 1.0 / (1.0 + math.exp(-value * 1e-3))
        slope = sigmoid * (1.0 - sigmoid)
        window = 0.02 / max(slope, 0.01) * 1e3 # clip when below 1% (queens, rooks)
        return window

#-----------------------------------------------------------------------
#       writeVector
#-----------------------------------------------------------------------

def writeVector(vector, filename):
        with open(filename, 'w') as fp:
                json.dump(dict(zip(names, vector)), fp, indent=1, separators=(',', ': '), sort_keys=True)
                fp.write('\n')

#-----------------------------------------------------------------------
#       main
#-----------------------------------------------------------------------

if __name__ == '__main__':

        # -- Step 0: Get vector definition from module

        vector, names = getVector()

        # -- Step 1: Parse command line arguments

        if len(sys.argv) == 1:
                print 'Usage: python tune.py [ <option> ... ] <vector> [ <parameter> ... ]'
                print 'Arguments:'
                print '    vector         - JSON file to tune (input/output)'
                print '    parameter ...  - names of parameter(s) to tune (empty means all)'
                print 'Options:'
                print '    -n <cpu>       - (not implemented)'
                print '    -d <depth>     - search depth per position (default 0)'
                print '    -s <steps>     - probe steps before window shrink (default 2)'
                print '    -m <count>     - active positions before short-cut evaluation (default +Inf)'
                print '    -q             - print the current residual and quit'
                sys.exit(0)

        argi = 1

        while len(sys.argv) >= argi and sys.argv[argi][0] == '-':

                if sys.argv[argi] == '-n':
                        cpu = int(sys.argv[argi+1])
                        argi += 2
                        continue

                if sys.argv[argi] == '-d':
                        depth = int(sys.argv[argi+1])
                        argi += 2
                        continue

                if sys.argv[argi] == '-s':
                        nrSteps = int(sys.argv[argi+1])
                        argi += 2
                        continue

                if sys.argv[argi] == '-m':
                        maxActive = int(sys.argv[argi+1])
                        argi += 2
                        continue

                if sys.argv[argi] == '-q':
                        quit = True
                        argi += 1
                        continue

                print 'No such option:', sys.argv[argi]
                sys.exit(1)

        # Read vector from file, if any
        filename = sys.argv[argi]
        argi += 1
        try:
                with open(filename, 'r') as fp:
                        values = dict(zip(names, vector))
                        values.update(json.load(fp))
                        vector = [values[name] for name in names]
                        for coef in range(len(vector)):
                                engine.setCoefficient(coef, vector[coef])
        except IOError as err:
                print err
                print 'continue'

        coefList = range(len(vector))
        if len(sys.argv) > argi:
                coefList = [coef for coef in coefList if names[coef] in sys.argv[argi:]]

        # -- Step 2: Read positions from stdin

        tests = readTestsFromEpd(sys.stdin)

        # -- Step 3: Prepare. Calculate initial scores and residual

        bestResidual, bestScores = evaluateVector(tests, {}, False)

        print 'vector filename %s residual %.9f positions %d depth %d' % (repr(filename), bestResidual, len(tests), depth)
        print

        if quit:
                sys.exit(0)

        # -- Step 4: Tune all, half the set and repeat tuning until no more halving

        nrRounds = 0
        exitValue = 1
        exhausted = False
        while len(coefList) > 0 and not exhausted:
                nrRounds += 1
                print 'round %d count %d' % (nrRounds, len(coefList))
                print

                deltas = {}
                exhausted = True
                for coef in coefList:
                        oldValue = vector[coef]
                        newValue, newResidual, newScores, active = tuneSingle(coef, tests, oldValue, bestResidual, bestScores)

                        deltaResidual = newResidual - bestResidual
                        deltas[coef] = deltaResidual

                        if newValue != oldValue:
                                print 'update id %s residual %.9f delta %.3e active %d oldValue %d newValue %d' % (
                                        names[coef], newResidual, deltaResidual, active, oldValue, newValue)
                                vector[coef] = newValue
                                writeVector(vector, filename)
                                exitValue = 0
                                bestResidual, bestScores = newResidual, newScores
                                exhausted = False
                        print

                # Keep the most volatile half for the next round
                coefList = sorted(coefList, key=lambda x:deltas[x])
                newLen = len(coefList) // 2
                coefList = coefList[:newLen]

        # -- Step 5: Report result and exit

        print 'vector filename %s residual %.9f positions %d depth %d' % (repr(filename), bestResidual, len(tests), depth)
        print

        sys.exit(exitValue)

#-----------------------------------------------------------------------
#
#-----------------------------------------------------------------------

<|endoftext|>


<commit_before>
################
# TurnToFace.py
# Has robot automatically turn to the last face it sees
####################

from Movement import Movement
from FaceRecognition import FaceRecognition
import time
import logging
import os

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

directory = '/home/pi/logs'

if not os.path.exists(directory):
    os.makedirs(directory)

handler = logging.FileHandler(directory + '/TurtleThoughts.log')
handler.setLevel(logging.INFO)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)

logger.addHandler(handler)

logger.info('Started Turn To Face')

m = Movement()
f = FaceRecognition(logger)
threshold = 5
targetWidth = 150

while(True):
	faceLocation = f.FindFace()

	if(faceLocation[2] == 0):
		faceLocation[2] = targetWidth

	if(faceLocation[0] != 0 and faceLocation[1] != 0):
		delta = faceLocation[0]-320

		logger.info("Face Delta from center is:" + str(delta))

		# turn left
		if(delta > threshold):
			m.turnDegrees(10)

		#turn right
		if(delta < -threshold):
			m.turnDegrees(10)

		# target is within threshold, move closer/farther
		if(delta <= threshold and delta > -threshold):

			logger.info("Inside Threshold")
			m.turnSpeed(0)

			if(faceLocation[2] < targetWidth-2):
				logger.info("Moving Forward")
				m.moveCM(3)

			if(faceLocation[2] > targetWidth+2):
				logger.info("Moving Backward")
				m.moveCM(-3)

			if(faceLocation[2] > targetWidth-2 and faceLocation[2] < targetWidth+2):
				logger.info("Staying Still")
				m.moveCM(0)

<commit_msg>Updating turn logic<commit_after>
################
# TurnToFace.py
# Has robot automatically turn to, and move towards/away from the last face it sees
####################

from Movement import Movement
from FaceRecognition import FaceRecognition
import time
import logging
import os

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

directory = '/home/pi/logs'

if not os.path.exists(directory):
    os.makedirs(directory)

handler = logging.FileHandler(directory + '/TurtleThoughts.log')
handler.setLevel(logging.INFO)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)

logger.addHandler(handler)

logger.info('Started Turn To Face')

m = Movement()
f = FaceRecognition(logger)
center = 320
turnThreshold = 15

moveThreshold = 20
targetWidth = 150

while(True):
	faceLocation = f.FindFace()

	if(faceLocation[2] == 0):
		faceLocation[2] = targetWidth

	if(faceLocation[0] != 0 and faceLocation[1] != 0):
		delta = faceLocation[0]-center)

		logger.info("Face Delta from center is:" + str(delta))

		# turn left
		if(delta > turnThreshold):
			m.turnDegrees(10)

		#turn right
		if(delta < -turnThreshold):
			m.turnDegrees(-10)

		# target is within turnThreshold, move closer/farther
		if(delta <= turnThreshold and delta > -turnThreshold):
			width = faceLocation[2]
			logger.info('Movement Width is:' + str(width))

			logger.info("Inside Turn Threshold")
			m.turnSpeed(0)

			if(width < targetWidth-moveThreshold):
				logger.info("Moving Forward")
				m.moveCM(3)

			if(width > targetWidth+moveThreshold):
				logger.info("Moving Backward")
				m.moveCM(-3)

			if(width > targetWidth-moveThreshold and width < targetWidth+moveThreshold):
				logger.info("Staying Still")
				m.moveCM(0)

<|endoftext|>


<commit_before>import curses
import re
from editor import Editor

nonwords = ' .,;-_'

def main():
    running = True

    import settings

    default_note = settings.args['default_note']
    editor = Editor(default_note)

    end_state = None

    while running:
        try:
            editor.draw_screen()
            cy, cx = editor.cursor

            c = editor.screen.getch()
            handled_key = False
            for key, shortcut in settings.shortcuts:
                if c == ord(key)-64:
                    if shortcut == 'quit':
                        if editor.altered:
                            result = editor.dialog('Save before quitting? (Y or N)', yesno=True)

                            if result is None:
                                pass
                            elif result:
                                editor.save_note(editor.current_note)
                                running = False
                            else: running = False
                        else: running = False

                    elif shortcut == 'save':
                        editor.save_note(editor.current_note)
                        if editor.current_note == '**settings**':
                            settings = reload(settings)

                    elif shortcut == 'goto note':
                        result = editor.dialog('Enter the name of the note to load (^C to cancel):')
                        if result:
                            editor.load_note(result)

                    elif shortcut == 'forward':
                        editor.forward()

                    elif shortcut == 'back':
                        editor.back()

                    elif shortcut == 'new note':
                        editor.load_note(None)

                    elif shortcut == 'settings':
                        editor.load_note('**settings**')

                    elif shortcut == 'paste':
                        if editor.cuts:
                            editor.alter()

                            if cy == len(editor.buffer): editor.buffer.append('')
                            editor.buffer = editor.buffer[:cy] + editor.cuts + editor.buffer[cy:]
                            editor.correct_cursor(cy+len(editor.cuts), cx)
                            editor.cutting = False

                    elif shortcut == 'cut':
                        if cy < len(editor.buffer):
                            editor.alter()

                            if editor.cutting:
                                editor.cuts += [editor.buffer[cy]]
                            else:
                                editor.cuts = [editor.buffer[cy]]

                            editor.buffer = editor.buffer[:cy] + (editor.buffer[cy+1:] if cy < len(editor.buffer)-1 else [])
                            editor.correct_cursor(cy, 0)
                            editor.cutting = True

                    handled_key = True
            if not handled_key:
                if c == curses.KEY_UP:
                    editor.correct_cursor(cy-1, min(cx, len(editor.buffer[cy-1]) if 0 <= cy-1 < len(editor.buffer) else 0))
                    editor.cutting = False
                elif c == curses.KEY_DOWN:
                    editor.correct_cursor(cy+1, min(cx, len(editor.buffer[cy+1]) if 0 < cy+1 < len(editor.buffer) else 0))
                    editor.cutting = False
                elif c == curses.KEY_LEFT:
                    editor.correct_cursor(cy, cx-1)
                elif c == curses.KEY_RIGHT:
                    editor.correct_cursor(cy, cx+1)
                elif c == curses.KEY_LEFT + 279:
                    # ctrl left
                    first = True
                    while first or not (cx==0 or (cy < len(editor.buffer) and len(editor.buffer[cy])>0 and editor.buffer[cy][cx-1] in nonwords
                        and (cx >= len(editor.buffer[cy]) or not editor.buffer[cy][cx] in nonwords))):
                        first = False
                        editor.correct_cursor(cy, cx-1)
                        cy, cx = editor.cursor
                elif c == curses.KEY_RIGHT + 293:
                    # ctrl right
                    first = True
                    while first or not (cx==0 or (cy < len(editor.buffer) and len(editor.buffer[cy])>0 and editor.buffer[cy][cx-1] in nonwords
                        and (cx >= len(editor.buffer[cy]) or not editor.buffer[cy][cx] in nonwords))):
                        first = False
                        editor.correct_cursor(cy, cx+1)
                        cy, cx = editor.cursor
                elif c == ord('\n'):
                    follow_link = False
                    for pos, text in editor.links:
                        if pos <= cx < pos+len(text):
                            follow_link = text[1:-1]
                            current_note = follow_link
                            editor.load_note(current_note)
                    if not follow_link:
                        editor.alter()
                        if cy == len(editor.buffer):
                            editor.buffer.append('')
                        else:
                            editor.buffer = (editor.buffer[:cy] + [editor.buffer[cy][:cx]] +
                                             [editor.buffer[cy][cx:]] + editor.buffer[cy+1:])
                        editor.correct_cursor(cy+1, 0)
                elif c == curses.KEY_BACKSPACE:
                    editor.correct_cursor(cy, cx-1)
                    editor.del_char(cy, cx-1)
                elif c == curses.KEY_DC:
                    editor.alter()
                    editor.del_char(cy, cx)
                elif c == ord('\t'):
                    # tab
                    tab = ' '*settings.args['tab_width']
                    if cy == len(editor.buffer):
                        editor.buffer.append(tab)
                    else:
                        editor.buffer[cy] = tab + editor.buffer[cy]
                    editor.correct_cursor(cy, cx+len(tab))
                elif c == 353:
                    # shift+tab
                    tab = ' '*settings.args['tab_width']
                    if cy < len(editor.buffer) and editor.buffer[cy].startswith(tab):
                        editor.buffer[cy] = editor.buffer[cy][len(tab):]
                        editor.correct_cursor(cy, cx-len(tab))
                elif c == curses.KEY_HOME:
                    if cx == 0 and cy < len(editor.buffer) and editor.buffer[cy]:
                        stripped = editor.buffer[cy].lstrip(' ')
                        cx = len(editor.buffer[cy]) - len(stripped)
                        editor.correct_cursor(cy, cx)
                    else:
                        editor.correct_cursor(cy, 0)
                elif c == curses.KEY_END:
                    if cy < len(editor.buffer):
                        editor.correct_cursor(cy, len(editor.buffer[cy]))
                elif c == curses.KEY_PPAGE:
                    editor.correct_cursor(cy-editor.height, 0)
                elif c == curses.KEY_NPAGE:
                    editor.correct_cursor(cy+editor.height, 0)
                # TODO: c<255? not all of those are good characters
                elif 0 < c < 255:
                    editor.alter()
                    if cy > len(editor.buffer)-1: editor.buffer += ['']
                    editor.buffer[cy] = editor.buffer[cy][:cx] + chr(c) + editor.buffer[cy][cx:]
                    editor.correct_cursor(cy, cx+1)

                editor.status = str(c)

        except KeyboardInterrupt:
            running = False

        except Exception as e:
            #raise
            editor.status = 'exception: %s' % e
            end_state = editor.status
            #running = False

    editor.end_app()
    if end_state: print end_state

if __name__ == '__main__':
    main()
<commit_msg>Removing debugging line.<commit_after>import curses
import re
from editor import Editor

nonwords = ' .,;-_'

def main():
    running = True

    import settings

    default_note = settings.args['default_note']
    editor = Editor(default_note)

    end_state = None

    while running:
        try:
            editor.draw_screen()
            cy, cx = editor.cursor

            c = editor.screen.getch()
            handled_key = False
            for key, shortcut in settings.shortcuts:
                if c == ord(key)-64:
                    if shortcut == 'quit':
                        if editor.altered:
                            result = editor.dialog('Save before quitting? (Y or N)', yesno=True)

                            if result is None:
                                pass
                            elif result:
                                editor.save_note(editor.current_note)
                                running = False
                            else: running = False
                        else: running = False

                    elif shortcut == 'save':
                        editor.save_note(editor.current_note)
                        if editor.current_note == '**settings**':
                            settings = reload(settings)

                    elif shortcut == 'goto note':
                        result = editor.dialog('Enter the name of the note to load (^C to cancel):')
                        if result:
                            editor.load_note(result)

                    elif shortcut == 'forward':
                        editor.forward()

                    elif shortcut == 'back':
                        editor.back()

                    elif shortcut == 'new note':
                        editor.load_note(None)

                    elif shortcut == 'settings':
                        editor.load_note('**settings**')

                    elif shortcut == 'paste':
                        if editor.cuts:
                            editor.alter()

                            if cy == len(editor.buffer): editor.buffer.append('')
                            editor.buffer = editor.buffer[:cy] + editor.cuts + editor.buffer[cy:]
                            editor.correct_cursor(cy+len(editor.cuts), cx)
                            editor.cutting = False

                    elif shortcut == 'cut':
                        if cy < len(editor.buffer):
                            editor.alter()

                            if editor.cutting:
                                editor.cuts += [editor.buffer[cy]]
                            else:
                                editor.cuts = [editor.buffer[cy]]

                            editor.buffer = editor.buffer[:cy] + (editor.buffer[cy+1:] if cy < len(editor.buffer)-1 else [])
                            editor.correct_cursor(cy, 0)
                            editor.cutting = True

                    handled_key = True
            if not handled_key:
                if c == curses.KEY_UP:
                    editor.correct_cursor(cy-1, min(cx, len(editor.buffer[cy-1]) if 0 <= cy-1 < len(editor.buffer) else 0))
                    editor.cutting = False
                elif c == curses.KEY_DOWN:
                    editor.correct_cursor(cy+1, min(cx, len(editor.buffer[cy+1]) if 0 < cy+1 < len(editor.buffer) else 0))
                    editor.cutting = False
                elif c == curses.KEY_LEFT:
                    editor.correct_cursor(cy, cx-1)
                elif c == curses.KEY_RIGHT:
                    editor.correct_cursor(cy, cx+1)
                elif c == curses.KEY_LEFT + 279:
                    # ctrl left
                    first = True
                    while first or not (cx==0 or (cy < len(editor.buffer) and len(editor.buffer[cy])>0 and editor.buffer[cy][cx-1] in nonwords
                        and (cx >= len(editor.buffer[cy]) or not editor.buffer[cy][cx] in nonwords))):
                        first = False
                        editor.correct_cursor(cy, cx-1)
                        cy, cx = editor.cursor
                elif c == curses.KEY_RIGHT + 293:
                    # ctrl right
                    first = True
                    while first or not (cx==0 or (cy < len(editor.buffer) and len(editor.buffer[cy])>0 and editor.buffer[cy][cx-1] in nonwords
                        and (cx >= len(editor.buffer[cy]) or not editor.buffer[cy][cx] in nonwords))):
                        first = False
                        editor.correct_cursor(cy, cx+1)
                        cy, cx = editor.cursor
                elif c == ord('\n'):
                    follow_link = False
                    for pos, text in editor.links:
                        if pos <= cx < pos+len(text):
                            follow_link = text[1:-1]
                            current_note = follow_link
                            editor.load_note(current_note)
                    if not follow_link:
                        editor.alter()
                        if cy == len(editor.buffer):
                            editor.buffer.append('')
                        else:
                            editor.buffer = (editor.buffer[:cy] + [editor.buffer[cy][:cx]] +
                                             [editor.buffer[cy][cx:]] + editor.buffer[cy+1:])
                        editor.correct_cursor(cy+1, 0)
                elif c == curses.KEY_BACKSPACE:
                    editor.correct_cursor(cy, cx-1)
                    editor.del_char(cy, cx-1)
                elif c == curses.KEY_DC:
                    editor.alter()
                    editor.del_char(cy, cx)
                elif c == ord('\t'):
                    # tab
                    tab = ' '*settings.args['tab_width']
                    if cy == len(editor.buffer):
                        editor.buffer.append(tab)
                    else:
                        editor.buffer[cy] = tab + editor.buffer[cy]
                    editor.correct_cursor(cy, cx+len(tab))
                elif c == 353:
                    # shift+tab
                    tab = ' '*settings.args['tab_width']
                    if cy < len(editor.buffer) and editor.buffer[cy].startswith(tab):
                        editor.buffer[cy] = editor.buffer[cy][len(tab):]
                        editor.correct_cursor(cy, cx-len(tab))
                elif c == curses.KEY_HOME:
                    if cx == 0 and cy < len(editor.buffer) and editor.buffer[cy]:
                        stripped = editor.buffer[cy].lstrip(' ')
                        cx = len(editor.buffer[cy]) - len(stripped)
                        editor.correct_cursor(cy, cx)
                    else:
                        editor.correct_cursor(cy, 0)
                elif c == curses.KEY_END:
                    if cy < len(editor.buffer):
                        editor.correct_cursor(cy, len(editor.buffer[cy]))
                elif c == curses.KEY_PPAGE:
                    editor.correct_cursor(cy-editor.height, 0)
                elif c == curses.KEY_NPAGE:
                    editor.correct_cursor(cy+editor.height, 0)
                # TODO: c<255? not all of those are good characters
                elif 0 < c < 255:
                    editor.alter()
                    if cy > len(editor.buffer)-1: editor.buffer += ['']
                    editor.buffer[cy] = editor.buffer[cy][:cx] + chr(c) + editor.buffer[cy][cx:]
                    editor.correct_cursor(cy, cx+1)

                #editor.status = str(c)

        except KeyboardInterrupt:
            running = False

        except Exception as e:
            #raise
            editor.status = 'exception: %s' % e
            end_state = editor.status
            #running = False

    editor.end_app()
    if end_state: print end_state

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>ELECTRUM_VERSION = "0.55a"
SEED_VERSION = 4  # bump this everytime the seed generation is modified
<commit_msg>version 0.56<commit_after>ELECTRUM_VERSION = "0.56"
SEED_VERSION = 4  # bump this everytime the seed generation is modified
<|endoftext|>


<commit_before>#http://legacy.python.org/dev/peps/pep-0440/
version_info = ('3', '0', '0b4')
<commit_msg>Bump version<commit_after>#http://legacy.python.org/dev/peps/pep-0440/
version_info = ('3', '0', '0b5')
<|endoftext|>


<commit_before>from plenum.server.node import Node
from plenum.server.replica import Replica
from plenum.server.suspicion_codes import Suspicions
from plenum.test.delayers import ppgDelay
from plenum.test.stasher import delay_rules
from stp_core.common.log import getlogger
from plenum.test.helper import sdk_send_random_and_check
from plenum.test.checkpoints.conftest import chkFreqPatched

logger = getlogger()
CHK_FREQ = 1

def test_replica_received_preprepare_with_ordered_request(looper,
                                                          txnPoolNodeSet,
                                                          sdk_pool_handle,
                                                          sdk_wallet_steward,
                                                          chkFreqPatched):
    sdk_send_random_and_check(looper, txnPoolNodeSet,
                              sdk_pool_handle, sdk_wallet_steward, 1)

    replica = txnPoolNodeSet[1].master_replica
    params = replica.spylog.getLastParams(Replica.processPrePrepare)
    pp = params["pre_prepare"]
    sender = params["sender"]
    start_request_propagate_count = replica.node.spylog.count(Node.request_propagates)

    def discard(offendingMsg, reason, logger, cliOutput=False):
        assert offendingMsg == pp
        assert Suspicions.DUPLICATE_PPR_SENT.reason == reason

    replica.node.discard = discard

    replica.processPrePrepare(pp, sender)

    assert 0 == replica.node.spylog.count(Node.request_propagates) - start_request_propagate_count
    assert (pp, sender, set(pp.reqIdr)) not in replica.prePreparesPendingFinReqs

def test_replica_received_preprepare_with_unknown_request(looper,
                                                          txnPoolNodeSet,
                                                          sdk_pool_handle,
                                                          sdk_wallet_steward,
                                                          chkFreqPatched):
    sdk_send_random_and_check(looper, txnPoolNodeSet,
                              sdk_pool_handle, sdk_wallet_steward, 1)
    replica = txnPoolNodeSet[1].master_replica
    start_request_propagate_count = replica.node.spylog.count(Node.request_propagates)
    with delay_rules(txnPoolNodeSet[1].nodeIbStasher, ppgDelay(delay=10)):
        sdk_send_random_and_check(looper, txnPoolNodeSet,
                                  sdk_pool_handle, sdk_wallet_steward, 1)

    params = replica.spylog.getLastParams(Replica.processPrePrepare)
    pp = params["pre_prepare"]
    sender = params["sender"]
    assert (pp, sender, set(pp.reqIdr)) not in replica.prePreparesPendingFinReqs
    assert 1 == replica.node.spylog.count(Node.request_propagates) - start_request_propagate_count
<commit_msg>INDY-1843: update test_replica_received_preprepare_with_ordered_request<commit_after>from plenum.server.node import Node
from plenum.server.replica import Replica
from plenum.server.suspicion_codes import Suspicions
from plenum.test.delayers import ppgDelay
from plenum.test.stasher import delay_rules
from stp_core.common.log import getlogger
from plenum.test.helper import sdk_send_random_and_check
from plenum.test.checkpoints.conftest import chkFreqPatched

logger = getlogger()
CHK_FREQ = 1

def test_replica_received_preprepare_with_ordered_request(looper,
                                                          txnPoolNodeSet,
                                                          sdk_pool_handle,
                                                          sdk_wallet_steward,
                                                          chkFreqPatched):
    sdk_send_random_and_check(looper, txnPoolNodeSet,
                              sdk_pool_handle, sdk_wallet_steward, 1)

    replica = txnPoolNodeSet[1].master_replica
    params = replica.spylog.getLastParams(Replica.processPrePrepare)
    pp = params["pre_prepare"]
    sender = params["sender"]
    start_request_propagate_count = replica.node.spylog.count(Node.request_propagates)

    def discard(offendingMsg, reason, logger, cliOutput=False):
        assert offendingMsg == pp
        assert Suspicions.PPR_WITH_ORDERED_REQUEST.reason == reason

    replica.node.discard = discard

    replica.processPrePrepare(pp, sender)

    assert 0 == replica.node.spylog.count(Node.request_propagates) - start_request_propagate_count
    assert (pp, sender, set(pp.reqIdr)) not in replica.prePreparesPendingFinReqs

def test_replica_received_preprepare_with_unknown_request(looper,
                                                          txnPoolNodeSet,
                                                          sdk_pool_handle,
                                                          sdk_wallet_steward,
                                                          chkFreqPatched):
    sdk_send_random_and_check(looper, txnPoolNodeSet,
                              sdk_pool_handle, sdk_wallet_steward, 1)
    replica = txnPoolNodeSet[1].master_replica
    start_request_propagate_count = replica.node.spylog.count(Node.request_propagates)
    with delay_rules(txnPoolNodeSet[1].nodeIbStasher, ppgDelay(delay=10)):
        sdk_send_random_and_check(looper, txnPoolNodeSet,
                                  sdk_pool_handle, sdk_wallet_steward, 1)

    params = replica.spylog.getLastParams(Replica.processPrePrepare)
    pp = params["pre_prepare"]
    sender = params["sender"]
    assert (pp, sender, set(pp.reqIdr)) not in replica.prePreparesPendingFinReqs
    assert 1 == replica.node.spylog.count(Node.request_propagates) - start_request_propagate_count
<|endoftext|>


<commit_before>from django import forms
from rest_framework import serializers
from .models import Content, Tag

class TagSerializer(serializers.ModelSerializer):
    class Meta:
        model = Tag

class ContentSerializer(serializers.ModelSerializer):
    url = serializers.HyperlinkedIdentityField(
        view_name='content-detail',
        lookup_field='pk'
    )
    class Meta:
        model = Content

<commit_msg>Added nested Tag serializer to Content<commit_after>from django import forms
from rest_framework import serializers
from .models import Content, Tag

class TagSerializer(serializers.ModelSerializer):
    class Meta:
        model = Tag

class ContentSerializer(serializers.ModelSerializer):
    url = serializers.HyperlinkedIdentityField(
        view_name='content-detail',
        lookup_field='pk'
    )
    tags = TagSerializer(many=True)

    class Meta:
        model = Content

<|endoftext|>


<commit_before>"""
Parameters that are the same for all planners.

"""
from __future__ import division
import numpy as np

#################################################

# Dimensionality
nstates = 6
ncontrols = 3

#################################################

# Tree growth
ss_buff = 10  # m
horizon = 2  # s
dt = 0.1  # s
max_nodes = 1E5
real_tol = [0.1, 0.1, np.deg2rad(5), 0.1, 0.1, np.deg2rad(5)]

#################################################

# Speed limits for the planner to use (so we don't move too fast for controller tracking)
velmax_pos_plan = np.array([1.1, 0.4, 0.2])  # (m/s, m/s, rad/s), body-frame forward
velmax_neg_plan = np.array([-0.7, -0.4, -0.2])  # (m/s, m/s, rad/s), body-frame backward

#################################################

# Mass and yaw inertia
m = 350  # kg
I = 400  # kg*m^2
invM = np.array([1/m, 1/m, 1/I])

#################################################

# Physical top speeds and thrusts
# velmax_pos = np.copy(velmax_pos_plan)
# velmax_neg = np.copy(velmax_neg_plan)
velmax_pos = np.array([2, 0.8, 0.5])  # (m/s, m/s, rad/s), body-frame forward
velmax_neg = np.array([-0.8, -0.8, -0.5])  # (m/s, m/s, rad/s), body-frame backward
thrust_max = np.array([220, 220, 220, 220])  # N, per thruster

#################################################

# Thruster layout, [back-left, back-right, front-left front-right] (m)
thruster_positions = np.array([[-1.9000,  1.0000, -0.0123],
							   [-1.9000, -1.0000, -0.0123],
							   [ 1.6000,  0.6000, -0.0123],
							   [ 1.6000, -0.6000, -0.0123]])

# Thruster heading vectors, [back-left, back-right, front-left front-right] (m)
thruster_directions = np.array([[ 0.7071,  0.7071,  0.0000],
								[ 0.7071, -0.7071,  0.0000],
								[ 0.7071, -0.7071,  0.0000],
								[ 0.7071,  0.7071,  0.0000]])

# Fresh sprinkles
thrust_levers = np.cross(thruster_positions, thruster_directions)
B = np.concatenate((thruster_directions.T, thrust_levers.T))[[0, 1, 5]]
invB = np.linalg.pinv(B)

#################################################

# Effective linear drag coefficients given thrust and speed limits
Fx_max = B.dot(thrust_max * [1, 1, 1, 1])[0]
Fy_max = B.dot(thrust_max * [1, -1, -1, 1])[1]
Mz_max = B.dot(thrust_max * [-1, 1, -1, 1])[2]
D_pos = np.abs([Fx_max, Fy_max, Mz_max] / velmax_pos)
D_neg = np.abs([Fx_max, Fy_max, Mz_max] / velmax_neg)

#################################################

# Boat shape
boat_length = 210 * 0.0254  # m
boat_width = 96 * 0.0254  # m
boat_buffer = 0.15  # m

# Grid of points defining boat
vps_spacing = 1.0  # m
vps_grid_x, vps_grid_y = np.mgrid[slice(-(boat_length+boat_buffer)/2, (boat_length+boat_buffer)/2+vps_spacing, vps_spacing),
								  slice(-(boat_width+boat_buffer)/2, (boat_width+boat_buffer)/2+vps_spacing, vps_spacing)]
vps_grid_x = vps_grid_x.reshape(vps_grid_x.size)
vps_grid_y = vps_grid_y.reshape(vps_grid_y.size)
vps = np.zeros((vps_grid_x.size, 2))
for i in range(len(vps)):
	vps[i] = [vps_grid_x[i], vps_grid_y[i]]
vps = vps.T

#################################################

def erf(xgoal, x):
	"""
	Returns error e given two states xgoal and x.
	Angle differences are taken properly on SO3.

	"""
	e = xgoal - x
	c = np.cos(x[2])
	s = np.sin(x[2])
	cg = np.cos(xgoal[2])
	sg = np.sin(xgoal[2])
	e[2] = np.arctan2(sg*c - cg*s, cg*c + sg*s)
	return e

#################################################

def unset(*args):
	raise AttributeError("This function has not been set yet!")
<commit_msg>DEMO PARAMS: change some defaults<commit_after>"""
Parameters that are the same for all planners.

"""
from __future__ import division
import numpy as np

#################################################

# Dimensionality
nstates = 6
ncontrols = 3

#################################################

# Tree growth
ss_buff = 10  # m
horizon = 2  # s
dt = 0.1  # s
max_nodes = 1E5
real_tol = [1.5, 1.5, np.deg2rad(10), np.inf, np.inf, np.inf]

#################################################

# Speed limits for the planner to use (so we don't move too fast for controller tracking)
velmax_pos_plan = np.array([1.1, 0.4, 0.2])  # (m/s, m/s, rad/s), body-frame forward
velmax_neg_plan = np.array([-0.7, -0.4, -0.2])  # (m/s, m/s, rad/s), body-frame backward

#################################################

# Mass and yaw inertia
m = 350  # kg
I = 400  # kg*m^2
invM = np.array([1/m, 1/m, 1/I])

#################################################

# Physical top speeds and thrusts
# velmax_pos = np.copy(velmax_pos_plan)
# velmax_neg = np.copy(velmax_neg_plan)
velmax_pos = np.array([1.5, 0.6, 0.25])  # (m/s, m/s, rad/s), body-frame forward
velmax_neg = np.array([-0.8, -0.6, -0.25])  # (m/s, m/s, rad/s), body-frame backward
thrust_max = np.array([220, 220, 220, 220])  # N, per thruster

#################################################

# Thruster layout, [back-left, back-right, front-left front-right] (m)
thruster_positions = np.array([[-1.9000,  1.0000, -0.0123],
							   [-1.9000, -1.0000, -0.0123],
							   [ 1.6000,  0.6000, -0.0123],
							   [ 1.6000, -0.6000, -0.0123]])

# Thruster heading vectors, [back-left, back-right, front-left front-right] (m)
thruster_directions = np.array([[ 0.7071,  0.7071,  0.0000],
								[ 0.7071, -0.7071,  0.0000],
								[ 0.7071, -0.7071,  0.0000],
								[ 0.7071,  0.7071,  0.0000]])

# Fresh sprinkles
thrust_levers = np.cross(thruster_positions, thruster_directions)
B = np.concatenate((thruster_directions.T, thrust_levers.T))[[0, 1, 5]]
invB = np.linalg.pinv(B)

#################################################

# Effective linear drag coefficients given thrust and speed limits
Fx_max = B.dot(thrust_max * [1, 1, 1, 1])[0]
Fy_max = B.dot(thrust_max * [1, -1, -1, 1])[1]
Mz_max = B.dot(thrust_max * [-1, 1, -1, 1])[2]
D_pos = np.abs([Fx_max, Fy_max, Mz_max] / velmax_pos)
D_neg = np.abs([Fx_max, Fy_max, Mz_max] / velmax_neg)

#################################################

# Boat shape
boat_length = 210 * 0.0254  # m
boat_width = 96 * 0.0254  # m
boat_buffer = 0.15  # m

# Grid of points defining boat
vps_spacing = 1.0  # m
vps_grid_x, vps_grid_y = np.mgrid[slice(-(boat_length+boat_buffer)/2, (boat_length+boat_buffer)/2+vps_spacing, vps_spacing),
								  slice(-(boat_width+boat_buffer)/2, (boat_width+boat_buffer)/2+vps_spacing, vps_spacing)]
vps_grid_x = vps_grid_x.reshape(vps_grid_x.size)
vps_grid_y = vps_grid_y.reshape(vps_grid_y.size)
vps = np.zeros((vps_grid_x.size, 2))
for i in range(len(vps)):
	vps[i] = [vps_grid_x[i], vps_grid_y[i]]
vps = vps.T

#################################################

def erf(xgoal, x):
	"""
	Returns error e given two states xgoal and x.
	Angle differences are taken properly on SO3.

	"""
	e = xgoal - x
	c = np.cos(x[2])
	s = np.sin(x[2])
	cg = np.cos(xgoal[2])
	sg = np.sin(xgoal[2])
	e[2] = np.arctan2(sg*c - cg*s, cg*c + sg*s)
	return e

#################################################

def unset(*args):
	raise AttributeError("This function has not been set yet!")
<|endoftext|>


<commit_before>from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = "E07000076"
    addresses_name = (
        "local.2019-05-02/Version 2/Tendring Eastings & Northings GIS References.TSV"
    )
    stations_name = (
        "local.2019-05-02/Version 2/Tendring Eastings & Northings GIS References.TSV"
    )
    elections = ["local.2019-05-02"]
    csv_delimiter = "\t"

    def address_record_to_dict(self, record):
        rec = super().address_record_to_dict(record)
        uprn = record.property_urn.strip().lstrip("0")

        if uprn in [
            "10007940599",  # CO168TA -> CO148TA : 106 Kirby Road, Walton-on-the-Naze, Essex
            "10007941701",  # CO169DT -> CO169BX : The Annexe, 5 Bentley Road, Weeley, Clacton-on-Sea, Essex
            "10007944266",  # CO151PT -> CO151QA : Flat 1 Moon and Starfish, 1 Marine Parade East, Clacton-on-Sea, Essex
            "10007944355",  # CO151PT -> CO151QA : Flat 2 Moon and Starfish, 1 Marine Parade East, Clacton-on-Sea, Essex
            "100090634962",  # CO123PJ -> CO123PF : 302A Main Road, Harwich, Essex
            "100090635741",  # CO154LJ -> CO123UB : 11 Nightingale Way, Clacton-on-Sea, Essex
            "100091272552",  # CO130AG -> CO130DT : 30 Walton Road, Frinton-on-Sea, Essex
            "100091460461",  # CO112UJ -> CO112TA : Spinnels Farm, Spinnels Lane, Wix, Manningtree, Essex
            "100091462057",  # CO130PA -> CO130PB : Melrose, Frinton Road, Kirby Cross, Frinton-on-Sea, Essex
            "100091463517",  # CO168ET -> CO151UG : 2 Beach Road, Lee-Over-Sands, Clacton-on-Sea, Essex
            "100091464541",  # CO153AZ -> CO153AT : 135, Old Road, Clacton-on-Sea, Essex
            "10024056898",  # CO78RF -> CO78RE : 2 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056899",  # CO78RF -> CO78RE : 9 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056900",  # CO78RF -> CO78RE : 15 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056901",  # CO78RF -> CO78RE : 19 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056902",  # CO78RF -> CO78RE : 10 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056903",  # CO78RF -> CO78RE : 24 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059148",  # CO78RF -> CO78RE : 17 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059227",  # CO78RF -> CO78RE : 7 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059229",  # CO78RF -> CO78RE : 4 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060309",  # CO78RF -> CO78RE : 1 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060310",  # CO78RF -> CO78RE : 2 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060311",  # CO78RF -> CO78RE : 15 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060312",  # CO78RF -> CO78RE : 21 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060345",  # CO78RF -> CO78RE : 19 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060445",  # CO78RF -> CO78RE : 7 Whitebeam, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090655999",  # CO78RF -> CO78RE : 1 Whitebeam, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090657509",  # CO78RF -> CO78RE : 21 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090657800",  # CO78RF -> CO78RE : 3 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10094246358",  # CO168SG -> CO168TA : 422 Seawick Holiday Park, Beach Road, St Osyth, Clacton-on-Sea, Essex
            "10094246534",  # CO156LY -> CO154BE : 406 Valley Farm Holiday Park, Valley Road, Clacton-on-Sea, Essex
            "200001487728",  # CO77DW -> CO77DJ : Brendon, Colchester Road, Frating, Colchester, Essex
        ]:
            rec["accept_suggestion"] = True

        if uprn in [
            "200001737111",  # CO168TA -> CO168SG : 101A Priory Close, Bel Air Chalet Estate, St Osyth, Clacton-on-Sea, Essex
            "100091274872",  # CO153DL -> CO111BU : 57 Empire Court, Warwick Road, Clacton-on-Sea, Essex
            "10090656718",  # CO125JE -> CO148BG : 52A Harwich Road, Little Oakley, Harwich, Essex
            "100091463343",  # CO123LD -> CO151PP : 18 Orwell Road, Harwich, Essex
            "100090623245",  # CO70AG -> CO111AH : 25A High Street, Brightlingsea, Colchester, Essex
        ]:
            rec["accept_suggestion"] = False

        return rec
<commit_msg>update Tendring (closes #1742)<commit_after>from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = "E07000076"
    addresses_name = "local.2019-05-02/Version 3/Democracy_Club__02May2019Tendring.tsv"
    stations_name = "local.2019-05-02/Version 3/Democracy_Club__02May2019Tendring.tsv"
    elections = ["local.2019-05-02"]
    csv_delimiter = "\t"

    def address_record_to_dict(self, record):
        rec = super().address_record_to_dict(record)
        uprn = record.property_urn.strip().lstrip("0")

        if uprn in [
            "10007940599",  # CO168TA -> CO148TA : 106 Kirby Road, Walton-on-the-Naze, Essex
            "10007941701",  # CO169DT -> CO169BX : The Annexe, 5 Bentley Road, Weeley, Clacton-on-Sea, Essex
            "10007944266",  # CO151PT -> CO151QA : Flat 1 Moon and Starfish, 1 Marine Parade East, Clacton-on-Sea, Essex
            "10007944355",  # CO151PT -> CO151QA : Flat 2 Moon and Starfish, 1 Marine Parade East, Clacton-on-Sea, Essex
            "100090634962",  # CO123PJ -> CO123PF : 302A Main Road, Harwich, Essex
            "100090635741",  # CO154LJ -> CO123UB : 11 Nightingale Way, Clacton-on-Sea, Essex
            "100091272552",  # CO130AG -> CO130DT : 30 Walton Road, Frinton-on-Sea, Essex
            "100091460461",  # CO112UJ -> CO112TA : Spinnels Farm, Spinnels Lane, Wix, Manningtree, Essex
            "100091462057",  # CO130PA -> CO130PB : Melrose, Frinton Road, Kirby Cross, Frinton-on-Sea, Essex
            "100091463517",  # CO168ET -> CO151UG : 2 Beach Road, Lee-Over-Sands, Clacton-on-Sea, Essex
            "100091464541",  # CO153AZ -> CO153AT : 135, Old Road, Clacton-on-Sea, Essex
            "10024056898",  # CO78RF -> CO78RE : 2 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056899",  # CO78RF -> CO78RE : 9 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056900",  # CO78RF -> CO78RE : 15 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056901",  # CO78RF -> CO78RE : 19 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056902",  # CO78RF -> CO78RE : 10 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024056903",  # CO78RF -> CO78RE : 24 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059148",  # CO78RF -> CO78RE : 17 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059227",  # CO78RF -> CO78RE : 7 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024059229",  # CO78RF -> CO78RE : 4 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060309",  # CO78RF -> CO78RE : 1 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060310",  # CO78RF -> CO78RE : 2 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060311",  # CO78RF -> CO78RE : 15 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060312",  # CO78RF -> CO78RE : 21 The Saltings, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060345",  # CO78RF -> CO78RE : 19 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10024060445",  # CO78RF -> CO78RE : 7 Whitebeam, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090655999",  # CO78RF -> CO78RE : 1 Whitebeam, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090657509",  # CO78RF -> CO78RE : 21 Riverside, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10090657800",  # CO78RF -> CO78RE : 3 Kingfisher Lake, Bentley Country Park, Flag Hill, Great Bentley, Colchester, Essex
            "10094246358",  # CO168SG -> CO168TA : 422 Seawick Holiday Park, Beach Road, St Osyth, Clacton-on-Sea, Essex
            "10094246534",  # CO156LY -> CO154BE : 406 Valley Farm Holiday Park, Valley Road, Clacton-on-Sea, Essex
            "200001487728",  # CO77DW -> CO77DJ : Brendon, Colchester Road, Frating, Colchester, Essex
        ]:
            rec["accept_suggestion"] = True

        if uprn in [
            "200001737111",  # CO168TA -> CO168SG : 101A Priory Close, Bel Air Chalet Estate, St Osyth, Clacton-on-Sea, Essex
            "100091274872",  # CO153DL -> CO111BU : 57 Empire Court, Warwick Road, Clacton-on-Sea, Essex
            "10090656718",  # CO125JE -> CO148BG : 52A Harwich Road, Little Oakley, Harwich, Essex
            "100091463343",  # CO123LD -> CO151PP : 18 Orwell Road, Harwich, Essex
            "100090623245",  # CO70AG -> CO111AH : 25A High Street, Brightlingsea, Colchester, Essex
        ]:
            rec["accept_suggestion"] = False

        return rec
<|endoftext|>


<commit_before># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Creates a GCE instance template for Forseti Security."""

def GenerateConfig(context):
    """Generate configuration."""

    if context.properties.get('branch-name'):
        DOWNLOAD_FORSETI = """
            git clone {}.git --branch {} --single-branch forseti-security
            cd forseti-security
        """.format(
            context.properties['src-path'],
            context.properties['branch-name'])
    else:
        DOWNLOAD_FORSETI = """
            wget -qO- {}/archive/v{}.tar.gz | tar xvz
            cd forseti-security-{}
        """.format(
            context.properties['src-path'],
            context.properties['release-version'],
            context.properties['release-version'])

    SQL_INSTANCE = context.properties['sql-instance']
    EXPLAIN_DATABASE_NAME = context.properties['database-name-forseti']
    FORSETI_DATABASE_NAME = context.properties['database-name-explain']

    resources = []

    resources.append({
        'name': '{}-explain-vm'.format(context.env['deployment']),
        'type': 'compute.v1.instance',
        'properties': {
            'zone': context.properties['zone'],
            'machineType': (
                'https://www.googleapis.com/compute/v1/projects/{}'
                '/zones/{}/machineTypes/{}'.format(
                context.env['project'], context.properties['zone'],
                context.properties['instance-type')),
            'disks': [{
                'deviceName': 'boot',
                'type': 'PERSISTENT',
                'boot': True,
                'autoDelete': True,
                'initializeParams': {
                    'sourceImage': (
                        'https://www.googleapis.com/compute/v1'
                        '/projects/{}/global/images/family/{}'.format(
                            'ubuntu-os-cloud',
                            'ubuntu-1604-lts',
                        )
                    )
                }
            }],
            'networkInterfaces': [{
                'network': (
                    'https://www.googleapis.com/compute/v1/'
                    'projects/{}/global/networks/default'.format(
                    context.env['project'])),
                'accessConfigs': [{
                    'name': 'External NAT',
                    'type': 'ONE_TO_ONE_NAT'
                }]
            }],
            'serviceAccounts': [{
                'email': context.properties['service-account'],
                'scopes': ['https://www.googleapis.com/auth/cloud-platform'],
            }],
            'metadata': {
                'dependsOn': ['db-instances'],
                'items': [{
                    'key': 'startup-script',
                    'value': """#!/bin/bash

exec > /tmp/deployment.log
exec 2>&1

# Ubuntu update
sudo apt-get update -y
sudo apt-get upgrade -y

# Forseti setup
sudo apt-get install -y git unzip
# Forseti dependencies
sudo apt-get install -y libmysqlclient-dev python-pip python-dev

USER_HOME=/home/ubuntu

# Install fluentd if necessary
FLUENTD=$(ls /usr/sbin/google-fluentd)
if [ -z "$FLUENTD" ]; then
      cd $USER_HOME
      curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh
      bash install-logging-agent.sh
fi

# Check whether Cloud SQL proxy is installed
CLOUD_SQL_PROXY=$(ls $USER_HOME/cloud_sql_proxy)
if [ -z "$CLOUD_SQL_PROXY" ]; then
        cd $USER_HOME
        wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64
        mv cloud_sql_proxy.linux.amd64 cloud_sql_proxy
        chmod +x cloud_sql_proxy
fi

# Install Forseti Security
cd $USER_HOME
rm -rf forseti-*
pip install --upgrade pip
pip install --upgrade setuptools
pip install google-apputils grpcio grpcio-tools protobuf

cd $USER_HOME

# Download Forseti src; see DOWNLOAD_FORSETI
{}
python setup.py install

# Create upstart script for API server
read -d '' API_SERVER << EOF
[Unit]
Description=Explain API Server
[Service]
Restart=always
RestartSec=3
ExecStart=/usr/local/bin/forseti_api '[::]:50051' 'mysql://root@127.0.0.1:3306/{}' 'mysql://root@127.0.0.1:3306/{}' playground explain
[Install]
WantedBy=multi-user.target
Wants=cloudsqlproxy.service
EOF
echo "$API_SERVER" > /lib/systemd/system/forseti.service

read -d '' SQL_PROXY << EOF
[Unit]
Description=Explain Cloud SQL Proxy
[Service]
Restart=always
RestartSec=3
ExecStart=/home/ubuntu/cloud_sql_proxy -instances={}=tcp:3306
[Install]
WantedBy=forseti.service
EOF
echo "$SQL_PROXY" > /lib/systemd/system/cloudsqlproxy.service

systemctl start cloudsqlproxy
sleep 1
systemctl start forseti

""".format(

    # install forseti
    DOWNLOAD_FORSETI,
    EXPLAIN_DATABASE_NAME.split(':')[-1],
    FORSETI_DATABASE_NAME.split(':')[-1],

    # cloud_sql_proxy
    SQL_INSTANCE,
)
                }]
            }
        }
    })

    return {'resources': resources}
<commit_msg>Fix broken template. (#382)<commit_after># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Creates a GCE instance template for Forseti Security."""

def GenerateConfig(context):
    """Generate configuration."""

    if context.properties.get('branch-name'):
        DOWNLOAD_FORSETI = """
            git clone {}.git --branch {} --single-branch forseti-security
            cd forseti-security
        """.format(
            context.properties['src-path'],
            context.properties['branch-name'])
    else:
        DOWNLOAD_FORSETI = """
            wget -qO- {}/archive/v{}.tar.gz | tar xvz
            cd forseti-security-{}
        """.format(
            context.properties['src-path'],
            context.properties['release-version'],
            context.properties['release-version'])

    SQL_INSTANCE = context.properties['sql-instance']
    EXPLAIN_DATABASE_NAME = context.properties['database-name-forseti']
    FORSETI_DATABASE_NAME = context.properties['database-name-explain']

    resources = []

    resources.append({
        'name': '{}-explain-vm'.format(context.env['deployment']),
        'type': 'compute.v1.instance',
        'properties': {
            'zone': context.properties['zone'],
            'machineType': (
                'https://www.googleapis.com/compute/v1/projects/{}'
                '/zones/{}/machineTypes/{}'.format(
                context.env['project'], context.properties['zone'],
                context.properties['instance-type'])),
            'disks': [{
                'deviceName': 'boot',
                'type': 'PERSISTENT',
                'boot': True,
                'autoDelete': True,
                'initializeParams': {
                    'sourceImage': (
                        'https://www.googleapis.com/compute/v1'
                        '/projects/{}/global/images/family/{}'.format(
                            'ubuntu-os-cloud',
                            'ubuntu-1604-lts',
                        )
                    )
                }
            }],
            'networkInterfaces': [{
                'network': (
                    'https://www.googleapis.com/compute/v1/'
                    'projects/{}/global/networks/default'.format(
                    context.env['project'])),
                'accessConfigs': [{
                    'name': 'External NAT',
                    'type': 'ONE_TO_ONE_NAT'
                }]
            }],
            'serviceAccounts': [{
                'email': context.properties['service-account'],
                'scopes': ['https://www.googleapis.com/auth/cloud-platform'],
            }],
            'metadata': {
                'dependsOn': ['db-instances'],
                'items': [{
                    'key': 'startup-script',
                    'value': """#!/bin/bash

exec > /tmp/deployment.log
exec 2>&1

# Ubuntu update
sudo apt-get update -y
sudo apt-get upgrade -y

# Forseti setup
sudo apt-get install -y git unzip
# Forseti dependencies
sudo apt-get install -y libmysqlclient-dev python-pip python-dev

USER_HOME=/home/ubuntu

# Install fluentd if necessary
FLUENTD=$(ls /usr/sbin/google-fluentd)
if [ -z "$FLUENTD" ]; then
      cd $USER_HOME
      curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh
      bash install-logging-agent.sh
fi

# Check whether Cloud SQL proxy is installed
CLOUD_SQL_PROXY=$(ls $USER_HOME/cloud_sql_proxy)
if [ -z "$CLOUD_SQL_PROXY" ]; then
        cd $USER_HOME
        wget https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64
        mv cloud_sql_proxy.linux.amd64 cloud_sql_proxy
        chmod +x cloud_sql_proxy
fi

# Install Forseti Security
cd $USER_HOME
rm -rf forseti-*
pip install --upgrade pip
pip install --upgrade setuptools
pip install google-apputils grpcio grpcio-tools protobuf

cd $USER_HOME

# Download Forseti src; see DOWNLOAD_FORSETI
{}
python setup.py install

# Create upstart script for API server
read -d '' API_SERVER << EOF
[Unit]
Description=Explain API Server
[Service]
Restart=always
RestartSec=3
ExecStart=/usr/local/bin/forseti_api '[::]:50051' 'mysql://root@127.0.0.1:3306/{}' 'mysql://root@127.0.0.1:3306/{}' playground explain
[Install]
WantedBy=multi-user.target
Wants=cloudsqlproxy.service
EOF
echo "$API_SERVER" > /lib/systemd/system/forseti.service

read -d '' SQL_PROXY << EOF
[Unit]
Description=Explain Cloud SQL Proxy
[Service]
Restart=always
RestartSec=3
ExecStart=/home/ubuntu/cloud_sql_proxy -instances={}=tcp:3306
[Install]
WantedBy=forseti.service
EOF
echo "$SQL_PROXY" > /lib/systemd/system/cloudsqlproxy.service

systemctl start cloudsqlproxy
sleep 1
systemctl start forseti

""".format(

    # install forseti
    DOWNLOAD_FORSETI,
    EXPLAIN_DATABASE_NAME.split(':')[-1],
    FORSETI_DATABASE_NAME.split(':')[-1],

    # cloud_sql_proxy
    SQL_INSTANCE,
)
                }]
            }
        }
    })

    return {'resources': resources}
<|endoftext|>