<commit_before>import constants

from copy import deepcopy

#floodfill is a tree which calculates possible paths.
#The order for a max depth of "1" is     1
#                                  /   |   |   \
#                               north east west south
# where each value holds the largest depth in that direction.
#If two branches are tied, should choose the safest direction.

def floodfill(depth, direction, length, pathlength, arena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height):

    localArena = deepcopy(arena)
#---------------------------------------------------------------------------------------
#Build local Arena, if not moving into food, then remove from tail.
#---------------------------------------------------------------------------------------

    #print 'floodfilling at depth ', depth, ' direction ', direction
    #print 'floodfilling at', localSnakeX, localSnakeY

    if direction == constants.NORTH:
        if invalidmove_wall(constants.NORTH, localArena, localSnakeX, localSnakeY, width, height) == constants.INVALID_MOVE:
            print 'invalid move at', localSnakeX, localSnakeY
            return constants.INVALID_MOVE
        #check if move is into food
        elif arena [localSnakeY - 1][localSnakeX] != 1:

            localArena [snakeData["coords"][length - depth - 1][1]][snakeData["coords"][length - depth - 1][0]] = 0

    elif direction == constants.EAST:
        #check if move is into food
        if invalidmove_wall(constants.EAST, localArena, localSnakeX, localSnakeY, width, height) == constants.INVALID_MOVE:
            print 'invalid move at', localSnakeX, localSnakeY
            return constants.INVALID_MOVE
        elif arena [localSnakeY][localSnakeX + 1] != 1:
            #remove the last tail
            localArena [snakeData["coords"][length - depth - 1][1]][snakeData["coords"][length - depth - 1][0]] = 0

    elif direction == constants.WEST:
        #check if move is into food
        if invalidmove_wall(constants.WEST, localArena, localSnakeX, localSnakeY, width, height) == constants.INVALID_MOVE:
            print 'invalid move at', localSnakeX, localSnakeY
            return constants.INVALID_MOVE
        elif arena [localSnakeY][localSnakeX - 1] != 1:
            #remove the last tail
            localArena [snakeData["coords"][length - depth - 1][1]][snakeData["coords"][length - depth - 1][0]] = 0

    elif direction == constants.SOUTH:
        #check if move is into food
        if invalidmove_wall(constants.SOUTH, localArena, localSnakeX, localSnakeY, width, height) == constants.INVALID_MOVE:
            print 'invalid move at', localSnakeX, localSnakeY
            return constants.INVALID_MOVE
        elif arena [localSnakeY + 1][localSnakeX] != 1:
            #remove the last tail
            localArena [snakeData["coords"][length - depth - 1][1]][snakeData["coords"][length - depth - 1][0]] = 0

#---------------------------------------------------------------------------------------
#Check if it is an invalid move
#---------------------------------------------------------------------------------------
    if invalidmove(localArena, localSnakeX, localSnakeY, direction, width, height):
        print 'invalid move at', localSnakeX, localSnakeY
        return constants.INVALID_MOVE
#---------------------------------------------------------------------------------------
#Add Move to the LocalArena
#---------------------------------------------------------------------------------------
    else:
        if direction == constants.NORTH:
            #add the move to the localArena
            localArena [localSnakeY - 1][localSnakeX] = snakeNumber

        elif direction == constants.EAST:
            #add the move to the localArena
            localArena [localSnakeY][localSnakeX + 1] = snakeNumber

        elif direction == constants.WEST:
            #add the move to the localArena
            localArena [localSnakeY][localSnakeX - 1] = snakeNumber

        elif direction == constants.SOUTH:
            #add the move to the localArena
            localArena [localSnakeY + 1][localSnakeX] = snakeNumber

#---------------------------------------------------------------------------------------
#Check if end of recursion
#---------------------------------------------------------------------------------------
        if depth == 0:
            print 'found path to', localSnakeX, localSnakeY
            print localArena
            print localArena[40]
            return pathlength + 1
        else:
#---------------------------------------------------------------------------------------
#Run Floodfill on on a move to the North
#---------------------------------------------------------------------------------------

            southval = 0
            westval = 0
            eastval = 0
            northval = 0

            if direction == constants.NORTH:
               northval = floodfill_start(depth, length, pathlength, localArena, localSnakeX, localSnakeY - 1, snakeData, snakeNumber, width, height)
#Run Floodfill on on a move to the East
#---------------------------------------------------------------------------------------
            elif direction == constants.EAST:
                eastval = floodfill_start(depth, length, pathlength, localArena, localSnakeX + 1, localSnakeY, snakeData, snakeNumber, width, height)
#---------------------------------------------------------------------------------------
#Run Floodfill on on a move to the West
#---------------------------------------------------------------------------------------
            elif direction == constants.WEST:
                westval = floodfill_start(depth, length, pathlength, localArena, localSnakeX - 1, localSnakeY, snakeData, snakeNumber, width, height)
#---------------------------------------------------------------------------------------
#Run Floodfill on on a move to the South
#---------------------------------------------------------------------------------------
            elif direction == constants.SOUTH:
                southval = floodfill_start(depth, length, pathlength, localArena, localSnakeX, localSnakeY + 1, snakeData, snakeNumber, width, height)

            return southval + westval + eastval + northval

            #find greatest length of sub trees.
#            if northVal > southVal:
#                if westVal > eastVal:
#                    if northVal > westVal:
#                        return northVal + 1
#                    else:
#                        return westVal + 1
#                else:
#                    if northVal > eastVal:
#                        return northVal + 1
#                    else:
#                        return eastVal + 1
#            else:
#                if westVal > eastVal:
#                    if southVal > westVal:
#                        return southVal + 1
#                    else:
#                        return westVal + 1
#                else:
#                    if southVal > eastVal:
#                        return southVal + 1
#                    else:
#                        return eastVal + 1

def floodfill_start (depth, length, pathlength, localArena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height):

    #print 'floodfill_start called here'
    #print 'to look at X Y'
    #print localSnakeX
    #print localSnakeY

    northVal = floodfill (depth - 1, constants.NORTH, length, pathlength, localArena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height)
    eastVal = floodfill (depth - 1, constants.EAST, length, pathlength, localArena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height)
    westVal = floodfill (depth - 1, constants.WEST, length, pathlength, localArena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height)
    southVal = floodfill (depth - 1, constants.SOUTH, length, pathlength, localArena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height)

    return northVal + eastVal + westVal + southVal

    #find greatest length of sub trees.
#    if northVal > southVal:
#        if westVal > eastVal:
#            if northVal > westVal:
#                return northVal + 1
#            else:
#                return westVal + 1
#        else:
#            if northVal > eastVal:
#                return northVal + 1
#            else:
#                return eastVal + 1
#    else:
#        if westVal > eastVal:
#            if southVal > westVal:
#                return southVal + 1
#            else:
#                return westVal + 1
#        else:
#            if southVal > eastVal:
#                return southVal + 1
#            else:
#                return eastVal + 1
<commit_msg>Forgot to add imports.<commit_after>import constants
from checks import invalidmove_wall, invalidmove

from copy import deepcopy

#floodfill is a tree which calculates possible paths.
#The order for a max depth of "1" is     1
#                                  /   |   |   \
#                               north east west south
# where each value holds the largest depth in that direction.
#If two branches are tied, should choose the safest direction.

def floodfill(depth, direction, length, pathlength, arena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height):

    localArena = deepcopy(arena)
#---------------------------------------------------------------------------------------
#Build local Arena, if not moving into food, then remove from tail.
#---------------------------------------------------------------------------------------

    #print 'floodfilling at depth ', depth, ' direction ', direction
    #print 'floodfilling at', localSnakeX, localSnakeY

    if direction == constants.NORTH:
        if invalidmove_wall(constants.NORTH, localArena, localSnakeX, localSnakeY, width, height) == constants.INVALID_MOVE:
            print 'invalid move at', localSnakeX, localSnakeY
            return constants.INVALID_MOVE
        #check if move is into food
        elif arena [localSnakeY - 1][localSnakeX] != 1:

            localArena [snakeData["coords"][length - depth - 1][1]][snakeData["coords"][length - depth - 1][0]] = 0

    elif direction == constants.EAST:
        #check if move is into food
        if invalidmove_wall(constants.EAST, localArena, localSnakeX, localSnakeY, width, height) == constants.INVALID_MOVE:
            print 'invalid move at', localSnakeX, localSnakeY
            return constants.INVALID_MOVE
        elif arena [localSnakeY][localSnakeX + 1] != 1:
            #remove the last tail
            localArena [snakeData["coords"][length - depth - 1][1]][snakeData["coords"][length - depth - 1][0]] = 0

    elif direction == constants.WEST:
        #check if move is into food
        if invalidmove_wall(constants.WEST, localArena, localSnakeX, localSnakeY, width, height) == constants.INVALID_MOVE:
            print 'invalid move at', localSnakeX, localSnakeY
            return constants.INVALID_MOVE
        elif arena [localSnakeY][localSnakeX - 1] != 1:
            #remove the last tail
            localArena [snakeData["coords"][length - depth - 1][1]][snakeData["coords"][length - depth - 1][0]] = 0

    elif direction == constants.SOUTH:
        #check if move is into food
        if invalidmove_wall(constants.SOUTH, localArena, localSnakeX, localSnakeY, width, height) == constants.INVALID_MOVE:
            print 'invalid move at', localSnakeX, localSnakeY
            return constants.INVALID_MOVE
        elif arena [localSnakeY + 1][localSnakeX] != 1:
            #remove the last tail
            localArena [snakeData["coords"][length - depth - 1][1]][snakeData["coords"][length - depth - 1][0]] = 0

#---------------------------------------------------------------------------------------
#Check if it is an invalid move
#---------------------------------------------------------------------------------------
    if invalidmove(localArena, localSnakeX, localSnakeY, direction, width, height):
        print 'invalid move at', localSnakeX, localSnakeY
        return constants.INVALID_MOVE
#---------------------------------------------------------------------------------------
#Add Move to the LocalArena
#---------------------------------------------------------------------------------------
    else:
        if direction == constants.NORTH:
            #add the move to the localArena
            localArena [localSnakeY - 1][localSnakeX] = snakeNumber

        elif direction == constants.EAST:
            #add the move to the localArena
            localArena [localSnakeY][localSnakeX + 1] = snakeNumber

        elif direction == constants.WEST:
            #add the move to the localArena
            localArena [localSnakeY][localSnakeX - 1] = snakeNumber

        elif direction == constants.SOUTH:
            #add the move to the localArena
            localArena [localSnakeY + 1][localSnakeX] = snakeNumber

#---------------------------------------------------------------------------------------
#Check if end of recursion
#---------------------------------------------------------------------------------------
        if depth == 0:
            print 'found path to', localSnakeX, localSnakeY
            print localArena
            print localArena[40]
            return pathlength + 1
        else:
#---------------------------------------------------------------------------------------
#Run Floodfill on on a move to the North
#---------------------------------------------------------------------------------------

            southval = 0
            westval = 0
            eastval = 0
            northval = 0

            if direction == constants.NORTH:
               northval = floodfill_start(depth, length, pathlength, localArena, localSnakeX, localSnakeY - 1, snakeData, snakeNumber, width, height)
#Run Floodfill on on a move to the East
#---------------------------------------------------------------------------------------
            elif direction == constants.EAST:
                eastval = floodfill_start(depth, length, pathlength, localArena, localSnakeX + 1, localSnakeY, snakeData, snakeNumber, width, height)
#---------------------------------------------------------------------------------------
#Run Floodfill on on a move to the West
#---------------------------------------------------------------------------------------
            elif direction == constants.WEST:
                westval = floodfill_start(depth, length, pathlength, localArena, localSnakeX - 1, localSnakeY, snakeData, snakeNumber, width, height)
#---------------------------------------------------------------------------------------
#Run Floodfill on on a move to the South
#---------------------------------------------------------------------------------------
            elif direction == constants.SOUTH:
                southval = floodfill_start(depth, length, pathlength, localArena, localSnakeX, localSnakeY + 1, snakeData, snakeNumber, width, height)

            return southval + westval + eastval + northval

            #find greatest length of sub trees.
#            if northVal > southVal:
#                if westVal > eastVal:
#                    if northVal > westVal:
#                        return northVal + 1
#                    else:
#                        return westVal + 1
#                else:
#                    if northVal > eastVal:
#                        return northVal + 1
#                    else:
#                        return eastVal + 1
#            else:
#                if westVal > eastVal:
#                    if southVal > westVal:
#                        return southVal + 1
#                    else:
#                        return westVal + 1
#                else:
#                    if southVal > eastVal:
#                        return southVal + 1
#                    else:
#                        return eastVal + 1

def floodfill_start (depth, length, pathlength, localArena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height):

    #print 'floodfill_start called here'
    #print 'to look at X Y'
    #print localSnakeX
    #print localSnakeY

    northVal = floodfill (depth - 1, constants.NORTH, length, pathlength, localArena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height)
    eastVal = floodfill (depth - 1, constants.EAST, length, pathlength, localArena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height)
    westVal = floodfill (depth - 1, constants.WEST, length, pathlength, localArena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height)
    southVal = floodfill (depth - 1, constants.SOUTH, length, pathlength, localArena, localSnakeX, localSnakeY, snakeData, snakeNumber, width, height)

    return northVal + eastVal + westVal + southVal

    #find greatest length of sub trees.
#    if northVal > southVal:
#        if westVal > eastVal:
#            if northVal > westVal:
#                return northVal + 1
#            else:
#                return westVal + 1
#        else:
#            if northVal > eastVal:
#                return northVal + 1
#            else:
#                return eastVal + 1
#    else:
#        if westVal > eastVal:
#            if southVal > westVal:
#                return southVal + 1
#            else:
#                return westVal + 1
#        else:
#            if southVal > eastVal:
#                return southVal + 1
#            else:
#                return eastVal + 1
<|endoftext|>


<commit_before>import ctypes
import unittest

from cupy.cuda import memory
from cupy import testing

class MockMemory(object):
    cur_ptr = 1

    def __init__(self, size):
        self.ptr = ctypes.c_void_p(MockMemory.cur_ptr)
        MockMemory.cur_ptr += size
        self.size = size
        self._device = None

    def __int__(self):
        return self.ptr.value or 0

    @property
    def device(self):
        return self._device

def mock_alloc(size):
    mem = MockMemory(size)
    return memory.MemoryPointer(mem, 0)

# -----------------------------------------------------------------------------
# Memory pointer

@testing.gpu
class TestMemoryPointer(unittest.TestCase):

    def test_int(self):
        pval = MockMemory.cur_ptr
        memptr = mock_alloc(1)
        self.assertEqual(pval, int(memptr))

    def test_add(self):
        pval = MockMemory.cur_ptr
        memptr = mock_alloc(8)

        memptr2 = memptr + 4
        self.assertIsInstance(memptr2, memory.MemoryPointer)
        self.assertEqual(pval + 4, int(memptr2))

        memptr3 = 4 + memptr
        self.assertIsInstance(memptr3, memory.MemoryPointer)
        self.assertEqual(pval + 4, int(memptr3))

        memptr += 4
        self.assertIsInstance(memptr, memory.MemoryPointer)
        self.assertEqual(pval + 4, int(memptr))

    def test_sub(self):
        pval = MockMemory.cur_ptr
        memptr = mock_alloc(8) + 4

        memptr2 = memptr - 4
        self.assertIsInstance(memptr2, memory.MemoryPointer)
        self.assertEqual(pval, int(memptr2))

        memptr -= 4
        self.assertIsInstance(memptr, memory.MemoryPointer)
        self.assertEqual(pval, int(memptr))

    def test_copy_to_and_from_host(self):
        a_gpu = memory.alloc(4)
        a_cpu = ctypes.c_int(100)
        a_gpu.copy_from(ctypes.byref(a_cpu), 4)
        b_cpu = ctypes.c_int()
        a_gpu.copy_to_host(ctypes.byref(b_cpu), 4)
        self.assertEqual(b_cpu.value, a_cpu.value)

    def test_copy_from_device(self):
        a_gpu = memory.alloc(4)
        a_cpu = ctypes.c_int(100)
        a_gpu.copy_from(ctypes.byref(a_cpu), 4)

        b_gpu = memory.alloc(4)
        b_gpu.copy_from(a_gpu, 4)
        b_cpu = ctypes.c_int()
        b_gpu.copy_to_host(ctypes.byref(b_cpu), 4)
        self.assertEqual(b_cpu.value, a_cpu.value)

    def test_memset(self):
        a_gpu = memory.alloc(4)
        a_gpu.memset(1, 4)
        a_cpu = ctypes.c_ubyte()
        for i in range(4):
            a_gpu.copy_to_host(ctypes.byref(a_cpu), 1)
            self.assertEqual(a_cpu.value, 1)
            a_gpu += 1

# -----------------------------------------------------------------------------
# Memory pool

@testing.gpu
class TestSingleDeviceMemoryPool(unittest.TestCase):

    def setUp(self):
        self.pool = memory.SingleDeviceMemoryPool(allocator=mock_alloc)

    def test_alloc(self):
        p1 = self.pool.malloc(10)
        p2 = self.pool.malloc(10)
        p3 = self.pool.malloc(20)
        self.assertNotEqual(p1.ptr.value, p2.ptr.value)
        self.assertNotEqual(p1.ptr.value, p3.ptr.value)
        self.assertNotEqual(p2.ptr.value, p3.ptr.value)

    def test_free(self):
        p1 = self.pool.malloc(10)
        ptr1 = p1.ptr
        del p1
        p2 = self.pool.malloc(10)
        self.assertEqual(ptr1.value, p2.ptr.value)

    def test_free_different_size(self):
        p1 = self.pool.malloc(10)
        ptr1 = p1.ptr
        del p1
        p2 = self.pool.malloc(20)
        self.assertNotEqual(ptr1.value, p2.ptr.value)

    def test_free_all_free(self):
        p1 = self.pool.malloc(10)
        ptr1 = p1.ptr
        del p1
        self.pool.free_all_free()
        p2 = self.pool.malloc(10)
        self.assertNotEqual(ptr1.value, p2.ptr.value)

@testing.gpu
class TestMemoryPool(unittest.TestCase):

    def setUp(self):
        self.pool = memory.SingleDeviceMemoryPool()

    def test_zero_size_alloc(self):
        mem = self.pool.malloc(0).mem
        self.assertIsInstance(mem, memory.Memory)
<commit_msg>Fix style<commit_after>import ctypes
import unittest

from cupy.cuda import memory
from cupy import testing

class MockMemory(object):
    cur_ptr = 1

    def __init__(self, size):
        self.ptr = ctypes.c_void_p(MockMemory.cur_ptr)
        MockMemory.cur_ptr += size
        self.size = size
        self._device = None

    def __int__(self):
        return self.ptr.value or 0

    @property
    def device(self):
        return self._device

def mock_alloc(size):
    mem = MockMemory(size)
    return memory.MemoryPointer(mem, 0)

# -----------------------------------------------------------------------------
# Memory pointer

@testing.gpu
class TestMemoryPointer(unittest.TestCase):

    def test_int(self):
        pval = MockMemory.cur_ptr
        memptr = mock_alloc(1)
        self.assertEqual(pval, int(memptr))

    def test_add(self):
        pval = MockMemory.cur_ptr
        memptr = mock_alloc(8)

        memptr2 = memptr + 4
        self.assertIsInstance(memptr2, memory.MemoryPointer)
        self.assertEqual(pval + 4, int(memptr2))

        memptr3 = 4 + memptr
        self.assertIsInstance(memptr3, memory.MemoryPointer)
        self.assertEqual(pval + 4, int(memptr3))

        memptr += 4
        self.assertIsInstance(memptr, memory.MemoryPointer)
        self.assertEqual(pval + 4, int(memptr))

    def test_sub(self):
        pval = MockMemory.cur_ptr
        memptr = mock_alloc(8) + 4

        memptr2 = memptr - 4
        self.assertIsInstance(memptr2, memory.MemoryPointer)
        self.assertEqual(pval, int(memptr2))

        memptr -= 4
        self.assertIsInstance(memptr, memory.MemoryPointer)
        self.assertEqual(pval, int(memptr))

    def test_copy_to_and_from_host(self):
        a_gpu = memory.alloc(4)
        a_cpu = ctypes.c_int(100)
        a_gpu.copy_from(ctypes.byref(a_cpu), 4)
        b_cpu = ctypes.c_int()
        a_gpu.copy_to_host(ctypes.byref(b_cpu), 4)
        self.assertEqual(b_cpu.value, a_cpu.value)

    def test_copy_from_device(self):
        a_gpu = memory.alloc(4)
        a_cpu = ctypes.c_int(100)
        a_gpu.copy_from(ctypes.byref(a_cpu), 4)

        b_gpu = memory.alloc(4)
        b_gpu.copy_from(a_gpu, 4)
        b_cpu = ctypes.c_int()
        b_gpu.copy_to_host(ctypes.byref(b_cpu), 4)
        self.assertEqual(b_cpu.value, a_cpu.value)

    def test_memset(self):
        a_gpu = memory.alloc(4)
        a_gpu.memset(1, 4)
        a_cpu = ctypes.c_ubyte()
        for i in range(4):
            a_gpu.copy_to_host(ctypes.byref(a_cpu), 1)
            self.assertEqual(a_cpu.value, 1)
            a_gpu += 1

# -----------------------------------------------------------------------------
# Memory pool

@testing.gpu
class TestSingleDeviceMemoryPool(unittest.TestCase):

    def setUp(self):
        self.pool = memory.SingleDeviceMemoryPool(allocator=mock_alloc)

    def test_alloc(self):
        p1 = self.pool.malloc(10)
        p2 = self.pool.malloc(10)
        p3 = self.pool.malloc(20)
        self.assertNotEqual(p1.ptr.value, p2.ptr.value)
        self.assertNotEqual(p1.ptr.value, p3.ptr.value)
        self.assertNotEqual(p2.ptr.value, p3.ptr.value)

    def test_free(self):
        p1 = self.pool.malloc(10)
        ptr1 = p1.ptr
        del p1
        p2 = self.pool.malloc(10)
        self.assertEqual(ptr1.value, p2.ptr.value)

    def test_free_different_size(self):
        p1 = self.pool.malloc(10)
        ptr1 = p1.ptr
        del p1
        p2 = self.pool.malloc(20)
        self.assertNotEqual(ptr1.value, p2.ptr.value)

    def test_free_all_free(self):
        p1 = self.pool.malloc(10)
        ptr1 = p1.ptr
        del p1
        self.pool.free_all_free()
        p2 = self.pool.malloc(10)
        self.assertNotEqual(ptr1.value, p2.ptr.value)

@testing.gpu
class TestMemoryPool(unittest.TestCase):

    def setUp(self):
        self.pool = memory.SingleDeviceMemoryPool()

    def test_zero_size_alloc(self):
        mem = self.pool.malloc(0).mem
        self.assertIsInstance(mem, memory.Memory)
<|endoftext|>


<commit_before># coding: utf-8

import re
import subprocess
import time

from ccmlib import common
from dtest import Tester
from tools import rows_to_list
from cassandra.util import sortedset

class TokenGenerator(Tester):
    """
    Basic tools/bin/token-generator test.
    """

    def call_token_generator(self, install_dir, randomPart, nodes):
        executable = install_dir + "/tools/bin/token-generator"
        if common.is_win():
            executable += ".bat"

        args = [executable]
        for n in nodes:
            args.append(str(n))

        if randomPart:
            if randomPart:
                args.insert(1, "--random")
            else:
                args.insert(1, "--murmur3")

        token_gen_output = subprocess.check_output(args)
        lines = token_gen_output.split("\n")
        dc_tokens = None
        generated_tokens = []
        for line in lines:
            if line.startswith("DC #"):
                if dc_tokens is not None:
                    self.assertGreater(dc_tokens.__len__(), 0, "dc_tokens is empty from token-generator %r" % args)
                    generated_tokens.append(dc_tokens)
                dc_tokens = []
            else:
                if line.__len__() > 0:
                    m = re.search("^  Node #(\d+): [ ]*([-]?\d+)$", line)
                    self.assertIsNotNone(m, "Line \"%r\" does not match pattern from token-generator %r" % (line, args))
                    node_num = int(m.group(1))
                    node_token = int(m.group(2))
                    dc_tokens.append(node_token)
                    self.assertEqual(node_num, dc_tokens.__len__(), "invalid token count from token-generator %r" % args)
        self.assertIsNotNone(dc_tokens, "No tokens from token-generator %r" % args)
        self.assertGreater(dc_tokens.__len__(), 0, "No tokens from token-generator %r" % args)
        generated_tokens.append(dc_tokens)

        return generated_tokens

    def prepare(self, randomPart=None, nodes=1):
        cluster = self.cluster

        install_dir = cluster.get_install_dir();

        generated_tokens = self.call_token_generator(install_dir, randomPart, [nodes])

        if not randomPart:
            cluster.set_partitioner("org.apache.cassandra.dht.Murmur3Partitioner")
        else:
            if randomPart:
                cluster.set_partitioner("org.apache.cassandra.dht.RandomPartitioner")
            else:
                cluster.set_partitioner("org.apache.cassandra.dht.Murmur3Partitioner")

        # remove these from cluster options - otherwise node's config would be overridden with cluster._config_options_
        cluster._config_options.__delitem__('num_tokens')
        cluster._config_options.__delitem__('initial_token')

        self.assertTrue(not cluster.nodelist(), "nodelist() already initialized")
        cluster.populate(nodes, use_vnodes=False, tokens=generated_tokens[0]).start(wait_for_binary_proto=True)
        time.sleep(0.2)

        node = cluster.nodelist()[0]
        session = self.patient_cql_connection(node)

        return generated_tokens, session

    def _token_gen_test(self, nodes, randomPart=None):
        generated_tokens, cursor = self.prepare(randomPart, nodes=nodes)
        dc_tokens = generated_tokens[0]

        tokens = []
        local_tokens = rows_to_list(cursor.execute("SELECT tokens FROM system.local"))[0];
        self.assertEqual(local_tokens.__len__(), 1, "too many tokens for peer")
        for tok in local_tokens:
            tokens += tok

        rows = rows_to_list(cursor.execute("SELECT tokens FROM system.peers"))
        self.assertEqual(rows.__len__(), nodes - 1)
        for row in rows:
            peer_tokens = row[0]
            self.assertEqual(peer_tokens.__len__(), 1, "too many tokens for peer")
            for tok in peer_tokens:
                tokens.append(tok)

        self.assertEqual(tokens.__len__(), dc_tokens.__len__())
        for cluster_token in tokens:
            tok = int(cluster_token)
            self.assertGreaterEqual(dc_tokens.index(tok), 0, "token in cluster does not match generated tokens")

    def token_gen_def_test(self, nodes=3):
        """ Validate token-generator with Murmur3Partitioner with default token-generator behavior """

        self._token_gen_test(nodes)

    def token_gen_murmur3_test(self, nodes=3):
        """ Validate token-generator with Murmur3Partitioner with explicit murmur3 """

        self._token_gen_test(nodes, False)

    def token_gen_random_test(self, nodes=3):
        """ Validate token-generator with Murmur3Partitioner with explicit random """

        self._token_gen_test(nodes, True)

    dc_nodes_combinations = [
        [3, 5],
        [3, 5, 5],
        [12, 5, 7],
        [50, 100, 250],
        [100, 100, 100],
        [250, 250, 250],
        [1000, 1000, 1000]
    ]

    def _multi_dc_tokens(self, random=None):
        t_min = 0
        t_max = 1<<127
        if random is None or not random:
            t_min = -1<<63
            t_max = 1<<63
        for dc_nodes in self.dc_nodes_combinations:
            all_tokens = sortedset()
            node_count = 0
            generated_tokens = self.call_token_generator(self.cluster.get_install_dir(), random, dc_nodes)
            self.assertEqual(dc_nodes.__len__(), generated_tokens.__len__())
            for n in range(0, dc_nodes.__len__()):
                nodes = dc_nodes[n]
                node_count += nodes
                tokens = generated_tokens[n]
                self.assertEqual(nodes, tokens.__len__())
                for tok in tokens:
                    self.assertTrue(t_min <= tok < t_max, "Generated token %r out of Murmur3Partitioner range %r..%r" % (tok, t_min, t_max-1))
                    self.assertTrue(not all_tokens.__contains__(tok), "Duplicate token %r for nodes-counts %r" % (tok, dc_nodes))
                    all_tokens.add(tok)
            self.assertEqual(all_tokens.__len__(), node_count, "Number of tokens %r and number of nodes %r does not match for %r" % (all_tokens.__len__(), node_count, dc_nodes))

    def multi_dc_tokens_default_test(self):
        self._multi_dc_tokens()

    def multi_dc_tokens_murmur3_test(self):
        self._multi_dc_tokens(False)

    def multi_dc_tokens_random_test(self):
        self._multi_dc_tokens(True)<commit_msg>Added require and fixed problem with randomPart<commit_after># coding: utf-8

import re
import subprocess
import time

from ccmlib import common
from dtest import Tester, debug
from tools import rows_to_list, require
from cassandra.util import sortedset

@require('9300')
class TokenGenerator(Tester):
    """
    Basic tools/bin/token-generator test.
    """

    def call_token_generator(self, install_dir, randomPart, nodes):
        executable = install_dir + "/tools/bin/token-generator"
        if common.is_win():
            executable += ".bat"

        args = [executable]

        if randomPart is not None:
            if randomPart:
                args.append("--random")
            else:
                args.append("--murmur3")

        for n in nodes:
            args.append(str(n))

        debug('Invoking %s' % (args,))
        token_gen_output = subprocess.check_output(args)
        lines = token_gen_output.split("\n")
        dc_tokens = None
        generated_tokens = []
        for line in lines:
            if line.startswith("DC #"):
                if dc_tokens is not None:
                    self.assertGreater(dc_tokens.__len__(), 0, "dc_tokens is empty from token-generator %r" % args)
                    generated_tokens.append(dc_tokens)
                dc_tokens = []
            else:
                if line.__len__() > 0:
                    m = re.search("^  Node #(\d+): [ ]*([-]?\d+)$", line)
                    self.assertIsNotNone(m, "Line \"%r\" does not match pattern from token-generator %r" % (line, args))
                    node_num = int(m.group(1))
                    node_token = int(m.group(2))
                    dc_tokens.append(node_token)
                    self.assertEqual(node_num, dc_tokens.__len__(), "invalid token count from token-generator %r" % args)
        self.assertIsNotNone(dc_tokens, "No tokens from token-generator %r" % args)
        self.assertGreater(dc_tokens.__len__(), 0, "No tokens from token-generator %r" % args)
        generated_tokens.append(dc_tokens)

        return generated_tokens

    def prepare(self, randomPart=None, nodes=1):
        cluster = self.cluster

        install_dir = cluster.get_install_dir();

        generated_tokens = self.call_token_generator(install_dir, randomPart, [nodes])

        if not randomPart:
            cluster.set_partitioner("org.apache.cassandra.dht.Murmur3Partitioner")
        else:
            if randomPart:
                cluster.set_partitioner("org.apache.cassandra.dht.RandomPartitioner")
            else:
                cluster.set_partitioner("org.apache.cassandra.dht.Murmur3Partitioner")

        # remove these from cluster options - otherwise node's config would be overridden with cluster._config_options_
        cluster._config_options.__delitem__('num_tokens')
        cluster._config_options.__delitem__('initial_token')

        self.assertTrue(not cluster.nodelist(), "nodelist() already initialized")
        cluster.populate(nodes, use_vnodes=False, tokens=generated_tokens[0]).start(wait_for_binary_proto=True)
        time.sleep(0.2)

        node = cluster.nodelist()[0]
        session = self.patient_cql_connection(node)

        return generated_tokens, session

    def _token_gen_test(self, nodes, randomPart=None):
        generated_tokens, cursor = self.prepare(randomPart, nodes=nodes)
        dc_tokens = generated_tokens[0]

        tokens = []
        local_tokens = rows_to_list(cursor.execute("SELECT tokens FROM system.local"))[0];
        self.assertEqual(local_tokens.__len__(), 1, "too many tokens for peer")
        for tok in local_tokens:
            tokens += tok

        rows = rows_to_list(cursor.execute("SELECT tokens FROM system.peers"))
        self.assertEqual(rows.__len__(), nodes - 1)
        for row in rows:
            peer_tokens = row[0]
            self.assertEqual(peer_tokens.__len__(), 1, "too many tokens for peer")
            for tok in peer_tokens:
                tokens.append(tok)

        self.assertEqual(tokens.__len__(), dc_tokens.__len__())
        for cluster_token in tokens:
            tok = int(cluster_token)
            self.assertGreaterEqual(dc_tokens.index(tok), 0, "token in cluster does not match generated tokens")

    def token_gen_def_test(self, nodes=3):
        """ Validate token-generator with Murmur3Partitioner with default token-generator behavior """

        self._token_gen_test(nodes)

    def token_gen_murmur3_test(self, nodes=3):
        """ Validate token-generator with Murmur3Partitioner with explicit murmur3 """

        self._token_gen_test(nodes, False)

    def token_gen_random_test(self, nodes=3):
        """ Validate token-generator with Murmur3Partitioner with explicit random """

        self._token_gen_test(nodes, True)

    dc_nodes_combinations = [
        [3, 5],
        [3, 5, 5],
        [12, 5, 7],
        [50, 100, 250],
        [100, 100, 100],
        [250, 250, 250],
        [1000, 1000, 1000]
    ]

    def _multi_dc_tokens(self, random=None):
        t_min = 0
        t_max = 1<<127
        if random is None or not random:
            t_min = -1<<63
            t_max = 1<<63
        for dc_nodes in self.dc_nodes_combinations:
            all_tokens = sortedset()
            node_count = 0
            generated_tokens = self.call_token_generator(self.cluster.get_install_dir(), random, dc_nodes)
            self.assertEqual(dc_nodes.__len__(), generated_tokens.__len__())
            for n in range(0, dc_nodes.__len__()):
                nodes = dc_nodes[n]
                node_count += nodes
                tokens = generated_tokens[n]
                self.assertEqual(nodes, tokens.__len__())
                for tok in tokens:
                    self.assertTrue(t_min <= tok < t_max, "Generated token %r out of Murmur3Partitioner range %r..%r" % (tok, t_min, t_max-1))
                    self.assertTrue(not all_tokens.__contains__(tok), "Duplicate token %r for nodes-counts %r" % (tok, dc_nodes))
                    all_tokens.add(tok)
            self.assertEqual(all_tokens.__len__(), node_count, "Number of tokens %r and number of nodes %r does not match for %r" % (all_tokens.__len__(), node_count, dc_nodes))

    def multi_dc_tokens_default_test(self):
        self._multi_dc_tokens()

    def multi_dc_tokens_murmur3_test(self):
        self._multi_dc_tokens(False)

    def multi_dc_tokens_random_test(self):
        self._multi_dc_tokens(True)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codauthor: :email:`Mike Place <mp@saltstack.com>`
'''

# Import Salt Testing libs
from salttesting import skipIf
from salttesting.helpers import (ensure_in_syspath, destructiveTest)
from salttesting.mock import patch, MagicMock, call, NO_MOCK, NO_MOCK_REASON
ensure_in_syspath('../')

# Import Python libs
import os
import shutil

# Import salt libs
import integration
from salt.fileserver import gitfs

gitfs.__opts__ = {'gitfs_remotes': [''],
                  'gitfs_root': os.path.join(integration.TMP, 'gitfs_root'),
                  'fileserver_backend': 'gitfs',
                  }

GITFS_AVAILABLE = None
try:
    import git
    GITFS_AVAILABLE = True
except ImportError:
    pass

if not gitfs.__virtual__():
    GITFS_AVAILABLE = False

@skipIf(not GITFS_AVAILABLE, "GitFS could not be loaded. Skipping GitFS tests!")
class GitFSTest(integration.ModuleCase):
    def setUp(self):
        '''
        We don't want to check in another .git dir into GH because that just gets messy.
        Instead, we'll create a temporary repo on the fly for the tests to examine.
        :return:
        '''
        integration_base_files = os.path.join(integration.FILES, 'file', 'base')
        tmp_repo_dir = os.path.join(integration.TMP, 'gitfs_root')
        tmp_repo_git = os.path.join(tmp_repo_dir, '.git')

        # Create the dir if it doesn't already exist
        os.mkdirs(tmp_repo_git)

        git.Repo.create(tmp_repo_git, bare=True)

        shutil.copytree(integration_base_files, tmp_repo_dir, symlinks=True)

        git_bin = git.Git(tmp_repo_git)
        git_bin.add(tmp_repo_dir)

    def test_fake(self):
        pass
<commit_msg>More gitfs integration testing.<commit_after># -*- coding: utf-8 -*-
'''
    :codauthor: :email:`Mike Place <mp@saltstack.com>`
'''

# Import Salt Testing libs
from salttesting import skipIf
from salttesting.helpers import (ensure_in_syspath, destructiveTest)
from salttesting.mock import patch, MagicMock, call, NO_MOCK, NO_MOCK_REASON

ensure_in_syspath('../')

# Import Python libs
import os
import shutil

# Import salt libs
import integration
from salt.fileserver import gitfs

gitfs.__opts__ = {'gitfs_remotes': [''],
		  'gitfs_root': '',
                  'fileserver_backend': 'gitfs',
		  'gitfs_base': 'master',
		  'fileserver_events': True,
}

load = {'saltenv': 'base'}

GITFS_AVAILABLE = None
try:
    import git

    GITFS_AVAILABLE = True
except ImportError:
    pass

if not gitfs.__virtual__():
    GITFS_AVAILABLE = False

# @skipIf(not GITFS_AVAILABLE, "GitFS could not be loaded. Skipping GitFS tests!")
class GitFSTest(integration.ModuleCase):
    def setUp(self):
        '''
        We don't want to check in another .git dir into GH because that just gets messy.
        Instead, we'll create a temporary repo on the fly for the tests to examine.
        :return:
        '''
	self.integration_base_files = os.path.join(integration.FILES, 'file', 'base')
	self.tmp_repo_dir = os.path.join(integration.TMP, 'gitfs_root')
	self.tmp_repo_git = os.path.join(self.tmp_repo_dir, '.git')

        # Create the dir if it doesn't already exist

	try:
	    shutil.copytree(self.integration_base_files, self.tmp_repo_dir + '/')
	except OSError as e:
	    # We probably caught an error because files already exist. Ignore
	    pass

	if not os.path.exists(self.tmp_repo_git):
	    os.makedirs(self.tmp_repo_git)
	try:
	    git_bin = git.Git(self.tmp_repo_git)
	    git_bin.init(self.tmp_repo_dir)
	    os.chdir(self.tmp_repo_dir)
	    git_bin.add('.', with_keep_cwd=True)  # Is there a way to pass in a .git repo?
	    git_bin.commit('-a', '-m', 'Test', with_keep_cwd=True)
	except git.GitCommandError:  # Will throw a command error if you try to init a repo that already exists
	    pass

	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    gitfs.update()

    def test_file_list(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.file_list(load)
	    self.assertIn('testfile', ret)

    def test_find_file(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.find_file('testfile')
	    expected_ret = {'path': '/tmp/salttest/cache/gitfs/refs/master/testfile',
			    'rel': 'testfile'}

	    self.assertDictEqual(ret, expected_ret)

    def test_dir_list(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.dir_list(load)
	    self.assertIn('grail', ret)

    @skipIf(True, 'This test is failing and for good reason! See #9193')
    def test_file_list_emptydirs(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.file_list_emptydirs(load)
	    self.assertIn('empty_dir', ret)

    def test_envs(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.envs()
	    self.assertIn('base', ret)

    def test_file_hash_sha1(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir'],
					 'hash_type': 'blob_sha1'}):
	    tmp_load = load
	    tmp_load['path'] = 'testfile'
	    fnd = {'rel': 'testfile',
		   'path': 'testfile'}
	    ret = gitfs.file_hash(load, fnd)
	    self.assertDictEqual({'hash_type': 'blob_sha1', 'hsum': '0d234303e6451128d756c5c259175de37d767742'}, ret)

    @skipIf(True, 'This test is failing and for good reason! See #9194')
    def test_file_hash_md5(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir'],
					 'hash_type': 'md5'}):
	    tmp_load = load
	    tmp_load['path'] = 'testfile'
	    fnd = {'rel': 'testfile',
		   'path': 'testfile'}
	    ret = gitfs.file_hash(load, fnd)
	    self.assertDictEqual({'hash_type': 'md5', 'hsum': '98aa509006628302ce38ce521a7f805f'}, ret)

    def test_serve_file(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir'],
					 'file_buffer_size': 262144}):
	    fnd = {'rel': 'testfile',
		   'path': 'testfile'}

	    tmp_load = load
	    tmp_load['loc'] = 0

	    ret = gitfs.serve_file(load, fnd)
	    self.assertDictEqual({'data': 'Scene 24\n\n \n  OLD MAN:  Ah, hee he he ha!\n  ARTHUR:  And this enchanter of whom you speak, he has seen the grail?\n  OLD MAN:  Ha ha he he he he!\n  ARTHUR:  Where does he live?  Old man, where does he live?\n  OLD MAN:  He knows of a cave, a cave which no man has entered.\n  ARTHUR:  And the Grail... The Grail is there?\n  OLD MAN:  Very much danger, for beyond the cave lies the Gorge\n      of Eternal Peril, which no man has ever crossed.\n  ARTHUR:  But the Grail!  Where is the Grail!?\n  OLD MAN:  Seek you the Bridge of Death.\n  ARTHUR:  The Bridge of Death, which leads to the Grail?\n  OLD MAN:  Hee hee ha ha!\n\n',
 'dest': 'testfile'}, ret)

if __name__ == '__main__':
    integration.run_tests(GitFSTest)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codauthor: :email:`Mike Place <mp@saltstack.com>`
'''

# Import Salt Testing libs
from salttesting import skipIf
from salttesting.helpers import (ensure_in_syspath, destructiveTest)
from salttesting.mock import patch, MagicMock, call, NO_MOCK, NO_MOCK_REASON

ensure_in_syspath('../')

# Import Python libs
import os
import shutil

# Import salt libs
import integration
from salt.fileserver import gitfs

gitfs.__opts__ = {'gitfs_remotes': [''],
		  'gitfs_root': '',
                  'fileserver_backend': 'gitfs',
		  'gitfs_base': 'master',
		  'fileserver_events': True,
}

load = {'saltenv': 'base'}

GITFS_AVAILABLE = None
try:
    import git

    GITFS_AVAILABLE = True
except ImportError:
    pass

if not gitfs.__virtual__():
    GITFS_AVAILABLE = False

# @skipIf(not GITFS_AVAILABLE, "GitFS could not be loaded. Skipping GitFS tests!")
class GitFSTest(integration.ModuleCase):
    def setUp(self):
        '''
        We don't want to check in another .git dir into GH because that just gets messy.
        Instead, we'll create a temporary repo on the fly for the tests to examine.
        :return:
        '''
	self.integration_base_files = os.path.join(integration.FILES, 'file', 'base')
	self.tmp_repo_dir = os.path.join(integration.TMP, 'gitfs_root')
	self.tmp_repo_git = os.path.join(self.tmp_repo_dir, '.git')

        # Create the dir if it doesn't already exist

	try:
	    shutil.copytree(self.integration_base_files, self.tmp_repo_dir + '/')
	except OSError as e:
	    # We probably caught an error because files already exist. Ignore
	    pass

	if not os.path.exists(self.tmp_repo_git):
	    os.makedirs(self.tmp_repo_git)
	try:
	    git_bin = git.Git(self.tmp_repo_git)
	    git_bin.init(self.tmp_repo_dir)
	    os.chdir(self.tmp_repo_dir)
	    git_bin.add('.', with_keep_cwd=True)  # Is there a way to pass in a .git repo?
	    git_bin.commit('-a', '-m', 'Test', with_keep_cwd=True)
	except git.GitCommandError:  # Will throw a command error if you try to init a repo that already exists
	    pass

	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    gitfs.update()

    def test_file_list(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.file_list(load)
	    self.assertIn('testfile', ret)

    def test_find_file(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.find_file('testfile')
	    expected_ret = {'path': '/tmp/salttest/cache/gitfs/refs/master/testfile',
			    'rel': 'testfile'}

	    self.assertDictEqual(ret, expected_ret)

    def test_dir_list(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.dir_list(load)
	    self.assertIn('grail', ret)

    @skipIf(True, 'This test is failing and for good reason! See #9193')
    def test_file_list_emptydirs(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.file_list_emptydirs(load)
	    self.assertIn('empty_dir', ret)

    def test_envs(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.envs()
	    self.assertIn('base', ret)

    def test_file_hash_sha1(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir'],
					 'hash_type': 'blob_sha1'}):
	    tmp_load = load
	    tmp_load['path'] = 'testfile'
	    fnd = {'rel': 'testfile',
		   'path': 'testfile'}
	    ret = gitfs.file_hash(load, fnd)
	    self.assertDictEqual({'hash_type': 'blob_sha1', 'hsum': '0d234303e6451128d756c5c259175de37d767742'}, ret)

    @skipIf(True, 'This test is failing and for good reason! See #9194')
    def test_file_hash_md5(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir'],
					 'hash_type': 'md5'}):
	    tmp_load = load
	    tmp_load['path'] = 'testfile'
	    fnd = {'rel': 'testfile',
		   'path': 'testfile'}
	    ret = gitfs.file_hash(load, fnd)
	    self.assertDictEqual({'hash_type': 'md5', 'hsum': '98aa509006628302ce38ce521a7f805f'}, ret)

    def test_serve_file(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir'],
					 'file_buffer_size': 262144}):
	    fnd = {'rel': 'testfile',
		   'path': 'testfile'}

	    tmp_load = load
	    tmp_load['loc'] = 0

	    ret = gitfs.serve_file(load, fnd)
	    self.assertDictEqual({'data': 'Scene 24\n\n \n  OLD MAN:  Ah, hee he he ha!\n  ARTHUR:  And this enchanter of whom you speak, he has seen the grail?\n  OLD MAN:  Ha ha he he he he!\n  ARTHUR:  Where does he live?  Old man, where does he live?\n  OLD MAN:  He knows of a cave, a cave which no man has entered.\n  ARTHUR:  And the Grail... The Grail is there?\n  OLD MAN:  Very much danger, for beyond the cave lies the Gorge\n      of Eternal Peril, which no man has ever crossed.\n  ARTHUR:  But the Grail!  Where is the Grail!?\n  OLD MAN:  Seek you the Bridge of Death.\n  ARTHUR:  The Bridge of Death, which leads to the Grail?\n  OLD MAN:  Hee hee ha ha!\n\n',
 'dest': 'testfile'}, ret)

if __name__ == '__main__':
    integration.run_tests(GitFSTest)
<commit_msg>Pylinting.<commit_after># -*- coding: utf-8 -*-
'''
    :codauthor: :email:`Mike Place <mp@saltstack.com>`
'''

# Import Salt Testing libs
from salttesting import skipIf
from salttesting.helpers import (ensure_in_syspath, destructiveTest)
from salttesting.mock import patch, MagicMock, call, NO_MOCK, NO_MOCK_REASON

ensure_in_syspath('../')

# Import Python libs
import os
import shutil

# Import salt libs
import integration
from salt.fileserver import gitfs

gitfs.__opts__ = {'gitfs_remotes': [''],
		  'gitfs_root': '',
                  'fileserver_backend': 'gitfs',
		  'gitfs_base': 'master',
		  'fileserver_events': True,
}

load = {'saltenv': 'base'}

GITFS_AVAILABLE = None
try:
    import git

    GITFS_AVAILABLE = True
except ImportError:
    pass

if not gitfs.__virtual__():
    GITFS_AVAILABLE = False

# @skipIf(not GITFS_AVAILABLE, "GitFS could not be loaded. Skipping GitFS tests!")
class GitFSTest(integration.ModuleCase):
    def setUp(self):
        '''
        We don't want to check in another .git dir into GH because that just gets messy.
        Instead, we'll create a temporary repo on the fly for the tests to examine.
        :return:
        '''
	self.integration_base_files = os.path.join(integration.FILES, 'file', 'base')
	self.tmp_repo_dir = os.path.join(integration.TMP, 'gitfs_root')
	self.tmp_repo_git = os.path.join(self.tmp_repo_dir, '.git')

        # Create the dir if it doesn't already exist

	try:
	    shutil.copytree(self.integration_base_files, self.tmp_repo_dir + '/')
	except OSError as e:
	    # We probably caught an error because files already exist. Ignore
	    pass

	if not os.path.exists(self.tmp_repo_git):
	    os.makedirs(self.tmp_repo_git)
	try:
	    git_bin = git.Git(self.tmp_repo_git)
	    git_bin.init(self.tmp_repo_dir)
	    os.chdir(self.tmp_repo_dir)
	    git_bin.add('.', with_keep_cwd=True)  # Is there a way to pass in a .git repo?
	    git_bin.commit('-a', '-m', 'Test', with_keep_cwd=True)
	except git.GitCommandError:  # Will throw a command error if you try to init a repo that already exists
	    pass

	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    gitfs.update()

    def test_file_list(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.file_list(load)
	    self.assertIn('testfile', ret)

    def test_find_file(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.find_file('testfile')
	    expected_ret = {'path': '/tmp/salttest/cache/gitfs/refs/master/testfile',
			    'rel': 'testfile'}

	    self.assertDictEqual(ret, expected_ret)

    def test_dir_list(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.dir_list(load)
	    self.assertIn('grail', ret)

    @skipIf(True, 'This test is failing and for good reason! See #9193')
    def test_file_list_emptydirs(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.file_list_emptydirs(load)
	    self.assertIn('empty_dir', ret)

    def test_envs(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir']}):
	    ret = gitfs.envs()
	    self.assertIn('base', ret)

    def test_file_hash_sha1(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir'],
					 'hash_type': 'blob_sha1'}):
	    tmp_load = load
	    tmp_load['path'] = 'testfile'
	    fnd = {'rel': 'testfile',
		   'path': 'testfile'}
	    ret = gitfs.file_hash(load, fnd)
	    self.assertDictEqual({'hash_type': 'blob_sha1', 'hsum': '0d234303e6451128d756c5c259175de37d767742'}, ret)

    @skipIf(True, 'This test is failing and for good reason! See #9194')
    def test_file_hash_md5(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir'],
					 'hash_type': 'md5'}):
	    tmp_load = load
	    tmp_load['path'] = 'testfile'
	    fnd = {'rel': 'testfile',
		   'path': 'testfile'}
	    ret = gitfs.file_hash(load, fnd)
	    self.assertDictEqual({'hash_type': 'md5', 'hsum': '98aa509006628302ce38ce521a7f805f'}, ret)

    def test_serve_file(self):
	with patch.dict(gitfs.__opts__, {'cachedir': self.master_opts['cachedir'],
					 'gitfs_remotes': ['file://' + self.tmp_repo_git],
					 'sock_dir': self.master_opts['sock_dir'],
					 'file_buffer_size': 262144}):
	    fnd = {'rel': 'testfile',
		   'path': 'testfile'}

	    tmp_load = load
	    tmp_load['loc'] = 0

	    ret = gitfs.serve_file(load, fnd)
	    self.assertDictEqual({'data': 'Scene 24\n\n \n  OLD MAN:  Ah, hee he he ha!\n  ARTHUR:  And this enchanter of whom you speak, he has seen the grail?\n  OLD MAN:  Ha ha he he he he!\n  ARTHUR:  Where does he live?  Old man, where does he live?\n  OLD MAN:  He knows of a cave, a cave which no man has entered.\n  ARTHUR:  And the Grail... The Grail is there?\n  OLD MAN:  Very much danger, for beyond the cave lies the Gorge\n      of Eternal Peril, which no man has ever crossed.\n  ARTHUR:  But the Grail!  Where is the Grail!?\n  OLD MAN:  Seek you the Bridge of Death.\n  ARTHUR:  The Bridge of Death, which leads to the Grail?\n  OLD MAN:  Hee hee ha ha!\n\n',
 'dest': 'testfile'}, ret)

if __name__ == '__main__':
    integration.run_tests(GitFSTest)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
from os.path import join
from os.path import dirname

import pytest
import dhtmlparser

from marcxml2mods import transformators
from marcxml2mods import xslt_transformer
from marcxml2mods.mods_postprocessor import monograph

# Variables ===================================================================
DIRNAME = join(dirname(dirname(__file__)), "data")
OAI_FILENAME = join(DIRNAME, "oai_example.oai")

@pytest.fixture
def postprocessed():
    fn = join(DIRNAME, "postprocessed_mods.xml")

    with open(fn) as f:
        return f.read()

# Tests =======================================================================
def test_postprocess_mods_mono(postprocessed):
    result = transformators.transform_to_mods_mono(OAI_FILENAME, "someid")

    # with open("xex.xml", "wt") as f:
        # f.write(result[0])

    assert result[0] == postprocessed

def test_fix_location_tag():
    XML = """
<mods:mods>
  <mods:identifier type="ccnb">cnb000003024</mods:identifier>
  <mods:location>
    <mods:physicalLocation authority="siglaADR">ABA001</mods:physicalLocation>
    <mods:url displayLabel="electronic resource" usage="primary display">http://edeposit-test.nkp.cz/producents/beta-nakladatelstvi/epublications/controlling-v-msp-nejen-o-cislech/controlling.pdf</mods:url>
    <mods:url displayLabel="electronic resource">http://edeposit.nkp.cz/</mods:url>
  </mods:location>
  <mods:identifier type="uuid">236b6283-22d1-4014-ae50-a303cfd15419</mods:identifier>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
  <mods:relatedItem>
    <mods:location>
      <mods:url>http://edeposit.nkp.cz/</mods:url>
    </mods:location>
  </mods:relatedItem>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
</mods:mods>
    """

    dom = dhtmlparser.parseString(XML)
    dhtmlparser.makeDoubleLinked(dom)

    monograph.fix_location_tag(dom)

    assert dom.__str__() == """
<mods:mods>
  <mods:identifier type="ccnb">cnb000003024</mods:identifier>
  <mods:location>
    <mods:holdingSimple>
      <mods:copyInformation>
        <mods:electronicLocator>http://edeposit-test.nkp.cz/producents/beta-nakladatelstvi/epublications/controlling-v-msp-nejen-o-cislech/controlling.pdf</mods:electronicLocator>
      </mods:copyInformation>
    </mods:holdingSimple>
  </mods:location>
  <mods:identifier type="uuid">236b6283-22d1-4014-ae50-a303cfd15419</mods:identifier>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
  <mods:relatedItem>
    <mods:location>
      <mods:url>http://edeposit.nkp.cz/</mods:url>
    </mods:location>
  </mods:relatedItem>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
</mods:mods>
    """

def test_fix_related_item_tag():
    XML = """
<mods:mods>
  <mods:identifier type="ccnb">cnb000003024</mods:identifier>
  <mods:identifier type="uuid">236b6283-22d1-4014-ae50-a303cfd15419</mods:identifier>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
  <mods:relatedItem>
    <mods:location>
      <mods:url>http://edeposit.nkp.cz/</mods:url>
    </mods:location>
  </mods:relatedItem>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
</mods:mods>
    """

    dom = dhtmlparser.parseString(XML)
    dhtmlparser.makeDoubleLinked(dom)

    monograph.fix_related_item_tag(dom)

    assert dom.prettify() == """<mods:mods>
  <mods:identifier type="ccnb">cnb000003024</mods:identifier>
  <mods:identifier type="uuid">236b6283-22d1-4014-ae50-a303cfd15419</mods:identifier>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
    <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
</mods:mods>
"""
<commit_msg>#6: Added tests of add_marccountry_tag().<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
from os.path import join
from os.path import dirname

import pytest
import dhtmlparser

from marcxml2mods import transformators
from marcxml2mods import xslt_transformer
from marcxml2mods.mods_postprocessor import monograph
from marcxml2mods.mods_postprocessor import shared_funcs

# Variables ===================================================================
DIRNAME = join(dirname(dirname(__file__)), "data")
OAI_FILENAME = join(DIRNAME, "oai_example.oai")

@pytest.fixture
def postprocessed():
    fn = join(DIRNAME, "postprocessed_mods.xml")

    with open(fn) as f:
        return f.read()

# Tests =======================================================================
def test_postprocess_mods_mono(postprocessed):
    result = transformators.transform_to_mods_mono(OAI_FILENAME, "someid")

    # with open("xex.xml", "wt") as f:
        # f.write(result[0])

    assert result[0] == postprocessed

def test_fix_location_tag():
    XML = """
<mods:mods>
  <mods:identifier type="ccnb">cnb000003024</mods:identifier>
  <mods:location>
    <mods:physicalLocation authority="siglaADR">ABA001</mods:physicalLocation>
    <mods:url displayLabel="electronic resource" usage="primary display">http://edeposit-test.nkp.cz/producents/beta-nakladatelstvi/epublications/controlling-v-msp-nejen-o-cislech/controlling.pdf</mods:url>
    <mods:url displayLabel="electronic resource">http://edeposit.nkp.cz/</mods:url>
  </mods:location>
  <mods:identifier type="uuid">236b6283-22d1-4014-ae50-a303cfd15419</mods:identifier>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
  <mods:relatedItem>
    <mods:location>
      <mods:url>http://edeposit.nkp.cz/</mods:url>
    </mods:location>
  </mods:relatedItem>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
</mods:mods>
    """
    dom = shared_funcs.double_linked_dom(XML)

    monograph.fix_location_tag(dom)

    assert dom.__str__() == """
<mods:mods>
  <mods:identifier type="ccnb">cnb000003024</mods:identifier>
  <mods:location>
    <mods:holdingSimple>
      <mods:copyInformation>
        <mods:electronicLocator>http://edeposit-test.nkp.cz/producents/beta-nakladatelstvi/epublications/controlling-v-msp-nejen-o-cislech/controlling.pdf</mods:electronicLocator>
      </mods:copyInformation>
    </mods:holdingSimple>
  </mods:location>
  <mods:identifier type="uuid">236b6283-22d1-4014-ae50-a303cfd15419</mods:identifier>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
  <mods:relatedItem>
    <mods:location>
      <mods:url>http://edeposit.nkp.cz/</mods:url>
    </mods:location>
  </mods:relatedItem>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
</mods:mods>
    """

def test_fix_related_item_tag():
    XML = """
<mods:mods>
  <mods:identifier type="ccnb">cnb000003024</mods:identifier>
  <mods:identifier type="uuid">236b6283-22d1-4014-ae50-a303cfd15419</mods:identifier>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
  <mods:relatedItem>
    <mods:location>
      <mods:url>http://edeposit.nkp.cz/</mods:url>
    </mods:location>
  </mods:relatedItem>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
</mods:mods>
    """
    dom = shared_funcs.double_linked_dom(XML)

    monograph.fix_related_item_tag(dom)

    assert dom.prettify() == """<mods:mods>
  <mods:identifier type="ccnb">cnb000003024</mods:identifier>
  <mods:identifier type="uuid">236b6283-22d1-4014-ae50-a303cfd15419</mods:identifier>
  <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
    <mods:identifier type="isbn">978-80-7408-086-9</mods:identifier>
</mods:mods>
"""

def test_add_marccountry_tag():
    XML = """<mods:mods>
  <mods:originInfo>
    <mods:place>
      <mods:placeTerm type="text">Brno</mods:placeTerm>
    </mods:place>
  </mods:originInfo>
</mods:mods>
"""
    dom = shared_funcs.double_linked_dom(XML)

    monograph.add_marccountry_tag(dom)

    assert dom.prettify() == """<mods:mods>
  <mods:originInfo>
    <mods:place>
      <mods:placeTerm type="code" authority="marccountry">xr-</mods:placeTerm>
    </mods:place>
    <mods:place>
      <mods:placeTerm type="text">Brno</mods:placeTerm>
    </mods:place>
  </mods:originInfo>
</mods:mods>
"""<|endoftext|>


<commit_before><commit_msg>Removed empty tests/regressiontests/forms/localflavor.py file left over after [6379]<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

import json
import operator
import time

from flask import current_app, redirect, render_template, request, url_for
from flask.ext.babel import gettext
from sqlalchemy import and_, func
from sqlalchemy.orm import undefer_group
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.sql.expression import select, desc

from cache import cache
from database import db_session
from models.bill import Bill
from models.candidacy import Candidacy
from models.cosponsorship import cosponsorship
from models.person import Person
from utils.jinja import breadcrumb

def register(app):

    app.views['person'] = 'person_main'
    gettext('person') # for babel extraction

    person_names_json = json.dumps(all_person_names())

    # 루트
    @app.route('/person/', methods=['GET'])
    @breadcrumb(app)
    def person_main():
        # FIXME: 19
        assembly_id = int(request.args.get('assembly_id', 19))
        officials = Person.query.order_by(desc(Person.id))\
                                .join(Candidacy)\
                                .filter(and_(Candidacy.age == assembly_id,
                                             Candidacy.is_elected == True))

        return render_template('people.html',
                                officials=officials,
                                assembly_id=assembly_id)

    @app.route('/person/all-names.json', methods=['GET'])
    def person_all_names():
        return person_names_json

    # 사람
    @app.route('/person/<int:id>', methods=['GET'])
    @breadcrumb(app, 'person')
    def person(id):
        try:
            person = Person.query\
                           .filter_by(id=id)\
                           .options(undefer_group('extra'),
                                    undefer_group('profile'))\
                           .one()
        except NoResultFound, e:
            return render_template('not-found.html'), 404

        try:
            person_extra_vars = json.loads(person.extra_vars or '{}')
            if type(person_extra_vars.get('experience', None)) in [str, unicode]:
                person_extra_vars['experience'] = [person_extra_vars['experience']]
        except ValueError, e:
            pass

        return render_template('person.html', person=person,
                distribution_of_cosponsorships=distribution_of_cosponsorships,
                person_extra_vars=person_extra_vars)

def all_person_names():
    name_tuples = (list(i) for i in db_session.query(
        Person.name,
        Person.name_en,
        ))
    all_names = list(set(reduce(operator.add, name_tuples)))
    return all_names

@cache.cached(timeout=60*60*4, key_prefix='distribution_of_cosponsorships')
def distribution_of_cosponsorships(age):
    bill_t = Bill.__table__
    stmt = select([func.count(cosponsorship.c.id)])\
            .select_from(cosponsorship.join(bill_t))\
            .where(bill_t.c.age == age)\
            .group_by(cosponsorship.c.person_id)
    distribution = db_session.execute(stmt).fetchall()
    distribution = map(lambda x: x[0], distribution)
    return distribution
<commit_msg>Fix buggy caching<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

import json
import operator
import time

from flask import current_app, redirect, render_template, request, url_for
from flask.ext.babel import gettext
from sqlalchemy import and_, func
from sqlalchemy.orm import undefer_group
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.sql.expression import select, desc

from cache import cache
from database import db_session
from models.bill import Bill
from models.candidacy import Candidacy
from models.cosponsorship import cosponsorship
from models.person import Person
from utils.jinja import breadcrumb

def register(app):

    app.views['person'] = 'person_main'
    gettext('person') # for babel extraction

    person_names_json = json.dumps(all_person_names())

    # 루트
    @app.route('/person/', methods=['GET'])
    @breadcrumb(app)
    def person_main():
        # FIXME: 19
        assembly_id = int(request.args.get('assembly_id', 19))
        officials = Person.query.order_by(desc(Person.id))\
                                .join(Candidacy)\
                                .filter(and_(Candidacy.age == assembly_id,
                                             Candidacy.is_elected == True))

        return render_template('people.html',
                                officials=officials,
                                assembly_id=assembly_id)

    @app.route('/person/all-names.json', methods=['GET'])
    def person_all_names():
        return person_names_json

    # 사람
    @app.route('/person/<int:id>', methods=['GET'])
    @breadcrumb(app, 'person')
    def person(id):
        try:
            person = Person.query\
                           .filter_by(id=id)\
                           .options(undefer_group('extra'),
                                    undefer_group('profile'))\
                           .one()
        except NoResultFound, e:
            return render_template('not-found.html'), 404

        try:
            person_extra_vars = json.loads(person.extra_vars or '{}')
            if type(person_extra_vars.get('experience', None)) in [str, unicode]:
                person_extra_vars['experience'] = [person_extra_vars['experience']]
        except ValueError, e:
            pass

        return render_template('person.html', person=person,
                distribution_of_cosponsorships=distribution_of_cosponsorships,
                person_extra_vars=person_extra_vars)

def all_person_names():
    name_tuples = (list(i) for i in db_session.query(
        Person.name,
        Person.name_en,
        ))
    all_names = list(set(reduce(operator.add, name_tuples)))
    return all_names

@cache.memoize(timeout=60*60*4)
def distribution_of_cosponsorships(age):
    bill_t = Bill.__table__
    stmt = select([func.count(cosponsorship.c.id)])\
            .select_from(cosponsorship.join(bill_t))\
            .where(bill_t.c.age == age)\
            .group_by(cosponsorship.c.person_id)
    distribution = db_session.execute(stmt).fetchall()
    distribution = map(lambda x: x[0], distribution)
    return distribution
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Message backends views
"""
import json, texttable

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

from django import http
from django.db.models.query import QuerySet
from django.contrib.sites.models import Site
from django.shortcuts import get_object_or_404

from djangotribune import TRIBUNE_MESSAGES_MAX_LIMIT, TRIBUNE_MESSAGES_DEFAULT_LIMIT, TRIBUNE_BAK_SESSION_NAME
from djangotribune.models import Channel, Message
from djangotribune.clocks import ClockIndice
from djangotribune.views import getmax_identity, BackendEncoder, LockView

class RemoteBaseMixin(object):
    """
    This Mixin implement all base stuff to generate a remote backend

    But a view must implement his correct ``build_backend`` methods.
    """
    http_method_names = ['get']
    clock_indexation = True
    mimetype = "text/plain; charset=utf-8"
    default_row_direction = "desc"
    # ``clock`` and ``created`` fields are required if ``clock_indexation`` is actived
    remote_fields = ('clock', 'created', 'author__username', 'user_agent', 'raw')

    def get_channel(self):
        """Get the channel to fetch messages"""
        memokey = '_cache_get_channel'
        if not hasattr(self, memokey):
            if self.request.GET.get('channel', None):
                channel = get_object_or_404(Channel, slug=self.request.GET['channel'])
            else:
                channel = None
            setattr(self, memokey, channel)
        return getattr(self, memokey)

    def get_last_id(self):
        """Get the id from wich to start row fetching"""
        last_id = self.kwargs.get('last_id', 0)
        last_id = self.request.GET.get('last_id', last_id)
        return last_id

    def get_row_direction(self):
        """Get row display direction from args if any"""
        direction = self.request.GET.get('direction', self.default_row_direction)
        if direction.lower() in ('asc', 'desc'):
            return direction
        return self.default_row_direction

    def get_row_limit(self):
        """Get the row limit number to fetch"""
        try:
            limit = int(self.request.GET.get('limit', 0))
        except ValueError:
            limit = TRIBUNE_MESSAGES_DEFAULT_LIMIT
        finally:
            if limit == 0 or limit > TRIBUNE_MESSAGES_MAX_LIMIT:
                limit = TRIBUNE_MESSAGES_DEFAULT_LIMIT
        return limit

    def get_backend_queryset(self, channel, last_id, direction, limit):
        """
        Get the queryset to fetch messages with options from args/kwargs

        If the user session contain a BaK controller, it will be used to extract filters
        and apply them.

        The returned queryset contains only dicts of messages (where each dict is a
        message), mapped key comes from ``self.remote_fields``.
        """
        bak = self.request.session.get(TRIBUNE_BAK_SESSION_NAME, None)
        filters = None
        if bak and bak.active:
            filters = bak.get_filters()
        return Message.objects.get_backend(channel=channel, filters=filters, last_id=last_id).values(*self.remote_fields)[:limit]

    def get_backend(self):
        """
        Return a messages list containing a dict for each message
        """
        limit = self.get_row_limit()
        last_id = self.get_last_id()
        direction = self.get_row_direction()
        channel = self.get_channel()
        # Build the queryset with filtering, base order, limit, etc..
        q = self.get_backend_queryset(channel, last_id, direction, limit)
        # Clock indexation
        if self.clock_indexation:
            q = self.clock_indexing(q)
        else:
            q = map(self.patch_row, q)
        # Change direction if option is "asc", default is desc
        if direction == 'asc':
            if isinstance(q, QuerySet):
                q = list(q)
            q.reverse()
        return q

    def get_redirect_url(self, url):
        """Return the URL 'redirect to' with the URL args if any"""
        args = self.request.META["QUERY_STRING"]
        if args:
            url = "{0}?{1}".format(url, args)
        return url

    def build_backend(self, messages):
        return 'Hello World'

    def clock_indexing(self, backend):
        """
        Same clock indexation

        Linear identical clocks are marked with an indice called "clock indice" (or
        *brother indice*), a brother clock recover the previous indice incremented.

        This method use actually a double pass through on the message list.
        """
        # Count all duplicate clock and store them in a registry
        duplicates = {}
        for item in backend:
            this_date = item['created'].strftime('%Y%m%d%H%M%S')
            if this_date in duplicates:
                duplicates[this_date] += 1
            else:
                duplicates[this_date] = 1
        # Apply the "real" indice to duplicates
        for item in backend:
            this_date = item['created'].strftime('%Y%m%d%H%M%S')
            this_clock = item['clock'].strftime('%H:%M:%S')
            if this_date in duplicates:
                item['clock_indice'] = ClockIndice(duplicates[this_date])
                duplicates[this_date] = duplicates[this_date]-1
            else:
                item['clock_indice'] = ClockIndice(1)
            self.patch_row(item)
        return backend

    def patch_row(self, row):
        """
        For patching rows

        Do nothing by default
        """
        return row

    def patch_response(self, response):
        """
        For patching response like headers injection, etc..

        By default this add cache information to avoid remotes caching by browsers
        """
        response['Pragma'] = "no-cache"
        response['Cache-Control'] = "no-cache, no-store, must-revalidate, max-age=0"
        return response

class RemotePlainMixin(RemoteBaseMixin):
    """
    Remote PLAIN TEXT mixin
    """
    default_row_direction = "asc"

    def patch_row(self, row):
        row['user_agent'] = row['user_agent'][:30]
        return row

    def build_backend(self, messages):
        """
        Plain/text backend is just a row list of messages with three columns (clock,
        identity, message) tabulated and optionnaly an header title

        * Clock column width is always fixed on 10 characters;
        * Identity column width is aligned on the biggest identity (username or
          user_agent);
        * Message column takes the size left in the table maximum width.
        """
        clock_col_width = 10
        default_table_width = 90
        max_table_width = 600

        try:
            table_width = int(self.request.GET.get('table_width', default_table_width))
        except ValueError:
            table_width = default_table_width
        else:
            if table_width > max_table_width:
                table_width = max_table_width

        username_modifier = lambda x: u"<{0}>".format(x)
        identity_col_width = reduce(lambda x,y: getmax_identity(x,y, username_cmp=username_modifier), messages, 10)

        table = texttable.Texttable()
        table.set_deco(texttable.Texttable.HEADER)
        table.set_cols_dtype(["t", "t", "t"])
        table.set_cols_align(["l", "l", "l"])
        table.set_cols_valign(["t", "t", "t"])
        table.set_cols_width([clock_col_width, identity_col_width, table_width-identity_col_width-clock_col_width])

        for item in messages:
            clock = item['clock'].strftime("%H:%M:%S")
            if int(item.get('clock_indice', 1)) > 1:
                clock += unicode(item['clock_indice'])

            identity = item['user_agent']
            if item['author__username']:
                identity = username_modifier(item['author__username'])

            message = item['raw']

            table.add_row( (clock.encode('utf-8'), identity.encode('utf-8'), message.encode('utf-8')) )

        return table.draw()

class RemoteJsonMixin(RemoteBaseMixin):
    """
    Remote JSON
    """
    mimetype = "application/json; charset=utf-8"
    remote_fields = ('id', 'created', 'clock', 'author__username', 'user_agent', 'web_render')

    def build_backend(self, messages):
        if isinstance(messages, QuerySet):
            messages = list(messages)
        return json.dumps(messages, cls=BackendEncoder)

    def patch_row(self, row):
        """
        Add a *clockclass* suitable in ``class=""``, it's a combination of ``clock`` and
        ``clock_indice`` (padded on two digits)
        """
        row['clockclass'] = row['clock'].strftime("%H%M%S") + str(row.get('clock_indice', 1))
        return row

class RemoteHtmlMixin(RemoteJsonMixin):
    """
    Remote Html for template usage

    There is no real builded backend as this is the queryset that is returned in the
    template. And some URL argument are ignored thus they have no sense in this context.
    """
    default_row_direction = "asc"
    def build_backend(self, messages):
        return messages

    def patch_row(self, row):
        row['user_agent'] = row['user_agent'][:30]
        return row

    def get_last_id(self):
        """Ignore 'last_id' option"""
        return 0

    def get_row_direction(self):
        """Ignore 'direction' option"""
        return self.default_row_direction

class RemoteXmlMixin(RemoteBaseMixin):
    """
    Remote XML
    """
    clock_indexation = False
    prettify_backend = False
    mimetype = "application/xml; charset=utf-8"
    remote_fields = ('id', 'created', 'author__username', 'user_agent', 'remote_render')

    def build_backend(self, messages):
        root = ET.Element("board")
        root.set('site', "http://{0}".format(Site.objects.get_current().domain))

        for item in messages:
            message_item = ET.SubElement(root, "post")
            message_item.set('id', str(item['id']))
            message_item.set('time', item['created'].strftime('%Y%m%d%H%M%S'))

            user_agent = ET.SubElement(message_item, "info")
            user_agent.text = item['user_agent']

            username = ET.SubElement(message_item, "login")
            if item['author__username']:
                username.text = item['author__username']

            content = ET.SubElement(message_item, "message")
            content.text = item['remote_render']

        if self.prettify_backend:
            self._prettify(root)

        return ET.tostring(root, 'UTF-8')

    def _prettify(self, elem, level=0):
        """Prettify an ElementTree with indent"""
        i = "\n" + level*" "
        if len(elem):
            if not elem.text or not elem.text.strip():
                elem.text = i + " "
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
            for elem in elem:
                self._prettify(elem, level+1)
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
        else:
            if level and (not elem.tail or not elem.tail.strip()):
                elem.tail = i

class RemoteBaseView(LockView):
    """
    Remote base view is not intended to be used as a real view, this is just the base
    implementation view to be inherited with a remote mixin
    """
    def get(self, request, *args, **kwargs):
        messages = self.get_backend()
        backend = self.build_backend(messages)
        return self.patch_response( http.HttpResponse(backend, mimetype=self.mimetype) )

class RemotePlainView(RemotePlainMixin, RemoteBaseView):
    """
    Remote PLAIN TEXT view
    """
    pass

class RemoteJsonView(RemoteJsonMixin, RemoteBaseView):
    """
    Remote JSON view
    """
    pass

class RemoteXmlView(RemoteXmlMixin, RemoteBaseView):
    """
    Remote XML view
    """
    pass

class RemoteCrapXmlView(RemoteXmlView):
    """
    Remote XML view with indent (for some very old client application)
    """
    prettify_backend = True
<commit_msg>Correct behavior with TRIBUNE_MESSAGES_MAX_LIMIT setting<commit_after># -*- coding: utf-8 -*-
"""
Message backends views
"""
import json, texttable

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

from django import http
from django.db.models.query import QuerySet
from django.contrib.sites.models import Site
from django.shortcuts import get_object_or_404

from djangotribune import TRIBUNE_MESSAGES_MAX_LIMIT, TRIBUNE_MESSAGES_DEFAULT_LIMIT, TRIBUNE_BAK_SESSION_NAME
from djangotribune.models import Channel, Message
from djangotribune.clocks import ClockIndice
from djangotribune.views import getmax_identity, BackendEncoder, LockView

class RemoteBaseMixin(object):
    """
    This Mixin implement all base stuff to generate a remote backend

    But a view must implement his correct ``build_backend`` methods.
    """
    http_method_names = ['get']
    clock_indexation = True
    mimetype = "text/plain; charset=utf-8"
    default_row_direction = "desc"
    # ``clock`` and ``created`` fields are required if ``clock_indexation`` is actived
    remote_fields = ('clock', 'created', 'author__username', 'user_agent', 'raw')

    def get_channel(self):
        """Get the channel to fetch messages"""
        memokey = '_cache_get_channel'
        if not hasattr(self, memokey):
            if self.request.GET.get('channel', None):
                channel = get_object_or_404(Channel, slug=self.request.GET['channel'])
            else:
                channel = None
            setattr(self, memokey, channel)
        return getattr(self, memokey)

    def get_last_id(self):
        """Get the id from wich to start row fetching"""
        last_id = self.kwargs.get('last_id', 0)
        last_id = self.request.GET.get('last_id', last_id)
        return last_id

    def get_row_direction(self):
        """Get row display direction from args if any"""
        direction = self.request.GET.get('direction', self.default_row_direction)
        if direction.lower() in ('asc', 'desc'):
            return direction
        return self.default_row_direction

    def get_row_limit(self):
        """Get the row limit number to fetch"""
        try:
            limit = int(self.request.GET.get('limit', 0))
        except ValueError:
            limit = TRIBUNE_MESSAGES_DEFAULT_LIMIT
        finally:
            if not limit:
                limit = TRIBUNE_MESSAGES_DEFAULT_LIMIT
            elif limit > TRIBUNE_MESSAGES_MAX_LIMIT:
                limit = TRIBUNE_MESSAGES_MAX_LIMIT
        return limit

    def get_backend_queryset(self, channel, last_id, direction, limit):
        """
        Get the queryset to fetch messages with options from args/kwargs

        If the user session contain a BaK controller, it will be used to extract filters
        and apply them.

        The returned queryset contains only dicts of messages (where each dict is a
        message), mapped key comes from ``self.remote_fields``.
        """
        bak = self.request.session.get(TRIBUNE_BAK_SESSION_NAME, None)
        filters = None
        if bak and bak.active:
            filters = bak.get_filters()
        return Message.objects.get_backend(channel=channel, filters=filters, last_id=last_id).values(*self.remote_fields)[:limit]

    def get_backend(self):
        """
        Return a messages list containing a dict for each message
        """
        limit = self.get_row_limit()
        last_id = self.get_last_id()
        direction = self.get_row_direction()
        channel = self.get_channel()
        # Build the queryset with filtering, base order, limit, etc..
        q = self.get_backend_queryset(channel, last_id, direction, limit)
        # Clock indexation
        if self.clock_indexation:
            q = self.clock_indexing(q)
        else:
            q = map(self.patch_row, q)
        # Change direction if option is "asc", default is desc
        if direction == 'asc':
            if isinstance(q, QuerySet):
                q = list(q)
            q.reverse()
        return q

    def get_redirect_url(self, url):
        """Return the URL 'redirect to' with the URL args if any"""
        args = self.request.META["QUERY_STRING"]
        if args:
            url = "{0}?{1}".format(url, args)
        return url

    def build_backend(self, messages):
        return 'Hello World'

    def clock_indexing(self, backend):
        """
        Same clock indexation

        Linear identical clocks are marked with an indice called "clock indice" (or
        *brother indice*), a brother clock recover the previous indice incremented.

        This method use actually a double pass through on the message list.
        """
        # Count all duplicate clock and store them in a registry
        duplicates = {}
        for item in backend:
            this_date = item['created'].strftime('%Y%m%d%H%M%S')
            if this_date in duplicates:
                duplicates[this_date] += 1
            else:
                duplicates[this_date] = 1
        # Apply the "real" indice to duplicates
        for item in backend:
            this_date = item['created'].strftime('%Y%m%d%H%M%S')
            this_clock = item['clock'].strftime('%H:%M:%S')
            if this_date in duplicates:
                item['clock_indice'] = ClockIndice(duplicates[this_date])
                duplicates[this_date] = duplicates[this_date]-1
            else:
                item['clock_indice'] = ClockIndice(1)
            self.patch_row(item)
        return backend

    def patch_row(self, row):
        """
        For patching rows

        Do nothing by default
        """
        return row

    def patch_response(self, response):
        """
        For patching response like headers injection, etc..

        By default this add cache information to avoid remotes caching by browsers
        """
        response['Pragma'] = "no-cache"
        response['Cache-Control'] = "no-cache, no-store, must-revalidate, max-age=0"
        return response

class RemotePlainMixin(RemoteBaseMixin):
    """
    Remote PLAIN TEXT mixin
    """
    default_row_direction = "asc"

    def patch_row(self, row):
        row['user_agent'] = row['user_agent'][:30]
        return row

    def build_backend(self, messages):
        """
        Plain/text backend is just a row list of messages with three columns (clock,
        identity, message) tabulated and optionnaly an header title

        * Clock column width is always fixed on 10 characters;
        * Identity column width is aligned on the biggest identity (username or
          user_agent);
        * Message column takes the size left in the table maximum width.
        """
        clock_col_width = 10
        default_table_width = 90
        max_table_width = 600

        try:
            table_width = int(self.request.GET.get('table_width', default_table_width))
        except ValueError:
            table_width = default_table_width
        else:
            if table_width > max_table_width:
                table_width = max_table_width

        username_modifier = lambda x: u"<{0}>".format(x)
        identity_col_width = reduce(lambda x,y: getmax_identity(x,y, username_cmp=username_modifier), messages, 10)

        table = texttable.Texttable()
        table.set_deco(texttable.Texttable.HEADER)
        table.set_cols_dtype(["t", "t", "t"])
        table.set_cols_align(["l", "l", "l"])
        table.set_cols_valign(["t", "t", "t"])
        table.set_cols_width([clock_col_width, identity_col_width, table_width-identity_col_width-clock_col_width])

        for item in messages:
            clock = item['clock'].strftime("%H:%M:%S")
            if int(item.get('clock_indice', 1)) > 1:
                clock += unicode(item['clock_indice'])

            identity = item['user_agent']
            if item['author__username']:
                identity = username_modifier(item['author__username'])

            message = item['raw']

            table.add_row( (clock.encode('utf-8'), identity.encode('utf-8'), message.encode('utf-8')) )

        return table.draw()

class RemoteJsonMixin(RemoteBaseMixin):
    """
    Remote JSON
    """
    mimetype = "application/json; charset=utf-8"
    remote_fields = ('id', 'created', 'clock', 'author__username', 'user_agent', 'web_render')

    def build_backend(self, messages):
        if isinstance(messages, QuerySet):
            messages = list(messages)
        return json.dumps(messages, cls=BackendEncoder)

    def patch_row(self, row):
        """
        Add a *clockclass* suitable in ``class=""``, it's a combination of ``clock`` and
        ``clock_indice`` (padded on two digits)
        """
        row['clockclass'] = row['clock'].strftime("%H%M%S") + str(row.get('clock_indice', 1))
        return row

class RemoteHtmlMixin(RemoteJsonMixin):
    """
    Remote Html for template usage

    There is no real builded backend as this is the queryset that is returned in the
    template. And some URL argument are ignored thus they have no sense in this context.
    """
    default_row_direction = "asc"
    def build_backend(self, messages):
        return messages

    def patch_row(self, row):
        row['user_agent'] = row['user_agent'][:30]
        return row

    def get_last_id(self):
        """Ignore 'last_id' option"""
        return 0

    def get_row_direction(self):
        """Ignore 'direction' option"""
        return self.default_row_direction

class RemoteXmlMixin(RemoteBaseMixin):
    """
    Remote XML
    """
    clock_indexation = False
    prettify_backend = False
    mimetype = "application/xml; charset=utf-8"
    remote_fields = ('id', 'created', 'author__username', 'user_agent', 'remote_render')

    def build_backend(self, messages):
        root = ET.Element("board")
        root.set('site', "http://{0}".format(Site.objects.get_current().domain))

        for item in messages:
            message_item = ET.SubElement(root, "post")
            message_item.set('id', str(item['id']))
            message_item.set('time', item['created'].strftime('%Y%m%d%H%M%S'))

            user_agent = ET.SubElement(message_item, "info")
            user_agent.text = item['user_agent']

            username = ET.SubElement(message_item, "login")
            if item['author__username']:
                username.text = item['author__username']

            content = ET.SubElement(message_item, "message")
            content.text = item['remote_render']

        if self.prettify_backend:
            self._prettify(root)

        return ET.tostring(root, 'UTF-8')

    def _prettify(self, elem, level=0):
        """Prettify an ElementTree with indent"""
        i = "\n" + level*" "
        if len(elem):
            if not elem.text or not elem.text.strip():
                elem.text = i + " "
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
            for elem in elem:
                self._prettify(elem, level+1)
            if not elem.tail or not elem.tail.strip():
                elem.tail = i
        else:
            if level and (not elem.tail or not elem.tail.strip()):
                elem.tail = i

class RemoteBaseView(LockView):
    """
    Remote base view is not intended to be used as a real view, this is just the base
    implementation view to be inherited with a remote mixin
    """
    def get(self, request, *args, **kwargs):
        messages = self.get_backend()
        backend = self.build_backend(messages)
        return self.patch_response( http.HttpResponse(backend, mimetype=self.mimetype) )

class RemotePlainView(RemotePlainMixin, RemoteBaseView):
    """
    Remote PLAIN TEXT view
    """
    pass

class RemoteJsonView(RemoteJsonMixin, RemoteBaseView):
    """
    Remote JSON view
    """
    pass

class RemoteXmlView(RemoteXmlMixin, RemoteBaseView):
    """
    Remote XML view
    """
    pass

class RemoteCrapXmlView(RemoteXmlView):
    """
    Remote XML view with indent (for some very old client application)
    """
    prettify_backend = True
<|endoftext|>