<commit_before>#
# test_zopfli
#
#   Copyright (c) 2015 Akinori Hattori <hattya@gmail.com>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

import io
import os
import shutil
import sys
import tempfile
import unittest
import zipfile

import zopfli

class ZopfliTestCase(unittest.TestCase):

    def test_format(self):
        self.assertEqual(zopfli.ZOPFLI_FORMAT_GZIP, 0)
        self.assertEqual(zopfli.ZOPFLI_FORMAT_ZLIB, 1)
        self.assertEqual(zopfli.ZOPFLI_FORMAT_DEFLATE, 2)

    def test_gzip(self):
        self._test_zopfli(zopfli.ZOPFLI_FORMAT_GZIP)

    def test_zlib(self):
        self._test_zopfli(zopfli.ZOPFLI_FORMAT_ZLIB)

    def test_deflate(self):
        self._test_zopfli(zopfli.ZOPFLI_FORMAT_DEFLATE)

    def _test_zopfli(self, fmt):
        for i in range(-1, 5):
            c = zopfli.ZopfliCompressor(fmt, block_splitting=i)
            b = b'Hello, world!'
            z = c.compress(b) + c.flush()
            self._test_decompress(fmt, z, b)

    def test_unknown(self):
        with self.assertRaises(ValueError):
            zopfli.ZopfliCompressor(-1)

        with self.assertRaises(ValueError):
            zopfli.ZopfliDecompressor(-1)

    def test_compressor(self):
        with self.assertRaises(TypeError):
            zopfli.ZopfliCompressor(None)

        c = zopfli.ZopfliCompressor()
        with self.assertRaises(TypeError):
            c.compress(None)

        c = zopfli.ZopfliCompressor()
        self.assertEqual(c.flush(), b'')
        with self.assertRaises(ValueError):
            c.compress(b'')
        with self.assertRaises(ValueError):
            c.flush()

    def test_deflater(self):
        for i in range(3):
            c = zopfli.ZopfliDeflater(block_splitting=i)
            b = b'Hello, world!'
            z = c.compress(b) + c.flush()
            self._test_decompress(zopfli.ZOPFLI_FORMAT_DEFLATE, z, b)

            c = zopfli.ZopfliDeflater(block_splitting=i)
            b = b'Hello, world!'
            z = c.compress(b) + c.compress(b) + c.compress(b) + c.flush()
            self._test_decompress(zopfli.ZOPFLI_FORMAT_DEFLATE, z, b * 3)

        with self.assertRaises(TypeError):
            zopfli.ZopfliDeflater(iterations=None)

        with self.assertRaises(TypeError):
            zopfli.ZopfliDeflater(block_splitting=None)

        with self.assertRaises(TypeError):
            zopfli.ZopfliDeflater(block_splitting_max=None)

        c = zopfli.ZopfliDeflater()
        c.compress(None)
        with self.assertRaises(TypeError):
            c.compress(None)

        c = zopfli.ZopfliDeflater()
        self.assertEqual(c.flush(), b'')
        with self.assertRaises(ValueError):
            c.compress(b'')
        with self.assertRaises(ValueError):
            c.flush()

    def _test_decompress(self, fmt, z, b):
        d = zopfli.ZopfliDecompressor(fmt)
        self.assertEqual(d.decompress(z) + d.flush(), b)
        self.assertEqual(d.unused_data, b'')
        self.assertEqual(d.unconsumed_tail, b'')
        if sys.version_info >= (3, 3):
            self.assertTrue(d.eof)

# 8-bit PNG (64x64 pixels)
black_png = (
    b'\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52\x00\x00'
    b'\x00\x40\x00\x00\x00\x40\x08\x00\x00\x00\x00\x8f\x02\x2e\x02\x00\x00\x00'
    b'\x1b\x49\x44\x41\x54\x78\x9c\xed\xc1\x81\x00\x00\x00\x00\xc3\xa0\xf9\x53'
    b'\xdf\xe0\x04\x55\x01\x00\x00\x00\x7c\x03\x10\x40\x00\x01\x46\x38\x0d\x1d'
    b'\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82'
)

class ZopfliPNGTestCase(unittest.TestCase):

    def test_png(self):
        png = zopfli.ZopfliPNG()
        self.assertFalse(png.verbose)
        self.assertFalse(png.lossy_transparent)
        self.assertFalse(png.lossy_8bit)
        self.assertEqual(png.filter_strategies, '')
        self.assertTrue(png.auto_filter_strategy)
        self.assertEqual(png.keep_chunks, ())
        self.assertTrue(png.use_zopfli)
        self.assertEqual(png.iterations, 15)
        self.assertEqual(png.iterations_large, 5)
        self.assertEqual(png.block_split_strategy, 1)

        png = zopfli.ZopfliPNG(verbose=True,
                               lossy_transparent=True,
                               lossy_8bit=True,
                               filter_strategies='01234mepb',
                               keep_chunks=['tEXt', 'zTXt', 'iTXt'],
                               use_zopfli=False,
                               iterations=30,
                               iterations_large=10,
                               block_split_strategy=3)
        self.assertTrue(png.verbose)
        self.assertTrue(png.lossy_transparent)
        self.assertTrue(png.lossy_8bit)
        self.assertEqual(png.filter_strategies, '01234mepb')
        self.assertFalse(png.auto_filter_strategy)
        self.assertEqual(png.keep_chunks, ('tEXt', 'zTXt', 'iTXt'))
        self.assertFalse(png.use_zopfli)
        self.assertEqual(png.iterations, 30)
        self.assertEqual(png.iterations_large, 10)
        self.assertEqual(png.block_split_strategy, 3)

        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(filter_strategies=None)
        with self.assertRaises(ValueError):
            zopfli.ZopfliPNG(filter_strategies=u'\u00B7')
        with self.assertRaises(ValueError):
            zopfli.ZopfliPNG(filter_strategies='z')

        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(keep_chunks=None)
        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(keep_chunks=[None])
        with self.assertRaises(ValueError):
            zopfli.ZopfliPNG(keep_chunks=[u'\u00B7'])

        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(iterations=None)

        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(iterations_large=None)

        png = zopfli.ZopfliPNG(block_split_strategy=-1)
        self.assertEqual(png.block_split_strategy, 1)
        png = zopfli.ZopfliPNG(block_split_strategy=4)
        self.assertEqual(png.block_split_strategy, 1)
        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(block_split_strategy=None)

    def test_optimize(self):
        png = zopfli.ZopfliPNG()
        self.assertGreater(len(black_png), len(png.optimize(black_png)))

        with self.assertRaises(TypeError):
            png.optimize(None)
        with self.assertRaises(ValueError):
            png.optimize(b'')

class ZipFileTest(unittest.TestCase):

    def setUp(self):
        super(ZipFileTest, self).setUp()
        self.path = tempfile.mkdtemp(prefix='zopfli-')

    def tearDown(self):
        super(ZipFileTest, self).tearDown()
        shutil.rmtree(self.path)

    def test_ascii(self):
        names = {
            'New Folder': 'New Folder',
            'spam': 'spam',
            'eggs': 'eggs',
            'ham': 'ham',
            'toast': 'toast',
            'beans': 'beans',
            'bacon': 'bacon',
            'sausage': 'sausage',
        }
        self._test_zip('ascii', names)

    def test_cp932(self):
        names = {
            'New Folder': u'\u65b0\u3057\u3044\u30d5\u30a9\u30eb\u30c0\u30fc',
            'spam': u'\u30b9\u30d1\u30e0',
            'eggs': u'\u30a8\u30c3\u30b0\u30b9',
            'ham': u'\u30cf\u30e0',
            'toast': u'\u30c8\u30fc\u30b9\u30c8',
            'beans': u'\u30d3\u30fc\u30f3\u30ba',
            'bacon': u'\u30d9\u30fc\u30b3\u30f3',
            'sausage': u'\u30bd\u30fc\u30bb\u30fc\u30b8',
        }
        self._test_zip('cp932', names)

    def test_utf_8(self):
        names = {
            'New Folder': u'\u65b0\u3057\u3044\u30d5\u30a9\u30eb\u30c0\u30fc',
            'spam': u'\u30b9\u30d1\u30e0',
            'eggs': u'\u30a8\u30c3\u30b0\u30b9',
            'ham': u'\u30cf\u30e0',
            'toast': u'\u30c8\u30fc\u30b9\u30c8',
            'beans': u'\u30d3\u30fc\u30f3\u30ba',
            'bacon': u'\u30d9\u30fc\u30b3\u30f3',
            'sausage': u'\u30bd\u30fc\u30bb\u30fc\u30b8',
        }
        self._test_zip('utf-8', names)

    def _test_zip(self, encoding, names):
        if sys.version_info < (3, 0):
            def u(s):
                return s.decode('utf-8').format(**names)
        else:
            def u(s):
                return s.format(**names)

        def write(zf, name, deflate=True):
            p = os.path.join(self.path, name)
            with io.open(p, 'w', encoding=encoding) as fp:
                fp.write(os.path.splitext(os.path.basename(name))[0])
            zf.write(p, name, zipfile.ZIP_DEFLATED if deflate else zipfile.ZIP_STORED)

        def writestr(zf, name, deflate=True):
            data = os.path.splitext(os.path.basename(name))[0].encode(encoding)
            zf.writestr(name, data, zipfile.ZIP_DEFLATED if deflate else zipfile.ZIP_STORED)

        path = os.path.join(self.path, '{}.zip'.format(encoding))
        folder = '{New Folder}'
        os.mkdir(u(os.path.join(self.path, folder)))
        with zopfli.ZipFile(path, 'w', encoding=encoding) as zf:
            write(zf, u(os.path.join(folder, '{spam}.txt')))
            write(zf, u(os.path.join(folder, '{eggs}.txt')), False)
            writestr(zf, u(os.path.join(folder, '{ham}.txt')))
            writestr(zf, u(os.path.join(folder, '{toast}.txt')), False)
            write(zf, u(os.path.join(folder, '{beans}.txt')))
            writestr(zf, u(os.path.join(folder, '{bacon}.txt')))
            write(zf, u(os.path.join(folder, '{sausage}.txt')), False)
        with zopfli.ZipFile(path, 'r', encoding=encoding) as zf:
            for n, compress_type in (('{spam}.txt', zipfile.ZIP_DEFLATED),
                                     ('{eggs}.txt', zipfile.ZIP_STORED),
                                     ('{ham}.txt', zipfile.ZIP_DEFLATED),
                                     ('{toast}.txt', zipfile.ZIP_STORED),
                                     ('{beans}.txt', zipfile.ZIP_DEFLATED),
                                     ('{bacon}.txt', zipfile.ZIP_DEFLATED),
                                     ('{sausage}.txt', zipfile.ZIP_STORED)):
                name = u(os.path.join(folder, n)).replace(os.path.sep, '/')
                raw_name = name.encode(encoding)
                if sys.version_info >= (3, 0):
                    raw_name = raw_name.decode('utf-8' if encoding == 'utf-8' else 'cp437')
                zi = zf.getinfo(name)
                self.assertEqual(zi.orig_filename, raw_name)
                self.assertEqual(zi.filename, name)
                self.assertEqual(zi.compress_type, compress_type)
                self.assertEqual(zi.flag_bits, 0x800 if encoding == 'utf-8' else 0)
                self.assertEqual(zf.read(zi), u(os.path.splitext(n)[0]).encode(encoding))
<commit_msg>Fix for Python 2.7<commit_after>#
# test_zopfli
#
#   Copyright (c) 2015 Akinori Hattori <hattya@gmail.com>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#

import io
import os
import shutil
import sys
import tempfile
import unittest
import zipfile

import zopfli

class ZopfliTestCase(unittest.TestCase):

    def test_format(self):
        self.assertEqual(zopfli.ZOPFLI_FORMAT_GZIP, 0)
        self.assertEqual(zopfli.ZOPFLI_FORMAT_ZLIB, 1)
        self.assertEqual(zopfli.ZOPFLI_FORMAT_DEFLATE, 2)

    def test_gzip(self):
        self._test_zopfli(zopfli.ZOPFLI_FORMAT_GZIP)

    def test_zlib(self):
        self._test_zopfli(zopfli.ZOPFLI_FORMAT_ZLIB)

    def test_deflate(self):
        self._test_zopfli(zopfli.ZOPFLI_FORMAT_DEFLATE)

    def _test_zopfli(self, fmt):
        for i in range(-1, 5):
            c = zopfli.ZopfliCompressor(fmt, block_splitting=i)
            b = b'Hello, world!'
            z = c.compress(b) + c.flush()
            self._test_decompress(fmt, z, b)

    def test_unknown(self):
        with self.assertRaises(ValueError):
            zopfli.ZopfliCompressor(-1)

        with self.assertRaises(ValueError):
            zopfli.ZopfliDecompressor(-1)

    def test_compressor(self):
        with self.assertRaises(TypeError):
            zopfli.ZopfliCompressor(None)

        c = zopfli.ZopfliCompressor()
        with self.assertRaises(TypeError):
            c.compress(None)

        c = zopfli.ZopfliCompressor()
        self.assertEqual(c.flush(), b'')
        with self.assertRaises(ValueError):
            c.compress(b'')
        with self.assertRaises(ValueError):
            c.flush()

    def test_deflater(self):
        for i in range(3):
            c = zopfli.ZopfliDeflater(block_splitting=i)
            b = b'Hello, world!'
            z = c.compress(b) + c.flush()
            self._test_decompress(zopfli.ZOPFLI_FORMAT_DEFLATE, z, b)

            c = zopfli.ZopfliDeflater(block_splitting=i)
            b = b'Hello, world!'
            z = c.compress(b) + c.compress(b) + c.compress(b) + c.flush()
            self._test_decompress(zopfli.ZOPFLI_FORMAT_DEFLATE, z, b * 3)

        with self.assertRaises(TypeError):
            zopfli.ZopfliDeflater(iterations=None)

        with self.assertRaises(TypeError):
            zopfli.ZopfliDeflater(block_splitting=None)

        with self.assertRaises(TypeError):
            zopfli.ZopfliDeflater(block_splitting_max=None)

        c = zopfli.ZopfliDeflater()
        c.compress(None)
        with self.assertRaises(TypeError):
            c.compress(None)

        c = zopfli.ZopfliDeflater()
        self.assertEqual(c.flush(), b'')
        with self.assertRaises(ValueError):
            c.compress(b'')
        with self.assertRaises(ValueError):
            c.flush()

    def _test_decompress(self, fmt, z, b):
        d = zopfli.ZopfliDecompressor(fmt)
        self.assertEqual(d.decompress(z) + d.flush(), b)
        self.assertEqual(d.unused_data, b'')
        self.assertEqual(d.unconsumed_tail, b'')
        if sys.version_info >= (3, 3):
            self.assertTrue(d.eof)

# 8-bit PNG (64x64 pixels)
black_png = (
    b'\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52\x00\x00'
    b'\x00\x40\x00\x00\x00\x40\x08\x00\x00\x00\x00\x8f\x02\x2e\x02\x00\x00\x00'
    b'\x1b\x49\x44\x41\x54\x78\x9c\xed\xc1\x81\x00\x00\x00\x00\xc3\xa0\xf9\x53'
    b'\xdf\xe0\x04\x55\x01\x00\x00\x00\x7c\x03\x10\x40\x00\x01\x46\x38\x0d\x1d'
    b'\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82'
)

class ZopfliPNGTestCase(unittest.TestCase):

    def test_png(self):
        png = zopfli.ZopfliPNG()
        self.assertFalse(png.verbose)
        self.assertFalse(png.lossy_transparent)
        self.assertFalse(png.lossy_8bit)
        self.assertEqual(png.filter_strategies, '')
        self.assertTrue(png.auto_filter_strategy)
        self.assertEqual(png.keep_chunks, ())
        self.assertTrue(png.use_zopfli)
        self.assertEqual(png.iterations, 15)
        self.assertEqual(png.iterations_large, 5)
        self.assertEqual(png.block_split_strategy, 1)

        png = zopfli.ZopfliPNG(verbose=True,
                               lossy_transparent=True,
                               lossy_8bit=True,
                               filter_strategies='01234mepb',
                               keep_chunks=['tEXt', 'zTXt', 'iTXt'],
                               use_zopfli=False,
                               iterations=30,
                               iterations_large=10,
                               block_split_strategy=3)
        self.assertTrue(png.verbose)
        self.assertTrue(png.lossy_transparent)
        self.assertTrue(png.lossy_8bit)
        self.assertEqual(png.filter_strategies, '01234mepb')
        self.assertFalse(png.auto_filter_strategy)
        self.assertEqual(png.keep_chunks, ('tEXt', 'zTXt', 'iTXt'))
        self.assertFalse(png.use_zopfli)
        self.assertEqual(png.iterations, 30)
        self.assertEqual(png.iterations_large, 10)
        self.assertEqual(png.block_split_strategy, 3)

        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(filter_strategies=None)
        with self.assertRaises(ValueError):
            zopfli.ZopfliPNG(filter_strategies=u'\u00B7')
        with self.assertRaises(ValueError):
            zopfli.ZopfliPNG(filter_strategies='z')

        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(keep_chunks=None)
        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(keep_chunks=[None])
        with self.assertRaises(ValueError):
            zopfli.ZopfliPNG(keep_chunks=[u'\u00B7'])

        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(iterations=None)

        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(iterations_large=None)

        png = zopfli.ZopfliPNG(block_split_strategy=-1)
        self.assertEqual(png.block_split_strategy, 1)
        png = zopfli.ZopfliPNG(block_split_strategy=4)
        self.assertEqual(png.block_split_strategy, 1)
        with self.assertRaises(TypeError):
            zopfli.ZopfliPNG(block_split_strategy=None)

    def test_optimize(self):
        png = zopfli.ZopfliPNG()
        self.assertGreater(len(black_png), len(png.optimize(black_png)))

        with self.assertRaises(TypeError):
            png.optimize(None)
        with self.assertRaises(ValueError):
            png.optimize(b'')

class ZipFileTest(unittest.TestCase):

    def setUp(self):
        super(ZipFileTest, self).setUp()
        self.path = tempfile.mkdtemp(prefix='zopfli-')
        if sys.version_info < (3, 0):
            self.path = self.path.decode(sys.getfilesystemencoding(), 'surrogateescape')

    def tearDown(self):
        super(ZipFileTest, self).tearDown()
        shutil.rmtree(self.path)

    def test_ascii(self):
        names = {
            'New Folder': 'New Folder',
            'spam': 'spam',
            'eggs': 'eggs',
            'ham': 'ham',
            'toast': 'toast',
            'beans': 'beans',
            'bacon': 'bacon',
            'sausage': 'sausage',
        }
        self._test_zip('ascii', names)

    def test_cp932(self):
        names = {
            'New Folder': u'\u65b0\u3057\u3044\u30d5\u30a9\u30eb\u30c0\u30fc',
            'spam': u'\u30b9\u30d1\u30e0',
            'eggs': u'\u30a8\u30c3\u30b0\u30b9',
            'ham': u'\u30cf\u30e0',
            'toast': u'\u30c8\u30fc\u30b9\u30c8',
            'beans': u'\u30d3\u30fc\u30f3\u30ba',
            'bacon': u'\u30d9\u30fc\u30b3\u30f3',
            'sausage': u'\u30bd\u30fc\u30bb\u30fc\u30b8',
        }
        self._test_zip('cp932', names)

    def test_utf_8(self):
        names = {
            'New Folder': u'\u65b0\u3057\u3044\u30d5\u30a9\u30eb\u30c0\u30fc',
            'spam': u'\u30b9\u30d1\u30e0',
            'eggs': u'\u30a8\u30c3\u30b0\u30b9',
            'ham': u'\u30cf\u30e0',
            'toast': u'\u30c8\u30fc\u30b9\u30c8',
            'beans': u'\u30d3\u30fc\u30f3\u30ba',
            'bacon': u'\u30d9\u30fc\u30b3\u30f3',
            'sausage': u'\u30bd\u30fc\u30bb\u30fc\u30b8',
        }
        self._test_zip('utf-8', names)

    def _test_zip(self, encoding, names):
        if sys.version_info < (3, 0):
            def u(s):
                return s.decode('utf-8').format(**names)
        else:
            def u(s):
                return s.format(**names)

        def write(zf, name, deflate=True):
            p = os.path.join(self.path, name)
            with io.open(p, 'w', encoding=encoding) as fp:
                fp.write(os.path.splitext(os.path.basename(name))[0])
            zf.write(p, name, zipfile.ZIP_DEFLATED if deflate else zipfile.ZIP_STORED)

        def writestr(zf, name, deflate=True):
            data = os.path.splitext(os.path.basename(name))[0].encode(encoding)
            zf.writestr(name, data, zipfile.ZIP_DEFLATED if deflate else zipfile.ZIP_STORED)

        path = os.path.join(self.path, '{}.zip'.format(encoding))
        folder = '{New Folder}'
        os.mkdir(u(os.path.join(self.path, folder)))
        with zopfli.ZipFile(path, 'w', encoding=encoding) as zf:
            write(zf, u(os.path.join(folder, '{spam}.txt')))
            write(zf, u(os.path.join(folder, '{eggs}.txt')), False)
            writestr(zf, u(os.path.join(folder, '{ham}.txt')))
            writestr(zf, u(os.path.join(folder, '{toast}.txt')), False)
            write(zf, u(os.path.join(folder, '{beans}.txt')))
            writestr(zf, u(os.path.join(folder, '{bacon}.txt')))
            write(zf, u(os.path.join(folder, '{sausage}.txt')), False)
        with zopfli.ZipFile(path, 'r', encoding=encoding) as zf:
            for n, compress_type in (('{spam}.txt', zipfile.ZIP_DEFLATED),
                                     ('{eggs}.txt', zipfile.ZIP_STORED),
                                     ('{ham}.txt', zipfile.ZIP_DEFLATED),
                                     ('{toast}.txt', zipfile.ZIP_STORED),
                                     ('{beans}.txt', zipfile.ZIP_DEFLATED),
                                     ('{bacon}.txt', zipfile.ZIP_DEFLATED),
                                     ('{sausage}.txt', zipfile.ZIP_STORED)):
                name = u(os.path.join(folder, n)).replace(os.path.sep, '/')
                raw_name = name.encode(encoding)
                if sys.version_info >= (3, 0):
                    raw_name = raw_name.decode('utf-8' if encoding == 'utf-8' else 'cp437')
                zi = zf.getinfo(name)
                self.assertEqual(zi.orig_filename, raw_name)
                self.assertEqual(zi.filename, name)
                self.assertEqual(zi.compress_type, compress_type)
                self.assertEqual(zi.flag_bits, 0x800 if encoding == 'utf-8' else 0)
                self.assertEqual(zf.read(zi), u(os.path.splitext(n)[0]).encode(encoding))
<|endoftext|>


<commit_before>from utils import console, parsing

class auditd():

    def __init__(self):
        self.auditd_config_file = '/etc/audit/auditd.conf'
        self.auditd_config = parsing.parse_config_file(self.auditd_config_file, '=')
        console.heading("auditd config")
        return

    def check_admin_space_left_action(self):
        # RHEL-06-000163
        if self.auditd_config['admin_space_left_action'].upper() != "SINGLE":
            console.error("Auditd admin_space_left_action is not set to 'SINGLE'!")
            return None

    def check_spece_left_action(self):
        # RHEL-06-000005
        if self.auditd_config['space_left_action'] == "EMAIL":
            console.ok("Free space warning enabled in auditd")
            return None
        elif self.auditd_config['space_left_action'] == "SYSLOG":
            console.warning("Free space warning is set to SYSLOG. Make sure this notifies the necessary individuals in a timely manner")
            return None
        else:
            console.error("Free space warning not enabled in auditd")
            return None

    def check_max_log_file_action(self):
        # RHEL-06-000161
        if self.auditd_config['max_log_file_action'].upper() != "ROTATE":
            console.error("Auditd max_log_file_actoin not set to 'ROTATE'!")

    def check_max_log_file(self):
        # RHEL-06-000160
        #  This rule may need to be altered because the STIG only defines this
        #  must be set large enough to meet the retention period required.
        if not int(self.auditd_config['max_log_file']) >= 6:
            console.error("Auditd max_log_file not set to at least 6!")

    def check_num_logs(self):
        # RHEL-06-000159
        #  This rule may need to be altered because the STIG only defines this
        #  must be set large enough to meet the retention period required.
        if not int(self.auditd_config['num_logs']) >= 5:
            console.error("Auditd num_logs not set to at least 5!")

def run_tests():

    audit = auditd()
    audit.check_admin_space_left_action()
    audit.check_spece_left_action()
    audit.check_max_log_file_action()
    audit.check_max_log_file()
    audit.check_num_logs()
<commit_msg>Added another auditd config test and imporoved the value checking functions to a less repetitive process.<commit_after>from utils import console, parsing
import operator

class auditd():

    def __init__(self):
        self.auditd_config_file = '/etc/audit/auditd.conf'
        self.auditd_config = parsing.parse_config_file(self.auditd_config_file, '=')
        console.heading("auditd config")
        return

    def check_value(self, key, value, operator_function):
        if not operator_function(self.auditd_config[key], value):
            console.error("Auditd %s is not set to %s!" % (key, value))

    def check_spece_left_action(self):
        # RHEL-06-000005
        if self.auditd_config['space_left_action'] == "EMAIL":
            console.ok("Free space warning enabled in auditd")
            return None
        elif self.auditd_config['space_left_action'] == "SYSLOG":
            console.warning("Free space warning is set to SYSLOG. Make sure this notifies the necessary individuals in a timely manner")
            return None
        else:
            console.error("Free space warning not enabled in auditd")
            return None

def run_tests():

    audit = auditd()
    audit.check_value('admin_space_left_action', 'SINGLE', operator.eq)  # RHEL-06-000163
    audit.check_spece_left_action()
    audit.check_value('max_log_file_action', 'ROTATE', operator.eq)  # RHEL-06-000161
    audit.check_value('max_log_file', '6', operator.eq)  # RHEL-06-000160
    audit.check_value('num_logs', '5', operator.ge)  # # RHEL-06-000159
<|endoftext|>


<commit_before>import uuid
import timeit
import lz4
import os
from timeit import Timer
import sys
import blosc

DATA = open(sys.argv[1], "rb").read()
LZ4_DATA = lz4.block.compress(DATA)
BLOSC_DATA = blosc.compress(DATA, cname='lz4', clevel=5, shuffle=True)
LOOPS = 100

print("Data Size:")
print("  Input: %d" % len(DATA))
print("  LZ4: %d (%.2f)" % (len(LZ4_DATA), len(LZ4_DATA) / float(len(DATA))))
print("  Blosc: %d (%.2f)" %
      (len(BLOSC_DATA), len(BLOSC_DATA) / float(len(DATA))))
print("  LZ4 / Blosc: %f" % (float(len(LZ4_DATA)) / float(len(BLOSC_DATA))))

print("Benchmark: %d calls" % LOOPS)
print("  LZ4 Compression: %fs" % (Timer("lz4.block.compress(DATA)",
                                        "from __main__ import DATA; import lz4").timeit(number=LOOPS)/LOOPS))
print("  Blosc Compression: %fs" % (Timer("blosc.compress(DATA, cname='lz4', clevel=5, shuffle=True)",
                                          "from __main__ import DATA; import blosc").timeit(number=LOOPS)/LOOPS))
print("  LZ4 Decompression: %fs" % (Timer("lz4.block.decompress(LZ4_DATA)",
                                          "from __main__ import LZ4_DATA; import lz4").timeit(number=LOOPS)/LOOPS))
print("  Blosc Decompression : %fs" % (Timer("blosc.decompress(BLOSC_DATA)",
                                             "from __main__ import BLOSC_DATA; import blosc").timeit(number=LOOPS)/LOOPS))
<commit_msg>Remove bench.py<commit_after><|endoftext|>


<commit_before>from __future__ import print_function

import multiprocessing
import time
from operator import mul
from os.path import join
import warnings

import h5py
import numpy as np

import PyGreentea as pygt

''' where this will be used:
* train()
  * getting unordered batches of a dataset, specified with offset, input size and output size
  * behavior should be like a queue. at the end of each training iteration, train() specifies a replacement
* process()
  * getting batches from a dataset, specified with offset and input size
'''

def update_shared_dataset(index_of_shared, index_of_which_dataset, input_slice, output_slice, transform=True):
    # print("in process id {}".format(os.getpid()))
    shared_dataset = shared_datasets[index_of_shared]
    original_dataset = datasets[index_of_which_dataset]
    # print("original_dataset: ", [
    #     (key,
    #      type(original_dataset[key]),
    #      original_dataset[key])
    #     for key in original_dataset.keys()])
    dataset_numpy = dict()
    # load inputs
    # print("original_dataset['data']: ", original_dataset['data'].shape, original_dataset['data'].dtype)
    data_slice = np.array(original_dataset['data'][input_slice], dtype=np.float32) / (2. ** 8)
    if transform:
        if 'transform' in original_dataset:
            # print('Pre:',(data_slice.min(),data_slice.mean(),data_slice.max()))
            lo, hi = original_dataset['transform']['scale']
            data_slice = 0.5 + (data_slice-0.5)*np.random.uniform(low=lo,high=hi)
            lo, hi = original_dataset['transform']['shift']
            data_slice = data_slice + np.random.uniform(low=lo,high=hi)
        else:
            print("WARNING: source data doesn't have 'transform' attribute.")
    dataset_numpy['data'] = data_slice
    # load outputs if desired
    if output_slice is not None:
        dataset_numpy['components'] = np.array(original_dataset['components'][output_slice])
        if 'label' in original_dataset:
            label_slice = (slice(None),) + output_slice
            dataset_numpy['label'] = np.array(original_dataset['label'][label_slice])
        else:
            # compute affinities from components
            dataset_numpy['label'] = pygt.malis.seg_to_affgraph(dataset_numpy['components'], original_dataset['nhood'])
            warnings.warn("Computing affinity labels because 'label' wasn't provided in data source.", UserWarning)
        if 'mask' in original_dataset:
            dataset_numpy['mask'] = np.array(original_dataset['mask'][output_slice], dtype=np.uint8)
        else:
            # assume no masking
            dataset_numpy['mask'] = np.ones_like(dataset_numpy['components'], dtype=np.uint8)
            warnings.warn("No mask provided. Setting to 1 everywhere.", UserWarning)
    for key in shared_dataset:
        source_array = dataset_numpy[key].astype(dtypes[key])
        target_mp_array = shared_dataset[key]
        if pygt.DEBUG:
            print("dataset_numpy['{0}']: dtype {1} and shape {2}".format(key, source_array.dtype, source_array.shape))
        # print("dataset_numpy['{0}'].flatten().size: {1}".format(key, source_array.flatten().size))
        # print(dir(target_mp_array))
        # print(target_mp_array._length_, target_mp_array._objects, target_mp_array._type_, target_mp_array._b_base_)
        target_mp_array[:] = source_array.flatten()
    return

class DataLoader(object):
    def __init__(self, size, datasets, input_shape, output_shape=None, n_workers=1):
        self.size = size
        self.datasets = datasets
        self.input_shape = input_shape
        self.outputs_are_ignored = output_shape is None
        self.output_shape = output_shape or (0, 0, 0)
        self._list = list()
        self.shapes = {
            'data': (1,) + self.input_shape,
            'components': (1,) + self.output_shape,
            'label': (3,) + self.output_shape,
            'mask': (1,) + self.output_shape,
        }
        self.dtypes = {
            'data': np.float32,
            'components': np.int32,
            'label': np.int32,
            'mask': np.uint8,
        }
        self.keys_to_ignore = []
        if self.outputs_are_ignored:
            # then ignore all outputs. (e.g. for test processing)
            self.keys_to_ignore = ['label', 'components', 'mask']
            for output_key in self.keys_to_ignore:
                self.dtypes.pop(output_key)
                self.shapes.pop(output_key)
        sizes = dict()
        for key, shape in self.shapes.iteritems():
            sizes[key] = reduce(mul, shape)
        if pygt.DEBUG:
            print("sizes: ", sizes)
        self.shared_datasets = []
        for n in range(size):
            shared_dataset = dict()
            for key in self.dtypes:
                size = sizes[key]
                dtype = self.dtypes[key]
                ctype = type(np.ctypeslib.as_ctypes(dtype(0)))
                if pygt.DEBUG:
                    print("creating {key}'s multiprocessing.Array with "
                          "ctype {c} and size {s}".format(key=key, c=ctype, s=size))
                shared_dataset[key] = multiprocessing.Array(ctype, size, lock=False)
            self.shared_datasets.append(shared_dataset)
        self.pool = multiprocessing.Pool(
            processes=n_workers,
            initializer=self._initialize_pool,
            initargs=(),
            maxtasksperchild=10
        )
        self.ready_shared_datasets = []
        return

    def _initialize_pool(self):
        global shared_datasets
        shared_datasets = self.shared_datasets
        global datasets
        datasets = self.datasets
        global dtypes
        dtypes = self.dtypes

    def get_dataset(self, copy=False):
        while len(self.ready_shared_datasets) < 1:
            print('\n', 'waiting for an available shared dataset', '\n')
            time.sleep(1)
            continue
        dataset_metadata = self.ready_shared_datasets.pop(0)
        index_of_shared_dataset = dataset_metadata['shared']
        index_of_given_dataset = dataset_metadata['real']
        new_dataset = dict()
        new_dataset['offset'] = dataset_metadata['offset']
        shared_dataset = self.shared_datasets[index_of_shared_dataset]
        given_dataset = self.datasets[index_of_given_dataset]
        for key in shared_dataset:
            # print("loading shared_dataset['{}']".format(key))
            # print("{}'s desired shape: {}".format(key, self.shapes[key]))
            # if key is 'data':
            #     dtype = np.float32
            # else:
            #     dtype = np.int32
            dtype = self.dtypes[key]
            new_dataset[key] = np.frombuffer(shared_dataset[key], dtype)
            if pygt.DEBUG:
                print(key, new_dataset[key].shape, self.shapes[key])
            new_dataset[key] = new_dataset[key].reshape(self.shapes[key])
            if copy:
                new_dataset[key] = new_dataset[key].copy()
        for key in given_dataset:
            # print("processing given_dataset['{}']".format(key))
            if key in shared_dataset or key in self.keys_to_ignore:
                # we already loaded it, or we want to ignore it
                # print("ignoring given_dataset['{key}']".format(key=key))
                pass
            else:
                # get the value from the original dataset dict
                new_dataset[key] = given_dataset[key]
        return new_dataset, index_of_shared_dataset

    def start_refreshing_shared_dataset(self, shared_dataset_index, offset, dataset_index, transform=True, wait=False):
        if self.outputs_are_ignored:
            output_slice = None
        else:
            borders = tuple([(in_ - out_) / 2 for (in_, out_) in zip(self.input_shape, self.output_shape)])
            # print("borders: ", borders)
            output_slice = tuple([slice(offset[i] + borders[i], offset[i] + borders[i] + self.output_shape[i])
                            for i in range(len(offset))])
        # print("output_slice: {}".format(output_slice))
        input_slice = tuple([slice(offset[i], offset[i] + self.input_shape[i])
                       for i in range(len(offset))])
        # print("input_slice: {}".format(input_slice))

        dataset_metadata = dict(real=dataset_index, shared=shared_dataset_index, offset=offset)
        def pool_callback(*args, **kwargs):
            # print(args, kwargs)
            return self.ready_shared_datasets.append(dataset_metadata)
        async_result = self.pool.apply_async(
            func=update_shared_dataset,
            kwds=dict(
                index_of_shared=shared_dataset_index,
                index_of_which_dataset=dataset_index,
                input_slice=input_slice,
                output_slice=output_slice,
                transform=transform
            ),
            callback=pool_callback
        )
        if wait:
            final_result = async_result.get()
            if final_result is not None:
                # probably an error
                print(final_result)
        return shared_dataset_index, async_result

if __name__ == '__main__':
    path = '/groups/turaga/home/turagas/data/FlyEM/fibsem_medulla_7col/'
    datasets = []
    for dname in ['trvol-250-1-h5', 'trvol-250-2-h5']:
        datasets.append(
            dict({
                'name': dname,
                'data': h5py.File(join(path, dname, 'im_uint8.h5'), 'r')['main'],
                'components': h5py.File(join(path, dname, 'groundtruth_seg_thick.h5'), 'r')['main'],
                'nhood': pygt.malis.mknhood3d(),
                'transform': dict({'scale': (0.8, 1.2), 'shift': (-0.2, 0.2)})
            })
        )
    queue_size = 1
    q = DataLoader(queue_size,
                   datasets=datasets,
                   input_shape=(80, 80, 80),
                   output_shape=(60, 60, 60),
                   n_workers=1
                   )
    for j in range(len(datasets)):
        i = 0  # index of shared dataset to use
        shared_dataset_index, async_result = q.start_refreshing_shared_dataset(i, (15, 25, 35), j, wait=True)
        print("{}'s async_result.get(): {}".format(datasets[j]['name'], async_result.get()))
        dataset_result, index_of_shared_dataset = q.get_dataset(copy=False)
        print('start - ********************************************************************************')
        for key, value in dataset_result.iteritems():
            try:
                print(key, value.dtype, value.shape, type(value), value[0, 5, 50, 20:30], np.mean(value))
            except:
                print(key, value)
        print('end   - ********************************************************************************')
<commit_msg>Reset numpy.random's seed in forked processes<commit_after>from __future__ import print_function

import multiprocessing
import time
from operator import mul
from os.path import join
import warnings

import h5py
import numpy as np

import PyGreentea as pygt

''' where this will be used:
* train()
  * getting unordered batches of a dataset, specified with offset, input size and output size
  * behavior should be like a queue. at the end of each training iteration, train() specifies a replacement
* process()
  * getting batches from a dataset, specified with offset and input size
'''

def update_shared_dataset(index_of_shared, index_of_which_dataset, input_slice, output_slice, transform=True):
    # print("in process id {}".format(os.getpid()))
    shared_dataset = shared_datasets[index_of_shared]
    original_dataset = datasets[index_of_which_dataset]
    # print("original_dataset: ", [
    #     (key,
    #      type(original_dataset[key]),
    #      original_dataset[key])
    #     for key in original_dataset.keys()])
    dataset_numpy = dict()
    # load inputs
    # print("original_dataset['data']: ", original_dataset['data'].shape, original_dataset['data'].dtype)
    data_slice = np.array(original_dataset['data'][input_slice], dtype=np.float32) / (2. ** 8)
    if transform:
        if 'transform' in original_dataset:
            # print('Pre:',(data_slice.min(),data_slice.mean(),data_slice.max()))
            lo, hi = original_dataset['transform']['scale']
            data_slice = 0.5 + (data_slice-0.5)*np.random.uniform(low=lo,high=hi)
            lo, hi = original_dataset['transform']['shift']
            data_slice = data_slice + np.random.uniform(low=lo,high=hi)
        else:
            print("WARNING: source data doesn't have 'transform' attribute.")
    dataset_numpy['data'] = data_slice
    # load outputs if desired
    if output_slice is not None:
        dataset_numpy['components'] = np.array(original_dataset['components'][output_slice])
        if 'label' in original_dataset:
            label_slice = (slice(None),) + output_slice
            dataset_numpy['label'] = np.array(original_dataset['label'][label_slice])
        else:
            # compute affinities from components
            dataset_numpy['label'] = pygt.malis.seg_to_affgraph(dataset_numpy['components'], original_dataset['nhood'])
            warnings.warn("Computing affinity labels because 'label' wasn't provided in data source.", UserWarning)
        if 'mask' in original_dataset:
            dataset_numpy['mask'] = np.array(original_dataset['mask'][output_slice], dtype=np.uint8)
        else:
            # assume no masking
            dataset_numpy['mask'] = np.ones_like(dataset_numpy['components'], dtype=np.uint8)
            warnings.warn("No mask provided. Setting to 1 everywhere.", UserWarning)
    for key in shared_dataset:
        source_array = dataset_numpy[key].astype(dtypes[key])
        target_mp_array = shared_dataset[key]
        if pygt.DEBUG:
            print("dataset_numpy['{0}']: dtype {1} and shape {2}".format(key, source_array.dtype, source_array.shape))
        # print("dataset_numpy['{0}'].flatten().size: {1}".format(key, source_array.flatten().size))
        # print(dir(target_mp_array))
        # print(target_mp_array._length_, target_mp_array._objects, target_mp_array._type_, target_mp_array._b_base_)
        target_mp_array[:] = source_array.flatten()
    return

class DataLoader(object):
    def __init__(self, size, datasets, input_shape, output_shape=None, n_workers=1):
        self.size = size
        self.datasets = datasets
        self.input_shape = input_shape
        self.outputs_are_ignored = output_shape is None
        self.output_shape = output_shape or (0, 0, 0)
        self._list = list()
        self.shapes = {
            'data': (1,) + self.input_shape,
            'components': (1,) + self.output_shape,
            'label': (3,) + self.output_shape,
            'mask': (1,) + self.output_shape,
        }
        self.dtypes = {
            'data': np.float32,
            'components': np.int32,
            'label': np.int32,
            'mask': np.uint8,
        }
        self.keys_to_ignore = []
        if self.outputs_are_ignored:
            # then ignore all outputs. (e.g. for test processing)
            self.keys_to_ignore = ['label', 'components', 'mask']
            for output_key in self.keys_to_ignore:
                self.dtypes.pop(output_key)
                self.shapes.pop(output_key)
        sizes = dict()
        for key, shape in self.shapes.iteritems():
            sizes[key] = reduce(mul, shape)
        if pygt.DEBUG:
            print("sizes: ", sizes)
        self.shared_datasets = []
        for n in range(size):
            shared_dataset = dict()
            for key in self.dtypes:
                size = sizes[key]
                dtype = self.dtypes[key]
                ctype = type(np.ctypeslib.as_ctypes(dtype(0)))
                if pygt.DEBUG:
                    print("creating {key}'s multiprocessing.Array with "
                          "ctype {c} and size {s}".format(key=key, c=ctype, s=size))
                shared_dataset[key] = multiprocessing.Array(ctype, size, lock=False)
            self.shared_datasets.append(shared_dataset)
        self.pool = multiprocessing.Pool(
            processes=n_workers,
            initializer=self._initialize_pool,
            initargs=(),
            maxtasksperchild=10
        )
        self.ready_shared_datasets = []
        return

    def _initialize_pool(self):
        np.random.seed()
        global shared_datasets
        shared_datasets = self.shared_datasets
        global datasets
        datasets = self.datasets
        global dtypes
        dtypes = self.dtypes

    def get_dataset(self, copy=False):
        while len(self.ready_shared_datasets) < 1:
            print('\n', 'waiting for an available shared dataset', '\n')
            time.sleep(1)
            continue
        dataset_metadata = self.ready_shared_datasets.pop(0)
        index_of_shared_dataset = dataset_metadata['shared']
        index_of_given_dataset = dataset_metadata['real']
        new_dataset = dict()
        new_dataset['offset'] = dataset_metadata['offset']
        shared_dataset = self.shared_datasets[index_of_shared_dataset]
        given_dataset = self.datasets[index_of_given_dataset]
        for key in shared_dataset:
            # print("loading shared_dataset['{}']".format(key))
            # print("{}'s desired shape: {}".format(key, self.shapes[key]))
            # if key is 'data':
            #     dtype = np.float32
            # else:
            #     dtype = np.int32
            dtype = self.dtypes[key]
            new_dataset[key] = np.frombuffer(shared_dataset[key], dtype)
            if pygt.DEBUG:
                print(key, new_dataset[key].shape, self.shapes[key])
            new_dataset[key] = new_dataset[key].reshape(self.shapes[key])
            if copy:
                new_dataset[key] = new_dataset[key].copy()
        for key in given_dataset:
            # print("processing given_dataset['{}']".format(key))
            if key in shared_dataset or key in self.keys_to_ignore:
                # we already loaded it, or we want to ignore it
                # print("ignoring given_dataset['{key}']".format(key=key))
                pass
            else:
                # get the value from the original dataset dict
                new_dataset[key] = given_dataset[key]
        return new_dataset, index_of_shared_dataset

    def start_refreshing_shared_dataset(self, shared_dataset_index, offset, dataset_index, transform=True, wait=False):
        if self.outputs_are_ignored:
            output_slice = None
        else:
            borders = tuple([(in_ - out_) / 2 for (in_, out_) in zip(self.input_shape, self.output_shape)])
            # print("borders: ", borders)
            output_slice = tuple([slice(offset[i] + borders[i], offset[i] + borders[i] + self.output_shape[i])
                            for i in range(len(offset))])
        # print("output_slice: {}".format(output_slice))
        input_slice = tuple([slice(offset[i], offset[i] + self.input_shape[i])
                       for i in range(len(offset))])
        # print("input_slice: {}".format(input_slice))

        dataset_metadata = dict(real=dataset_index, shared=shared_dataset_index, offset=offset)
        def pool_callback(*args, **kwargs):
            # print(args, kwargs)
            return self.ready_shared_datasets.append(dataset_metadata)
        async_result = self.pool.apply_async(
            func=update_shared_dataset,
            kwds=dict(
                index_of_shared=shared_dataset_index,
                index_of_which_dataset=dataset_index,
                input_slice=input_slice,
                output_slice=output_slice,
                transform=transform
            ),
            callback=pool_callback
        )
        if wait:
            final_result = async_result.get()
            if final_result is not None:
                # probably an error
                print(final_result)
        return shared_dataset_index, async_result

if __name__ == '__main__':
    path = '/groups/turaga/home/turagas/data/FlyEM/fibsem_medulla_7col/'
    datasets = []
    for dname in ['trvol-250-1-h5', 'trvol-250-2-h5']:
        datasets.append(
            dict({
                'name': dname,
                'data': h5py.File(join(path, dname, 'im_uint8.h5'), 'r')['main'],
                'components': h5py.File(join(path, dname, 'groundtruth_seg_thick.h5'), 'r')['main'],
                'nhood': pygt.malis.mknhood3d(),
                'transform': dict({'scale': (0.8, 1.2), 'shift': (-0.2, 0.2)})
            })
        )
    queue_size = 1
    q = DataLoader(queue_size,
                   datasets=datasets,
                   input_shape=(80, 80, 80),
                   output_shape=(60, 60, 60),
                   n_workers=1
                   )
    for j in range(len(datasets)):
        i = 0  # index of shared dataset to use
        shared_dataset_index, async_result = q.start_refreshing_shared_dataset(i, (15, 25, 35), j, wait=True)
        print("{}'s async_result.get(): {}".format(datasets[j]['name'], async_result.get()))
        dataset_result, index_of_shared_dataset = q.get_dataset(copy=False)
        print('start - ********************************************************************************')
        for key, value in dataset_result.iteritems():
            try:
                print(key, value.dtype, value.shape, type(value), value[0, 5, 50, 20:30], np.mean(value))
            except:
                print(key, value)
        print('end   - ********************************************************************************')
<|endoftext|>


<commit_before>from django.core.exceptions import ObjectDoesNotExist
from django.http import HttpResponse
from tastypie.authorization import DjangoAuthorization
from tastypie.authentication import Authentication
from tastypie.resources import ModelResource, Resource, ALL
from tastypie.cache import SimpleCache

try:
    from django.views.decorators.csrf import csrf_exempt
except ImportError:
    def csrf_exempt(func):
        return func

from datapoints.models import RegionType,Region,RegionPermission

class CustomAuthentication(Authentication):
    '''
    Super Simple permissions check to ensure user is logged in.  Futher
    permissions are appled for GET and POST for the datapoint resource
    based on the regional and indicator level permission of the user found
    in the request.
    '''

    def is_authenticated(self, request, **kwargs):
        '''
        If the user is logged in, return True else return False...
        '''

        if request.user.id:
            return True

        return False

class CustomCache(SimpleCache):
    '''
    Set up to override the simple cache method in order to customize the
    behavior of the cache control headers.
    '''

    def cache_control(self):
        '''
        Instatiate the cache_control instance, and add the headers needed.
        '''
        control = super(CustomCache, self).cache_control()
        control.update({'must_revalidate':True, 'max_age': 3600})
        return control

class BaseModelResource(ModelResource):
    '''
    This applies to only the V1 API.  This method inherits from Tastypie's
    model resource.

    This resource strips down almost all of the tastypie functions which
    drastically slow down the API performance.

    IMPORTANT: if you note, all of the resources use the .values() option for
    each queryset.  That returns the model as JSON, so the idea is that the
    API does not need to serialize or dehydrate the resource.

    The models are set up so that the API does as little transformation as
    possible.  That means however, that a few of our metadata models ( see
    campaign / indicator ) are cached and contain related information making
    the job of the API easy.
    '''

    class Meta:
        authentication = CustomAuthentication()
        authorization = DjangoAuthorization()
        always_return_data = True
        allowed_methods = ['get','post','put','patch', 'delete']
        filtering = {
            "id": ALL,
        }
        cache = CustomCache()

    def dispatch(self, request_type, request, **kwargs):
        """
        Overrides Tastypie and calls get_list.  For now, that makes these
        resources GET only. ( no POST / PUT / PATCH ).
        """

        response = self.get_list(request, **kwargs)

        if not isinstance(response, HttpResponse):
            return http.HttpNoContent()

        return response

    def get_list(self, request, **kwargs):
        """
        Overriden from Tastypie..
        """

        base_bundle = self.build_bundle(request=request)
        objects = self.obj_get_list(bundle=base_bundle, **self.remove_api_resource_names(kwargs))
        sorted_objects = self.apply_sorting(objects, options=request.GET)

        paginator = self._meta.paginator_class(request.GET, sorted_objects, resource_uri=self.get_resource_uri(), limit=self._meta.limit, max_limit=self._meta.max_limit, collection_name=self._meta.collection_name)
        to_be_serialized = paginator.page()

        # Dehydrate the bundles in preparation for serialization.
        bundles = [obj for obj in to_be_serialized[self._meta.collection_name]]
        response_meta = {
            'limit':paginator.get_limit(),
            'offset':paginator.get_offset(),
            'total_count':len(objects),
        }

        response_data = {
            'objects': bundles,
            'meta': response_meta, ## add paginator info here..
            'error': None
        }

        return self.create_response(request, response_data)

class BaseNonModelResource(Resource):
    '''
    NOTE: This applies to only the V1 API.  This is only used for the
    /api/v1/datapoint endpoint.

    This is the top level class all other Resource Classes inherit from this.
    The API Key authentication is defined here and thus is required by all
    other resources.  This class enherits fro the Tastyppie "ModelResource"

    See Here: http://django-tastypie.readthedocs.org/en/latest/resources.html?highlight=modelresource
    '''

    class Meta():
        authentication = CustomAuthentication()
        authorization = DjangoAuthorization()
        always_return_data = True
        cache = CustomCache()

    def parse_url_strings(self,query_dict):
        '''
        As the geo endpoint is based off of the region/parent_region paremeter
        we go through a pretty hacky try / except frenzy in order to find the
        parameters necessary to get region/shape data for the front end.  The
        parameters here for region_in,level, and parent_region in were
        constructed in accordance to the request from the front end team.
        '''

        self.region__in, self.region_type_id, self.parent_region__in = \
            None, None, None

        ## REGION_ID
        try:
            self.region__in = [int(r) for r in query_dict['region__in']\
                .split(',')]
        except KeyError:
            pass
        except ValueError:
            pass

        ## REGION TYPE ##
        try:
            self.region_type_id = RegionType.objects.get(name = query_dict\
                ['level'].lower()).id
        except KeyError:
            pass
        except ObjectDoesNotExist:
            all_r_types = RegionType.objects.all().values_list('name',flat=True)
            err = 'region type doesnt exist. options are:  %s' % all_r_types
            return err, []

        try:
            self.parent_region__in = [int(r) for r in query_dict['parent_region__in']\
                .split(',')]
        except KeyError:
            pass
        except ValueError:
            pass

        return None

    def get_regions_to_return_from_url(self,request):
        '''
        This method is used in both the /geo and /datapoint endpoints.  Based
        on the values parsed from the URL parameters find the regions needed
        to fulfill the request based on the four rules below.
        1  region__in returns geo data for the regions requested
        2. passing only parent_region__in  should return the shapes for all the
           immediate children in that region if no level parameter is supplied
        3. no params - return regions at the top of the tree ( no parent_id).
        After the four steps are worked through in this method, we apply the
        permissions function in order to determine the final list of regions
        to return based on the user making the request.
        '''

        ## attach these to self and return only error #
        err = self.parse_url_strings(request.GET)

        if err:
            self.err = err
            return err, []

        ## CASE 1 ##
        if self.region__in is not None:

            region_ids = Region.objects.filter(id__in = self.region__in)\
                .values_list('id',flat=True)

        ## CASE 2 #
        elif self.parent_region__in is not None and self.region_type_id is None:

            region_ids = Region.objects.filter(parent_region__in = \
                self.parent_region__in).values_list('id',flat=True)

        else:
            region_ids = Region.objects.filter(parent_region_id__isnull=True).\
                values_list('id',flat=True)

        ## now apply regional permissions ##

        # permitted_region_qs = RegionTree.objects.filter(parent_region_id__in =\
        #     region_ids).values_list('region_id',flat=True)

        # final_region_ids = list(set(region_ids).intersection(set\
        #     (permitted_region_ids)))

        return None, region_ids
<commit_msg>remove paginator in base_model api<commit_after>from django.core.exceptions import ObjectDoesNotExist
from django.http import HttpResponse
from tastypie.authorization import DjangoAuthorization
from tastypie.authentication import Authentication
from tastypie.resources import ModelResource, Resource, ALL
from tastypie.cache import SimpleCache

try:
    from django.views.decorators.csrf import csrf_exempt
except ImportError:
    def csrf_exempt(func):
        return func

from datapoints.models import RegionType,Region,RegionPermission

class CustomAuthentication(Authentication):
    '''
    Super Simple permissions check to ensure user is logged in.  Futher
    permissions are appled for GET and POST for the datapoint resource
    based on the regional and indicator level permission of the user found
    in the request.
    '''

    def is_authenticated(self, request, **kwargs):
        '''
        If the user is logged in, return True else return False...
        '''

        if request.user.id:
            return True

        return False

class CustomCache(SimpleCache):
    '''
    Set up to override the simple cache method in order to customize the
    behavior of the cache control headers.
    '''

    def cache_control(self):
        '''
        Instatiate the cache_control instance, and add the headers needed.
        '''
        control = super(CustomCache, self).cache_control()
        control.update({'must_revalidate':True, 'max_age': 3600})
        return control

class BaseModelResource(ModelResource):
    '''
    This applies to only the V1 API.  This method inherits from Tastypie's
    model resource.

    This resource strips down almost all of the tastypie functions which
    drastically slow down the API performance.

    IMPORTANT: if you note, all of the resources use the .values() option for
    each queryset.  That returns the model as JSON, so the idea is that the
    API does not need to serialize or dehydrate the resource.

    The models are set up so that the API does as little transformation as
    possible.  That means however, that a few of our metadata models ( see
    campaign / indicator ) are cached and contain related information making
    the job of the API easy.
    '''

    class Meta:
        authentication = CustomAuthentication()
        authorization = DjangoAuthorization()
        always_return_data = True
        allowed_methods = ['get','post','put','patch', 'delete']
        filtering = {
            "id": ALL,
        }
        cache = CustomCache()

    def dispatch(self, request_type, request, **kwargs):
        """
        Overrides Tastypie and calls get_list.  For now, that makes these
        resources GET only. ( no POST / PUT / PATCH ).
        """

        response = self.get_list(request, **kwargs)

        if not isinstance(response, HttpResponse):
            return http.HttpNoContent()

        return response

    def get_list(self, request, **kwargs):
        """
        Overriden from Tastypie..
        """

        base_bundle = self.build_bundle(request=request)
        objects = self.obj_get_list(bundle=base_bundle, **self.remove_api_resource_names(kwargs))
        # sorted_objects = self.apply_sorting(objects, options=request.GET)

        # paginator = self._meta.paginator_class(request.GET, sorted_objects, resource_uri=self.get_resource_uri(), limit=self._meta.limit, max_limit=self._meta.max_limit, collection_name=self._meta.collection_name)
        # to_be_serialized = paginator.page()

        # Dehydrate the bundles in preparation for serialization.
        bundles = [obj for obj in objects]#[self._meta.collection_name]]
        response_meta = {
            'limit':None, ## paginator.get_limit(),
            'offset': None, ## paginator.get_offset(),
            'total_count':len(objects),
        }

        response_data = {
            'objects': bundles,
            'meta': response_meta, ## add paginator info here..
            'error': None
        }

        return self.create_response(request, response_data)

class BaseNonModelResource(Resource):
    '''
    NOTE: This applies to only the V1 API.  This is only used for the
    /api/v1/datapoint endpoint.

    This is the top level class all other Resource Classes inherit from this.
    The API Key authentication is defined here and thus is required by all
    other resources.  This class enherits fro the Tastyppie "ModelResource"

    See Here: http://django-tastypie.readthedocs.org/en/latest/resources.html?highlight=modelresource
    '''

    class Meta():
        authentication = CustomAuthentication()
        authorization = DjangoAuthorization()
        always_return_data = True
        cache = CustomCache()

    def parse_url_strings(self,query_dict):
        '''
        As the geo endpoint is based off of the region/parent_region paremeter
        we go through a pretty hacky try / except frenzy in order to find the
        parameters necessary to get region/shape data for the front end.  The
        parameters here for region_in,level, and parent_region in were
        constructed in accordance to the request from the front end team.
        '''

        self.region__in, self.region_type_id, self.parent_region__in = \
            None, None, None

        ## REGION_ID
        try:
            self.region__in = [int(r) for r in query_dict['region__in']\
                .split(',')]
        except KeyError:
            pass
        except ValueError:
            pass

        ## REGION TYPE ##
        try:
            self.region_type_id = RegionType.objects.get(name = query_dict\
                ['level'].lower()).id
        except KeyError:
            pass
        except ObjectDoesNotExist:
            all_r_types = RegionType.objects.all().values_list('name',flat=True)
            err = 'region type doesnt exist. options are:  %s' % all_r_types
            return err, []

        try:
            self.parent_region__in = [int(r) for r in query_dict['parent_region__in']\
                .split(',')]
        except KeyError:
            pass
        except ValueError:
            pass

        return None

    def get_regions_to_return_from_url(self,request):
        '''
        This method is used in both the /geo and /datapoint endpoints.  Based
        on the values parsed from the URL parameters find the regions needed
        to fulfill the request based on the four rules below.
        1  region__in returns geo data for the regions requested
        2. passing only parent_region__in  should return the shapes for all the
           immediate children in that region if no level parameter is supplied
        3. no params - return regions at the top of the tree ( no parent_id).
        After the four steps are worked through in this method, we apply the
        permissions function in order to determine the final list of regions
        to return based on the user making the request.
        '''

        ## attach these to self and return only error #
        err = self.parse_url_strings(request.GET)

        if err:
            self.err = err
            return err, []

        ## CASE 1 ##
        if self.region__in is not None:

            region_ids = Region.objects.filter(id__in = self.region__in)\
                .values_list('id',flat=True)

        ## CASE 2 #
        elif self.parent_region__in is not None and self.region_type_id is None:

            region_ids = Region.objects.filter(parent_region__in = \
                self.parent_region__in).values_list('id',flat=True)

        else:
            region_ids = Region.objects.filter(parent_region_id__isnull=True).\
                values_list('id',flat=True)

        ## now apply regional permissions ##

        # permitted_region_qs = RegionTree.objects.filter(parent_region_id__in =\
        #     region_ids).values_list('region_id',flat=True)

        # final_region_ids = list(set(region_ids).intersection(set\
        #     (permitted_region_ids)))

        return None, region_ids
<|endoftext|>


<commit_before>import unittest
from pyuntl.untl_structure import PYUNTL_DISPATCH, UNTLStructureException
from tests import UNNORMALIZED_DICT, NORMALIZED_DICT, UNNORMALIZED_UNTLBS, \
    UNNORMALIZED_LCSH, NORMALIZED_UNTLBS, NORMALIZED_LCSH
from pyuntl.util import untldict_normalizer, normalize_UNTL, normalize_LCSH

class FieldTest(unittest.TestCase):

    def setUp(self):
        self.field = PYUNTL_DISPATCH['title'](content=None)

    def test_quick_access(self):
        self.assertEqual(self.field.tag, 'title')

    def test_set_qualifier(self):
        self.field.set_qualifier('test')
        self.assertEqual(self.field.qualifier, 'test')

    def test_set_qualifier_wrong_element(self):
        self.field = PYUNTL_DISPATCH['collection'](content=None)
        self.assertRaises(UNTLStructureException,
                          lambda: self.field.set_qualifier('test'))

    def test_set_content(self):
        self.field.set_content('test')
        self.assertEqual(self.field.content, 'test')

    def test_add_unspecified_attribute(self):
        self.field.name = 'Donald Ronald'
        self.assertTrue(self.field.name)

    def test_add_content_non_ascii(self):
        self.field.set_content('test \xc2')
        self.assertEqual(self.field.content, 'test \xc2')

    def test_add_content_removes_leading_trailing_space(self):
        self.field.set_content('   test   ')
        self.assertEqual(self.field.content, 'test')

    def test_qualifier_removes_trailing_space(self):
        self.field.set_qualifier('test ')
        self.assertEqual(self.field.qualifier, 'test')

    def testLCSHFieldNormalization(self, lcsh_content=UNNORMALIZED_LCSH,
                                   normalized_content=NORMALIZED_LCSH):
        """Tests normalizing an LCSH string"""
        norm = normalize_LCSH(lcsh_content)
        self.assertEqual(norm, normalized_content)

    def testUNTLBSFieldNormalization(self, untlbs_content=UNNORMALIZED_UNTLBS,
                                     normalized_content=NORMALIZED_UNTLBS):
        """Tests normalizing a UNTL-BS string"""
        norm = normalize_UNTL(untlbs_content)
        self.assertEqual(norm, normalized_content)

    def testSingleNormalization(self, untl_dict=UNNORMALIZED_DICT,
                                normalized_dict=NORMALIZED_DICT):
        """Test normalizing LCSH untl data"""
        normalize_required = {
            'subject': ['LCSH'],
            }
        norm = untldict_normalizer(untl_dict, normalize_required)
        self.assertEqual(norm, normalized_dict)

    def testAllNormalizations(self, untl_dict=UNNORMALIZED_DICT,
                              normalized_dict=NORMALIZED_DICT):
        """Tests all the normalizations for untl"""
        normalize_required = {
            'subject': ['LCSH', 'UNTL-BS'],
            }
        norm = untldict_normalizer(untl_dict, normalize_required)
        self.assertEqual(norm, normalized_dict)

def suite():
    test_suite = unittest.makeSuite(FieldTest, 'test')
    return test_suite

if __name__ == '__main__':
    unittest.main()
<commit_msg>Digi style changes to tests/field.py<commit_after>import unittest

from pyuntl.untl_structure import PYUNTL_DISPATCH, UNTLStructureException
from pyuntl.util import untldict_normalizer, normalize_UNTL, normalize_LCSH
from tests import (UNNORMALIZED_DICT, NORMALIZED_DICT, UNNORMALIZED_UNTLBS,
                   UNNORMALIZED_LCSH, NORMALIZED_UNTLBS, NORMALIZED_LCSH)

class FieldTest(unittest.TestCase):
    def setUp(self):
        self.field = PYUNTL_DISPATCH['title'](content=None)

    def test_quick_access(self):
        self.assertEqual(self.field.tag, 'title')

    def test_set_qualifier(self):
        self.field.set_qualifier('test')
        self.assertEqual(self.field.qualifier, 'test')

    def test_set_qualifier_wrong_element(self):
        self.field = PYUNTL_DISPATCH['collection'](content=None)
        self.assertRaises(UNTLStructureException,
                          lambda: self.field.set_qualifier('test'))

    def test_set_content(self):
        self.field.set_content('test')
        self.assertEqual(self.field.content, 'test')

    def test_add_unspecified_attribute(self):
        self.field.name = 'Donald Ronald'
        self.assertTrue(self.field.name)

    def test_add_content_non_ascii(self):
        self.field.set_content('test \xc2')
        self.assertEqual(self.field.content, 'test \xc2')

    def test_add_content_removes_leading_trailing_space(self):
        self.field.set_content('   test   ')
        self.assertEqual(self.field.content, 'test')

    def test_qualifier_removes_trailing_space(self):
        self.field.set_qualifier('test ')
        self.assertEqual(self.field.qualifier, 'test')

    def testLCSHFieldNormalization(self, lcsh_content=UNNORMALIZED_LCSH,
                                   normalized_content=NORMALIZED_LCSH):
        """Test normalizing an LCSH string."""
        norm = normalize_LCSH(lcsh_content)
        self.assertEqual(norm, normalized_content)

    def testUNTLBSFieldNormalization(self, untlbs_content=UNNORMALIZED_UNTLBS,
                                     normalized_content=NORMALIZED_UNTLBS):
        """Test normalizing a UNTL-BS string."""
        norm = normalize_UNTL(untlbs_content)
        self.assertEqual(norm, normalized_content)

    def testSingleNormalization(self, untl_dict=UNNORMALIZED_DICT,
                                normalized_dict=NORMALIZED_DICT):
        """Test normalizing LCSH UNTL data."""
        normalize_required = {
            'subject': ['LCSH'],
            }
        norm = untldict_normalizer(untl_dict, normalize_required)
        self.assertEqual(norm, normalized_dict)

    def testAllNormalizations(self, untl_dict=UNNORMALIZED_DICT,
                              normalized_dict=NORMALIZED_DICT):
        """Test all the normalizations for UNTL."""
        normalize_required = {
            'subject': ['LCSH', 'UNTL-BS'],
            }
        norm = untldict_normalizer(untl_dict, normalize_required)
        self.assertEqual(norm, normalized_dict)

def suite():
    test_suite = unittest.makeSuite(FieldTest, 'test')
    return test_suite

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>__version__ = '0.0.14'
<commit_msg>Bump version number<commit_after>__version__ = '0.0.15'
<|endoftext|>


<commit_before>import inspect
import hashlib
import logging

from django.core.cache import caches
from django.conf import settings

log = logging.getLogger(__name__)

class cache_types(object):
    NONE = None
    DEFAULT = 'default'
    SIMPLE = 'simple'  # Stores queryset objects directly in cache
    PK_LIST = 'pk_list'  # Stores queryset pks in cache for later expansion back to queryset

try:
    CACHE_BACKEND = settings.DATATABLEVIEW_CACHE_BACKEND
except AttributeError:
    CACHE_BACKEND = 'default'

try:
    CACHE_PREFIX = settings.DATATABLEVIEW_CACHE_PREFIX
except AttributeError:
    CACHE_PREFIX = 'datatableview_'

try:
    DEFAULT_CACHE_TYPE = settings.DATATABLEVIEW_DEFAULT_CACHE_TYPE
except AttributeError:
    DEFAULT_CACHE_TYPE = cache_types.SIMPLE

try:
    CACHE_KEY_HASH = settings.DATATABLEVIEW_CACHE_KEY_HASH
except AttributeError:
    CACHE_KEY_HASH = True

try:
    CACHE_KEY_HASH_LENGTH = settings.DATATABLEVIEW_CACHE_KEY_HASH_LENGTH
except AttributeError:
    CACHE_KEY_HASH_LENGTH = None

cache = caches[CACHE_BACKEND]

hash_slice = None
if CACHE_KEY_HASH:
    hash_slice = slice(None, CACHE_KEY_HASH_LENGTH)

def _hash_key_component(s):
    return hashlib.sha1(s.encode('utf-8')).hexdigest()[hash_slice]

def get_cache_key(datatable_class, view=None, user=None, **kwargs):
    """
    Returns a cache key unique to the current table, and (if available) the request user.

    The ``view`` argument should be the class reference itself, since it is easily obtainable
    in contexts where the instance is not available.
    """

    datatable_name = datatable_class.__name__
    if datatable_name.endswith('_Synthesized'):
        datatable_name = datatable_name[:-12]
    datatable_id = '%s.%s' % (datatable_class.__module__, datatable_name)
    if CACHE_KEY_HASH:
        datatable_id = _hash_key_component(datatable_id)

    cache_key = 'datatable_%s' % (datatable_id,)

    if view:
        if not inspect.isclass(view):
            # Reduce view to its class
            view = view.__class__

        view_id = '%s.%s' % (view.__module__, view.__name__)
        if CACHE_KEY_HASH:
            view_id = _hash_key_component(view_id)
        cache_key += '__view_%s' % (view_id,)

    if user and user.is_authenticated():
        cache_key += '__user_%s' % (user.pk,)

    # All other kwargs are used directly to create a hashed suffix
    # Order the kwargs by key name, then convert them to their repr() values.
    items = sorted(kwargs.items(), key=lambda item: item[0])
    values = []
    for k, v in items:
        values.append('%r:%r' % (k, v))

    if values:
        kwargs_id = '__'.join(values)
        kwargs_id = _hash_key_component(kwargs_id)
        cache_key += '__kwargs_%s' % (kwargs_id,)

    log.debug("Cache key derived for %r: %r (from kwargs %r)", datatable_class, cache_key, values)

    return cache_key

def get_cached_data(datatable, **kwargs):
    """ Returns the cached object list under the appropriate key, or None if not set. """
    cache_key = '%s%s' % (CACHE_PREFIX, datatable.get_cache_key(**kwargs))
    data = cache.get(cache_key)
    log.debug("Reading data from cache at %r: %r", cache_key, data)
    return data

def cache_data(datatable, data, **kwargs):
    """ Stores the object list in the cache under the appropriate key. """
    cache_key = '%s%s' % (CACHE_PREFIX, datatable.get_cache_key(**kwargs))
    log.debug("Setting data to cache at %r: %r", cache_key, data)
    cache.set(cache_key, data)
<commit_msg>Bug fix is_authenticated<commit_after>import inspect
import hashlib
import logging

from django.core.cache import caches
from django.conf import settings

log = logging.getLogger(__name__)

class cache_types(object):
    NONE = None
    DEFAULT = 'default'
    SIMPLE = 'simple'  # Stores queryset objects directly in cache
    PK_LIST = 'pk_list'  # Stores queryset pks in cache for later expansion back to queryset

try:
    CACHE_BACKEND = settings.DATATABLEVIEW_CACHE_BACKEND
except AttributeError:
    CACHE_BACKEND = 'default'

try:
    CACHE_PREFIX = settings.DATATABLEVIEW_CACHE_PREFIX
except AttributeError:
    CACHE_PREFIX = 'datatableview_'

try:
    DEFAULT_CACHE_TYPE = settings.DATATABLEVIEW_DEFAULT_CACHE_TYPE
except AttributeError:
    DEFAULT_CACHE_TYPE = cache_types.SIMPLE

try:
    CACHE_KEY_HASH = settings.DATATABLEVIEW_CACHE_KEY_HASH
except AttributeError:
    CACHE_KEY_HASH = True

try:
    CACHE_KEY_HASH_LENGTH = settings.DATATABLEVIEW_CACHE_KEY_HASH_LENGTH
except AttributeError:
    CACHE_KEY_HASH_LENGTH = None

cache = caches[CACHE_BACKEND]

hash_slice = None
if CACHE_KEY_HASH:
    hash_slice = slice(None, CACHE_KEY_HASH_LENGTH)

def _hash_key_component(s):
    return hashlib.sha1(s.encode('utf-8')).hexdigest()[hash_slice]

def get_cache_key(datatable_class, view=None, user=None, **kwargs):
    """
    Returns a cache key unique to the current table, and (if available) the request user.

    The ``view`` argument should be the class reference itself, since it is easily obtainable
    in contexts where the instance is not available.
    """

    datatable_name = datatable_class.__name__
    if datatable_name.endswith('_Synthesized'):
        datatable_name = datatable_name[:-12]
    datatable_id = '%s.%s' % (datatable_class.__module__, datatable_name)
    if CACHE_KEY_HASH:
        datatable_id = _hash_key_component(datatable_id)

    cache_key = 'datatable_%s' % (datatable_id,)

    if view:
        if not inspect.isclass(view):
            # Reduce view to its class
            view = view.__class__

        view_id = '%s.%s' % (view.__module__, view.__name__)
        if CACHE_KEY_HASH:
            view_id = _hash_key_component(view_id)
        cache_key += '__view_%s' % (view_id,)

    if user and user.is_authenticated:
        cache_key += '__user_%s' % (user.pk,)

    # All other kwargs are used directly to create a hashed suffix
    # Order the kwargs by key name, then convert them to their repr() values.
    items = sorted(kwargs.items(), key=lambda item: item[0])
    values = []
    for k, v in items:
        values.append('%r:%r' % (k, v))

    if values:
        kwargs_id = '__'.join(values)
        kwargs_id = _hash_key_component(kwargs_id)
        cache_key += '__kwargs_%s' % (kwargs_id,)

    log.debug("Cache key derived for %r: %r (from kwargs %r)", datatable_class, cache_key, values)

    return cache_key

def get_cached_data(datatable, **kwargs):
    """ Returns the cached object list under the appropriate key, or None if not set. """
    cache_key = '%s%s' % (CACHE_PREFIX, datatable.get_cache_key(**kwargs))
    data = cache.get(cache_key)
    log.debug("Reading data from cache at %r: %r", cache_key, data)
    return data

def cache_data(datatable, data, **kwargs):
    """ Stores the object list in the cache under the appropriate key. """
    cache_key = '%s%s' % (CACHE_PREFIX, datatable.get_cache_key(**kwargs))
    log.debug("Setting data to cache at %r: %r", cache_key, data)
    cache.set(cache_key, data)
<|endoftext|>