<commit_before>from __future__ import absolute_import

__author__ = 'andrews'

import unittest
import uuid
import array
from binascii import hexlify

from libpebble2.protocol.voice import *

class TestVoiceProtocol(unittest.TestCase):

    def test_session_setup_command(self):
        # this test is a bit of a sanity check to ensure that the voice control endpoint description works with the
        # pebble protocol definition methodology

        attributes = []
        app_uuid = uuid.uuid4()
        attributes.append(Attribute(data=AppUuid(uuid=app_uuid)))
        encoder_info = Attribute(data=SpeexEncoderInfo(version="1.2rc1",
                                                       sample_rate=16000,
                                                       bit_rate=12800,
                                                       bitstream_version=4,
                                                       frame_size=320
                                                       ))
        attributes.append(encoder_info)
        attr_list = AttributeList(dictionary=attributes)
        packet = VoiceControlCommand(flags=1, data=SessionSetupCommand(session_type=SessionType.Dictation,
                                                                       session_id=0x55, attributes=attr_list))
        data = [0x01,                    # Message ID: Session setup
                0x01, 0x00, 0x00, 0x00,  # flags
                0x01,                    # Dictation session
                0x55, 0x00,              # Audio streaming session ID

                0x02,                    # attribute list - num attributes

                0x03,                    # attribute id - app uuid
                16, 0x00,
                ]
        serial = array.array('B', data).tostring() + app_uuid.bytes
        data = [0x01,                    # attribute id - attribute info speex
                0x1D, 0x00,              # attribute length
                ord('1'), ord('.'), ord('2'), ord('r'), ord('c'), ord('1'), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                0x80, 0x3e, 0x00, 0x00,
                0x00, 0x32,
                0x04,
                0x40, 0x01]
        serial += array.array('B', data).tostring()
        self.assertEqual(serial, packet.serialise())

        cmd = VoiceControlCommand.parse(packet.serialise())[0]

        self.assertEqual(cmd.flags, 1)
        self.assertEqual(cmd.command, Command.SessionSetup)
        self.assertIsInstance(cmd.data, SessionSetupCommand)
        setup = cmd.data
        self.assertEqual(setup.session_type, SessionType.Dictation)
        self.assertEqual(setup.session_id, 0x55)
        self.assertEqual(setup.attributes.count, 2)
        attributes = setup.attributes.dictionary
        self.assertIsInstance(attributes[0].data, AppUuid)
        self.assertIsInstance(attributes[1].data, SpeexEncoderInfo)
        self.assertEqual(attributes[0].data.uuid, app_uuid)
        self.assertEqual(attributes[1].data.version, "1.2rc1")
        self.assertEqual(attributes[1].data.sample_rate, 16000)
        self.assertEqual(attributes[1].data.bit_rate, 12800)
        self.assertEqual(attributes[1].data.bitstream_version, 4)
        self.assertEqual(attributes[1].data.frame_size, 320)

    def test_session_setup_result(self):
        data = [0x01,                    # Message ID: Session setup
                0x01, 0x00, 0x00, 0x00,  # flags
                0x01,                    # Dictation session
                0x00
                ]
        serial = array.array('B', data).tostring()

        msg = VoiceControlResult(flags=1, data=SessionSetupResult(session_type=SessionType.Dictation,
                                                                  result=Result.Success))
        self.assertEqual(serial, msg.serialise())

    def test_transcription(self):
        expected = [
            # Transcription
            0x01,         # Transcription type
            0x02,         # Sentence count

            # Sentence #1
            0x02, 0x00,   # Word count

            # Word #1
            85,           # Confidence
            0x05, 0x00,   # Word length
            ord('H'), ord('e'), ord('l'), ord('l'), ord('o'),

            # Word #2
            74,           # Confidence
            0x08, 0x00,   # Word length
            ord('c'), ord('o'), ord('m'), ord('p'), ord('u'), ord('t'), ord('e'), ord('r'),

            # Sentence #2
            0x03, 0x00,   # Word count

            # Word #1
            13,           # Confidence
            0x04, 0x00,   # Word length
            ord('h'), ord('e'), ord('l'), ord('l'),

            # Word #1
            3,           # Confidence
            0x02, 0x00,   # Word length
            ord('o'), ord('h'),

            # Word #2
            0,           # Confidence
            0x07, 0x00,   # Word length
            ord('c'), ord('o'), ord('m'), ord('p'), ord('u'), ord('t'), ord('a')
        ]

        expected = array.array('B', expected).tostring()
        s1 = [Word(confidence=85, data="Hello"), Word(confidence=74, data="computer")]
        s2 = [Word(confidence=13, data="hell"), Word(confidence=3, data="oh"), Word(confidence=0, data="computa")]

        t = Transcription(transcription=SentenceList(sentences=[Sentence(words=s1), Sentence(words=s2)]))
        self.assertEqual(expected, t.serialise(default_endianness='<'))

    def test_dictation_result(self):
        app_uuid = uuid.uuid4()
        expected = [
            0x02,         # Message ID: Dictation result
            0x01, 0x00, 0x00, 0x00,  # flags
            0x11, 0x22,   # Audio streaming session ID
            0x00,         # Dictation result - success

            0x02,         # attribute list - num attributes

            0x03,         # attribute type - app uuid
            0x10, 0x00,   # attribute length
        ]
        expected = array.array('B', expected).tostring() + app_uuid.bytes
        transcription = [
            0x02,         # attribute type - transcription
            0x2F, 0x00,   # attribute length

            # Transcription
            0x01,         # Transcription type
            0x02,         # Sentence count

            # Sentence #1
            0x02, 0x00,   # Word count

            # Word #1
            85,           # Confidence
            0x05, 0x00,   # Word length
            ord('H'), ord('e'), ord('l'), ord('l'), ord('o'),

            # Word #2
            74,           # Confidence
            0x08, 0x00,   # Word length
            ord('c'), ord('o'), ord('m'), ord('p'), ord('u'), ord('t'), ord('e'), ord('r'),

            # Sentence #2
            0x03, 0x00,   # Word count

            # Word #1
            13,           # Confidence
            0x04, 0x00,   # Word length
            ord('h'), ord('e'), ord('l'), ord('l'),

            # Word #1
            3,           # Confidence
            0x02, 0x00,   # Word length
            ord('o'), ord('h'),

            # Word #2
            0,           # Confidence
            0x07, 0x00,   # Word length
            ord('c'), ord('o'), ord('m'), ord('p'), ord('u'), ord('t'), ord('a')
        ]
        expected += array.array('B', transcription).tostring()

        s1 = [Word(confidence=85, data="Hello"), Word(confidence=74, data="computer")]
        s2 = [Word(confidence=13, data="hell"), Word(confidence=3, data="oh"), Word(confidence=0, data="computa")]

        t = Transcription(type=TranscriptionType.SentenceList,
                          transcription=SentenceList(sentences=[Sentence(words=s1), Sentence(words=s2)]))
        attr_list = AttributeList(dictionary=[Attribute(data=AppUuid(uuid=app_uuid)), Attribute(data=t)])
        msg = VoiceControlResult(flags=1,
                                 data=DictationResult(session_id=0x2211, result=Result.Success, attributes=attr_list))
        self.assertEqual(expected, msg.serialise())

if __name__ == '__main__':
    unittest.main()
<commit_msg>remove unused import<commit_after>from __future__ import absolute_import

__author__ = 'andrews'

import unittest
import uuid
import array

from libpebble2.protocol.voice import *

class TestVoiceProtocol(unittest.TestCase):

    def test_session_setup_command(self):
        # this test is a bit of a sanity check to ensure that the voice control endpoint description works with the
        # pebble protocol definition methodology

        attributes = []
        app_uuid = uuid.uuid4()
        attributes.append(Attribute(data=AppUuid(uuid=app_uuid)))
        encoder_info = Attribute(data=SpeexEncoderInfo(version="1.2rc1",
                                                       sample_rate=16000,
                                                       bit_rate=12800,
                                                       bitstream_version=4,
                                                       frame_size=320
                                                       ))
        attributes.append(encoder_info)
        attr_list = AttributeList(dictionary=attributes)
        packet = VoiceControlCommand(flags=1, data=SessionSetupCommand(session_type=SessionType.Dictation,
                                                                       session_id=0x55, attributes=attr_list))
        data = [0x01,                    # Message ID: Session setup
                0x01, 0x00, 0x00, 0x00,  # flags
                0x01,                    # Dictation session
                0x55, 0x00,              # Audio streaming session ID

                0x02,                    # attribute list - num attributes

                0x03,                    # attribute id - app uuid
                16, 0x00,
                ]
        serial = array.array('B', data).tostring() + app_uuid.bytes
        data = [0x01,                    # attribute id - attribute info speex
                0x1D, 0x00,              # attribute length
                ord('1'), ord('.'), ord('2'), ord('r'), ord('c'), ord('1'), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                0x80, 0x3e, 0x00, 0x00,
                0x00, 0x32,
                0x04,
                0x40, 0x01]
        serial += array.array('B', data).tostring()
        self.assertEqual(serial, packet.serialise())

        cmd = VoiceControlCommand.parse(packet.serialise())[0]

        self.assertEqual(cmd.flags, 1)
        self.assertEqual(cmd.command, Command.SessionSetup)
        self.assertIsInstance(cmd.data, SessionSetupCommand)
        setup = cmd.data
        self.assertEqual(setup.session_type, SessionType.Dictation)
        self.assertEqual(setup.session_id, 0x55)
        self.assertEqual(setup.attributes.count, 2)
        attributes = setup.attributes.dictionary
        self.assertIsInstance(attributes[0].data, AppUuid)
        self.assertIsInstance(attributes[1].data, SpeexEncoderInfo)
        self.assertEqual(attributes[0].data.uuid, app_uuid)
        self.assertEqual(attributes[1].data.version, "1.2rc1")
        self.assertEqual(attributes[1].data.sample_rate, 16000)
        self.assertEqual(attributes[1].data.bit_rate, 12800)
        self.assertEqual(attributes[1].data.bitstream_version, 4)
        self.assertEqual(attributes[1].data.frame_size, 320)

    def test_session_setup_result(self):
        data = [0x01,                    # Message ID: Session setup
                0x01, 0x00, 0x00, 0x00,  # flags
                0x01,                    # Dictation session
                0x00
                ]
        serial = array.array('B', data).tostring()

        msg = VoiceControlResult(flags=1, data=SessionSetupResult(session_type=SessionType.Dictation,
                                                                  result=Result.Success))
        self.assertEqual(serial, msg.serialise())

    def test_transcription(self):
        expected = [
            # Transcription
            0x01,         # Transcription type
            0x02,         # Sentence count

            # Sentence #1
            0x02, 0x00,   # Word count

            # Word #1
            85,           # Confidence
            0x05, 0x00,   # Word length
            ord('H'), ord('e'), ord('l'), ord('l'), ord('o'),

            # Word #2
            74,           # Confidence
            0x08, 0x00,   # Word length
            ord('c'), ord('o'), ord('m'), ord('p'), ord('u'), ord('t'), ord('e'), ord('r'),

            # Sentence #2
            0x03, 0x00,   # Word count

            # Word #1
            13,           # Confidence
            0x04, 0x00,   # Word length
            ord('h'), ord('e'), ord('l'), ord('l'),

            # Word #1
            3,           # Confidence
            0x02, 0x00,   # Word length
            ord('o'), ord('h'),

            # Word #2
            0,           # Confidence
            0x07, 0x00,   # Word length
            ord('c'), ord('o'), ord('m'), ord('p'), ord('u'), ord('t'), ord('a')
        ]

        expected = array.array('B', expected).tostring()
        s1 = [Word(confidence=85, data="Hello"), Word(confidence=74, data="computer")]
        s2 = [Word(confidence=13, data="hell"), Word(confidence=3, data="oh"), Word(confidence=0, data="computa")]

        t = Transcription(transcription=SentenceList(sentences=[Sentence(words=s1), Sentence(words=s2)]))
        self.assertEqual(expected, t.serialise(default_endianness='<'))

    def test_dictation_result(self):
        app_uuid = uuid.uuid4()
        expected = [
            0x02,         # Message ID: Dictation result
            0x01, 0x00, 0x00, 0x00,  # flags
            0x11, 0x22,   # Audio streaming session ID
            0x00,         # Dictation result - success

            0x02,         # attribute list - num attributes

            0x03,         # attribute type - app uuid
            0x10, 0x00,   # attribute length
        ]
        expected = array.array('B', expected).tostring() + app_uuid.bytes
        transcription = [
            0x02,         # attribute type - transcription
            0x2F, 0x00,   # attribute length

            # Transcription
            0x01,         # Transcription type
            0x02,         # Sentence count

            # Sentence #1
            0x02, 0x00,   # Word count

            # Word #1
            85,           # Confidence
            0x05, 0x00,   # Word length
            ord('H'), ord('e'), ord('l'), ord('l'), ord('o'),

            # Word #2
            74,           # Confidence
            0x08, 0x00,   # Word length
            ord('c'), ord('o'), ord('m'), ord('p'), ord('u'), ord('t'), ord('e'), ord('r'),

            # Sentence #2
            0x03, 0x00,   # Word count

            # Word #1
            13,           # Confidence
            0x04, 0x00,   # Word length
            ord('h'), ord('e'), ord('l'), ord('l'),

            # Word #1
            3,           # Confidence
            0x02, 0x00,   # Word length
            ord('o'), ord('h'),

            # Word #2
            0,           # Confidence
            0x07, 0x00,   # Word length
            ord('c'), ord('o'), ord('m'), ord('p'), ord('u'), ord('t'), ord('a')
        ]
        expected += array.array('B', transcription).tostring()

        s1 = [Word(confidence=85, data="Hello"), Word(confidence=74, data="computer")]
        s2 = [Word(confidence=13, data="hell"), Word(confidence=3, data="oh"), Word(confidence=0, data="computa")]

        t = Transcription(type=TranscriptionType.SentenceList,
                          transcription=SentenceList(sentences=[Sentence(words=s1), Sentence(words=s2)]))
        attr_list = AttributeList(dictionary=[Attribute(data=AppUuid(uuid=app_uuid)), Attribute(data=t)])
        msg = VoiceControlResult(flags=1,
                                 data=DictationResult(session_id=0x2211, result=Result.Success, attributes=attr_list))
        self.assertEqual(expected, msg.serialise())

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Copyright (c) 2013-2015, Freja Nordsiek
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import sys
import copy
import os
import os.path
import math
import random
import collections

import numpy as np
import numpy.random

import hdf5storage

from nose.tools import raises

from asserts import *
from make_randoms import *

random.seed()

class TestPythonMatlabFormat(object):
    # Test for the ability to write python types to an HDF5 file that
    # type information and matlab information are stored in, and then
    # read it back and have it be the same.
    def __init__(self):
        self.filename = 'data.mat'
        self.options = hdf5storage.Options()

        # Need a list of the supported numeric dtypes to test, excluding
        # those not supported by MATLAB. 'S' and 'U' dtype chars have to
        # be used for the bare byte and unicode string dtypes since the
        # dtype strings (but not chars) are not the same in Python 2 and
        # 3.
        self.dtypes = ['bool', 'uint8', 'uint16', 'uint32', 'uint64',
                       'int8', 'int16', 'int32', 'int64',
                       'float32', 'float64', 'complex64', 'complex128',
                       'S', 'U']

    def write_readback(self, data, name, options):
        # Write the data to the proper file with the given name, read it
        # back, and return the result. The file needs to be deleted
        # before and after to keep junk from building up.
        if os.path.exists(self.filename):
            os.remove(self.filename)
        try:
            hdf5storage.write(data, path=name, filename=self.filename,
                              options=options)
            out = hdf5storage.read(path=name, filename=self.filename,
                                   options=options)
        except:
            raise
        finally:
            if os.path.exists(self.filename):
                os.remove(self.filename)
        return out

    def assert_equal(self, a, b):
        assert_equal(a, b)

    def check_numpy_scalar(self, dtype):
        # Makes a random numpy scalar of the given type, writes it and
        # reads it back, and then compares it.
        data = random_numpy_scalar(dtype)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def check_numpy_array(self, dtype, dimensions):
        # Makes a random numpy array of the given type, writes it and
        # reads it back, and then compares it.
        shape = random_numpy_shape(dimensions,
                                   max_array_axis_length)
        data = random_numpy(shape, dtype)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def check_numpy_empty(self, dtype):
        # Makes an empty numpy array of the given type, writes it and
        # reads it back, and then compares it.
        data = np.array([], dtype)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def check_numpy_structured_array(self, dimensions):
        # Makes a random structured ndarray of the given type, writes it
        # and reads it back, and then compares it.
        shape = random_numpy_shape(dimensions, \
            max_structured_ndarray_axis_length)
        data = random_structured_numpy_array(shape)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def check_numpy_structured_array_empty(self, dimensions):
        # Makes a random structured ndarray of the given type, writes it
        # and reads it back, and then compares it.
        shape = random_numpy_shape(dimensions, \
            max_structured_ndarray_axis_length)
        data = random_structured_numpy_array(shape, (1, 0))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def check_python_collection(self, tp):
        # Makes a random collection of the specified type, writes it and
        # reads it back, and then compares it.
        if tp in (set, frozenset):
            data = tp(random_list(max_list_length,
                      python_or_numpy='python'))
        else:
            data = tp(random_list(max_list_length,
                      python_or_numpy='numpy'))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_None(self):
        data = None
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bool_True(self):
        data = True
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bool_False(self):
        data = False
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_int(self):
        data = random_int()
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    # Only relevant in Python 2.x.
    def test_long(self):
        if sys.hexversion < 0x03000000:
            data = long(random_int())
            out = self.write_readback(data, random_name(),
                                      self.options)
            self.assert_equal(out, data)

    def test_int_or_long_too_big(self):
        if sys.hexversion >= 0x03000000:
            data = 2**64 * random_int()
        else:
            data = long(2)**64 * long(random_int())
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_float(self):
        data = random_float()
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_float_inf(self):
        data = float(np.inf)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_float_ninf(self):
        data = float(-np.inf)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_float_nan(self):
        data = float(np.nan)
        out = self.write_readback(data, random_name(),
                                  self.options)
        assert math.isnan(out)

    def test_complex(self):
        data = random_float() + 1j*random_float()
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_complex_real_nan(self):
        data = complex(np.nan, random_float())
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_complex_imaginary_nan(self):
        data = complex(random_float(), np.nan)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_str_ascii(self):
        data = random_str_ascii(random.randint(1,
                                max_string_length))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_str_ascii_encoded_utf8(self):
        data = random_str_some_unicode(random.randint(1, \
            max_string_length)).encode('UTF-8')
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_str_with_null(self):
        strs = [random_str_ascii(
                random.randint(1, max_string_length))
                for i in range(2)]
        if sys.hexversion < 0x03000000:
            data = u'\x00'.join(strs)
        else:
            data = '\x00'.join(strs)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_str_unicode(self):
        data = random_str_some_unicode(random.randint(1,
                                       max_string_length))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_str_empty(self):
        data = ''
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bytes(self):
        data = random_bytes(random.randint(1,
                            max_string_length))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bytes_empty(self):
        data = b''
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bytes_with_null(self):
        strs = [random_bytes(
                random.randint(1, max_string_length))
                for i in range(2)]
        if sys.hexversion < 0x03000000:
            data = '\x00'.join(strs)
        else:
            data = b'\x00'.join(strs)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bytearray(self):
        data = bytearray(random_bytes(random.randint(1,
                         max_string_length)))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bytearray_empty(self):
        data = bytearray(b'')
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_numpy_scalar(self):
        for dt in self.dtypes:
            yield self.check_numpy_scalar, dt

    def test_numpy_array_1d(self):
        dtypes = copy.deepcopy(self.dtypes)
        dtypes.append('object')
        for dt in dtypes:
            yield self.check_numpy_array, dt, 1

    def test_numpy_array_2d(self):
        dtypes = copy.deepcopy(self.dtypes)
        dtypes.append('object')
        for dt in dtypes:
            yield self.check_numpy_array, dt, 2

    def test_numpy_array_3d(self):
        dtypes = copy.deepcopy(self.dtypes)
        dtypes.append('object')
        for dt in dtypes:
            yield self.check_numpy_array, dt, 3

    def test_numpy_empty(self):
        for dt in self.dtypes:
            yield self.check_numpy_empty, dt

    def test_numpy_structured_array(self):
        for i in range(1, 4):
            yield self.check_numpy_structured_array, i

    def test_numpy_structured_array_empty(self):
        for i in range(1, 4):
            yield self.check_numpy_structured_array_empty, i

    def test_numpy_structured_array_unicode_fields(self):
        # Makes a random 1d structured ndarray with non-ascii characters
        # in its fields, writes it and reads it back, and then compares
        # it.
        shape = random_numpy_shape(1, \
            max_structured_ndarray_axis_length)
        data = random_structured_numpy_array(shape,
                                             nonascii_fields=True)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_python_collection(self):
        for tp in (list, tuple, set, frozenset, collections.deque):
            yield self.check_python_collection, tp

    def test_dict(self):
        data = random_dict()
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

class TestPythonFormat(TestPythonMatlabFormat):
    def __init__(self):
        # The parent does most of the setup. All that has to be changed
        # is turning MATLAB compatibility off and changing the file
        # name.
        TestPythonMatlabFormat.__init__(self)
        self.options = hdf5storage.Options(matlab_compatible=False)
        self.filename = 'data.h5'

class TestNoneFormat(TestPythonMatlabFormat):
    def __init__(self):
        # The parent does most of the setup. All that has to be changed
        # is turning off the storage of type information as well as
        # MATLAB compatibility.
        TestPythonMatlabFormat.__init__(self)
        self.options = hdf5storage.Options(store_python_metadata=False,
                                           matlab_compatible=False)

        # Add in float16 to the set of types tested.
        self.dtypes.append('float16')

    def assert_equal(self, a, b):
        assert_equal_none_format(a, b)

class TestMatlabFormat(TestPythonMatlabFormat):
    def __init__(self):
        # The parent does most of the setup. All that has to be changed
        # is turning on the matlab compatibility, and changing the
        # filename.
        TestPythonMatlabFormat.__init__(self)
        self.options = hdf5storage.Options(store_python_metadata=False,
                                           matlab_compatible=True)
        self.filename = 'data.mat'

    def assert_equal(self, a, b):
        assert_equal_matlab_format(a, b)
<commit_msg>Fixed issue where a local variable having the same name as a global was confusting.<commit_after># Copyright (c) 2013-2015, Freja Nordsiek
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import sys
import copy
import os
import os.path
import math
import random
import collections

import numpy as np
import numpy.random

import hdf5storage

from nose.tools import raises

from asserts import *
from make_randoms import *

random.seed()

class TestPythonMatlabFormat(object):
    # Test for the ability to write python types to an HDF5 file that
    # type information and matlab information are stored in, and then
    # read it back and have it be the same.
    def __init__(self):
        self.filename = 'data.mat'
        self.options = hdf5storage.Options()

        # Need a list of the supported numeric dtypes to test, excluding
        # those not supported by MATLAB. 'S' and 'U' dtype chars have to
        # be used for the bare byte and unicode string dtypes since the
        # dtype strings (but not chars) are not the same in Python 2 and
        # 3.
        self.dtypes = ['bool', 'uint8', 'uint16', 'uint32', 'uint64',
                       'int8', 'int16', 'int32', 'int64',
                       'float32', 'float64', 'complex64', 'complex128',
                       'S', 'U']

    def write_readback(self, data, name, options):
        # Write the data to the proper file with the given name, read it
        # back, and return the result. The file needs to be deleted
        # before and after to keep junk from building up.
        if os.path.exists(self.filename):
            os.remove(self.filename)
        try:
            hdf5storage.write(data, path=name, filename=self.filename,
                              options=options)
            out = hdf5storage.read(path=name, filename=self.filename,
                                   options=options)
        except:
            raise
        finally:
            if os.path.exists(self.filename):
                os.remove(self.filename)
        return out

    def assert_equal(self, a, b):
        assert_equal(a, b)

    def check_numpy_scalar(self, dtype):
        # Makes a random numpy scalar of the given type, writes it and
        # reads it back, and then compares it.
        data = random_numpy_scalar(dtype)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def check_numpy_array(self, dtype, dimensions):
        # Makes a random numpy array of the given type, writes it and
        # reads it back, and then compares it.
        shape = random_numpy_shape(dimensions,
                                   max_array_axis_length)
        data = random_numpy(shape, dtype)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def check_numpy_empty(self, dtype):
        # Makes an empty numpy array of the given type, writes it and
        # reads it back, and then compares it.
        data = np.array([], dtype)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def check_numpy_structured_array(self, dimensions):
        # Makes a random structured ndarray of the given type, writes it
        # and reads it back, and then compares it.
        shape = random_numpy_shape(dimensions, \
            max_structured_ndarray_axis_length)
        data = random_structured_numpy_array(shape)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def check_numpy_structured_array_empty(self, dimensions):
        # Makes a random structured ndarray of the given type, writes it
        # and reads it back, and then compares it.
        shape = random_numpy_shape(dimensions, \
            max_structured_ndarray_axis_length)
        data = random_structured_numpy_array(shape, (1, 0))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def check_python_collection(self, tp):
        # Makes a random collection of the specified type, writes it and
        # reads it back, and then compares it.
        if tp in (set, frozenset):
            data = tp(random_list(max_list_length,
                      python_or_numpy='python'))
        else:
            data = tp(random_list(max_list_length,
                      python_or_numpy='numpy'))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_None(self):
        data = None
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bool_True(self):
        data = True
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bool_False(self):
        data = False
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_int(self):
        data = random_int()
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    # Only relevant in Python 2.x.
    def test_long(self):
        if sys.hexversion < 0x03000000:
            data = long(random_int())
            out = self.write_readback(data, random_name(),
                                      self.options)
            self.assert_equal(out, data)

    def test_int_or_long_too_big(self):
        if sys.hexversion >= 0x03000000:
            data = 2**64 * random_int()
        else:
            data = long(2)**64 * long(random_int())
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_float(self):
        data = random_float()
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_float_inf(self):
        data = float(np.inf)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_float_ninf(self):
        data = float(-np.inf)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_float_nan(self):
        data = float(np.nan)
        out = self.write_readback(data, random_name(),
                                  self.options)
        assert math.isnan(out)

    def test_complex(self):
        data = random_float() + 1j*random_float()
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_complex_real_nan(self):
        data = complex(np.nan, random_float())
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_complex_imaginary_nan(self):
        data = complex(random_float(), np.nan)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_str_ascii(self):
        data = random_str_ascii(random.randint(1,
                                max_string_length))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_str_ascii_encoded_utf8(self):
        data = random_str_some_unicode(random.randint(1, \
            max_string_length)).encode('UTF-8')
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_str_with_null(self):
        strs = [random_str_ascii(
                random.randint(1, max_string_length))
                for i in range(2)]
        if sys.hexversion < 0x03000000:
            data = u'\x00'.join(strs)
        else:
            data = '\x00'.join(strs)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_str_unicode(self):
        data = random_str_some_unicode(random.randint(1,
                                       max_string_length))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_str_empty(self):
        data = ''
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bytes(self):
        data = random_bytes(random.randint(1,
                            max_string_length))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bytes_empty(self):
        data = b''
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bytes_with_null(self):
        strs = [random_bytes(
                random.randint(1, max_string_length))
                for i in range(2)]
        if sys.hexversion < 0x03000000:
            data = '\x00'.join(strs)
        else:
            data = b'\x00'.join(strs)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bytearray(self):
        data = bytearray(random_bytes(random.randint(1,
                         max_string_length)))
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_bytearray_empty(self):
        data = bytearray(b'')
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_numpy_scalar(self):
        for dt in self.dtypes:
            yield self.check_numpy_scalar, dt

    def test_numpy_array_1d(self):
        dts = copy.deepcopy(self.dtypes)
        dts.append('object')
        for dt in dts:
            yield self.check_numpy_array, dt, 1

    def test_numpy_array_2d(self):
        dts = copy.deepcopy(self.dtypes)
        dts.append('object')
        for dt in dts:
            yield self.check_numpy_array, dt, 2

    def test_numpy_array_3d(self):
        dts = copy.deepcopy(self.dtypes)
        dts.append('object')
        for dt in dts:
            yield self.check_numpy_array, dt, 3

    def test_numpy_empty(self):
        for dt in self.dtypes:
            yield self.check_numpy_empty, dt

    def test_numpy_structured_array(self):
        for i in range(1, 4):
            yield self.check_numpy_structured_array, i

    def test_numpy_structured_array_empty(self):
        for i in range(1, 4):
            yield self.check_numpy_structured_array_empty, i

    def test_numpy_structured_array_unicode_fields(self):
        # Makes a random 1d structured ndarray with non-ascii characters
        # in its fields, writes it and reads it back, and then compares
        # it.
        shape = random_numpy_shape(1, \
            max_structured_ndarray_axis_length)
        data = random_structured_numpy_array(shape,
                                             nonascii_fields=True)
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

    def test_python_collection(self):
        for tp in (list, tuple, set, frozenset, collections.deque):
            yield self.check_python_collection, tp

    def test_dict(self):
        data = random_dict()
        out = self.write_readback(data, random_name(),
                                  self.options)
        self.assert_equal(out, data)

class TestPythonFormat(TestPythonMatlabFormat):
    def __init__(self):
        # The parent does most of the setup. All that has to be changed
        # is turning MATLAB compatibility off and changing the file
        # name.
        TestPythonMatlabFormat.__init__(self)
        self.options = hdf5storage.Options(matlab_compatible=False)
        self.filename = 'data.h5'

class TestNoneFormat(TestPythonMatlabFormat):
    def __init__(self):
        # The parent does most of the setup. All that has to be changed
        # is turning off the storage of type information as well as
        # MATLAB compatibility.
        TestPythonMatlabFormat.__init__(self)
        self.options = hdf5storage.Options(store_python_metadata=False,
                                           matlab_compatible=False)

        # Add in float16 to the set of types tested.
        self.dtypes.append('float16')

    def assert_equal(self, a, b):
        assert_equal_none_format(a, b)

class TestMatlabFormat(TestPythonMatlabFormat):
    def __init__(self):
        # The parent does most of the setup. All that has to be changed
        # is turning on the matlab compatibility, and changing the
        # filename.
        TestPythonMatlabFormat.__init__(self)
        self.options = hdf5storage.Options(store_python_metadata=False,
                                           matlab_compatible=True)
        self.filename = 'data.mat'

    def assert_equal(self, a, b):
        assert_equal_matlab_format(a, b)
<|endoftext|>


<commit_before>import pytest
import sqlalchemy as sa

from sqlalchemy_utils.types import timezone

@pytest.fixture
def Visitor(Base):
    class Visitor(Base):
        __tablename__ = 'visitor'
        id = sa.Column(sa.Integer, primary_key=True)
        timezone_dateutil = sa.Column(
            timezone.TimezoneType(backend='dateutil')
        )
        timezone_pytz = sa.Column(
            timezone.TimezoneType(backend='pytz')
        )

        def __repr__(self):
            return 'Visitor(%r)' % self.id
    return Visitor

@pytest.fixture
def init_models(Visitor):
    pass

class TestTimezoneType(object):

    def test_parameter_processing(self, session, Visitor):
        visitor = Visitor(
            timezone_dateutil=u'America/Los_Angeles',
            timezone_pytz=u'America/Los_Angeles'
        )

        session.add(visitor)
        session.commit()

        visitor_dateutil = session.query(Visitor).filter_by(
            timezone_dateutil=u'America/Los_Angeles'
        ).first()
        visitor_pytz = session.query(Visitor).filter_by(
            timezone_pytz=u'America/Los_Angeles'
        ).first()

        assert visitor_dateutil is not None
        assert visitor_pytz is not None
<commit_msg>add tests to coerce StaticTzInfo and DstTzInfo<commit_after>import pytest
import pytz
import sqlalchemy as sa
from dateutil.zoneinfo import getzoneinfofile_stream, tzfile, ZoneInfoFile

from sqlalchemy_utils.types import timezone, TimezoneType

@pytest.fixture
def Visitor(Base):
    class Visitor(Base):
        __tablename__ = 'visitor'
        id = sa.Column(sa.Integer, primary_key=True)
        timezone_dateutil = sa.Column(
            timezone.TimezoneType(backend='dateutil')
        )
        timezone_pytz = sa.Column(
            timezone.TimezoneType(backend='pytz')
        )

        def __repr__(self):
            return 'Visitor(%r)' % self.id
    return Visitor

@pytest.fixture
def init_models(Visitor):
    pass

class TestTimezoneType(object):

    def test_parameter_processing(self, session, Visitor):
        visitor = Visitor(
            timezone_dateutil=u'America/Los_Angeles',
            timezone_pytz=u'America/Los_Angeles'
        )

        session.add(visitor)
        session.commit()

        visitor_dateutil = session.query(Visitor).filter_by(
            timezone_dateutil=u'America/Los_Angeles'
        ).first()
        visitor_pytz = session.query(Visitor).filter_by(
            timezone_pytz=u'America/Los_Angeles'
        ).first()

        assert visitor_dateutil is not None
        assert visitor_pytz is not None

TIMEZONE_BACKENDS = ['dateutil', 'pytz']

def test_can_coerce_pytz_DstTzInfo():
    tzcol = TimezoneType(backend='pytz')
    tz = pytz.timezone('America/New_York')
    assert isinstance(tz, pytz.tzfile.DstTzInfo)
    assert tzcol._coerce(tz) is tz

def test_can_coerce_pytz_StaticTzInfo():
    tzcol = TimezoneType(backend='pytz')
    tz = pytz.timezone('Pacific/Truk')
    assert isinstance(tz, pytz.tzfile.StaticTzInfo)
    assert tzcol._coerce(tz) is tz

@pytest.mark.parametrize('zone', pytz.all_timezones)
def test_can_coerce_string_for_pytz_zone(zone):
    tzcol = TimezoneType(backend='pytz')
    assert tzcol._coerce(zone).zone == zone

@pytest.mark.parametrize(
    'zone', ZoneInfoFile(getzoneinfofile_stream()).zones.keys())
def test_can_coerce_string_for_dateutil_zone(zone):
    tzcol = TimezoneType(backend='dateutil')
    assert isinstance(tzcol._coerce(zone), tzfile)

@pytest.mark.parametrize('backend', TIMEZONE_BACKENDS)
def test_can_coerce_and_raise_UnknownTimeZoneError_or_ValueError(backend):
    tzcol = TimezoneType(backend=backend)
    with pytest.raises((ValueError, pytz.exceptions.UnknownTimeZoneError)):
        tzcol._coerce('SolarSystem/Mars')
    with pytest.raises((ValueError, pytz.exceptions.UnknownTimeZoneError)):
        tzcol._coerce('')

@pytest.mark.parametrize('backend', TIMEZONE_BACKENDS)
def test_can_coerce_None(backend):
    tzcol = TimezoneType(backend=backend)
    assert tzcol._coerce(None) is None
<|endoftext|>


<commit_before>import re
import functools
import operator

from modularodm import Q
from rest_framework.filters import OrderingFilter
from rest_framework import serializers as ser

from api.base.exceptions import InvalidFilterError
from api.base import utils

class ODMOrderingFilter(OrderingFilter):
    """Adaptation of rest_framework.filters.OrderingFilter to work with modular-odm."""

    # override
    def filter_queryset(self, request, queryset, view):
        ordering = self.get_ordering(request, queryset, view)
        if ordering:
            return queryset.sort(*ordering)
        return queryset

query_pattern = re.compile(r'filter\[\s*(?P<field>\S*)\s*\]\s*')

def query_params_to_fields(query_params):
    return {
        query_pattern.match(key).groupdict()['field']: value
        for key, value in query_params.items()
        if query_pattern.match(key)
    }

class FilterMixin(object):
    """ View mixin with helper functions for filtering. """

    DEFAULT_OPERATOR = 'eq'

    def __init__(self, *args, **kwargs):
        super(FilterMixin, self).__init__(*args, **kwargs)
        if not self.serializer_class:
            raise NotImplementedError()

    def is_filterable_field(self, key):
        try:
            return key.strip() in self.serializer_class.filterable_fields
        except AttributeError:
            return key.strip() in self.serializer_class._declared_fields

    # Used so that that queries by _id will work
    def convert_key(self, key):
        key = key.strip()
        if self.serializer_class._declared_fields[key].source:
            return self.serializer_class._declared_fields[key].source
        return key

    # Used to convert string values from query params to Python booleans when necessary
    def convert_value(self, value, field):
        field_type = type(self.serializer_class._declared_fields[field])
        value = value.strip()
        if field_type == ser.BooleanField:
            if utils.is_truthy(value):
                return True
            elif utils.is_falsy(value):
                return False
            # TODO Should we handle if the value is neither TRUTHY nor FALSY (first add test for how we'd expect it to
            # work, then ensure that it works that way).
        else:
            return value

class ODMFilterMixin(FilterMixin):
    """View mixin that adds a get_query_from_request method which converts query params
    of the form `filter[field_name]=value` into an ODM Query object.

    Subclasses must define `get_default_odm_query()`.

    Serializers that want to restrict which fields are used for filtering need to have a variable called
    filterable_fields which is a frozenset of strings representing the field names as they appear in the serialization.
    """

    # TODO Handle simple and complex non-standard fields

    field_comparison_operators = {
        ser.CharField: 'icontains',
        ser.ListField: 'contains',
    }

    def __init__(self, *args, **kwargs):
        super(FilterMixin, self).__init__(*args, **kwargs)
        if not self.serializer_class:
            raise NotImplementedError()

    def get_comparison_operator(self, key):
        field_type = type(self.serializer_class._declared_fields[key])
        if field_type in self.field_comparison_operators:
            return self.field_comparison_operators[field_type]
        else:
            return self.DEFAULT_OPERATOR

    def get_default_odm_query(self):
        """Return the default MODM query for the result set.

        NOTE: If the client provides additional filters in query params, the filters
        will intersected with this query.
        """
        raise NotImplementedError('Must define get_default_odm_query')

    def get_query_from_request(self):
        param_query = self.query_params_to_odm_query(self.request.QUERY_PARAMS)
        default_query = self.get_default_odm_query()

        if param_query:
            query = param_query & default_query
        else:
            query = default_query

        return query

    def query_params_to_odm_query(self, query_params):
        """Convert query params to a modularodm Query object."""

        fields_dict = query_params_to_fields(query_params)
        if fields_dict:
            query_parts = []
            for key, value in fields_dict.items():
                if self.is_filterable_field(key=key):
                    query = Q(self.convert_key(key=key), self.get_comparison_operator(key=key), self.convert_value(value=value, field=key))
                    query_parts.append(query)
                else:
                    raise InvalidFilterError
            try:
                query = functools.reduce(operator.and_, query_parts)
            except TypeError:
                query = None
        else:
            query = None
        return query

class ListFilterMixin(FilterMixin):
    """View mixin that adds a get_queryset_from_request method which uses query params
    of the form `filter[field_name]=value` to filter a list of objects.

    Subclasses must define `get_default_queryset()`.

    Serializers that want to restrict which fields are used for filtering need to have a variable called
    filterable_fields which is a frozenset of strings representing the field names as they appear in the serialization.
    """

    def __init__(self, *args, **kwargs):
        super(FilterMixin, self).__init__(*args, **kwargs)
        if not self.serializer_class:
            raise NotImplementedError()

    def get_default_queryset(self):
        raise NotImplementedError('Must define get_default_queryset')

    def get_queryset_from_request(self):
        default_queryset = self.get_default_queryset()
        if self.request.QUERY_PARAMS:
            param_queryset = self.param_queryset(self.request.QUERY_PARAMS, default_queryset)
            return param_queryset
        else:
            return default_queryset

    def param_queryset(self, query_params, default_queryset):
        """filters default queryset based on query parameters"""
        fields_dict = query_params_to_fields(query_params)
        queryset = set(default_queryset)
        if fields_dict:
            for field_name, value in fields_dict.items():
                if self.is_filterable_field(key=field_name):
                    queryset = queryset.intersection(set(self.get_filtered_queryset(field_name, value, default_queryset)))
                else:
                    raise InvalidFilterError
        return list(queryset)

    def get_filtered_queryset(self, field_name, value, default_queryset):
        """filters default queryset based on the serializer field type"""
        field = self.serializer_class._declared_fields[field_name]
        field_name = field.source or field_name

        if isinstance(field, ser.SerializerMethodField):
            return_val = [item for item in default_queryset if self.get_serializer_method(field_name)(item) == self.convert_value(value, field_name)]
        elif isinstance(field, ser.BooleanField):
            return_val = [item for item in default_queryset if getattr(item, field_name, None) == self.convert_value(value, field_name)]
        elif isinstance(field, ser.CharField):
            return_val = [item for item in default_queryset if value.lower() in getattr(item, field_name, None).lower()]
        else:
            # TODO Ensure that if you try to filter on an invalid field, it returns a useful error.
            return_val = [item for item in default_queryset if value in getattr(item, field_name, None)]

        return return_val

    def get_serializer_method(self, field_name):
        """
        :param field_name: The name of a SerializerMethodField
        :return: The function attached to the SerializerMethodField to get its value
        """
        serializer = self.get_serializer()
        serializer_method_name = 'get_' + field_name
        return getattr(serializer, serializer_method_name)
<commit_msg>Fix ListFilterMixin for serializer & boolean fields<commit_after>import re
import functools
import operator

from modularodm import Q
from rest_framework.filters import OrderingFilter
from rest_framework import serializers as ser

from api.base.exceptions import InvalidFilterError
from api.base import utils

class ODMOrderingFilter(OrderingFilter):
    """Adaptation of rest_framework.filters.OrderingFilter to work with modular-odm."""

    # override
    def filter_queryset(self, request, queryset, view):
        ordering = self.get_ordering(request, queryset, view)
        if ordering:
            return queryset.sort(*ordering)
        return queryset

query_pattern = re.compile(r'filter\[\s*(?P<field>\S*)\s*\]\s*')

def query_params_to_fields(query_params):
    return {
        query_pattern.match(key).groupdict()['field']: value
        for key, value in query_params.items()
        if query_pattern.match(key)
    }

class FilterMixin(object):
    """ View mixin with helper functions for filtering. """

    DEFAULT_OPERATOR = 'eq'

    def __init__(self, *args, **kwargs):
        super(FilterMixin, self).__init__(*args, **kwargs)
        if not self.serializer_class:
            raise NotImplementedError()

    def is_filterable_field(self, key):
        try:
            return key.strip() in self.serializer_class.filterable_fields
        except AttributeError:
            return key.strip() in self.serializer_class._declared_fields

    # Used so that that queries by _id will work
    def convert_key(self, key):
        key = key.strip()
        if self.serializer_class._declared_fields[key].source:
            return self.serializer_class._declared_fields[key].source
        return key

    # Used to convert string values from query params to Python booleans when necessary
    def convert_value(self, value, field):
        field_type = type(self.serializer_class._declared_fields[field])
        value = value.strip()
        if field_type == ser.BooleanField:
            if utils.is_truthy(value):
                return True
            elif utils.is_falsy(value):
                return False
            # TODO Should we handle if the value is neither TRUTHY nor FALSY (first add test for how we'd expect it to
            # work, then ensure that it works that way).
        else:
            return value

class ODMFilterMixin(FilterMixin):
    """View mixin that adds a get_query_from_request method which converts query params
    of the form `filter[field_name]=value` into an ODM Query object.

    Subclasses must define `get_default_odm_query()`.

    Serializers that want to restrict which fields are used for filtering need to have a variable called
    filterable_fields which is a frozenset of strings representing the field names as they appear in the serialization.
    """

    # TODO Handle simple and complex non-standard fields

    field_comparison_operators = {
        ser.CharField: 'icontains',
        ser.ListField: 'contains',
    }

    def __init__(self, *args, **kwargs):
        super(FilterMixin, self).__init__(*args, **kwargs)
        if not self.serializer_class:
            raise NotImplementedError()

    def get_comparison_operator(self, key):
        field_type = type(self.serializer_class._declared_fields[key])
        if field_type in self.field_comparison_operators:
            return self.field_comparison_operators[field_type]
        else:
            return self.DEFAULT_OPERATOR

    def get_default_odm_query(self):
        """Return the default MODM query for the result set.

        NOTE: If the client provides additional filters in query params, the filters
        will intersected with this query.
        """
        raise NotImplementedError('Must define get_default_odm_query')

    def get_query_from_request(self):
        param_query = self.query_params_to_odm_query(self.request.QUERY_PARAMS)
        default_query = self.get_default_odm_query()

        if param_query:
            query = param_query & default_query
        else:
            query = default_query

        return query

    def query_params_to_odm_query(self, query_params):
        """Convert query params to a modularodm Query object."""

        fields_dict = query_params_to_fields(query_params)
        if fields_dict:
            query_parts = []
            for key, value in fields_dict.items():
                if self.is_filterable_field(key=key):
                    query = Q(self.convert_key(key=key), self.get_comparison_operator(key=key), self.convert_value(value=value, field=key))
                    query_parts.append(query)
                else:
                    raise InvalidFilterError
            try:
                query = functools.reduce(operator.and_, query_parts)
            except TypeError:
                query = None
        else:
            query = None
        return query

class ListFilterMixin(FilterMixin):
    """View mixin that adds a get_queryset_from_request method which uses query params
    of the form `filter[field_name]=value` to filter a list of objects.

    Subclasses must define `get_default_queryset()`.

    Serializers that want to restrict which fields are used for filtering need to have a variable called
    filterable_fields which is a frozenset of strings representing the field names as they appear in the serialization.
    """

    def __init__(self, *args, **kwargs):
        super(FilterMixin, self).__init__(*args, **kwargs)
        if not self.serializer_class:
            raise NotImplementedError()

    def get_default_queryset(self):
        raise NotImplementedError('Must define get_default_queryset')

    def get_queryset_from_request(self):
        default_queryset = self.get_default_queryset()
        if self.request.QUERY_PARAMS:
            param_queryset = self.param_queryset(self.request.QUERY_PARAMS, default_queryset)
            return param_queryset
        else:
            return default_queryset

    def param_queryset(self, query_params, default_queryset):
        """filters default queryset based on query parameters"""
        fields_dict = query_params_to_fields(query_params)
        queryset = set(default_queryset)
        if fields_dict:
            for field_name, value in fields_dict.items():
                if self.is_filterable_field(key=field_name):
                    queryset = queryset.intersection(set(self.get_filtered_queryset(field_name, value, default_queryset)))
                else:
                    raise InvalidFilterError
        return list(queryset)

    def get_filtered_queryset(self, field_name, value, default_queryset):
        """filters default queryset based on the serializer field type"""
        field = self.serializer_class._declared_fields[field_name]

        if isinstance(field, ser.SerializerMethodField):
            return_val = [item for item in default_queryset if self.get_serializer_method(field_name)(item) == self.convert_value(value, field_name)]
        elif isinstance(field, ser.BooleanField):
            return_val = [item for item in default_queryset if getattr(item, field.source, None) == self.convert_value(value, field_name)]
        elif isinstance(field, ser.CharField):
            return_val = [item for item in default_queryset if value.lower() in getattr(item, field.source, None).lower()]
        else:
            # TODO Ensure that if you try to filter on an invalid field, it returns a useful error.
            return_val = [item for item in default_queryset if value in getattr(item, field.source, None)]

        return return_val

    def get_serializer_method(self, field_name):
        """
        :param field_name: The name of a SerializerMethodField
        :return: The function attached to the SerializerMethodField to get its value
        """
        serializer = self.get_serializer()
        serializer_method_name = 'get_' + field_name
        return getattr(serializer, serializer_method_name)
<|endoftext|>


<commit_before>import re
import functools

from modularodm import Q
from rest_framework.filters import OrderingFilter
from rest_framework import serializers as ser

class ODMOrderingFilter(OrderingFilter):
    """Adaptation of rest_framework.filters.OrderingFilter to work with modular-odm."""

    # override
    def filter_queryset(self, request, queryset, view):
        ordering = self.get_ordering(request, queryset, view)
        if ordering:
            return queryset.sort(*ordering)
        return queryset

query_pattern = re.compile(r'filter\[\s*(?P<field>\S*)\s*\]\s*')

def query_params_to_fields(query_params):
    return {
        query_pattern.match(key).groupdict()['field']: value
        for key, value in query_params.items()
        if query_pattern.match(key)
    }

# Used to make intersection "reduce-able"
def intersect(x, y):
    return x & y

class FilterMixin(object):
    """ View mixin with helper functions for filtering. """

    TRUTHY = set(['true', 'True', 1, '1'])
    FALSY = set(['false', 'False', 0, '0'])
    DEFAULT_OPERATOR = 'eq'

    def is_filterable_field(self, key):
        try:
            return key.strip() in self.serializer_class.filterable_fields
        except AttributeError:
            return key.strip() in self.serializer_class._declared_fields

    # Used so that that queries by _id will work
    def convert_key(self, key):
        key = key.strip()
        if self.serializer_class._declared_fields[key].source:
            return self.serializer_class._declared_fields[key].source
        return key

    # Used to convert string values from query params to Python booleans when necessary
    def convert_value(self, value):
        value = value.strip()
        if value in self.TRUTHY:
            return True
        elif value in self.FALSY:
            return False
        # Convert me to current user's pk
        elif value == 'me' and not self.request.user.is_anonymous():
            return self.request.user.pk
        else:
            return value

class ODMFilterMixin(FilterMixin):
    """View mixin that adds a get_query_from_request method which converts query params
    of the form `filter[field_name]=value` into an ODM Query object.

    Subclasses must define `get_default_odm_query()`.

    Serializers that want to restrict which fields are used for filtering need to have a variable called
    filterable_fields which is a frozenset of strings representing the field names as they appear in the serialization.
    """

    # TODO Handle simple and complex non-standard fields

    # For the field_comparison_operators, instances can be a class or a tuple of classes
    field_comparison_operators = {
        ser.CharField: 'icontains',
        ser.ListField: 'in',
        }

    def get_comparison_operator(self, key):
        for field_type, comparison_operator in self.field_comparison_operators.items():
            if isinstance(self.serializer_class._declared_fields[key], field_type):
                return comparison_operator

        return self.DEFAULT_OPERATOR

    def get_default_odm_query(self):
        raise NotImplementedError('Must define get_default_odm_query')

    def get_query_from_request(self):
        param_query = self.query_params_to_odm_query(self.request.QUERY_PARAMS)
        default_query = self.get_default_odm_query()

        if param_query:
            query = param_query & default_query
        else:
            query = default_query

        return query

    def query_params_to_odm_query(self, query_params):
        """Convert query params to a modularodm Query object."""

        fields_dict = query_params_to_fields(query_params)
        if fields_dict:
            query_parts = [
                Q(self.convert_key(key=key), self.get_comparison_operator(key=key), self.convert_value(value=value))
                for key, value in fields_dict.items() if self.is_filterable_field(key=key)
            ]
            # TODO Ensure that if you try to filter on an invalid field, it returns a useful error. Fix related test.
            try:
                query = functools.reduce(intersect, query_parts)
            except TypeError:
                query = None
        else:
            query = None
        return query

class ListFilterMixin(FilterMixin):
    """View mixin that adds a get_queryset_from_request method which uses query params
    of the form `filter[field_name]=value` to filter a list of objects.

    Subclasses must define `get_default_queryset()`.

    Serializers that want to restrict which fields are used for filtering need to have a variable called
    filterable_fields which is a frozenset of strings representing the field names as they appear in the serialization.
    """

    def get_default_queryset(self):
        raise NotImplementedError('Must define get_default_queryset')

    def get_queryset_from_request(self):
        default_queryset = self.get_default_queryset()
        if self.request.QUERY_PARAMS:
            param_queryset = self.param_queryset(self.request.QUERY_PARAMS, default_queryset)
            return param_queryset
        else:
            return default_queryset

    def param_queryset(self, query_params, default_queryset):
        """filters default queryset based on query parameters"""
        fields_dict = query_params_to_fields(query_params)
        return_val = default_queryset
        if fields_dict:
            for field_name, value in fields_dict.items():
                if self.is_filterable_field(key=field_name):
                    if isinstance(self.serializer_class._declared_fields[field_name], ser.SerializerMethodField):
                        return_val = self.get_serializer_field_value(field_name, value, default_queryset)
                    elif isinstance(self.serializer_class._declared_fields[field_name], ser.BooleanField):
                        return_val = [item for item in default_queryset if item.get('key', None) == value]
                    else:
                        # TODO Ensure that if you try to filter on an invalid field, it returns a useful error.
                        return_val = [item for item in default_queryset if value in item.get('key', None)]
        return return_val

    def get_serializer_field_value(self, key, value, default_queryset):
        serializer = self.get_serializer()
        serializer_function = {
            'bibliographic': serializer.get_bibliographic
        }
        return [item for item in default_queryset if serializer_function[key](item) == self.convert_value(value)]
<commit_msg>Move logic for getting queryset based on field type<commit_after>import re
import functools

from modularodm import Q
from rest_framework.filters import OrderingFilter
from rest_framework import serializers as ser

class ODMOrderingFilter(OrderingFilter):
    """Adaptation of rest_framework.filters.OrderingFilter to work with modular-odm."""

    # override
    def filter_queryset(self, request, queryset, view):
        ordering = self.get_ordering(request, queryset, view)
        if ordering:
            return queryset.sort(*ordering)
        return queryset

query_pattern = re.compile(r'filter\[\s*(?P<field>\S*)\s*\]\s*')

def query_params_to_fields(query_params):
    return {
        query_pattern.match(key).groupdict()['field']: value
        for key, value in query_params.items()
        if query_pattern.match(key)
    }

# Used to make intersection "reduce-able"
def intersect(x, y):
    return x & y

class FilterMixin(object):
    """ View mixin with helper functions for filtering. """

    TRUTHY = set(['true', 'True', 1, '1'])
    FALSY = set(['false', 'False', 0, '0'])
    DEFAULT_OPERATOR = 'eq'

    def is_filterable_field(self, key):
        try:
            return key.strip() in self.serializer_class.filterable_fields
        except AttributeError:
            return key.strip() in self.serializer_class._declared_fields

    # Used so that that queries by _id will work
    def convert_key(self, key):
        key = key.strip()
        if self.serializer_class._declared_fields[key].source:
            return self.serializer_class._declared_fields[key].source
        return key

    # Used to convert string values from query params to Python booleans when necessary
    def convert_value(self, value):
        value = value.strip()
        if value in self.TRUTHY:
            return True
        elif value in self.FALSY:
            return False
        # Convert me to current user's pk
        elif value == 'me' and not self.request.user.is_anonymous():
            return self.request.user.pk
        else:
            return value

class ODMFilterMixin(FilterMixin):
    """View mixin that adds a get_query_from_request method which converts query params
    of the form `filter[field_name]=value` into an ODM Query object.

    Subclasses must define `get_default_odm_query()`.

    Serializers that want to restrict which fields are used for filtering need to have a variable called
    filterable_fields which is a frozenset of strings representing the field names as they appear in the serialization.
    """

    # TODO Handle simple and complex non-standard fields

    # For the field_comparison_operators, instances can be a class or a tuple of classes
    field_comparison_operators = {
        ser.CharField: 'icontains',
        ser.ListField: 'in',
        }

    def get_comparison_operator(self, key):
        for field_type, comparison_operator in self.field_comparison_operators.items():
            if isinstance(self.serializer_class._declared_fields[key], field_type):
                return comparison_operator

        return self.DEFAULT_OPERATOR

    def get_default_odm_query(self):
        raise NotImplementedError('Must define get_default_odm_query')

    def get_query_from_request(self):
        param_query = self.query_params_to_odm_query(self.request.QUERY_PARAMS)
        default_query = self.get_default_odm_query()

        if param_query:
            query = param_query & default_query
        else:
            query = default_query

        return query

    def query_params_to_odm_query(self, query_params):
        """Convert query params to a modularodm Query object."""

        fields_dict = query_params_to_fields(query_params)
        if fields_dict:
            query_parts = [
                Q(self.convert_key(key=key), self.get_comparison_operator(key=key), self.convert_value(value=value))
                for key, value in fields_dict.items() if self.is_filterable_field(key=key)
            ]
            # TODO Ensure that if you try to filter on an invalid field, it returns a useful error. Fix related test.
            try:
                query = functools.reduce(intersect, query_parts)
            except TypeError:
                query = None
        else:
            query = None
        return query

class ListFilterMixin(FilterMixin):
    """View mixin that adds a get_queryset_from_request method which uses query params
    of the form `filter[field_name]=value` to filter a list of objects.

    Subclasses must define `get_default_queryset()`.

    Serializers that want to restrict which fields are used for filtering need to have a variable called
    filterable_fields which is a frozenset of strings representing the field names as they appear in the serialization.
    """

    def get_default_queryset(self):
        raise NotImplementedError('Must define get_default_queryset')

    def get_queryset_from_request(self):
        default_queryset = self.get_default_queryset()
        if self.request.QUERY_PARAMS:
            param_queryset = self.param_queryset(self.request.QUERY_PARAMS, default_queryset)
            return param_queryset
        else:
            return default_queryset

    def param_queryset(self, query_params, default_queryset):
        """filters default queryset based on query parameters"""
        fields_dict = query_params_to_fields(query_params)
        return_val = default_queryset
        if fields_dict:
            for field_name, value in fields_dict.items():
                if self.is_filterable_field(key=field_name):
                    return_val = self.get_serializer_field_value(field_name, value, default_queryset)
        return return_val

    def get_serializer_field_value(self, field_name, value, default_queryset):
        serializer = self.get_serializer()
        field = self.serializer_class._declared_fields[field_name]
        serializer_function = {
            'bibliographic': serializer.get_bibliographic
        }

        if isinstance(field, ser.SerializerMethodField):
            return_val = [item for item in default_queryset if serializer_function[field_name](item) == self.convert_value(value)]
        elif isinstance(field, ser.BooleanField):
            return_val = [item for item in default_queryset if item.get(field_name, None) == value]
        else:
            # TODO Ensure that if you try to filter on an invalid field, it returns a useful error.
            return_val = [item for item in default_queryset if value in item.get(field_name, None)]

        return return_val<|endoftext|>


<commit_before>from rest_framework.parsers import JSONParser
from rest_framework.exceptions import ParseError

from api.base.renderers import JSONAPIRenderer
from api.base.exceptions import JSONAPIException

NO_ATTRIBUTES_ERROR = 'Request must include /data/attributes.'
NO_DATA_ERROR = 'Request must include /data.'

class JSONAPIParser(JSONParser):
    """
    Parses JSON-serialized data. Overrides media_type.
    """
    media_type = 'application/vnd.api+json'
    renderer_class = JSONAPIRenderer

    def data_flattener(self, resource_object, stream):
        """
        Flattens data objects, making attributes fields the same level as id and type.
        """
        if "attributes" not in resource_object and stream.method != 'DELETE':
            raise JSONAPIException(source={'pointer': '/data/attributes'}, detail=NO_ATTRIBUTES_ERROR)
        id = resource_object.get('id')
        object_type = resource_object.get('type')
        attributes = resource_object.get('attributes')

        parsed = {'id': id, 'type': object_type}
        if attributes:
            parsed.update(attributes)

        return parsed

    # Overrides JSONParser
    def parse(self, stream, media_type=None, parser_context=None):
        """
        Parses the incoming bytestream as JSON and returns the resulting data.
        """
        result = super(JSONAPIParser, self).parse(stream, media_type=media_type, parser_context=parser_context)
        if not isinstance(result, dict):
            raise ParseError()
        data = result.get('data', {})

        if data:
            if isinstance(data, list):
                data_collection = []
                for data_object in data:
                    parsed_data = self.data_flattener(data_object, stream)
                    data_collection.append(parsed_data)
                return data_collection

            else:
                return self.data_flattener(data, stream)

        else:
            raise JSONAPIException(source={'pointer': '/data'}, detail=NO_DATA_ERROR)

class JSONAPIParserForRegularJSON(JSONAPIParser):
    """
    Allows same processing as JSONAPIParser to occur for requests with application/json media type.
    """
    media_type = 'application/json'
<commit_msg>Add space<commit_after>from rest_framework.parsers import JSONParser
from rest_framework.exceptions import ParseError

from api.base.renderers import JSONAPIRenderer
from api.base.exceptions import JSONAPIException

NO_ATTRIBUTES_ERROR = 'Request must include /data/attributes.'
NO_DATA_ERROR = 'Request must include /data.'

class JSONAPIParser(JSONParser):
    """
    Parses JSON-serialized data. Overrides media_type.
    """
    media_type = 'application/vnd.api+json'
    renderer_class = JSONAPIRenderer

    def data_flattener(self, resource_object, stream):
        """
        Flattens data objects, making attributes fields the same level as id and type.
        """
        if "attributes" not in resource_object and stream.method != 'DELETE':
            raise JSONAPIException(source={'pointer': '/data/attributes'}, detail=NO_ATTRIBUTES_ERROR)
        id = resource_object.get('id')
        object_type = resource_object.get('type')
        attributes = resource_object.get('attributes')

        parsed = {'id': id, 'type': object_type}
        if attributes:
            parsed.update(attributes)

        return parsed

    # Overrides JSONParser
    def parse(self, stream, media_type=None, parser_context=None):
        """
        Parses the incoming bytestream as JSON and returns the resulting data.
        """
        result = super(JSONAPIParser, self).parse(stream, media_type=media_type, parser_context=parser_context)
        if not isinstance(result, dict):
            raise ParseError()
        data = result.get('data', {})

        if data:
            if isinstance(data, list):
                data_collection = []
                for data_object in data:
                    parsed_data = self.data_flattener(data_object, stream)
                    data_collection.append(parsed_data)
                return data_collection

            else:
                return self.data_flattener(data, stream)

        else:
            raise JSONAPIException(source={'pointer': '/data'}, detail=NO_DATA_ERROR)

class JSONAPIParserForRegularJSON(JSONAPIParser):
    """
    Allows same processing as JSONAPIParser to occur for requests with application/json media type.
    """
    media_type = 'application/json'
<|endoftext|>