<commit_before>from . import constants
from . import helpers

class StrikeKit(object):
    """
    File/Kit Header
    52 bytes
    ------------------
    03 byte 0x4b4954        - Begin KIT header
    02 byte 0x202c          - 32, 44
    04 byte 0x00            - Padding?
    02 byte 0x0063          - 00, 99
    04 byte                 - Flag data?
    04 byte                 - kit settings - reverb type, size, color, level
    33 byte Kit data?       - Appears to be kit-specific settings, needs decoding.
    """
    def __init__(self, *args, **kwargs):
        self._kit_settings = None
        self._instruments = None
        self._samples = None

        raw_data = kwargs.get("raw_data") or args[0]
        # parse raw data if it's there
        if raw_data:
            # Yea assignment by side effect
            self._parse_raw_kit(raw_data)

    def __str__(self):
        return "\n".join(map(str, self.instruments))

    def csv(self):
        return "\n".join(map(str, map(StrikeInstrument.csv, self.instruments)))

    @property
    def kit_settings(self):
        return self._kit_settings

    @property
    def instruments(self):
        return self._instruments

    @property
    def samples(self):
        return self._samples

    def _parse_raw_kit(self, data):
        raw_header = data[0:constants.KIT_HEADER_SIZE]
        inst_begin = constants.KIT_HEADER_SIZE
        inst_end = constants.KIT_HEADER_SIZE + (constants.INSTRUMENT_COUNT * constants.INSTRUMENT_SIZE)
        raw_instruments = data[inst_begin:inst_end]
        raw_sampledata = data[inst_end:]

        # Samples have to be parsed before instruments so we know who uses what.
        self._parse_samples(raw_sampledata)
        self._parse_instruments(raw_instruments)

    def _parse_samples(self, data):
        self._samples = StrikeSamples(data)

    def _parse_instruments(self, data):
        result = []

        for x in range(0, constants.INSTRUMENT_COUNT):
            start_index = x * constants.INSTRUMENT_SIZE
            end_index = start_index + constants.INSTRUMENT_SIZE
            raw_instrument = data[start_index:end_index]
            instrument = StrikeInstrument(raw_instrument, samples=self.samples)
            result.append(instrument)
        self._instruments = result

class StrikeInstrument(object):
    """
        Instrument section
        80 bytes

        offset      12 (that's weird) byte header
                    ------------------
        0           4 byte 0x696e7374       - Begin "inst" header
        4           1 byte 0x48             - "H" ???
        5           3 byte 0x00             - Padding
        8           3 byte trigger spec     - (K|S|T|H|C|R)([1-4])(H|R|F|B|E|D)
                                                Kick/Snare/Tom/Hat/Crash/Ride
                                                1-4 for Toms, 1-3 Crash, 1 everything else
                                                Head/Rim for pads
                                                Foot/Edge/Bow for Hat
                                                Edge/Bow for Crashes
                                                D/B/E for Rides, not sure which is bow and bell
        11          1 byte                  - 0x20 (SPC) terminator ?     FF if not set

        -- LAYER A DATA
        12          1 byte sample ref       - offset of sample as listed in sample section? FF if not set
        13          1 byte                  - 0x00 padding?
        14          1 byte level            - 0x00 to 0x63
        15          1 byte pan              - this is fun. Values > 128 are panned left. 255 - pan. Values < 128 are panned right.
        16          1 byte decay            - 0x00 to 0x63
        17          2 byte pad?             - 0x002
        19          1 byte Tune             - 0-12 = +, 243-255 = -
        20          1 byte fine             - 0-127 positive, 255-* negative
        21          1 byte cutoff           - 0-127 positive, 255-* negative
        22          1 byte vel filtertype   - 0 lo, 1 hi
        23          1 byte vel decay        -
        24          1 byte vel pitch        -
        25          1 byte vel filter       -
        26          1 byte vel level        -
        27          1 byte 0 pad
        28          1 byte 7f terminator?
        29          3 byte 0 pad
        -- LAYER B DATA
        32          1 byte sample ref       - offset of sample as listed in sample section? FF if not set
        33          1 byte                  - 0x00 padding?
        34          1 byte level            - 0x00 to 0x63
        34          1 byte pan              - this is fun. Values > 128 are panned left. 255 - pan. Values < 128 are panned right.
        35          1 byte decay            - 0x00 to 0x63
        36          2 byte pad?             - 0x00
        38          1 byte Tune             - 0-12 = +, 243-255 = -
        39          1 byte fine             - 0-127 positive, 255-* negative
        40          1 byte cutoff           - 0-127 positive, 255-* negative
        41          1 byte vel filtertype   - 0 lo, 1 hi
        42          1 byte vel decay        -
        43          1 byte vel pitch        -
        44          1 byte vel filter       -
        45          1 byte vel level        -
        46          1 byte 0 pad
        47          1 byte 7f terminator?
        48          3 byte 0 pad

        -- VOICE DATA (MIDI, sends, etc)
        51          1 byte reverb send
        52          1 byte FX send
        53          2 byte 0 pad?
        55          1 byte (Note off or Priority? need to investigate)
        56          1 byte mute group
        57          1 byte playback?
        58          1 byte MIDI chan
        59          1 byte MIDI note
        60          1 byte gate time
        61          1 byte note off?
        62          1 byte 0 pad?
        63          5 byte FF terminator
        68         11 byte zero pad?
        """

    #
    def __init__(self, raw_data=None, samples=[]):
        if raw_data:
            self._parse(raw_data, samples)

    def __str__(self):
        return """{0}
        Layer A: {1}
        Layer B: {2}
        """.format(self.trigger_spec, self.layer_a, self.layer_b)

    def csv(self):
        return ",".join([self.trigger_spec.csv(), self.layer_a.csv(), self.layer_b.csv()])

    @property
    def trigger_spec(self):
        return self._trigger_spec

    @property
    def layer_a(self):
        return self._layer_a

    @property
    def layer_b(self):
        return self._layer_b

    def _parse(self, data, samples=[]):
        raw_header = data[0:constants.INSTRUMENT_HEADER_SIZE]
        # throwaway data.
        inst_header = raw_header[0:8]
        # but make sure it's the right throwaway data.
        assert inst_header == constants.SENTINEL_INSTRUMENT_HEADER
        self._trigger_spec = StrikeInstrumentTriggerSpec(raw_header[8:11])
        raw_layers = data[constants.INSTRUMENT_HEADER_SIZE:constants.INSTRUMENT_HEADER_SIZE+(constants.INSTRUMENT_LAYER_SIZE*2)]
        assert len(raw_layers) == constants.INSTRUMENT_LAYER_SIZE*2

        self._layer_a = StrikeInstrumentLayer(raw_data=raw_layers[0:constants.INSTRUMENT_LAYER_SIZE], samples=samples)
        self._layer_b = StrikeInstrumentLayer(raw_data=raw_layers[constants.INSTRUMENT_LAYER_SIZE:], samples=samples)
        raw_voice = data[constants.INSTRUMENT_HEADER_SIZE+(2*constants.INSTRUMENT_LAYER_SIZE):]
        assert len(raw_voice) == constants.INSTRUMENT_VOICE_SIZE
        self.instrument_settings = StrikeInstrumentSettings(raw_data=raw_voice)

class StrikeInstrumentTriggerSpec(object):
    def __init__(self, raw_data=None, *args, **kwargs):
        if raw_data:
            self._parse(raw_data)

    def _parse(self, data):
        self.input_type = constants.INPUT_TYPES.get(chr(data[0]))
        self.input_index = chr(data[1])
        self.input_pin = constants.INPUT_PINS.get(chr(data[2]))

    def __str__(self):
        return "{0}{1} {2}".format(self.input_type, self.input_index, self.input_pin)

    def csv(self):
        return str(self)

class StrikeInstrumentLayer(object):
    def __init__(self, raw_data=None, samples=None, *args, **kwargs):
        if raw_data:
            self._parse(raw_data, samples)

    @property
    def sample_name(self):
        return self._sample_name

    @property
    def settings_str(self):
        return """
LEVEL:
    Level:  {0}
    Pan:    {1}
    Decay:  {2}

TONE:
    Tune:   {3}
    Fine:   {4}
    Cutoff: {5}

FILTER:
    Type:   {6}
    Decay:  {7}
    Pitch:  {8}
    Filter: {9}
    Level:  {10}
        """.format(self.lvl_level, helpers.pretty_pan(self.lvl_pan), self.lvl_decay,
                   self.tone_tune, self.tone_fine, self.tone_cutoff,
                   helpers.pretty_filter_type(self.vel_filtertype),
                   self.vel_decay, self.vel_pitch, self.vel_filter, self.vel_level)

    def __str__(self):
        return "Sample {0}: {1}".format(self._sample_index, self.sample_name)

    def csv(self):
        return "{0}".format(self.sample_name)

    def _parse(self, data, samples=None):
        # 0-47 if there's a sample. FF if not
        self._sample_index = helpers.parse_signed_byte(data[0])
        if self._sample_index >= 0 and self._sample_index != 255:
            self._sample_name = samples.get_sample_by_index(self._sample_index)
        else:
            self._sample_name = ""
        # mystery pad byte.
        byte2 = data[1]
        self.lvl_level = helpers.parse_signed_byte(data[2])
        self.lvl_pan = helpers.parse_signed_byte(data[3])
        self.lvl_decay = helpers.parse_signed_byte(data[4])
        pad0 = data[5:8]
        self.tone_tune = helpers.parse_signed_byte(data[8])
        self.tone_fine = helpers.parse_signed_byte(data[9])
        self.tone_cutoff = helpers.parse_signed_byte(data[10])
        self.vel_filtertype = helpers.parse_signed_byte(data[11])
        self.vel_decay = helpers.parse_signed_byte(data[12])
        self.vel_pitch = helpers.parse_signed_byte(data[13])
        self.vel_filter = helpers.parse_signed_byte(data[14])
        self.vel_level = helpers.parse_signed_byte(data[15])
        self.pad1 = data[16]
        self.term_pad = data[17:20]

class StrikeInstrumentSettings(object):
    def __init__(self, *args, **kwargs):
        raw_data = kwargs.get("raw_data")

        if raw_data:
            self._parse(raw_data)

    def _parse(self, data):
        self.send_reverb = helpers.parse_signed_byte(data[0])
        self.send_fx =  helpers.parse_signed_byte(data[1])
        pad0 = data[2:4]
        self.priority =  helpers.parse_signed_byte(data[4])
        self.mutegroup =  helpers.parse_signed_byte(data[5])
        self.playback =  helpers.parse_signed_byte(data[6])
        self.midi_channel = helpers.parse_signed_byte(data[7])
        self.midi_note =  helpers.parse_signed_byte(data[8])
        self.midi_gate = helpers.parse_signed_byte(data[7])
        self.midi_noteoff = helpers.parse_signed_byte(data[8])
        pad1 = data[9]
        ffterminator = data[10:15]
        pad2 = data[15:]

class StrikeSamples(object):
    def __init__(self, raw_data=None, *args, **kwargs):
        if raw_data:
            self._parse(raw_data)

    @property
    def sample_table(self):
        return self._sample_table

    def get_sample_by_index(self, index):

        return self._sample_table[index]

    def _parse(self, data):
        """
        Sample table is at the end of file. This makes things a little easier
        starts with a "str" type indicator, followed by space.

        The following 4 bytes are a 32-bit integer with the bytes stored in reverse order
        Since we get them as whole bytes, we can reverse them and parse the resulting hex.
        This number is our string length.

        Read that size til end of string. Alesis was nice and split them all on NUL boundaries
        """

        str_header = data[0:4]
        size_bytes = data[4:8]
        # Hey one of our more complicated functions, we only get to use it once

        table_size = helpers.parse_dword(size_bytes)
        raw_samples = data[8:]
        split_samples = map(lambda x:x.decode("utf-8"), raw_samples.split(b"\0"))
        self._sample_table = list([str(x) for x in split_samples if x != ""])

class StrikeKitSettings(object):
    def __init__(self, *args, **kwargs):
        # TODO(future) - Enum-type thing for reverb_type, map values to names
        pass
        self._reverb_type = kwargs.get("reverb_type")
        self._reverb_size = kwargs.get("reverb_size")
        self._reverb_level = kwargs.get("reverb_level")

    def parse(self, data):
        # Parse method is responsible for populating private vars.
        raise NotImplementedError("Abstract class method requires implementation")
<commit_msg>Even more pretty printing<commit_after>from strikeparse import constants
from strikeparse import helpers

class StrikeKit(object):
    """
    File/Kit Header
    52 bytes
    ------------------
    03 byte 0x4b4954        - Begin KIT header
    02 byte 0x202c          - 32, 44
    04 byte 0x00            - Padding?
    02 byte 0x0063          - 00, 99
    04 byte                 - Flag data?
    04 byte                 - kit settings - reverb type, size, color, level
    33 byte Kit data?       - Appears to be kit-specific settings, needs decoding.
    """
    def __init__(self, *args, **kwargs):
        self._kit_settings = None
        self._instruments = None
        self._samples = None

        raw_data = kwargs.get("raw_data") or args[0]
        # parse raw data if it's there
        if raw_data:
            # Yea assignment by side effect
            self._parse_raw_kit(raw_data)

    def __str__(self):
        return "\n".join(map(str, self.instruments))

    def csv(self):
        return "\n".join(map(str, map(StrikeInstrument.csv, self.instruments)))

    @property
    def kit_settings(self):
        return self._kit_settings

    @property
    def instruments(self):
        return self._instruments

    @property
    def samples(self):
        return self._samples

    def _parse_raw_kit(self, data):
        raw_header = data[0:constants.KIT_HEADER_SIZE]
        inst_begin = constants.KIT_HEADER_SIZE
        inst_end = constants.KIT_HEADER_SIZE + (constants.INSTRUMENT_COUNT * constants.INSTRUMENT_SIZE)
        raw_instruments = data[inst_begin:inst_end]
        raw_sampledata = data[inst_end:]

        # Samples have to be parsed before instruments so we know who uses what.
        self._parse_samples(raw_sampledata)
        self._parse_instruments(raw_instruments)

    def _parse_samples(self, data):
        self._samples = StrikeSamples(data)

    def _parse_instruments(self, data):
        result = []

        for x in range(0, constants.INSTRUMENT_COUNT):
            start_index = x * constants.INSTRUMENT_SIZE
            end_index = start_index + constants.INSTRUMENT_SIZE
            raw_instrument = data[start_index:end_index]
            instrument = StrikeInstrument(raw_instrument, samples=self.samples)
            result.append(instrument)
        self._instruments = result

class StrikeInstrument(object):
    """
        Instrument section
        80 bytes

        offset      12 (that's weird) byte header
                    ------------------
        0           4 byte 0x696e7374       - Begin "inst" header
        4           1 byte 0x48             - "H" ???
        5           3 byte 0x00             - Padding
        8           3 byte trigger spec     - (K|S|T|H|C|R)([1-4])(H|R|F|B|E|D)
                                                Kick/Snare/Tom/Hat/Crash/Ride
                                                1-4 for Toms, 1-3 Crash, 1 everything else
                                                Head/Rim for pads
                                                Foot/Edge/Bow for Hat
                                                Edge/Bow for Crashes
                                                D/B/E for Rides, not sure which is bow and bell
        11          1 byte                  - 0x20 (SPC) terminator ?     FF if not set

        -- LAYER A DATA
        12          1 byte sample ref       - offset of sample as listed in sample section? FF if not set
        13          1 byte                  - 0x00 padding?
        14          1 byte level            - 0x00 to 0x63
        15          1 byte pan              - this is fun. Values > 128 are panned left. 255 - pan. Values < 128 are panned right.
        16          1 byte decay            - 0x00 to 0x63
        17          2 byte pad?             - 0x002
        19          1 byte Tune             - 0-12 = +, 243-255 = -
        20          1 byte fine             - 0-127 positive, 255-* negative
        21          1 byte cutoff           - 0-127 positive, 255-* negative
        22          1 byte vel filtertype   - 0 lo, 1 hi
        23          1 byte vel decay        -
        24          1 byte vel pitch        -
        25          1 byte vel filter       -
        26          1 byte vel level        -
        27          1 byte 0 pad
        28          1 byte 7f terminator?
        29          3 byte 0 pad
        -- LAYER B DATA
        32          1 byte sample ref       - offset of sample as listed in sample section? FF if not set
        33          1 byte                  - 0x00 padding?
        34          1 byte level            - 0x00 to 0x63
        34          1 byte pan              - this is fun. Values > 128 are panned left. 255 - pan. Values < 128 are panned right.
        35          1 byte decay            - 0x00 to 0x63
        36          2 byte pad?             - 0x00
        38          1 byte Tune             - 0-12 = +, 243-255 = -
        39          1 byte fine             - 0-127 positive, 255-* negative
        40          1 byte cutoff           - 0-127 positive, 255-* negative
        41          1 byte vel filtertype   - 0 lo, 1 hi
        42          1 byte vel decay        -
        43          1 byte vel pitch        -
        44          1 byte vel filter       -
        45          1 byte vel level        -
        46          1 byte 0 pad
        47          1 byte 7f terminator?
        48          3 byte 0 pad

        -- VOICE DATA (MIDI, sends, etc)
        51          1 byte reverb send
        52          1 byte FX send
        53          2 byte 0 pad?
        55          1 byte (Note off or Priority? need to investigate)
        56          1 byte mute group
        57          1 byte playback?
        58          1 byte MIDI chan
        59          1 byte MIDI note
        60          1 byte gate time
        61          1 byte note off?
        62          1 byte 0 pad?
        63          5 byte FF terminator
        68         11 byte zero pad?
        """

    #
    def __init__(self, raw_data=None, samples=[]):
        if raw_data:
            self._parse(raw_data, samples)

    def __str__(self):
        return """{0}
        Layer A: {1}
        Layer B: {2}
        """.format(self.trigger_spec, self.layer_a, self.layer_b)

    def csv(self):
        return ",".join([self.trigger_spec.csv(), self.layer_a.csv(), self.layer_b.csv()])

    @property
    def trigger_spec(self):
        return self._trigger_spec

    @property
    def layer_a(self):
        return self._layer_a

    @property
    def layer_b(self):
        return self._layer_b

    def _parse(self, data, samples=[]):
        raw_header = data[0:constants.INSTRUMENT_HEADER_SIZE]
        # throwaway data.
        inst_header = raw_header[0:8]
        # but make sure it's the right throwaway data.
        assert inst_header == constants.SENTINEL_INSTRUMENT_HEADER
        self._trigger_spec = StrikeInstrumentTriggerSpec(raw_header[8:11])
        raw_layers = data[constants.INSTRUMENT_HEADER_SIZE:constants.INSTRUMENT_HEADER_SIZE+(constants.INSTRUMENT_LAYER_SIZE*2)]
        assert len(raw_layers) == constants.INSTRUMENT_LAYER_SIZE*2

        self._layer_a = StrikeInstrumentLayer(raw_data=raw_layers[0:constants.INSTRUMENT_LAYER_SIZE], samples=samples)
        self._layer_b = StrikeInstrumentLayer(raw_data=raw_layers[constants.INSTRUMENT_LAYER_SIZE:], samples=samples)
        raw_voice = data[constants.INSTRUMENT_HEADER_SIZE+(2*constants.INSTRUMENT_LAYER_SIZE):]
        assert len(raw_voice) == constants.INSTRUMENT_VOICE_SIZE
        self.instrument_settings = StrikeInstrumentSettings(raw_data=raw_voice)

class StrikeInstrumentTriggerSpec(object):
    def __init__(self, raw_data=None, *args, **kwargs):
        if raw_data:
            self._parse(raw_data)

    def _parse(self, data):
        self.input_type = constants.INPUT_TYPES.get(chr(data[0]))
        self.input_index = chr(data[1])
        self.input_pin = constants.INPUT_PINS.get(chr(data[2]))

    def __str__(self):
        return "{0}{1} {2}".format(self.input_type, self.input_index, self.input_pin)

    def csv(self):
        return str(self)

class StrikeInstrumentLayer(object):
    def __init__(self, raw_data=None, samples=None, *args, **kwargs):
        if raw_data:
            self._parse(raw_data, samples)

    @property
    def sample_name(self):
        return self._sample_name

    @property
    def settings_str(self):
        return """
LEVEL:
    Level:  {0}
    Pan:    {1}
    Decay:  {2}

TONE:
    Tune:   {3}
    Fine:   {4}
    Cutoff: {5}

FILTER:
    Type:   {6}
    Decay:  {7}
    Pitch:  {8}
    Filter: {9}
    Level:  {10}
        """.format(self.lvl_level, helpers.pretty_pan(self.lvl_pan), self.lvl_decay,
                   self.tone_tune, self.tone_fine, self.tone_cutoff,
                   helpers.pretty_filter_type(self.vel_filtertype),
                   self.vel_decay, self.vel_pitch, self.vel_filter, self.vel_level)

    def __str__(self):
        return "Sample {0}: {1}".format(self._sample_index, self.sample_name)

    def csv(self):
        return "{0}".format(self.sample_name)

    def _parse(self, data, samples=None):
        # 0-47 if there's a sample. FF if not
        self._sample_index = helpers.parse_signed_byte(data[0])
        if self._sample_index >= 0 and self._sample_index != 255:
            self._sample_name = samples.get_sample_by_index(self._sample_index)
        else:
            self._sample_name = ""
        # mystery pad byte.
        byte2 = data[1]
        self.lvl_level = helpers.parse_signed_byte(data[2])
        self.lvl_pan = helpers.parse_signed_byte(data[3])
        self.lvl_decay = helpers.parse_signed_byte(data[4])
        pad0 = data[5:8]
        self.tone_tune = helpers.parse_signed_byte(data[8])
        self.tone_fine = helpers.parse_signed_byte(data[9])
        self.tone_cutoff = helpers.parse_signed_byte(data[10])
        self.vel_filtertype = helpers.parse_signed_byte(data[11])
        self.vel_decay = helpers.parse_signed_byte(data[12])
        self.vel_pitch = helpers.parse_signed_byte(data[13])
        self.vel_filter = helpers.parse_signed_byte(data[14])
        self.vel_level = helpers.parse_signed_byte(data[15])
        self.pad1 = data[16]
        self.term_pad = data[17:20]

class StrikeInstrumentSettings(object):
    def __init__(self, *args, **kwargs):
        raw_data = kwargs.get("raw_data")

        if raw_data:
            self._parse(raw_data)

    def __str__(self):
        return """
Reverb Send:    {0}
FX Send:        {1}

Priority:       {2}
Mute Group:     {3}
Playback:       {4}

MIDI Channel:   {5}
MIDI Note:      {6}
MIDI Gate Time: {7}
MIDI Note Off:  {8}
        """.format(self.send_reverb, self.send_fx, helpers.pretty_priority(self.priority),
                   helpers.pretty_mute_group(self.mutegroup),
                   helpers.pretty_priority(self.priority),
                   helpers.pretty_playback(self.playback),
                   self.midi_channel, self.midi_note, self.midi_gate,
                   helpers.pretty_note_off(self.midi_noteoff))

    def _parse(self, data):
        self.send_reverb = helpers.parse_signed_byte(data[0])
        self.send_fx =  helpers.parse_signed_byte(data[1])
        pad0 = data[2:4]
        self.priority =  helpers.parse_signed_byte(data[4])
        self.mutegroup =  helpers.parse_signed_byte(data[5])
        self.playback =  helpers.parse_signed_byte(data[6])
        self.midi_channel = helpers.parse_signed_byte(data[7])
        self.midi_note =  helpers.parse_signed_byte(data[8])
        self.midi_gate = helpers.parse_signed_byte(data[7])
        self.midi_noteoff = helpers.parse_signed_byte(data[8])
        pad1 = data[9]
        ffterminator = data[10:15]
        pad2 = data[15:]

class StrikeSamples(object):
    def __init__(self, raw_data=None, *args, **kwargs):
        if raw_data:
            self._parse(raw_data)

    @property
    def sample_table(self):
        return self._sample_table

    def get_sample_by_index(self, index):

        return self._sample_table[index]

    def _parse(self, data):
        """
        Sample table is at the end of file. This makes things a little easier
        starts with a "str" type indicator, followed by space.

        The following 4 bytes are a 32-bit integer with the bytes stored in reverse order
        Since we get them as whole bytes, we can reverse them and parse the resulting hex.
        This number is our string length.

        Read that size til end of string. Alesis was nice and split them all on NUL boundaries
        """

        str_header = data[0:4]
        size_bytes = data[4:8]
        # Hey one of our more complicated functions, we only get to use it once

        table_size = helpers.parse_dword(size_bytes)
        raw_samples = data[8:]
        split_samples = map(lambda x:x.decode("utf-8"), raw_samples.split(b"\0"))
        self._sample_table = list([str(x) for x in split_samples if x != ""])

class StrikeKitSettings(object):
    def __init__(self, *args, **kwargs):
        # TODO(future) - Enum-type thing for reverb_type, map values to names
        pass
        self._reverb_type = kwargs.get("reverb_type")
        self._reverb_size = kwargs.get("reverb_size")
        self._reverb_level = kwargs.get("reverb_level")

    def parse(self, data):
        # Parse method is responsible for populating private vars.
        raise NotImplementedError("Abstract class method requires implementation")
<|endoftext|>


<commit_before>##########################################################################
#
#  Copyright (c) 2011-2012, Image Engine Design Inc. All rights reserved.
#  Copyright (c) 2011, John Haddon. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

from __future__ import with_statement

import IECore

import Gaffer
import GafferUI

class CompoundVectorParameterValueWidget( GafferUI.CompoundParameterValueWidget ) :

	def __init__( self, parameterHandler, collapsible=None, **kw ) :

		self.__vectorDataWidget = None

		GafferUI.CompoundParameterValueWidget.__init__( self, parameterHandler, collapsible, **kw )

		self.__plugSetConnection = parameterHandler.plug().node().plugSetSignal().connect( Gaffer.WeakMethod( self.__plugChanged ) )

	def _buildChildParameterUIs( self, column ) :

		del column[:]

		header = [ IECore.CamelCase.toSpaced( x ) for x in self.parameterHandler().parameter().keys() ]

		self.__vectorDataWidget = GafferUI.VectorDataWidget( header = header )
		self.__dataChangedConnection = self.__vectorDataWidget.dataChangedSignal().connect( Gaffer.WeakMethod( self.__dataChanged ) )

		column.append( self.__vectorDataWidget )

		self.__updateFromPlug()

	def __updateFromPlug( self ) :

		if self.__vectorDataWidget is None :
			return

		data = []
		for plug in self.parameterHandler().plug().children() :
			data.append( plug.getValue() )

		self.__vectorDataWidget.setData( data )

	def __plugChanged( self, plug ) :

		if plug.parent().isSame( self.parameterHandler().plug() ) :
			self.__updateFromPlug()

	def __dataChanged( self, vectorDataWidget ) :

		data = vectorDataWidget.getData()
		with Gaffer.BlockedConnection( self.__plugSetConnection ) :
			for d, p in zip( data, self.parameterHandler().plug().children() ) :
				p.setValue( d )

GafferUI.ParameterValueWidget.registerType( IECore.CompoundVectorParameter.staticTypeId(), CompoundVectorParameterValueWidget )
<commit_msg>CompoundVectorParameterValueWidget works again.<commit_after>##########################################################################
#
#  Copyright (c) 2011-2012, Image Engine Design Inc. All rights reserved.
#  Copyright (c) 2011, John Haddon. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

from __future__ import with_statement

import IECore

import Gaffer
import GafferUI

class CompoundVectorParameterValueWidget( GafferUI.CompoundParameterValueWidget ) :

	def __init__( self, parameterHandler, collapsible=None, **kw ) :

		GafferUI.CompoundParameterValueWidget.__init__( self, parameterHandler, collapsible, _plugValueWidgetClass=_PlugValueWidget, **kw )

class _PlugValueWidget( GafferUI.CompoundParameterValueWidget._PlugValueWidget ) :

	def __init__( self, parameterHandler, collapsed ) :

		GafferUI.CompoundParameterValueWidget._PlugValueWidget.__init__( self, parameterHandler, collapsed )

		self.__vectorDataWidget = None

	def _headerWidget( self ) :

		if self.__vectorDataWidget is not None :
			return self.__vectorDataWidget

		header = [ IECore.CamelCase.toSpaced( x ) for x in self._parameter().keys() ]

		self.__vectorDataWidget = GafferUI.VectorDataWidget( header = header )
		self.__dataChangedConnection = self.__vectorDataWidget.dataChangedSignal().connect( Gaffer.WeakMethod( self.__dataChanged ) )

		self._updateFromPlug()

		return self.__vectorDataWidget

	def _childPlugs( self ) :

		# because we represent everything in the header we don't
		# need any plug widgets made by the base class.
		return []

	def _updateFromPlug( self ) :

		GafferUI.CompoundParameterValueWidget._PlugValueWidget._updateFromPlug( self )

		data = []
		for plug in self._parameterHandler().plug().children() :
			plugData = plug.getValue()
			if len( data ) and len( plugData ) != len( data[0] ) :
				# in __dataChanged we have to update the child plug values
				# one at a time. when adding or removing rows, this means that the
				# columns will have differing lengths until the last plug
				# has been set. in this case we shortcut ourselves, and wait
				# for the final plug to be set before updating the VectorDataWidget.
				return
			data.append( plugData )

		self.__vectorDataWidget.setData( data )

	def __dataChanged( self, vectorDataWidget ) :

		data = vectorDataWidget.getData()

		with Gaffer.UndoContext( self.getPlug().ancestor( Gaffer.ScriptNode.staticTypeId() ) ) :
			for d, p in zip( data, self._parameterHandler().plug().children() ) :
				p.setValue( d )

GafferUI.ParameterValueWidget.registerType( IECore.CompoundVectorParameter.staticTypeId(), CompoundVectorParameterValueWidget )
<|endoftext|>


<commit_before># coding: utf-8

import unittest

from message.forms import SimpleRequestForm
from test.utils import lorem_ipsum, generate_string
from test.factories import UserFactory

class TestSimpleRequestForm(unittest.TestCase):

    def setUp(self):
        self.form = SimpleRequestForm()
        self.user = UserFactory.build()

    def tearDown(self):
        self.form = None
        self.user = None

    def test_form_type(self):
        self.assertEqual(1, self.form.fields['messageType'].initial)

    def test_send_data(self):
        data = {
            'message': lorem_ipsum(),
            'contact_first_name': self.user.first_name,
            'contact_last_name': self.user.last_name,
            'contact_mail': self.user.email
        }
        form = SimpleRequestForm(data)
        self.assertTrue(form.is_bound)
        self.assertTrue(form.is_valid())
<commit_msg>Testing for required form fields<commit_after># coding: utf-8

import unittest

from message.forms import SimpleRequestForm
from test.utils import lorem_ipsum, generate_string
from test.factories import UserFactory

class TestSimpleRequestForm(unittest.TestCase):

    def setUp(self):
        self.form = SimpleRequestForm()
        self.user = UserFactory.build()

    def tearDown(self):
        self.form = None
        self.user = None

    def test_form_type(self):
        self.assertEqual(1, self.form.fields['messageType'].initial)

    def test_send_data(self):
        data = {
            'message': lorem_ipsum(),
            'contact_first_name': self.user.first_name,
            'contact_last_name': self.user.last_name,
            'contact_mail': self.user.email,
            'address': lorem_ipsum(words_count=4),
        }
        form = SimpleRequestForm(data)
        self.assertTrue(form.is_bound)
        self.assertTrue(form.is_valid())

class TestRequiredFields(unittest.TestCase):

    def setUp(self):
        self.user = UserFactory.build()
        self.data = {
            'message': lorem_ipsum(),
            'contact_first_name': self.user.first_name,
            'contact_last_name': self.user.last_name,
            'contact_mail': self.user.email,
            'address': lorem_ipsum(words_count=4),
        }

    def tearDown(self):
        self.user = None
        self.data = None

    def test_lost_address(self):
        self.data['address'] = ''
        form = SimpleRequestForm(self.data)
        self.assertTrue(form.is_bound)
        self.assertFalse(form.is_valid())

    def test_lost_message(self):
        self.data['message'] = ''
        form = SimpleRequestForm(self.data)
        self.assertTrue(form.is_bound)
        self.assertFalse(form.is_valid())

    def test_lost_first_name(self):
        self.data['contact_first_name'] = ''
        form = SimpleRequestForm(self.data)
        self.assertTrue(form.is_bound)
        self.assertFalse(form.is_valid())

    def test_lost_last_name(self):
        self.data['contact_last_name'] = ''
        form = SimpleRequestForm(self.data)
        self.assertTrue(form.is_bound)
        self.assertFalse(form.is_valid())

<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# File: group.py
# Author: Yuxin Wu <ppwwyyxx@gmail.com>

import tensorflow as tf
from contextlib import contextmanager
import time

from .base import Callback, TrainCallbackType, TestCallbackType
from .summary import *
from ..utils import *

__all__ = ['Callbacks']

# --- Test-Callback related stuff seems not very useful.
@contextmanager
def create_test_graph(trainer):
    model = trainer.model
    with tf.Graph().as_default() as Gtest:
        # create a global step var in test graph
        global_step_var = tf.Variable(
            0, trainable=False, name=GLOBAL_STEP_OP_NAME)
        input_vars = model.get_input_vars()
        model.build_graph(input_vars, False)
        cost = model.get_cost()
        yield Gtest

@contextmanager
def create_test_session(trainer):
    """ create a test-time session from trainer"""
    with create_test_graph(trainer):
        with tf.Session() as sess:
            yield sess

class TestCallbackContext(object):
    """
    A class holding the context needed for running TestCallback
    """
    def __init__(self):
        self.sess = None

    @contextmanager
    def create_context(self, trainer):
        if self.sess is None:
            with create_test_session(trainer) as sess:
                self.sess = sess
                self.graph = sess.graph
                # no tower in test graph. just keep it as what it is
                self.saver = tf.train.Saver()
        with self.graph.as_default(), self.sess.as_default():
            yield

    # TODO also do this for after_train?

    def restore_checkpoint(self):
        ckpt = tf.train.get_checkpoint_state(logger.LOG_DIR)
        if ckpt is None:
            raise RuntimeError(
                "Cannot find a checkpoint state. Do you forget to use ModelSaver before all TestCallback?")
        logger.info(
            "Restore checkpoint from {}".format(ckpt.model_checkpoint_path))
        self.saver.restore(self.sess, ckpt.model_checkpoint_path)

    @contextmanager
    def test_context(self):
        with self.graph.as_default(), self.sess.as_default():
            yield
# ---

class CallbackTimeLogger(object):
    def __init__(self):
        self.times = []
        self.tot = 0

    def add(self, name, time):
        self.tot += time
        self.times.append((name, time))

    @contextmanager
    def timed_callback(self, name):
        s = time.time()
        yield
        self.add(name, time.time() - s)

    def log(self):
        """ log the time of some heavy callbacks """
        if self.tot < 3:
            return
        msgs = []
        for name, t in self.times:
            if t / self.tot > 0.3 and t > 1:
                msgs.append("{}: {:.3f}sec".format(name, t))
        logger.info(
            "Callbacks took {:.3f} sec in total. {}".format(
                self.tot, '; '.join(msgs)))

class Callbacks(Callback):
    """
    A container to hold all callbacks, and execute them in the right order and proper session.
    """
    def __init__(self, cbs):
        """
        :param cbs: a list of `Callbacks`
        """
        # check type
        for cb in cbs:
            assert isinstance(cb, Callback), cb.__class__
            if not isinstance(cb.type, (TrainCallbackType, TestCallbackType)):
                raise ValueError(
                    "Unknown callback running graph {}!".format(str(cb.type)))
        # move "StatPrinter" to the last
        for cb in cbs:
            if isinstance(cb, StatPrinter):
                sp = cb
                cbs.remove(sp)
                cbs.append(sp)
                break
        print(cbs)

        self.cbs = cbs
        self.test_callback_context = TestCallbackContext()

    def _setup_graph(self):
        with tf.name_scope(None):
            for cb in self.cbs:
                if isinstance(cb.type, TrainCallbackType):
                    cb.setup_graph(self.trainer)
                else:
                    with self.test_callback_context.create_context(self.trainer):
                        cb.setup_graph(self.trainer)

    def _before_train(self):
        for cb in self.cbs:
            if isinstance(cb.type, TrainCallbackType):
                cb.before_train()
            else:
                with self.test_callback_context.test_context():
                    cb.before_train()

    def _after_train(self):
        for cb in self.cbs:
            cb.after_train()

    def trigger_step(self):
        for cb in self.cbs:
            if isinstance(cb.type, TrainCallbackType):
                cb.trigger_step()
        # test callback don't have trigger_step

    def _trigger_epoch(self):
        tm = CallbackTimeLogger()

        test_sess_restored = False
        for cb in self.cbs:
            display_name = str(cb)
            if isinstance(cb.type, TrainCallbackType):
                with tm.timed_callback(display_name):
                    cb.trigger_epoch()
            else:
                if not test_sess_restored:
                    with tm.timed_callback('restore checkpoint'):
                        self.test_callback_context.restore_checkpoint()
                    test_sess_restored = True
                with self.test_callback_context.test_context(), \
                        tm.timed_callback(display_name):
                    cb.trigger_epoch()
        tm.log()
<commit_msg>bug fix<commit_after># -*- coding: UTF-8 -*-
# File: group.py
# Author: Yuxin Wu <ppwwyyxx@gmail.com>

import tensorflow as tf
from contextlib import contextmanager
import time

from .base import Callback, TrainCallbackType, TestCallbackType
from .stat import *
from ..utils import *

__all__ = ['Callbacks']

# --- Test-Callback related stuff seems not very useful.
@contextmanager
def create_test_graph(trainer):
    model = trainer.model
    with tf.Graph().as_default() as Gtest:
        # create a global step var in test graph
        global_step_var = tf.Variable(
            0, trainable=False, name=GLOBAL_STEP_OP_NAME)
        input_vars = model.get_input_vars()
        model.build_graph(input_vars, False)
        cost = model.get_cost()
        yield Gtest

@contextmanager
def create_test_session(trainer):
    """ create a test-time session from trainer"""
    with create_test_graph(trainer):
        with tf.Session() as sess:
            yield sess

class TestCallbackContext(object):
    """
    A class holding the context needed for running TestCallback
    """
    def __init__(self):
        self.sess = None

    @contextmanager
    def create_context(self, trainer):
        if self.sess is None:
            with create_test_session(trainer) as sess:
                self.sess = sess
                self.graph = sess.graph
                # no tower in test graph. just keep it as what it is
                self.saver = tf.train.Saver()
        with self.graph.as_default(), self.sess.as_default():
            yield

    # TODO also do this for after_train?

    def restore_checkpoint(self):
        ckpt = tf.train.get_checkpoint_state(logger.LOG_DIR)
        if ckpt is None:
            raise RuntimeError(
                "Cannot find a checkpoint state. Do you forget to use ModelSaver before all TestCallback?")
        logger.info(
            "Restore checkpoint from {}".format(ckpt.model_checkpoint_path))
        self.saver.restore(self.sess, ckpt.model_checkpoint_path)

    @contextmanager
    def test_context(self):
        with self.graph.as_default(), self.sess.as_default():
            yield
# ---

class CallbackTimeLogger(object):
    def __init__(self):
        self.times = []
        self.tot = 0

    def add(self, name, time):
        self.tot += time
        self.times.append((name, time))

    @contextmanager
    def timed_callback(self, name):
        s = time.time()
        yield
        self.add(name, time.time() - s)

    def log(self):
        """ log the time of some heavy callbacks """
        if self.tot < 3:
            return
        msgs = []
        for name, t in self.times:
            if t / self.tot > 0.3 and t > 1:
                msgs.append("{}: {:.3f}sec".format(name, t))
        logger.info(
            "Callbacks took {:.3f} sec in total. {}".format(
                self.tot, '; '.join(msgs)))

class Callbacks(Callback):
    """
    A container to hold all callbacks, and execute them in the right order and proper session.
    """
    def __init__(self, cbs):
        """
        :param cbs: a list of `Callbacks`
        """
        # check type
        for cb in cbs:
            assert isinstance(cb, Callback), cb.__class__
            if not isinstance(cb.type, (TrainCallbackType, TestCallbackType)):
                raise ValueError(
                    "Unknown callback running graph {}!".format(str(cb.type)))
        # move "StatPrinter" to the last
        for cb in cbs:
            if isinstance(cb, StatPrinter):
                sp = cb
                cbs.remove(sp)
                cbs.append(sp)
                break
        print(cbs)

        self.cbs = cbs
        self.test_callback_context = TestCallbackContext()

    def _setup_graph(self):
        with tf.name_scope(None):
            for cb in self.cbs:
                if isinstance(cb.type, TrainCallbackType):
                    cb.setup_graph(self.trainer)
                else:
                    with self.test_callback_context.create_context(self.trainer):
                        cb.setup_graph(self.trainer)

    def _before_train(self):
        for cb in self.cbs:
            if isinstance(cb.type, TrainCallbackType):
                cb.before_train()
            else:
                with self.test_callback_context.test_context():
                    cb.before_train()

    def _after_train(self):
        for cb in self.cbs:
            cb.after_train()

    def trigger_step(self):
        for cb in self.cbs:
            if isinstance(cb.type, TrainCallbackType):
                cb.trigger_step()
        # test callback don't have trigger_step

    def _trigger_epoch(self):
        tm = CallbackTimeLogger()

        test_sess_restored = False
        for cb in self.cbs:
            display_name = str(cb)
            if isinstance(cb.type, TrainCallbackType):
                with tm.timed_callback(display_name):
                    cb.trigger_epoch()
            else:
                if not test_sess_restored:
                    with tm.timed_callback('restore checkpoint'):
                        self.test_callback_context.restore_checkpoint()
                    test_sess_restored = True
                with self.test_callback_context.test_context(), \
                        tm.timed_callback(display_name):
                    cb.trigger_epoch()
        tm.log()
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

from collections import OrderedDict

from django.conf.urls import url
from django.core.urlresolvers import reverse
from django.http import Http404
from rest_framework import status
from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer
from rest_framework.response import Response
from rest_framework.viewsets import GenericViewSet

from wagtail.wagtailcore.models import Page
from wagtail.wagtailcore.utils import resolve_model_string
from wagtail.wagtaildocs.models import get_document_model
from wagtail.wagtailimages import get_image_model

from .filters import ChildOfFilter, DescendantOfFilter, FieldsFilter, OrderingFilter, SearchFilter
from .pagination import WagtailPagination
from .serializers import (
    BaseSerializer, DocumentSerializer, ImageSerializer, PageSerializer, get_serializer_class)
from .utils import BadRequestError

class BaseAPIEndpoint(GenericViewSet):
    renderer_classes = [JSONRenderer, BrowsableAPIRenderer]
    pagination_class = WagtailPagination
    base_serializer_class = BaseSerializer
    filter_backends = []
    model = None  # Set on subclass

    known_query_parameters = frozenset([
        'limit',
        'offset',
        'fields',
        'order',
        'search',

        # Used by jQuery for cache-busting. See #1671
        '_',

        # Required by BrowsableAPIRenderer
        'format',
    ])
    extra_api_fields = []
    name = None  # Set on subclass.

    def get_queryset(self):
        return self.model.objects.all().order_by('id')

    def listing_view(self, request):
        queryset = self.get_queryset()
        self.check_query_parameters(queryset)
        queryset = self.filter_queryset(queryset)
        queryset = self.paginate_queryset(queryset)
        serializer = self.get_serializer(queryset, many=True)
        return self.get_paginated_response(serializer.data)

    def detail_view(self, request, pk):
        instance = self.get_object()
        serializer = self.get_serializer(instance)
        return Response(serializer.data)

    def handle_exception(self, exc):
        if isinstance(exc, Http404):
            data = {'message': str(exc)}
            return Response(data, status=status.HTTP_404_NOT_FOUND)
        elif isinstance(exc, BadRequestError):
            data = {'message': str(exc)}
            return Response(data, status=status.HTTP_400_BAD_REQUEST)
        return super(BaseAPIEndpoint, self).handle_exception(exc)

    def get_api_fields(self, model):
        """
        This returns a list of field names that are allowed to
        be used in the API (excluding the id field).
        """
        api_fields = self.extra_api_fields[:]

        if hasattr(model, 'api_fields'):
            api_fields.extend(model.api_fields)

        return api_fields

    def check_query_parameters(self, queryset):
        """
        Ensure that only valid query paramters are included in the URL.
        """
        query_parameters = set(self.request.GET.keys())

        # All query paramters must be either a field or an operation
        allowed_parameters = set(self.get_api_fields(queryset.model))
        allowed_parameters = allowed_parameters.union(self.known_query_parameters)
        allowed_parameters.add('id')
        unknown_parameters = query_parameters - allowed_parameters
        if unknown_parameters:
            raise BadRequestError(
                "query parameter is not an operation or a recognised field: %s"
                % ', '.join(sorted(unknown_parameters))
            )

    def get_serializer_class(self):
        request = self.request

        # Get model
        if self.action == 'listing_view':
            model = self.get_queryset().model
        else:
            model = type(self.get_object())

        # Get all available fields
        all_fields = self.get_api_fields(model)
        # Removes any duplicates in case the developer put "title" in api_fields
        all_fields = list(OrderedDict.fromkeys(all_fields))

        if self.action == 'listing_view':
            # Listing views just show the title field and any other allowed field the user specified
            if 'fields' in request.GET:
                fields = set(request.GET['fields'].split(','))
            else:
                fields = {'title'}

            unknown_fields = fields - set(all_fields)

            if unknown_fields:
                raise BadRequestError("unknown fields: %s" % ', '.join(sorted(unknown_fields)))

            # Reorder fields so it matches the order of all_fields
            fields = [field for field in all_fields if field in fields]
        else:
            # Detail views show all fields all the time
            fields = all_fields

        # Always show id and meta first
        fields = ['id', 'meta'] + fields

        # If showing details, add the parent field
        if isinstance(self, PagesAPIEndpoint) and self.get_serializer_context().get('show_details', False):
            fields.insert(2, 'parent')

        return get_serializer_class(model, fields, base=self.base_serializer_class)

    def get_serializer_context(self):
        """
        The serialization context differs between listing and detail views.
        """
        context = {
            'request': self.request,
            'view': self,
            'router': self.request.wagtailapi_router
        }

        if self.action == 'detail_view':
            context['show_details'] = True

        return context

    def get_renderer_context(self):
        context = super(BaseAPIEndpoint, self).get_renderer_context()
        context['indent'] = 4
        return context

    @classmethod
    def get_urlpatterns(cls):
        """
        This returns a list of URL patterns for the endpoint
        """
        return [
            url(r'^$', cls.as_view({'get': 'listing_view'}), name='listing'),
            url(r'^(?P<pk>\d+)/$', cls.as_view({'get': 'detail_view'}), name='detail'),
        ]

    @classmethod
    def get_object_detail_urlpath(cls, model, pk, namespace=''):
        if namespace:
            url_name = namespace + ':detail'
        else:
            url_name = 'detail'

        return reverse(url_name, args=(pk, ))

class PagesAPIEndpoint(BaseAPIEndpoint):
    base_serializer_class = PageSerializer
    filter_backends = [
        FieldsFilter,
        ChildOfFilter,
        DescendantOfFilter,
        OrderingFilter,
        SearchFilter
    ]
    known_query_parameters = BaseAPIEndpoint.known_query_parameters.union([
        'type',
        'child_of',
        'descendant_of',
    ])
    extra_api_fields = ['title']
    name = 'pages'
    model = Page

    def get_queryset(self):
        request = self.request

        # Allow pages to be filtered to a specific type
        if 'type' not in request.GET:
            model = Page
        else:
            model_name = request.GET['type']
            try:
                model = resolve_model_string(model_name)
            except LookupError:
                raise BadRequestError("type doesn't exist")
            if not issubclass(model, Page):
                raise BadRequestError("type doesn't exist")

        # Get live pages that are not in a private section
        queryset = model.objects.public().live()

        # Filter by site
        queryset = queryset.descendant_of(request.site.root_page, inclusive=True)

        return queryset

    def get_object(self):
        base = super(PagesAPIEndpoint, self).get_object()
        return base.specific

class ImagesAPIEndpoint(BaseAPIEndpoint):
    base_serializer_class = ImageSerializer
    filter_backends = [FieldsFilter, OrderingFilter, SearchFilter]
    extra_api_fields = ['title', 'tags', 'width', 'height']
    name = 'images'
    model = get_image_model()

class DocumentsAPIEndpoint(BaseAPIEndpoint):
    base_serializer_class = DocumentSerializer
    filter_backends = [FieldsFilter, OrderingFilter, SearchFilter]
    extra_api_fields = ['title', 'tags']
    name = 'documents'
    model = get_document_model()
<commit_msg>Fix API v1 tests<commit_after>from __future__ import absolute_import, unicode_literals

import warnings
from collections import OrderedDict

from django.conf.urls import url
from django.core.urlresolvers import reverse
from django.http import Http404
from rest_framework import status
from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer
from rest_framework.response import Response
from rest_framework.viewsets import GenericViewSet

from wagtail.api import APIField
from wagtail.wagtailcore.models import Page
from wagtail.wagtailcore.utils import resolve_model_string
from wagtail.wagtaildocs.models import get_document_model
from wagtail.wagtailimages import get_image_model

from .filters import ChildOfFilter, DescendantOfFilter, FieldsFilter, OrderingFilter, SearchFilter
from .pagination import WagtailPagination
from .serializers import (
    BaseSerializer, DocumentSerializer, ImageSerializer, PageSerializer, get_serializer_class)
from .utils import BadRequestError

class BaseAPIEndpoint(GenericViewSet):
    renderer_classes = [JSONRenderer, BrowsableAPIRenderer]
    pagination_class = WagtailPagination
    base_serializer_class = BaseSerializer
    filter_backends = []
    model = None  # Set on subclass

    known_query_parameters = frozenset([
        'limit',
        'offset',
        'fields',
        'order',
        'search',

        # Used by jQuery for cache-busting. See #1671
        '_',

        # Required by BrowsableAPIRenderer
        'format',
    ])
    extra_api_fields = []
    name = None  # Set on subclass.

    def get_queryset(self):
        return self.model.objects.all().order_by('id')

    def listing_view(self, request):
        queryset = self.get_queryset()
        self.check_query_parameters(queryset)
        queryset = self.filter_queryset(queryset)
        queryset = self.paginate_queryset(queryset)
        serializer = self.get_serializer(queryset, many=True)
        return self.get_paginated_response(serializer.data)

    def detail_view(self, request, pk):
        instance = self.get_object()
        serializer = self.get_serializer(instance)
        return Response(serializer.data)

    def handle_exception(self, exc):
        if isinstance(exc, Http404):
            data = {'message': str(exc)}
            return Response(data, status=status.HTTP_404_NOT_FOUND)
        elif isinstance(exc, BadRequestError):
            data = {'message': str(exc)}
            return Response(data, status=status.HTTP_400_BAD_REQUEST)
        return super(BaseAPIEndpoint, self).handle_exception(exc)

    def get_api_fields(self, model):
        """
        This returns a list of field names that are allowed to
        be used in the API (excluding the id field).
        """
        api_fields = self.extra_api_fields[:]

        if hasattr(model, 'api_fields'):
            api_fields.extend(model.api_fields)

        # Remove any new-style API field configs (only supported in v2)
        def convert_api_fields(fields):
            for field in fields:
                if isinstance(field, APIField):
                    warnings.warn(
                        "class-based api_fields are not supported by the v1 API module. "
                        "Please update the .api_fields attribute of {}.{} or update to the "
                        "v2 API.".format(model._meta.app_label, model.__name__)
                    )

                    # Ignore fields with custom serializers
                    if field.serializer is None:
                        yield field.name
                else:
                    yield field

        return list(convert_api_fields(api_fields))

    def check_query_parameters(self, queryset):
        """
        Ensure that only valid query paramters are included in the URL.
        """
        query_parameters = set(self.request.GET.keys())

        # All query paramters must be either a field or an operation
        allowed_parameters = set(self.get_api_fields(queryset.model))
        allowed_parameters = allowed_parameters.union(self.known_query_parameters)
        allowed_parameters.add('id')
        unknown_parameters = query_parameters - allowed_parameters
        if unknown_parameters:
            raise BadRequestError(
                "query parameter is not an operation or a recognised field: %s"
                % ', '.join(sorted(unknown_parameters))
            )

    def get_serializer_class(self):
        request = self.request

        # Get model
        if self.action == 'listing_view':
            model = self.get_queryset().model
        else:
            model = type(self.get_object())

        # Get all available fields
        all_fields = self.get_api_fields(model)
        # Removes any duplicates in case the developer put "title" in api_fields
        all_fields = list(OrderedDict.fromkeys(all_fields))

        if self.action == 'listing_view':
            # Listing views just show the title field and any other allowed field the user specified
            if 'fields' in request.GET:
                fields = set(request.GET['fields'].split(','))
            else:
                fields = {'title'}

            unknown_fields = fields - set(all_fields)

            if unknown_fields:
                raise BadRequestError("unknown fields: %s" % ', '.join(sorted(unknown_fields)))

            # Reorder fields so it matches the order of all_fields
            fields = [field for field in all_fields if field in fields]
        else:
            # Detail views show all fields all the time
            fields = all_fields

        # Always show id and meta first
        fields = ['id', 'meta'] + fields

        # If showing details, add the parent field
        if isinstance(self, PagesAPIEndpoint) and self.get_serializer_context().get('show_details', False):
            fields.insert(2, 'parent')

        return get_serializer_class(model, fields, base=self.base_serializer_class)

    def get_serializer_context(self):
        """
        The serialization context differs between listing and detail views.
        """
        context = {
            'request': self.request,
            'view': self,
            'router': self.request.wagtailapi_router
        }

        if self.action == 'detail_view':
            context['show_details'] = True

        return context

    def get_renderer_context(self):
        context = super(BaseAPIEndpoint, self).get_renderer_context()
        context['indent'] = 4
        return context

    @classmethod
    def get_urlpatterns(cls):
        """
        This returns a list of URL patterns for the endpoint
        """
        return [
            url(r'^$', cls.as_view({'get': 'listing_view'}), name='listing'),
            url(r'^(?P<pk>\d+)/$', cls.as_view({'get': 'detail_view'}), name='detail'),
        ]

    @classmethod
    def get_object_detail_urlpath(cls, model, pk, namespace=''):
        if namespace:
            url_name = namespace + ':detail'
        else:
            url_name = 'detail'

        return reverse(url_name, args=(pk, ))

class PagesAPIEndpoint(BaseAPIEndpoint):
    base_serializer_class = PageSerializer
    filter_backends = [
        FieldsFilter,
        ChildOfFilter,
        DescendantOfFilter,
        OrderingFilter,
        SearchFilter
    ]
    known_query_parameters = BaseAPIEndpoint.known_query_parameters.union([
        'type',
        'child_of',
        'descendant_of',
    ])
    extra_api_fields = ['title']
    name = 'pages'
    model = Page

    def get_queryset(self):
        request = self.request

        # Allow pages to be filtered to a specific type
        if 'type' not in request.GET:
            model = Page
        else:
            model_name = request.GET['type']
            try:
                model = resolve_model_string(model_name)
            except LookupError:
                raise BadRequestError("type doesn't exist")
            if not issubclass(model, Page):
                raise BadRequestError("type doesn't exist")

        # Get live pages that are not in a private section
        queryset = model.objects.public().live()

        # Filter by site
        queryset = queryset.descendant_of(request.site.root_page, inclusive=True)

        return queryset

    def get_object(self):
        base = super(PagesAPIEndpoint, self).get_object()
        return base.specific

class ImagesAPIEndpoint(BaseAPIEndpoint):
    base_serializer_class = ImageSerializer
    filter_backends = [FieldsFilter, OrderingFilter, SearchFilter]
    extra_api_fields = ['title', 'tags', 'width', 'height']
    name = 'images'
    model = get_image_model()

class DocumentsAPIEndpoint(BaseAPIEndpoint):
    base_serializer_class = DocumentSerializer
    filter_backends = [FieldsFilter, OrderingFilter, SearchFilter]
    extra_api_fields = ['title', 'tags']
    name = 'documents'
    model = get_document_model()
<|endoftext|>


<commit_before>
import datetime
import sys
import unittest

from .. import Limit, RawLimit

if sys.version_info > (3, 0):
    import unittest.mock as mock
else:
    import mock

class LimitTestCase(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls._initial_date = datetime.datetime(2013, 4, 30, 1, 1, 1)
        cls._changed_date = datetime.datetime(2013, 6, 13, 1, 1, 1)

    def test_initial_start_time_correct(self):
        lim = Limit()
        self.assertIsNone(lim.start_time)

    def test_initial_duration_correct(self):
        lim = Limit()
        self.assertEqual(0, lim.duration)

    def test_initial_count_correct(self):
        lim = Limit()
        self.assertEqual(0, lim.count)

    def test_initial_limit_correct(self):
        lim = Limit()
        self.assertEqual(0, lim.limit)

    def test_set_raw_limit_first_set(self):
        lim = Limit()

        raw = RawLimit(50, 100, 10)

        now = datetime.datetime.now()
        lim.set_raw_limit(raw)

        self.assertAlmostEqual(now, lim.start_time)

        self.assertEqual(50, lim.count)
        self.assertEqual(100, lim.limit)
        self.assertEqual(10, lim.duration)

    @mock.patch(Limit.__module__ + '.datetime.datetime')
    def test_limit_duration_changed(self, mock_datetime):
        mock_datetime.now.return_value = LimitTestCase._initial_date

        lim = Limit()

        initial = RawLimit(50, 100, 10)
        changed = RawLimit(50, 100, 20)

        lim.set_raw_limit(initial)

        initial_start_time = lim.start_time
        mock_datetime.now.return_value = LimitTestCase._changed_date

        lim.set_raw_limit(changed)

        self.assertNotEqual(initial_start_time, lim.start_time)

        self.assertEqual(20, lim.duration)

    @mock.patch(Limit.__module__ + '.datetime.datetime')
    def test_resets_when_count_is_1(self, mock_datetime):
        mock_datetime.now.return_value = LimitTestCase._initial_date

        lim = Limit()

        initial = RawLimit(50, 100, 10)
        changed = RawLimit(1, 100, 10)

        lim.set_raw_limit(initial)

        initial_start_time = lim.start_time
        mock_datetime.now.return_value = LimitTestCase._changed_date

        lim.set_raw_limit(changed)

        self.assertNotEqual(initial_start_time, lim.start_time)

        self.assertEqual(1, lim.count)

    @mock.patch(Limit.__module__ + '.datetime.datetime')
    def test_start_time_stays_constant(self, mock_datetime):
        mock_datetime.now.return_value = LimitTestCase._initial_date

        lim = Limit()

        initial = RawLimit(1, 100, 10)

        lim.set_raw_limit(initial)

        mock_datetime.now.return_value = LimitTestCase._changed_date

        for count in range(2, 200):
            changed = RawLimit(count, 100, 10)

            lim.set_raw_limit(changed)

            self.assertEqual(LimitTestCase._initial_date, lim.start_time)

            self.assertEqual(count, lim.count)
            self.assertEqual(100, lim.limit)
            self.assertEqual(10, lim.duration)

    def test_setting_lower_count(self):
        lim = Limit()

        initial = RawLimit(50, 100, 10)
        changed = RawLimit(10, 100, 10)

        lim.set_raw_limit(initial)
        lim.set_raw_limit(changed)

        self.assertEqual(50, lim.count)

    def test_wait_time_under_limit(self):
        lim = Limit()

        lim.set_raw_limit(RawLimit(50, 100, 10))

        until = lim.wait_until()

        self.assertEqual(datetime.datetime(datetime.MINYEAR, 1, 1), until)

    @mock.patch(Limit.__module__ + '.datetime.datetime')
    def test_wait_time_over_limit(self, mock_datetime):
        initial_date = LimitTestCase._initial_date
        limit_duration = 200

        mock_datetime.now.return_value = initial_date

        lim = Limit()

        lim.set_raw_limit(RawLimit(10, 10, limit_duration))

        expected_time = initial_date + datetime.timedelta(seconds=limit_duration)
        actual_time = lim.wait_until()

        self.assertEqual(expected_time, actual_time)
<commit_msg>fixed slightly messed up unit test<commit_after>
import datetime
import sys
import unittest

from .. import Limit, RawLimit

if sys.version_info > (3, 0):
    import unittest.mock as mock
else:
    import mock

class LimitTestCase(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls._initial_date = datetime.datetime(2013, 4, 30, 1, 1, 1)
        cls._changed_date = datetime.datetime(2013, 6, 13, 1, 1, 1)

    def test_initial_start_time_correct(self):
        lim = Limit()
        self.assertIsNone(lim.start_time)

    def test_initial_duration_correct(self):
        lim = Limit()
        self.assertEqual(0, lim.duration)

    def test_initial_count_correct(self):
        lim = Limit()
        self.assertEqual(0, lim.count)

    def test_initial_limit_correct(self):
        lim = Limit()
        self.assertEqual(0, lim.limit)

    @mock.patch(Limit.__module__ + '.datetime.datetime')
    def test_set_raw_limit_first_set(self, mock_datetime):
        mock_datetime.now.return_value = LimitTestCase._initial_date

        lim = Limit()

        raw = RawLimit(50, 100, 10)

        lim.set_raw_limit(raw)

        self.assertEqual(LimitTestCase._initial_date, lim.start_time)

        self.assertEqual(50, lim.count)
        self.assertEqual(100, lim.limit)
        self.assertEqual(10, lim.duration)

    @mock.patch(Limit.__module__ + '.datetime.datetime')
    def test_limit_duration_changed(self, mock_datetime):
        mock_datetime.now.return_value = LimitTestCase._initial_date

        lim = Limit()

        initial = RawLimit(50, 100, 10)
        changed = RawLimit(50, 100, 20)

        lim.set_raw_limit(initial)

        initial_start_time = lim.start_time
        mock_datetime.now.return_value = LimitTestCase._changed_date

        lim.set_raw_limit(changed)

        self.assertNotEqual(initial_start_time, lim.start_time)

        self.assertEqual(20, lim.duration)

    @mock.patch(Limit.__module__ + '.datetime.datetime')
    def test_resets_when_count_is_1(self, mock_datetime):
        mock_datetime.now.return_value = LimitTestCase._initial_date

        lim = Limit()

        initial = RawLimit(50, 100, 10)
        changed = RawLimit(1, 100, 10)

        lim.set_raw_limit(initial)

        initial_start_time = lim.start_time
        mock_datetime.now.return_value = LimitTestCase._changed_date

        lim.set_raw_limit(changed)

        self.assertNotEqual(initial_start_time, lim.start_time)

        self.assertEqual(1, lim.count)

    @mock.patch(Limit.__module__ + '.datetime.datetime')
    def test_start_time_stays_constant(self, mock_datetime):
        mock_datetime.now.return_value = LimitTestCase._initial_date

        lim = Limit()

        initial = RawLimit(1, 100, 10)

        lim.set_raw_limit(initial)

        mock_datetime.now.return_value = LimitTestCase._changed_date

        for count in range(2, 200):
            changed = RawLimit(count, 100, 10)

            lim.set_raw_limit(changed)

            self.assertEqual(LimitTestCase._initial_date, lim.start_time)

            self.assertEqual(count, lim.count)
            self.assertEqual(100, lim.limit)
            self.assertEqual(10, lim.duration)

    def test_setting_lower_count(self):
        lim = Limit()

        initial = RawLimit(50, 100, 10)
        changed = RawLimit(10, 100, 10)

        lim.set_raw_limit(initial)
        lim.set_raw_limit(changed)

        self.assertEqual(50, lim.count)

    def test_wait_time_under_limit(self):
        lim = Limit()

        lim.set_raw_limit(RawLimit(50, 100, 10))

        until = lim.wait_until()

        self.assertEqual(datetime.datetime(datetime.MINYEAR, 1, 1), until)

    @mock.patch(Limit.__module__ + '.datetime.datetime')
    def test_wait_time_over_limit(self, mock_datetime):
        initial_date = LimitTestCase._initial_date
        limit_duration = 200

        mock_datetime.now.return_value = initial_date

        lim = Limit()

        lim.set_raw_limit(RawLimit(10, 10, limit_duration))

        expected_time = initial_date + datetime.timedelta(seconds=limit_duration)
        actual_time = lim.wait_until()

        self.assertEqual(expected_time, actual_time)
<|endoftext|>


<commit_before>from datetime import date
import tornado.escape
import tornado.ioloop
import tornado.web
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import json
import urlparse
import random
import string
import itertools
from datetime import datetime
import requests
import os

def get_field(l, field):
    results = []
    for d in l:
        if field in d:
            results.append(d[field])
    return results

def handle_query(payload, run=True):
    if run:
        r.db('public').table('queries').insert({'datetime': r.expr(datetime.now(r.make_timezone('-07:00'))), 'payload': payload}).run()
    dbobj = r.db('public').table(payload['table'])
    for key in payload.keys():
        if key in ['get', 'has_fields', 'doesnt_have_fields', 'match', 'has_string', 'match_any_field', 'has_string_in_any_field']:
            if type(payload[key]) is list and key in ['has_fields']:
                dbobj = getattr(dbobj, key)(*payload[key])
            elif type(payload[key]) is list and key in ['doesnt_have_fields']:
                # 'doesnt_have_fields',
                dbobj.filter(lambda row: row.pluck(*payload[key]).eq({}))
            elif key in ['doesnt_have_fields']:
                dbobj.filter(lambda row: row.pluck(payload[key]).eq({}))
            elif key == 'match':
                dbobj = getattr(dbobj, 'filter')(lambda case: case[payload['match']['field']].match(payload['match']['value']))
            elif key == 'has_string':
                dbobj = getattr(dbobj, 'filter')(lambda case: case[payload['has_string']['field']].match('.*?'+payload['has_string']['value']+'.*?'))
            elif key == 'match_any_field':
                dbobj = getattr(dbobj, 'filter')(lambda doc: doc.coerce_to('string').match(payload['match_any_field']))
            elif key == 'has_string_in_any_field':
                dbobj = getattr(dbobj, 'filter')(lambda doc: doc.coerce_to('string').match('(?i).*?'+payload['has_string_in_any_field']+'.*?'))
            else:
                dbobj = getattr(dbobj, key)(payload[key])

    if 'pluck' in payload:
        if type(payload['pluck']) is list:
            dbobj = getattr(dbobj, 'pluck')(*payload['pluck'])
        else:
            dbobj = getattr(dbobj, 'pluck')(payload['pluck'])

    if 'order_by' in payload:
        for o in payload['order_by']:
            if o['direction'] == 'desc':
                dbobj = getattr(dbobj, 'order_by')(index=r.desc(o['field']))
            else:
                dbobj = getattr(dbobj, 'order_by')(index=o['field'])

    if 'page' in payload:
        page = int(payload['page'])
        if 'rows_per_page' in payload:
            rows_per_page = int(payload['rows_per_page'])
        else:
            rows_per_page = int(10)
        dbobj = dbobj.slice((page - 1) * rows_per_page, page * rows_per_page)
        print list(dbobj.run())
    else:
        dbobj = dbobj.limit(1000)

    results_for_fields = list(dbobj.run())
    fields = [row.keys() for row in results_for_fields]
    fields = list(itertools.chain.from_iterable(fields))
    fields = sorted(list(set(fields)))

    #joined_data = list(r.db("public").table("police_internal_affairs_allegations").eq_join("organization_id", r.db("public").table("organizations")).map({"right":{
    #        "organization_id": r.row["right"]["id"],
    #        "organization_name": r.row["right"]["name"]
    #    }, "left": r.row["left"]}).zip().run())
    ids_for_other_tables = [field for field in fields if field.endswith('_id')]
    modified_joined_data = []
    special_names = {'person': 'people'}
    for field in ids_for_other_tables:

        #print field
        #print field[:-3]+'s'
        # get the fields of the table
        #results_for_fields = r.db('public').table(field[:-3]+'s').run()
        #right_fields = [row.keys() for row in results_for_fields]
        #right_fields = list(itertools.chain.from_iterable(right_fields))
        #right_fields = sorted(list(set(right_fields)))

        t = special_names[field[:-3]] if field[:-3] in special_names else field[:-3]+'s'
        if not t in r.db('public').table_list().run():
            continue
        print field, t
        dbobj = dbobj.eq_join(field, r.db("public").table(t))
        #d = {"left": r.row["left"], "right": {}}
        #for right_field in right_fields:
        #    d["right"][field[:-2]+right_field] = r.row["right"][right_field]
        #dbobj = dbobj.map(d)
        dbobj = dbobj.merge(  lambda row: {'left': row['right'].coerce_to('array').map(
                      lambda pair: [r.expr(field[:-2]) + pair[0], pair[1]]
                    ).coerce_to('object')}).without({'right': True}).zip()
        #dbobj = dbobj

    if 'filter' in payload:
        key = 'filter'
        dbobj = getattr(dbobj, key)(payload[key])
    print dbobj
    #for data in joined_data:
    #    d = {}
    #    d.update(data['left'])
    #    new_right_side = data['right'].items()
    #    new_right_side = dict([('organization_'+item[0], item[1]) for item in new_right_side])
    #    d.update(new_right_side)
    #    modified_joined_data.append(d)

    if 'action' in payload:
        if payload['action'] == 'get_fields':
            results = list(dbobj.run())
            fields = [row.keys() for row in results]
            fields = list(itertools.chain.from_iterable(fields))
            results = sorted(list(set(fields)))
        elif payload['action'] == 'count':
            results = {'count': dbobj.count().run()}
        elif payload['action'] == 'percentage_simple_matching':
            if 'match' in payload:
                base = r.db('public').table(payload['table']).filter(lambda case: case[payload['match']['field']].match(payload['match']['value']))

            else:
                base = r.db('public').table(payload['table']).filter(lambda case: case[payload['has_string']['field']].match('.*?'+payload['has_string']['value']+'.*?'))
            denominator = base.count().run()

            numerator = base.filter({payload['numerator']['field']: payload['numerator']['value']}).count().run()

            if denominator:
                percentage = float(numerator)/denominator
                percentage = "{:.0%}".format(percentage)+' (%s/%s)' % (numerator, denominator)
            else:
                percentage = 'Error: No denominator'
            results = {'numerator': numerator, 'denominator': denominator, 'percentage': percentage}
        elif payload['action'] == 'get_list':
            results = [item[payload['field']] for item in list(r.db('public').table(payload['table']).pluck(payload['field']).run())]
        elif payload['action'] == 'get_set':
            results = list(set([item[payload['field']] for item in list(r.db('public').table(payload['table']).pluck(payload['field']).run())]))
        elif payload['action'] == 'do_basic_mapping':
            dbobj = getattr(dbobj, 'pluck')(payload['field_for_key'], payload['field_for_value'])
            items = dbobj.run()
            d = {}
            for item in items:
                if type(item[payload['field_for_key']]) is list:
                    for k in item[payload['field_for_key']]:
                        if not k in d:
                            d[k] = [item[payload['field_for_value']]]
                        else:
                            d[k].append(item[payload['field_for_value']])
                else:
                    if not item[payload['field_for_key']] in d:
                        d[item[payload['field_for_key']]] = [item[payload['field_for_value']]]
                    else:
                        d[item[payload['field_for_key']]].append(item[payload['field_for_value']])
            results = d
        elif payload['action'] == 'do_row_mapping':
            print 'doing ', payload['action']
            items = list(dbobj.run())
            table_fields = [row.keys() for row in items]
            table_fields = list(set(list(itertools.chain.from_iterable(table_fields))))
            d = {}
            for item in items:
                if type(item[payload['field_for_key']]) is list:
                    for k in item[payload['field_for_key']]:
                        if not k in d:
                            d[k] = [item]
                        else:
                            d[k].append(item)
                else:
                    if not item[payload['field_for_key']] in d:
                        d[item[payload['field_for_key']]] = [item]
                    else:
                        d[item[payload['field_for_key']]].append(item)
            results = {'data': d, 'table_fields': table_fields, 'keys': d.keys()}
            return results
    else:
        if not run:
            return dbobj
        # if 'pluck' in payload:
        #if type(payload['pluck']) is list:
        #    dbobj = getattr(dbobj, 'pluck')(*payload['pluck'])
        results = {}
        results['table'] = r.db('public').table('tables').get(payload['table']).run()
        if 'default_order_by' in results['table']:
            dbobj = getattr(dbobj, 'order_by')(r.desc(results['table']['default_order_by']['field']))
        special_names_reversed = {value: key for key, value in special_names.items()}
        t = payload['table']
        t = special_names_reversed[t] if t in special_names_reversed else t[:-1]
        t = t + '_id'
        results['data'] = list(dbobj.run(time_format="raw"))
        if 'linked_tables' in results['table']:

            for linked_table in results['table']['linked_tables']:
                linked_table_data = list(r.db('public').table(linked_table).run(time_format="raw"))
                for i, row in enumerate(results['data']):
                    print i
                    row[linked_table] = [item for item in linked_table_data if item.get(t) == row['id']]
                    # consider using group e.g. r.db('public').table('police_internal_affairs_allegations').group('person_id').run()
                    #row[linked_table] = list(r.db('public').table(linked_table).filter({t: row['id']}).run(time_format="raw"))

        results['fields'] = [row.keys() for row in results['data']]
        results['fields'] = list(set(list(itertools.chain.from_iterable(results['fields']))))
        results['number_of_rows'] = len(results['data'])
        results['percentages'] = []
        results['group_counts'] = []
        for field in results['fields']:
            if not field.endswith('_id'):
                continue
            t = special_names[field[:-3]] if field[:-3] in special_names else field[:-3]+'s'
            if not t in r.db('public').table_list().run():
                continue
            g = r.db('public').table(t).group('id').run()

            results['group_counts'].append([t, [[g[item[0]], item[1]] for item in list(sorted(dbobj.group(field).count().run().items(), key=lambda x:x[1], reverse=True))][:10]])
        results['group_counts'].sort(key=lambda x:x[0])

        likely_boolean_fields = [field for field in results['fields'] if field.startswith('is_')]
        # remove if the field in a filter
        if 'filter' in payload:
            for field in payload['filter'].keys():
                if field in likely_boolean_fields:
                    likely_boolean_fields.remove(field)
        for field in likely_boolean_fields:
            items_in_field = get_field(results['data'], field)
            numerator = items_in_field.count(True)
            denominator = len(items_in_field)
            percentage = float(numerator)/denominator
            percentage = "{:.0%}".format(percentage)+' (%s/%s)' % (numerator, denominator)
            row_name = payload['table'].replace('_', ' ')
            if 'has_string_in_any_field' in payload:
                sentence = '%s of %s mentioning "%s" are %s.' % (percentage, row_name, payload['has_string_in_any_field'], field[3:])
            else:
                sentence = '%s of %s are %s.' % (percentage, row_name, field[3:])
            results['percentages'].append({'field': field, 'value': True, 'percentage': percentage, 'sentence': sentence})
        results['payload'] = payload
        results['name_for_rows'] = payload['table'].split('_')[-1]
        results['field_selectors'] = []
        for field in results['fields']:
            if field == 'id':
                continue
            elif field.startswith('is_'):
                results['field_selectors'].append({'selector': 'checkbox', 'name': field, 'display_name': field[3:].capitalize()})
            else:
                items = sorted(list(set([row[field] if isinstance(row[field], basestring) else '' for row in results['data'] if field in row and len(unicode(row.get(field)).encode('utf-8')) < 50])))
                if len(items) < 100 and not items == ['']:
                    results['field_selectors'].append({'selector': 'dropdown', 'name': field, 'display_name': field.replace('_', ' ').capitalize(), 'items': items})
        if run:
            return results
        return dbobj
        <commit_msg>auto commit<commit_after>from datetime import date
import tornado.escape
import tornado.ioloop
import tornado.web
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import json
import urlparse
import random
import string
import itertools
from datetime import datetime
import requests
import os

def get_field(l, field):
    results = []
    for d in l:
        if field in d:
            results.append(d[field])
    return results

def handle_query(payload, run=True):
    if run:
        r.db('public').table('queries').insert({'datetime': r.expr(datetime.now(r.make_timezone('-07:00'))), 'payload': payload}).run()
    dbobj = r.db('public').table(payload['table'])
    for key in payload.keys():
        if key in ['get', 'has_fields', 'doesnt_have_fields', 'match', 'has_string', 'match_any_field', 'has_string_in_any_field']:
            if type(payload[key]) is list and key in ['has_fields']:
                dbobj = getattr(dbobj, key)(*payload[key])
            elif type(payload[key]) is list and key in ['doesnt_have_fields']:
                # 'doesnt_have_fields',
                dbobj.filter(lambda row: row.pluck(*payload[key]).eq({}))
            elif key in ['doesnt_have_fields']:
                dbobj.filter(lambda row: row.pluck(payload[key]).eq({}))
            elif key == 'match':
                dbobj = getattr(dbobj, 'filter')(lambda case: case[payload['match']['field']].match(payload['match']['value']))
            elif key == 'has_string':
                dbobj = getattr(dbobj, 'filter')(lambda case: case[payload['has_string']['field']].match('.*?'+payload['has_string']['value']+'.*?'))
            elif key == 'match_any_field':
                dbobj = getattr(dbobj, 'filter')(lambda doc: doc.coerce_to('string').match(payload['match_any_field']))
            elif key == 'has_string_in_any_field':
                dbobj = getattr(dbobj, 'filter')(lambda doc: doc.coerce_to('string').match('(?i).*?'+payload['has_string_in_any_field']+'.*?'))
            else:
                dbobj = getattr(dbobj, key)(payload[key])

    if 'pluck' in payload:
        if type(payload['pluck']) is list:
            dbobj = getattr(dbobj, 'pluck')(*payload['pluck'])
        else:
            dbobj = getattr(dbobj, 'pluck')(payload['pluck'])

    if 'order_by' in payload:
        for o in payload['order_by']:
            if o['direction'] == 'desc':
                dbobj = getattr(dbobj, 'order_by')(index=r.desc(o['field']))
            else:
                dbobj = getattr(dbobj, 'order_by')(index=o['field'])
    rows_count = dbob
    if 'page' in payload:
        page = int(payload['page'])
        if 'rows_per_page' in payload:
            rows_per_page = int(payload['rows_per_page'])
        else:
            rows_per_page = int(10)
        dbobj = dbobj.slice((page - 1) * rows_per_page, page * rows_per_page)
        print list(dbobj.run())
    else:
        dbobj = dbobj.limit(1000)

    results_for_fields = list(dbobj.run())
    fields = [row.keys() for row in results_for_fields]
    fields = list(itertools.chain.from_iterable(fields))
    fields = sorted(list(set(fields)))

    #joined_data = list(r.db("public").table("police_internal_affairs_allegations").eq_join("organization_id", r.db("public").table("organizations")).map({"right":{
    #        "organization_id": r.row["right"]["id"],
    #        "organization_name": r.row["right"]["name"]
    #    }, "left": r.row["left"]}).zip().run())
    ids_for_other_tables = [field for field in fields if field.endswith('_id')]
    modified_joined_data = []
    special_names = {'person': 'people'}
    for field in ids_for_other_tables:

        #print field
        #print field[:-3]+'s'
        # get the fields of the table
        #results_for_fields = r.db('public').table(field[:-3]+'s').run()
        #right_fields = [row.keys() for row in results_for_fields]
        #right_fields = list(itertools.chain.from_iterable(right_fields))
        #right_fields = sorted(list(set(right_fields)))

        t = special_names[field[:-3]] if field[:-3] in special_names else field[:-3]+'s'
        if not t in r.db('public').table_list().run():
            continue
        print field, t
        dbobj = dbobj.eq_join(field, r.db("public").table(t))
        #d = {"left": r.row["left"], "right": {}}
        #for right_field in right_fields:
        #    d["right"][field[:-2]+right_field] = r.row["right"][right_field]
        #dbobj = dbobj.map(d)
        dbobj = dbobj.merge(  lambda row: {'left': row['right'].coerce_to('array').map(
                      lambda pair: [r.expr(field[:-2]) + pair[0], pair[1]]
                    ).coerce_to('object')}).without({'right': True}).zip()
        #dbobj = dbobj

    if 'filter' in payload:
        key = 'filter'
        dbobj = getattr(dbobj, key)(payload[key])
    print dbobj
    #for data in joined_data:
    #    d = {}
    #    d.update(data['left'])
    #    new_right_side = data['right'].items()
    #    new_right_side = dict([('organization_'+item[0], item[1]) for item in new_right_side])
    #    d.update(new_right_side)
    #    modified_joined_data.append(d)

    if 'action' in payload:
        if payload['action'] == 'get_fields':
            results = list(dbobj.run())
            fields = [row.keys() for row in results]
            fields = list(itertools.chain.from_iterable(fields))
            results = sorted(list(set(fields)))
        elif payload['action'] == 'count':
            results = {'count': dbobj.count().run()}
        elif payload['action'] == 'percentage_simple_matching':
            if 'match' in payload:
                base = r.db('public').table(payload['table']).filter(lambda case: case[payload['match']['field']].match(payload['match']['value']))

            else:
                base = r.db('public').table(payload['table']).filter(lambda case: case[payload['has_string']['field']].match('.*?'+payload['has_string']['value']+'.*?'))
            denominator = base.count().run()

            numerator = base.filter({payload['numerator']['field']: payload['numerator']['value']}).count().run()

            if denominator:
                percentage = float(numerator)/denominator
                percentage = "{:.0%}".format(percentage)+' (%s/%s)' % (numerator, denominator)
            else:
                percentage = 'Error: No denominator'
            results = {'numerator': numerator, 'denominator': denominator, 'percentage': percentage}
        elif payload['action'] == 'get_list':
            results = [item[payload['field']] for item in list(r.db('public').table(payload['table']).pluck(payload['field']).run())]
        elif payload['action'] == 'get_set':
            results = list(set([item[payload['field']] for item in list(r.db('public').table(payload['table']).pluck(payload['field']).run())]))
        elif payload['action'] == 'do_basic_mapping':
            dbobj = getattr(dbobj, 'pluck')(payload['field_for_key'], payload['field_for_value'])
            items = dbobj.run()
            d = {}
            for item in items:
                if type(item[payload['field_for_key']]) is list:
                    for k in item[payload['field_for_key']]:
                        if not k in d:
                            d[k] = [item[payload['field_for_value']]]
                        else:
                            d[k].append(item[payload['field_for_value']])
                else:
                    if not item[payload['field_for_key']] in d:
                        d[item[payload['field_for_key']]] = [item[payload['field_for_value']]]
                    else:
                        d[item[payload['field_for_key']]].append(item[payload['field_for_value']])
            results = d
        elif payload['action'] == 'do_row_mapping':
            print 'doing ', payload['action']
            items = list(dbobj.run())
            table_fields = [row.keys() for row in items]
            table_fields = list(set(list(itertools.chain.from_iterable(table_fields))))
            d = {}
            for item in items:
                if type(item[payload['field_for_key']]) is list:
                    for k in item[payload['field_for_key']]:
                        if not k in d:
                            d[k] = [item]
                        else:
                            d[k].append(item)
                else:
                    if not item[payload['field_for_key']] in d:
                        d[item[payload['field_for_key']]] = [item]
                    else:
                        d[item[payload['field_for_key']]].append(item)
            results = {'data': d, 'table_fields': table_fields, 'keys': d.keys()}
            return results
    else:
        if not run:
            return dbobj
        # if 'pluck' in payload:
        #if type(payload['pluck']) is list:
        #    dbobj = getattr(dbobj, 'pluck')(*payload['pluck'])
        results = {}
        results['table'] = r.db('public').table('tables').get(payload['table']).run()
        if 'default_order_by' in results['table']:
            dbobj = getattr(dbobj, 'order_by')(r.desc(results['table']['default_order_by']['field']))
        special_names_reversed = {value: key for key, value in special_names.items()}
        t = payload['table']
        t = special_names_reversed[t] if t in special_names_reversed else t[:-1]
        t = t + '_id'
        results['data'] = list(dbobj.run(time_format="raw"))
        if 'linked_tables' in results['table']:

            for linked_table in results['table']['linked_tables']:
                linked_table_data = list(r.db('public').table(linked_table).run(time_format="raw"))
                for i, row in enumerate(results['data']):
                    print i
                    row[linked_table] = [item for item in linked_table_data if item.get(t) == row['id']]
                    # consider using group e.g. r.db('public').table('police_internal_affairs_allegations').group('person_id').run()
                    #row[linked_table] = list(r.db('public').table(linked_table).filter({t: row['id']}).run(time_format="raw"))

        results['fields'] = [row.keys() for row in results['data']]
        results['fields'] = list(set(list(itertools.chain.from_iterable(results['fields']))))
        results['number_of_rows'] = len(results['data'])
        results['percentages'] = []
        results['group_counts'] = []
        for field in results['fields']:
            if not field.endswith('_id'):
                continue
            t = special_names[field[:-3]] if field[:-3] in special_names else field[:-3]+'s'
            if not t in r.db('public').table_list().run():
                continue
            g = r.db('public').table(t).group('id').run()

            results['group_counts'].append([t, [[g[item[0]], item[1]] for item in list(sorted(dbobj.group(field).count().run().items(), key=lambda x:x[1], reverse=True))][:10]])
        results['group_counts'].sort(key=lambda x:x[0])

        likely_boolean_fields = [field for field in results['fields'] if field.startswith('is_')]
        # remove if the field in a filter
        if 'filter' in payload:
            for field in payload['filter'].keys():
                if field in likely_boolean_fields:
                    likely_boolean_fields.remove(field)
        for field in likely_boolean_fields:
            items_in_field = get_field(results['data'], field)
            numerator = items_in_field.count(True)
            denominator = len(items_in_field)
            percentage = float(numerator)/denominator
            percentage = "{:.0%}".format(percentage)+' (%s/%s)' % (numerator, denominator)
            row_name = payload['table'].replace('_', ' ')
            if 'has_string_in_any_field' in payload:
                sentence = '%s of %s mentioning "%s" are %s.' % (percentage, row_name, payload['has_string_in_any_field'], field[3:])
            else:
                sentence = '%s of %s are %s.' % (percentage, row_name, field[3:])
            results['percentages'].append({'field': field, 'value': True, 'percentage': percentage, 'sentence': sentence})
        results['payload'] = payload
        results['name_for_rows'] = payload['table'].split('_')[-1]
        results['field_selectors'] = []
        for field in results['fields']:
            if field == 'id':
                continue
            elif field.startswith('is_'):
                results['field_selectors'].append({'selector': 'checkbox', 'name': field, 'display_name': field[3:].capitalize()})
            else:
                items = sorted(list(set([row[field] if isinstance(row[field], basestring) else '' for row in results['data'] if field in row and len(unicode(row.get(field)).encode('utf-8')) < 50])))
                if len(items) < 100 and not items == ['']:
                    results['field_selectors'].append({'selector': 'dropdown', 'name': field, 'display_name': field.replace('_', ' ').capitalize(), 'items': items})
        if run:
            return results
        return dbobj
        <|endoftext|>