<commit_before>use std;
use std::path::Path;

use aes;
use aes::{Aes128, MODE};
use aes::BLOCK_SIZE;
use aes::unpad_inplace;

use byteorder::{ByteOrder, NativeEndian};

use set1::decrypt_single_xor;

use mersenne;
use mersenne::MersenneTwister;

use serialize::from_base64;
use serialize::from_base64_lines;

use unstable_features::all_bytes;

use xor::XOR;

use rand;
use rand::Rng;

use helper::ceil_div;

use errors::*;

fn encoder_3_17(key: &[u8]) -> Result<(Vec<u8>, Vec<u8>)> {
    let inputs = [
        "MDAwMDAwTm93IHRoYXQgdGhlIHBhcnR5IGlzIGp1bXBpbmc=",
        "MDAwMDAxV2l0aCB0aGUgYmFzcyBraWNrZWQgaW4gYW5kIHRoZSBWZWdhJ3MgYXJlIHB1bXBpbic=",
        "MDAwMDAyUXVpY2sgdG8gdGhlIHBvaW50LCB0byB0aGUgcG9pbnQsIG5vIGZha2luZw==",
        "MDAwMDAzQ29va2luZyBNQydzIGxpa2UgYSBwb3VuZCBvZiBiYWNvbg==",
        "MDAwMDA0QnVybmluZyAnZW0sIGlmIHlvdSBhaW4ndCBxdWljayBhbmQgbmltYmxl",
        "MDAwMDA1SSBnbyBjcmF6eSB3aGVuIEkgaGVhciBhIGN5bWJhbA==",
        "MDAwMDA2QW5kIGEgaGlnaCBoYXQgd2l0aCBhIHNvdXBlZCB1cCB0ZW1wbw==",
        "MDAwMDA3SSdtIG9uIGEgcm9sbCwgaXQncyB0aW1lIHRvIGdvIHNvbG8=",
        "MDAwMDA4b2xsaW4nIGluIG15IGZpdmUgcG9pbnQgb2g=",
        "MDAwMDA5aXRoIG15IHJhZy10b3AgZG93biBzbyBteSBoYWlyIGNhbiBibG93",
    ];
    let mut rng = rand::thread_rng();
    let input_index = rng.gen_range(0, 10);
    let input = from_base64(inputs[input_index]).unwrap();
    let iv = random_block();
    let ciphertext = input.encrypt(key, Some(&iv), MODE::CBC)?;
    Ok((ciphertext, iv))
}

fn random_block() -> Vec<u8> {
    let mut rng = rand::thread_rng();
    rng.gen_iter().take(BLOCK_SIZE).collect()
}

fn matasano3_17() -> Result<()> {
    let key = random_block();
    let oracle = |iv: &[u8], ciphertext: &[u8]| -> Result<bool> {
        match ciphertext.decrypt(&key, Some(iv), MODE::CBC) {
            Err(aes::Error(aes::ErrorKind::InvalidPadding, _)) => Ok(false),
            Err(e) => Err(e.into()),
            _ => Ok(true),
        }
    };

    let (ciphertext, iv) = encoder_3_17(&key)?;
    let mut cleartext = vec![0; ciphertext.len()];
    let mut prev = iv.clone();
    for (block_count, block) in ciphertext.chunks(BLOCK_SIZE).enumerate() {
        let block_offset = block_count * BLOCK_SIZE;
        for i in (0..BLOCK_SIZE).rev() {
            let padding = (BLOCK_SIZE - i) as u8;
            prev[i + 1..].xor_inplace(&[(padding - 1) ^ padding]);
            //Awaits replacement for range_inclusive
            for u in all_bytes() {
                prev[i] ^= u;
                //let foo: Result<bool>= oracle(&prev, block);
                if oracle(&prev, block)? && (i < BLOCK_SIZE - 1 || {
                    prev[i - 1] ^= 1;
                    let result = oracle(&prev, block)?;
                    prev[i - 1] ^= 1;
                    result
                }) {
                    cleartext[block_offset + i] = padding ^ u;
                    break;
                }
                prev[i] ^= u;
            }
        }
        prev = block.to_vec();
    }
    unpad_inplace(&mut cleartext, BLOCK_SIZE as u8)?;
    compare(cleartext, ciphertext.decrypt(&key, Some(&iv), MODE::CBC)?)
}

fn matasano3_18() -> Result<()> {
    let ciphertext =
        from_base64("L77na/nrFsKvynd6HzOoG7GHTLXsTVu9qvY/2syLXzhPweyyMTJULu/6/kXX0KSvoOLSFQ==")?;
    let cleartext = ciphertext.decrypt(b"YELLOW SUBMARINE", None, MODE::CTR)?;
    compare(
        b"Yo, VIP Let's kick it Ice, Ice, baby Ice, Ice, baby ".as_ref(),
        &cleartext,
    )
}

fn matasano3_19_20(input_file: &Path) -> Result<()> {
    let cleartexts = from_base64_lines(input_file)?;
    let secret_key = random_block();
    let ciphertexts = cleartexts
        .iter()
        .map(|c| {
            c.encrypt(&secret_key, None, MODE::CTR)
                .map_err(|err| err.into())
        })
        .collect::<Result<Vec<Vec<u8>>>>()?;
    let size = ciphertexts.iter().map(|c| c.len()).min().unwrap();
    let mut transposed_blocks: Vec<Vec<u8>> = (0..size)
        .map(|_| Vec::with_capacity(ciphertexts.len()))
        .collect();
    for ciphertext in ciphertexts {
        for (&u, bt) in ciphertext[..size].iter().zip(transposed_blocks.iter_mut()) {
            bt.push(u);
        }
    }
    let key = transposed_blocks
        .iter()
        .map(|b| decrypt_single_xor(b))
        .collect::<Vec<u8>>();
    //TODO: The first entry of the recovered key is wrong because the distribution of first letters
    //of sentences is very different from the overall distribution of letters in a text.
    compare(
        &vec![0; size].encrypt(&secret_key, None, MODE::CTR)?[1..],
        &key[1..],
    ) // encrypt or decrypt?
}

fn matasano3_21() -> Result<()> {
    let mt = MersenneTwister::initialize(1);
    compare(
        vec![
            1_791_095_845,
            4_282_876_139,
            3_093_770_124,
            4_005_303_368,
            491_263,
            550_290_313,
            1_298_508_491,
            4_290_846_341,
            630_311_759,
            1_013_994_432,
        ],
        mt.take(10).collect::<Vec<u32>>(),
    )
}

fn matasano3_22() -> Result<()> {
    let mut rng = rand::thread_rng();
    let seed: u16 = rng.gen();
    let mut mt = MersenneTwister::initialize(u32::from(seed));
    compare(
        Some(u32::from(seed)),
        mersenne::crack_seed_from_nth(mt.nth(0).unwrap(), 0, 0..(u32::from(std::u16::MAX))),
    )
}

fn matasano3_23() -> Result<()> {
    let mut rng = rand::thread_rng();
    let seed = rng.gen();
    let mut mt = MersenneTwister::initialize(seed);
    let mut state = [0; mersenne::STATE_SIZE];
    for entry in state.iter_mut() {
        *entry = mersenne::untemper(mt.next().unwrap());
    }
    let mut mt_clone = MersenneTwister::initialize_with_state(state);
    for _ in 0..mersenne::STATE_SIZE {
        compare(mt.next(), mt_clone.next())?;
    }
    Ok(())
}

fn matasano3_24() -> Result<()> {
    let mut rng = rand::thread_rng();
    let prefix_len: u8 = rng.gen();
    let cleartext = {
        let mut v: Vec<u8> = rng.gen_iter().take(prefix_len as usize).collect();
        v.extend_from_slice(&[b'A'; 14]);
        v
    };
    let seed: u16 = rng.gen();
    let mt = MersenneTwister::initialize(u32::from(seed));

    let mut ciphertext: Vec<u8> = Vec::new();
    let buffer = &mut [0u8; 4];

    for (c, m) in cleartext.chunks(4).zip(mt) {
        //Get underlying bytes from u32 value
        <NativeEndian as ByteOrder>::write_u32(buffer, m);
        ciphertext.append(&mut c.xor(buffer));
    }

    let (used_random_numbers_count, _) = ceil_div(ciphertext.len(), 4);
    let index = used_random_numbers_count - 2;
    // To recover the seed, we use the fact that the block ciphertext[4*index..4*(index+1)] was
    // encrypted with the index-th random number (counting as always from 0). As we also know the
    // plaintext at this position we can recover the random number.
    let random_number = <NativeEndian as ByteOrder>::read_u32(
        &ciphertext[4 * index..4 * (index + 1)].xor(&[b'A'; 4]),
    );
    compare(
        Some(u32::from(seed)),
        mersenne::crack_seed_from_nth(random_number, index, 0..(u32::from(std::u16::MAX))),
    )
}

pub fn run() {
    println!("Set 3");
    run_exercise(matasano3_17, 17);
    run_exercise(matasano3_18, 18);
    run_exercise(|| matasano3_19_20(Path::new("data/19.txt")), 19);
    run_exercise(|| matasano3_19_20(Path::new("data/20.txt")), 20);
    run_exercise(matasano3_21, 21);
    run_exercise(matasano3_22, 22);
    run_exercise(matasano3_23, 23);
    run_exercise(matasano3_24, 24);
}
<commit_msg>Start to replace unidiomatic closures with structs<commit_after>use std;
use std::path::Path;

use aes;
use aes::{Aes128, MODE};
use aes::BLOCK_SIZE;
use aes::unpad_inplace;

use byteorder::{ByteOrder, NativeEndian};

use set1::decrypt_single_xor;

use mersenne;
use mersenne::MersenneTwister;

use serialize::from_base64;
use serialize::from_base64_lines;

use unstable_features::all_bytes;

use xor::XOR;

use rand;
use rand::Rng;

use helper::ceil_div;

use errors::*;

struct Server17 {
    key: Vec<u8>
}

impl Server17 {
    fn new() -> Self {
        Self { key: random_block() }
    }

    fn get_session_token(&self) -> Result<(Vec<u8>, Vec<u8>)> {
        let inputs = [
            "MDAwMDAwTm93IHRoYXQgdGhlIHBhcnR5IGlzIGp1bXBpbmc=",
            "MDAwMDAxV2l0aCB0aGUgYmFzcyBraWNrZWQgaW4gYW5kIHRoZSBWZWdhJ3MgYXJlIHB1bXBpbic=",
            "MDAwMDAyUXVpY2sgdG8gdGhlIHBvaW50LCB0byB0aGUgcG9pbnQsIG5vIGZha2luZw==",
            "MDAwMDAzQ29va2luZyBNQydzIGxpa2UgYSBwb3VuZCBvZiBiYWNvbg==",
            "MDAwMDA0QnVybmluZyAnZW0sIGlmIHlvdSBhaW4ndCBxdWljayBhbmQgbmltYmxl",
            "MDAwMDA1SSBnbyBjcmF6eSB3aGVuIEkgaGVhciBhIGN5bWJhbA==",
            "MDAwMDA2QW5kIGEgaGlnaCBoYXQgd2l0aCBhIHNvdXBlZCB1cCB0ZW1wbw==",
            "MDAwMDA3SSdtIG9uIGEgcm9sbCwgaXQncyB0aW1lIHRvIGdvIHNvbG8=",
            "MDAwMDA4b2xsaW4nIGluIG15IGZpdmUgcG9pbnQgb2g=",
            "MDAwMDA5aXRoIG15IHJhZy10b3AgZG93biBzbyBteSBoYWlyIGNhbiBibG93",
        ];
        let mut rng = rand::thread_rng();
        let input_index = rng.gen_range(0, 10);
        let input = from_base64(inputs[input_index]).unwrap();
        let iv = random_block();
        let ciphertext = input.encrypt(&self.key, Some(&iv), MODE::CBC)?;
        Ok((ciphertext, iv))
    }

    fn is_padding_valid(&self, iv: &[u8], ciphertext: &[u8]) -> Result<bool> {
        match ciphertext.decrypt(&self.key, Some(iv), MODE::CBC) {
            Err(aes::Error(aes::ErrorKind::InvalidPadding, _)) => Ok(false),
            Err(e) => Err(e.into()),
            _ => Ok(true),
        }
    }

    fn verify_solution(&self, cleartext: &[u8], iv: &[u8], ciphertext: &[u8]) -> Result<()> {
        compare(cleartext, &ciphertext.decrypt(&self.key, Some(&iv), MODE::CBC)?)
    }
}

fn random_block() -> Vec<u8> {
    let mut rng = rand::thread_rng();
    rng.gen_iter().take(BLOCK_SIZE).collect()
}

fn matasano3_17() -> Result<()> {
    let server = Server17::new();
    let (ciphertext, iv) = server.get_session_token()?;
    let mut cleartext = vec![0; ciphertext.len()];
    let mut prev = iv.clone();
    for (block_count, block) in ciphertext.chunks(BLOCK_SIZE).enumerate() {
        let block_offset = block_count * BLOCK_SIZE;
        for i in (0..BLOCK_SIZE).rev() {
            let padding = (BLOCK_SIZE - i) as u8;
            prev[i + 1..].xor_inplace(&[(padding - 1) ^ padding]);
            //Awaits replacement for range_inclusive
            for u in all_bytes() {
                prev[i] ^= u;
                if server.is_padding_valid(&prev, block)? && (i < BLOCK_SIZE - 1 || {
                    prev[i - 1] ^= 1;
                    let result = server.is_padding_valid(&prev, block)?;
                    prev[i - 1] ^= 1;
                    result
                }) {
                    cleartext[block_offset + i] = padding ^ u;
                    break;
                }
                prev[i] ^= u;
            }
        }
        prev = block.to_vec();
    }
    unpad_inplace(&mut cleartext, BLOCK_SIZE as u8)?;
    server.verify_solution(&cleartext, &ciphertext, &iv)
}

fn matasano3_18() -> Result<()> {
    let ciphertext =
        from_base64("L77na/nrFsKvynd6HzOoG7GHTLXsTVu9qvY/2syLXzhPweyyMTJULu/6/kXX0KSvoOLSFQ==")?;
    let cleartext = ciphertext.decrypt(b"YELLOW SUBMARINE", None, MODE::CTR)?;
    compare(
        b"Yo, VIP Let's kick it Ice, Ice, baby Ice, Ice, baby ".as_ref(),
        &cleartext,
    )
}

fn matasano3_19_20(input_file: &Path) -> Result<()> {
    let cleartexts = from_base64_lines(input_file)?;
    let secret_key = random_block();
    let ciphertexts = cleartexts
        .iter()
        .map(|c| {
            c.encrypt(&secret_key, None, MODE::CTR)
                .map_err(|err| err.into())
        })
        .collect::<Result<Vec<Vec<u8>>>>()?;
    let size = ciphertexts.iter().map(|c| c.len()).min().unwrap();
    let mut transposed_blocks: Vec<Vec<u8>> = (0..size)
        .map(|_| Vec::with_capacity(ciphertexts.len()))
        .collect();
    for ciphertext in ciphertexts {
        for (&u, bt) in ciphertext[..size].iter().zip(transposed_blocks.iter_mut()) {
            bt.push(u);
        }
    }
    let key = transposed_blocks
        .iter()
        .map(|b| decrypt_single_xor(b))
        .collect::<Vec<u8>>();
    //TODO: The first entry of the recovered key is wrong because the distribution of first letters
    //of sentences is very different from the overall distribution of letters in a text.
    compare(
        &vec![0; size].encrypt(&secret_key, None, MODE::CTR)?[1..],
        &key[1..],
    ) // encrypt or decrypt?
}

fn matasano3_21() -> Result<()> {
    let mt = MersenneTwister::initialize(1);
    compare(
        vec![
            1_791_095_845,
            4_282_876_139,
            3_093_770_124,
            4_005_303_368,
            491_263,
            550_290_313,
            1_298_508_491,
            4_290_846_341,
            630_311_759,
            1_013_994_432,
        ],
        mt.take(10).collect::<Vec<u32>>(),
    )
}

fn matasano3_22() -> Result<()> {
    let mut rng = rand::thread_rng();
    let seed: u16 = rng.gen();
    let mut mt = MersenneTwister::initialize(u32::from(seed));
    compare(
        Some(u32::from(seed)),
        mersenne::crack_seed_from_nth(mt.nth(0).unwrap(), 0, 0..(u32::from(std::u16::MAX))),
    )
}

fn matasano3_23() -> Result<()> {
    let mut rng = rand::thread_rng();
    let seed = rng.gen();
    let mut mt = MersenneTwister::initialize(seed);
    let mut state = [0; mersenne::STATE_SIZE];
    for entry in state.iter_mut() {
        *entry = mersenne::untemper(mt.next().unwrap());
    }
    let mut mt_clone = MersenneTwister::initialize_with_state(state);
    for _ in 0..mersenne::STATE_SIZE {
        compare(mt.next(), mt_clone.next())?;
    }
    Ok(())
}

fn matasano3_24() -> Result<()> {
    let mut rng = rand::thread_rng();
    let prefix_len: u8 = rng.gen();
    let cleartext = {
        let mut v: Vec<u8> = rng.gen_iter().take(prefix_len as usize).collect();
        v.extend_from_slice(&[b'A'; 14]);
        v
    };
    let seed: u16 = rng.gen();
    let mt = MersenneTwister::initialize(u32::from(seed));

    let mut ciphertext: Vec<u8> = Vec::new();
    let buffer = &mut [0u8; 4];

    for (c, m) in cleartext.chunks(4).zip(mt) {
        //Get underlying bytes from u32 value
        <NativeEndian as ByteOrder>::write_u32(buffer, m);
        ciphertext.append(&mut c.xor(buffer));
    }

    let (used_random_numbers_count, _) = ceil_div(ciphertext.len(), 4);
    let index = used_random_numbers_count - 2;
    // To recover the seed, we use the fact that the block ciphertext[4*index..4*(index+1)] was
    // encrypted with the index-th random number (counting as always from 0). As we also know the
    // plaintext at this position we can recover the random number.
    let random_number = <NativeEndian as ByteOrder>::read_u32(
        &ciphertext[4 * index..4 * (index + 1)].xor(&[b'A'; 4]),
    );
    compare(
        Some(u32::from(seed)),
        mersenne::crack_seed_from_nth(random_number, index, 0..(u32::from(std::u16::MAX))),
    )
}

pub fn run() {
    println!("Set 3");
    run_exercise(matasano3_17, 17);
    run_exercise(matasano3_18, 18);
    run_exercise(|| matasano3_19_20(Path::new("data/19.txt")), 19);
    run_exercise(|| matasano3_19_20(Path::new("data/20.txt")), 20);
    run_exercise(matasano3_21, 21);
    run_exercise(matasano3_22, 22);
    run_exercise(matasano3_23, 23);
    run_exercise(matasano3_24, 24);
}
<|endoftext|>


<commit_before>// This is a part of rust-encoding.
// Copyright (c) 2013, Kang Seonghoon.
// See README.md and LICENSE.txt for details.

//! Common codec implementation for single-byte encodings.

use std::str;
use util::StrCharIndex;
use types::*;

pub struct SingleByteEncoding {
    name: &'static str,
    index_forward: extern "Rust" fn(u8) -> u16,
    index_backward: extern "Rust" fn(u16) -> u8,
}

impl Clone for SingleByteEncoding {
    fn clone(&self) -> SingleByteEncoding {
        SingleByteEncoding { name: self.name,
                             index_forward: self.index_forward,
                             index_backward: self.index_backward }
    }
}

impl Encoding for SingleByteEncoding {
    pub fn name(&self) -> ~str { self.name.to_owned() }
    pub fn encoder(&self) -> ~Encoder { ~SingleByteEncoder { encoding: self.clone() } as ~Encoder }
    pub fn decoder(&self) -> ~Decoder { ~SingleByteDecoder { encoding: self.clone() } as ~Decoder }
    pub fn preferred_replacement_seq(&self) -> ~[u8] { ~[0x3f] /* "?" */ }
}

#[deriving(Clone)]
pub struct SingleByteEncoder {
    encoding: SingleByteEncoding,
}

impl Encoder for SingleByteEncoder {
    pub fn encoding(&self) -> ~Encoding { ~self.encoding.clone() as ~Encoding }

    pub fn feed<'r>(&mut self, input: &'r str) -> (~[u8],Option<EncoderError<'r>>) {
        let mut ret = ~[];
        let mut err = None;
        for input.index_iter().advance |((_,j), ch)| {
            if ch <= '\u007f' {
                ret.push(ch as u8);
            } else if ch <= '\uffff' {
                let index = (self.encoding.index_backward)(ch as u16);
                if index != 0xff {
                    ret.push((index + 0x80) as u8);
                } else {
                    err = Some(CodecError {
                        remaining: input.slice_from(j),
                        problem: str::from_char(ch),
                        cause: ~"unrepresentable character",
                    });
                    break;
                }
            }
        }
        (ret, err)
    }

    pub fn flush(~self) -> (~[u8],Option<EncoderError<'static>>) {
        (~[], None)
    }
}

#[deriving(Clone)]
pub struct SingleByteDecoder {
    encoding: SingleByteEncoding,
}

impl Decoder for SingleByteDecoder {
    pub fn encoding(&self) -> ~Encoding { ~self.encoding.clone() as ~Encoding }

    pub fn feed<'r>(&mut self, input: &'r [u8]) -> (~str,Option<DecoderError<'r>>) {
        let mut ret = ~"";
        let mut i = 0;
        let len = input.len();
        while i < len {
            if input[i] <= 0x7f {
                ret.push_char(input[i] as char);
            } else {
                let ch = (self.encoding.index_forward)(input[i] - 0x80);
                if ch != 0xffff {
                    ret.push_char(ch as char);
                } else {
                    return (ret, Some(CodecError {
                        remaining: input.slice(i+1, input.len()),
                        problem: ~[input[i]],
                        cause: ~"invalid sequence",
                    }));
                }
            }
            i += 1;
        }
        (ret, None)
    }

    pub fn flush(~self) -> (~str,Option<DecoderError<'static>>) {
        (~"", None)
    }
}

<commit_msg>Fix SingleByteEncoder for code points above U+FFFF.<commit_after>// This is a part of rust-encoding.
// Copyright (c) 2013, Kang Seonghoon.
// See README.md and LICENSE.txt for details.

//! Common codec implementation for single-byte encodings.

use std::str;
use util::StrCharIndex;
use types::*;

pub struct SingleByteEncoding {
    name: &'static str,
    index_forward: extern "Rust" fn(u8) -> u16,
    index_backward: extern "Rust" fn(u16) -> u8,
}

impl Clone for SingleByteEncoding {
    fn clone(&self) -> SingleByteEncoding {
        SingleByteEncoding { name: self.name,
                             index_forward: self.index_forward,
                             index_backward: self.index_backward }
    }
}

impl Encoding for SingleByteEncoding {
    pub fn name(&self) -> ~str { self.name.to_owned() }
    pub fn encoder(&self) -> ~Encoder { ~SingleByteEncoder { encoding: self.clone() } as ~Encoder }
    pub fn decoder(&self) -> ~Decoder { ~SingleByteDecoder { encoding: self.clone() } as ~Decoder }
    pub fn preferred_replacement_seq(&self) -> ~[u8] { ~[0x3f] /* "?" */ }
}

#[deriving(Clone)]
pub struct SingleByteEncoder {
    encoding: SingleByteEncoding,
}

impl Encoder for SingleByteEncoder {
    pub fn encoding(&self) -> ~Encoding { ~self.encoding.clone() as ~Encoding }

    pub fn feed<'r>(&mut self, input: &'r str) -> (~[u8],Option<EncoderError<'r>>) {
        let mut ret = ~[];
        let mut err = None;
        for input.index_iter().advance |((_,j), ch)| {
            if ch <= '\u007f' {
                ret.push(ch as u8);
                loop
            }
            if ch <= '\uffff' {
                let index = (self.encoding.index_backward)(ch as u16);
                if index != 0xff {
                    ret.push((index + 0x80) as u8);
                    loop
                }
            }
            err = Some(CodecError {
                remaining: input.slice_from(j),
                problem: str::from_char(ch),
                cause: ~"unrepresentable character",
            });
            break;
        }
        (ret, err)
    }

    pub fn flush(~self) -> (~[u8],Option<EncoderError<'static>>) {
        (~[], None)
    }
}

#[deriving(Clone)]
pub struct SingleByteDecoder {
    encoding: SingleByteEncoding,
}

impl Decoder for SingleByteDecoder {
    pub fn encoding(&self) -> ~Encoding { ~self.encoding.clone() as ~Encoding }

    pub fn feed<'r>(&mut self, input: &'r [u8]) -> (~str,Option<DecoderError<'r>>) {
        let mut ret = ~"";
        let mut i = 0;
        let len = input.len();
        while i < len {
            if input[i] <= 0x7f {
                ret.push_char(input[i] as char);
            } else {
                let ch = (self.encoding.index_forward)(input[i] - 0x80);
                if ch != 0xffff {
                    ret.push_char(ch as char);
                } else {
                    return (ret, Some(CodecError {
                        remaining: input.slice(i+1, input.len()),
                        problem: ~[input[i]],
                        cause: ~"invalid sequence",
                    }));
                }
            }
            i += 1;
        }
        (ret, None)
    }

    pub fn flush(~self) -> (~str,Option<DecoderError<'static>>) {
        (~"", None)
    }
}

#[test]
fn test_non_bmp() {
    use all;
    assert_eq!(all::ISO_8859_2.encode("A\uFFFFB", Replace).unwrap(), ~[0x41, 0x3F, 0x42]);
    assert_eq!(all::ISO_8859_2.encode("A\U00010000B", Replace).unwrap(), ~[0x41, 0x3F, 0x42]);
}
<|endoftext|>


<commit_before>use bot::LalafellBot;

use lalafell::bot::Bot;
use lalafell::commands::prelude::*;
use lalafell::commands::ChannelOrId;
use lalafell::error::*;

use discord::GetMessages;
use discord::model::{permissions, Message, PublicChannel};

use serde_json;

use std::sync::Arc;
use std::fs::{self, File};
use std::path::Path;

const USAGE: &'static str = "!archive <channel>";

pub struct ArchiveCommand {
  bot: Arc<LalafellBot>
}

impl ArchiveCommand {
  pub fn new(bot: Arc<LalafellBot>) -> ArchiveCommand {
    ArchiveCommand {
      bot: bot
    }
  }
}

impl HasBot for ArchiveCommand {
  fn bot(&self) -> &Bot {
    self.bot.as_ref()
  }
}

#[derive(Debug, Deserialize)]
pub struct Params {
  channel: ChannelOrId
}

impl HasParams for ArchiveCommand {
  type Params = Params;
}

impl<'a> PublicChannelCommand<'a> for ArchiveCommand {
  fn run(&self, message: &Message, server: &LiveServer, _: &PublicChannel, params: &[&str]) -> CommandResult<'a> {
    let params = self.params(USAGE, params)?;
    if !server.channels.iter().any(|c| c.id == *params.channel) {
      return Err("This command must be run in the server the channel is in.".into());
    }
    let can_manage_chans = server.permissions_for(*params.channel, message.author.id).contains(permissions::MANAGE_CHANNELS);
    if !can_manage_chans {
      return Err(ExternalCommandFailure::default()
        .message(|e: EmbedBuilder| e
          .title("Not enough permissions.")
          .description("You don't have enough permissions to use this command."))
        .wrap());
    }

    let archive_path = Path::new("./archives")
      .join(&server.id.to_string())
      .join(&params.channel.to_string())
      .with_extension("json");
    if let Some(parent) = archive_path.parent() {
      if !parent.exists() {
        fs::create_dir_all(&parent)
          .chain_err(|| "could not create archive directory")?;
      }
    }
    if archive_path.exists() {
      return Err("This channel is already archived.".into());
    }

    let mut messages = self.bot.discord.get_messages(*params.channel, GetMessages::MostRecent, Some(100))
      .chain_err(|| "could not download first set of messages")?;
    if messages.len() >= 100 {
      loop {
        let last_message_id = messages[messages.len() - 1].id;
        let mut next_batch = self.bot.discord.get_messages(*params.channel, GetMessages::Before(last_message_id), Some(100))
          .chain_err(|| "could not download more messages")?;
        if next_batch.is_empty() {
          break;
        }
        messages.append(&mut next_batch);
      }
    }

    let file = File::create(archive_path)
      .chain_err(|| "could not create archive file")?;
    serde_json::to_writer(file, &messages)
      .chain_err(|| "could not serialize messages")?;

    Ok(format!("Archived {} message{}.", messages.len(), if messages.len() == 1 { "" } else { "s" }).into())
  }
}
<commit_msg>feat: include more information in archive files<commit_after>use bot::LalafellBot;

use lalafell::bot::Bot;
use lalafell::commands::prelude::*;
use lalafell::commands::ChannelOrId;
use lalafell::error::*;

use discord::GetMessages;
use discord::model::{permissions, Message, PublicChannel, Member, Emoji, Role};

use serde_json;

use std::sync::Arc;
use std::fs::{self, File};
use std::path::Path;

const USAGE: &'static str = "!archive <channel>";

pub struct ArchiveCommand {
  bot: Arc<LalafellBot>
}

impl ArchiveCommand {
  pub fn new(bot: Arc<LalafellBot>) -> ArchiveCommand {
    ArchiveCommand {
      bot: bot
    }
  }
}

impl HasBot for ArchiveCommand {
  fn bot(&self) -> &Bot {
    self.bot.as_ref()
  }
}

#[derive(Debug, Deserialize)]
pub struct Params {
  channel: ChannelOrId
}

impl HasParams for ArchiveCommand {
  type Params = Params;
}

impl<'a> PublicChannelCommand<'a> for ArchiveCommand {
  fn run(&self, message: &Message, server: &LiveServer, channel: &PublicChannel, params: &[&str]) -> CommandResult<'a> {
    let params = self.params(USAGE, params)?;
    if !server.channels.iter().any(|c| c.id == *params.channel) {
      return Err("This command must be run in the server the channel is in.".into());
    }
    let can_manage_chans = server.permissions_for(*params.channel, message.author.id).contains(permissions::MANAGE_CHANNELS);
    if !can_manage_chans {
      return Err(ExternalCommandFailure::default()
        .message(|e: EmbedBuilder| e
          .title("Not enough permissions.")
          .description("You don't have enough permissions to use this command."))
        .wrap());
    }

    let archive_path = Path::new("./archives")
      .join(&server.id.to_string())
      .join(&params.channel.to_string())
      .with_extension("json");
    if let Some(parent) = archive_path.parent() {
      if !parent.exists() {
        fs::create_dir_all(&parent)
          .chain_err(|| "could not create archive directory")?;
      }
    }
    if archive_path.exists() {
      return Err("This channel is already archived.".into());
    }

    let mut messages = self.bot.discord.get_messages(*params.channel, GetMessages::MostRecent, Some(100))
      .chain_err(|| "could not download first set of messages")?;
    if messages.len() >= 100 {
      loop {
        let last_message_id = messages[messages.len() - 1].id;
        let mut next_batch = self.bot.discord.get_messages(*params.channel, GetMessages::Before(last_message_id), Some(100))
          .chain_err(|| "could not download more messages")?;
        if next_batch.is_empty() {
          break;
        }
        messages.append(&mut next_batch);
      }
    }

    let num_messages = messages.len();

    let archive = Archive {
      server: ArchiveServer {
        name: server.name.clone(),
        roles: server.roles.clone(),
        members: server.members.clone(),
        icon: server.icon.clone(),
        emojis: server.emojis.clone()
      },
      channel: ArchiveChannel {
        name: channel.name.clone(),
        topic: channel.topic.clone()
      },
      messages: messages
    };

    let file = File::create(archive_path)
      .chain_err(|| "could not create archive file")?;
    serde_json::to_writer(file, &archive)
      .chain_err(|| "could not serialize messages")?;

    Ok(format!("Archived {} message{}.", num_messages, if num_messages == 1 { "" } else { "s" }).into())
  }
}

#[derive(Debug, Serialize)]
struct Archive {
  server: ArchiveServer,
  channel: ArchiveChannel,
  messages: Vec<Message>
}

#[derive(Debug, Serialize)]
struct ArchiveServer {
  name: String,
  roles: Vec<Role>,
  members: Vec<Member>,
  icon: Option<String>,
  emojis: Vec<Emoji>
}

#[derive(Debug, Serialize)]
struct ArchiveChannel {
  name: String,
  topic: Option<String>
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2015 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

use any_pointer;
use MessageSize;
use capability::{CallContext, Request, ResultFuture, Server};

pub trait ResponseHook:Send + ::std::any::Any {
    fn get<'a>(&'a mut self) -> any_pointer::Reader<'a>;
}

pub trait RequestHook {
    fn message<'a>(&'a mut self) -> &'a mut ::message::Builder<::message::HeapAllocator>;
    fn send<'a>(self : Box<Self>) -> ResultFuture<any_pointer::Reader<'a>, any_pointer::Pipeline>;
}

pub trait ClientHook : Send + ::std::any::Any {
    fn copy(&self) -> Box<ClientHook+Send>;
    fn new_call(&self,
                interface_id : u64,
                method_id : u16,
                size_hint : Option<MessageSize>)
                -> Request<any_pointer::Builder, any_pointer::Reader, any_pointer::Pipeline>;
    fn call(&self, interface_id : u64, method_id : u16, context : Box<CallContextHook+Send>);

    // HACK
    fn get_descriptor(&self) -> Box<::std::any::Any>;
}

pub trait ServerHook : 'static {
    fn new_client(unused : Option<Self>, server : Box<Server+Send>) -> Client;
}

pub struct Client {
    pub hook : Box<ClientHook+Send>
}

impl Client {
    pub fn new(hook : Box<ClientHook+Send>) -> Client {
        Client { hook : hook }
    }

    pub fn new_call<Params, Results, Pipeline>(&self,
                                               interface_id : u64,
                                               method_id : u16,
                                               size_hint : Option<MessageSize>)
                                               -> Request<Params, Results, Pipeline> {
        let typeless = self.hook.new_call(interface_id, method_id, size_hint);
        Request { hook : typeless.hook, marker : ::std::marker::PhantomData }
    }
}

pub trait CallContextHook {
    fn get<'a>(&'a mut self) -> (any_pointer::Reader<'a>, any_pointer::Builder<'a>);
    fn fail(self : Box<Self>, message : String);
    fn done(self : Box<Self>);
}

// Where should this live?
pub fn internal_get_typed_context<Params, Results>(
    typeless : CallContext<any_pointer::Reader, any_pointer::Builder>)
    -> CallContext<Params, Results> {
    CallContext { hook : typeless.hook, marker : ::std::marker::PhantomData }
}

pub trait PipelineHook {
    fn copy(&self) -> Box<PipelineHook+Send>;
    fn get_pipelined_cap(&self, ops : Vec<PipelineOp>) -> Box<ClientHook+Send>;
}

#[derive(Clone, Copy)]
pub enum PipelineOp {
    Noop,
    GetPointerField(u16),
}

<commit_msg>Stop using archaic 'unused self' pattern.<commit_after>// Copyright (c) 2013-2015 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

use any_pointer;
use MessageSize;
use capability::{CallContext, Request, ResultFuture, Server};

pub trait ResponseHook:Send + ::std::any::Any {
    fn get<'a>(&'a mut self) -> any_pointer::Reader<'a>;
}

pub trait RequestHook {
    fn message<'a>(&'a mut self) -> &'a mut ::message::Builder<::message::HeapAllocator>;
    fn send<'a>(self : Box<Self>) -> ResultFuture<any_pointer::Reader<'a>, any_pointer::Pipeline>;
}

pub trait ClientHook : Send + ::std::any::Any {
    fn copy(&self) -> Box<ClientHook+Send>;
    fn new_call(&self,
                interface_id : u64,
                method_id : u16,
                size_hint : Option<MessageSize>)
                -> Request<any_pointer::Builder, any_pointer::Reader, any_pointer::Pipeline>;
    fn call(&self, interface_id : u64, method_id : u16, context : Box<CallContextHook+Send>);

    // HACK
    fn get_descriptor(&self) -> Box<::std::any::Any>;
}

pub trait ServerHook : 'static {
    fn new_client(server : Box<Server+Send>) -> Client;
}

pub struct Client {
    pub hook : Box<ClientHook+Send>
}

impl Client {
    pub fn new(hook : Box<ClientHook+Send>) -> Client {
        Client { hook : hook }
    }

    pub fn new_call<Params, Results, Pipeline>(&self,
                                               interface_id : u64,
                                               method_id : u16,
                                               size_hint : Option<MessageSize>)
                                               -> Request<Params, Results, Pipeline> {
        let typeless = self.hook.new_call(interface_id, method_id, size_hint);
        Request { hook : typeless.hook, marker : ::std::marker::PhantomData }
    }
}

pub trait CallContextHook {
    fn get<'a>(&'a mut self) -> (any_pointer::Reader<'a>, any_pointer::Builder<'a>);
    fn fail(self : Box<Self>, message : String);
    fn done(self : Box<Self>);
}

// Where should this live?
pub fn internal_get_typed_context<Params, Results>(
    typeless : CallContext<any_pointer::Reader, any_pointer::Builder>)
    -> CallContext<Params, Results> {
    CallContext { hook : typeless.hook, marker : ::std::marker::PhantomData }
}

pub trait PipelineHook {
    fn copy(&self) -> Box<PipelineHook+Send>;
    fn get_pipelined_cap(&self, ops : Vec<PipelineOp>) -> Box<ClientHook+Send>;
}

#[derive(Clone, Copy)]
pub enum PipelineOp {
    Noop,
    GetPointerField(u16),
}

<|endoftext|>


<commit_before>//! A partially ordered measure of progress at each timely dataflow location.

use std::fmt::Debug;
use std::any::Any;
use std::default::Default;
use std::fmt::Formatter;
use std::fmt::Error;
use std::hash::Hash;

use order::PartialOrder;
use progress::nested::product::Product;

use abomonation::Abomonation;

// TODO : Change Copy requirement to Clone;
/// A composite trait for types that serve as timestamps in timely dataflow.
pub trait Timestamp: Clone+Eq+PartialOrder+Default+Debug+Send+Any+Abomonation+Hash+Ord {
    /// A type summarizing action on a timestamp along a dataflow path.
    type Summary : PathSummary<Self> + 'static;
}

// TODO : Change Copy requirement to Clone
// TODO : Change `results_in` and perhaps `followed_by` to return an `Option`, indicating no path.
// TODO : This can be important when a summary would "overflow", as we want neither to overflow,
// TODO : nor wrap around, nor saturate.
/// A summary of how a timestamp advances along a timely dataflow path.
pub trait PathSummary<T> : Clone+'static+Eq+PartialOrder+Debug+Default {
    /// Advances a timestamp according to the timestamp actions on the path.
    ///
    /// The path may advance the timestamp sufficiently that it is no longer valid, for example if
    /// incrementing fields would result in integer overflow. In this case, `results_in` should
    /// return `None`.
    ///
    /// The `feedback` operator, apparently the only point where timestamps are actually incremented
    /// in computation, uses this method and will drop messages with timestamps that when advanced
    /// result in `None`. Ideally, all other timestamp manipulation should behave similarly.
    ///
    /// #Examples
    /// ```
    /// use timely::progress::timestamp::PathSummary;
    ///
    /// let timestamp = 3;
    ///
    /// let summary1 = 5;
    /// let summary2 = usize::max_value() - 2;
    ///
    /// assert_eq!(summary1.results_in(&timestamp), Some(8));
    /// assert_eq!(summary2.results_in(&timestamp), None);
    /// ```
    fn results_in(&self, src: &T) -> Option<T>;
    /// Composes this path summary with another path summary.
    ///
    /// It is possible that the two composed paths result in an invalid summary, for example when
    /// integer additions overflow. If it is correct that all timestamps moved along these paths
    /// would also result in overflow and be discarded, `followed_by` can return `None. It is very
    /// important that this not be used casually, as this does not prevent the actual movement of
    /// data.
    ///
    /// #Examples
    /// ```
    /// use timely::progress::timestamp::PathSummary;
    ///
    /// let summary1 = 5;
    /// let summary2 = usize::max_value() - 3;
    ///
    /// assert_eq!(summary1.followed_by(&summary2), None);
    /// ```
    fn followed_by(&self, other: &Self) -> Option<Self>;
}

/// An empty timestamp used by the root scope.
#[derive(Copy, Clone, Hash, Eq, Ord, PartialOrd, PartialEq, Default)]
pub struct RootTimestamp;
impl Timestamp for RootTimestamp { type Summary = RootSummary; }
impl Debug for RootTimestamp {
    #[inline]
    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
        f.write_str("Root")
    }
}

impl PartialOrder for RootTimestamp { #[inline(always)] fn less_equal(&self, _other: &Self) -> bool { true } }

impl Abomonation for RootTimestamp { }
impl RootTimestamp {
    /// Constructs a new `Product<RootTimestamp,T>`.
    #[inline]
    pub fn new<T: Timestamp>(t: T) -> Product<RootTimestamp, T> {
        Product::new(RootTimestamp, t)
    }
}

/// An empty path summary for root timestamps.
#[derive(Copy, Clone, Eq, Ord, PartialOrd, PartialEq, Debug, Default)]
pub struct RootSummary;
impl PathSummary<RootTimestamp> for RootSummary {
    #[inline]
    fn results_in(&self, _: &RootTimestamp) -> Option<RootTimestamp> { Some(RootTimestamp) }
    #[inline]
    fn followed_by(&self, _: &RootSummary) -> Option<RootSummary> { Some(RootSummary) }
}

impl PartialOrder for RootSummary { #[inline(always)] fn less_equal(&self, _other: &Self) -> bool { true } }

impl Timestamp for () { type Summary = (); }
impl PathSummary<()> for () {
    fn results_in(&self, _src: &()) -> Option<()> { Some(()) }
    fn followed_by(&self, _other: &()) -> Option<()> { Some(()) }
}

impl Timestamp for usize { type Summary = usize; }
impl PathSummary<usize> for usize {
    #[inline]
    fn results_in(&self, src: &usize) -> Option<usize> { self.checked_add(*src) }
    #[inline]
    fn followed_by(&self, other: &usize) -> Option<usize> { self.checked_add(*other) }
}

impl Timestamp for u64 { type Summary = u64; }
impl PathSummary<u64> for u64 {
    #[inline]
    fn results_in(&self, src: &u64) -> Option<u64> { self.checked_add(*src) }
    #[inline]
    fn followed_by(&self, other: &u64) -> Option<u64> { self.checked_add(*other) }
}

impl Timestamp for u32 { type Summary = u32; }
impl PathSummary<u32> for u32 {
    #[inline]
    fn results_in(&self, src: &u32) -> Option<u32> { self.checked_add(*src) }
    #[inline]
    fn followed_by(&self, other: &u32) -> Option<u32> { self.checked_add(*other) }
}

impl Timestamp for i32 { type Summary = i32; }
impl PathSummary<i32> for i32 {
    #[inline]
    fn results_in(&self, src: &i32) -> Option<i32> { self.checked_add(*src) }
    #[inline]
    fn followed_by(&self, other: &i32) -> Option<i32> { self.checked_add(*other) }
}
<commit_msg>Remove implemented TODOs<commit_after>//! A partially ordered measure of progress at each timely dataflow location.

use std::fmt::Debug;
use std::any::Any;
use std::default::Default;
use std::fmt::Formatter;
use std::fmt::Error;
use std::hash::Hash;

use order::PartialOrder;
use progress::nested::product::Product;

use abomonation::Abomonation;

/// A composite trait for types that serve as timestamps in timely dataflow.
pub trait Timestamp: Clone+Eq+PartialOrder+Default+Debug+Send+Any+Abomonation+Hash+Ord {
    /// A type summarizing action on a timestamp along a dataflow path.
    type Summary : PathSummary<Self> + 'static;
}

/// A summary of how a timestamp advances along a timely dataflow path.
pub trait PathSummary<T> : Clone+'static+Eq+PartialOrder+Debug+Default {
    /// Advances a timestamp according to the timestamp actions on the path.
    ///
    /// The path may advance the timestamp sufficiently that it is no longer valid, for example if
    /// incrementing fields would result in integer overflow. In this case, `results_in` should
    /// return `None`.
    ///
    /// The `feedback` operator, apparently the only point where timestamps are actually incremented
    /// in computation, uses this method and will drop messages with timestamps that when advanced
    /// result in `None`. Ideally, all other timestamp manipulation should behave similarly.
    ///
    /// #Examples
    /// ```
    /// use timely::progress::timestamp::PathSummary;
    ///
    /// let timestamp = 3;
    ///
    /// let summary1 = 5;
    /// let summary2 = usize::max_value() - 2;
    ///
    /// assert_eq!(summary1.results_in(&timestamp), Some(8));
    /// assert_eq!(summary2.results_in(&timestamp), None);
    /// ```
    fn results_in(&self, src: &T) -> Option<T>;
    /// Composes this path summary with another path summary.
    ///
    /// It is possible that the two composed paths result in an invalid summary, for example when
    /// integer additions overflow. If it is correct that all timestamps moved along these paths
    /// would also result in overflow and be discarded, `followed_by` can return `None. It is very
    /// important that this not be used casually, as this does not prevent the actual movement of
    /// data.
    ///
    /// #Examples
    /// ```
    /// use timely::progress::timestamp::PathSummary;
    ///
    /// let summary1 = 5;
    /// let summary2 = usize::max_value() - 3;
    ///
    /// assert_eq!(summary1.followed_by(&summary2), None);
    /// ```
    fn followed_by(&self, other: &Self) -> Option<Self>;
}

/// An empty timestamp used by the root scope.
#[derive(Copy, Clone, Hash, Eq, Ord, PartialOrd, PartialEq, Default)]
pub struct RootTimestamp;
impl Timestamp for RootTimestamp { type Summary = RootSummary; }
impl Debug for RootTimestamp {
    #[inline]
    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
        f.write_str("Root")
    }
}

impl PartialOrder for RootTimestamp { #[inline(always)] fn less_equal(&self, _other: &Self) -> bool { true } }

impl Abomonation for RootTimestamp { }
impl RootTimestamp {
    /// Constructs a new `Product<RootTimestamp,T>`.
    #[inline]
    pub fn new<T: Timestamp>(t: T) -> Product<RootTimestamp, T> {
        Product::new(RootTimestamp, t)
    }
}

/// An empty path summary for root timestamps.
#[derive(Copy, Clone, Eq, Ord, PartialOrd, PartialEq, Debug, Default)]
pub struct RootSummary;
impl PathSummary<RootTimestamp> for RootSummary {
    #[inline]
    fn results_in(&self, _: &RootTimestamp) -> Option<RootTimestamp> { Some(RootTimestamp) }
    #[inline]
    fn followed_by(&self, _: &RootSummary) -> Option<RootSummary> { Some(RootSummary) }
}

impl PartialOrder for RootSummary { #[inline(always)] fn less_equal(&self, _other: &Self) -> bool { true } }

impl Timestamp for () { type Summary = (); }
impl PathSummary<()> for () {
    fn results_in(&self, _src: &()) -> Option<()> { Some(()) }
    fn followed_by(&self, _other: &()) -> Option<()> { Some(()) }
}

impl Timestamp for usize { type Summary = usize; }
impl PathSummary<usize> for usize {
    #[inline]
    fn results_in(&self, src: &usize) -> Option<usize> { self.checked_add(*src) }
    #[inline]
    fn followed_by(&self, other: &usize) -> Option<usize> { self.checked_add(*other) }
}

impl Timestamp for u64 { type Summary = u64; }
impl PathSummary<u64> for u64 {
    #[inline]
    fn results_in(&self, src: &u64) -> Option<u64> { self.checked_add(*src) }
    #[inline]
    fn followed_by(&self, other: &u64) -> Option<u64> { self.checked_add(*other) }
}

impl Timestamp for u32 { type Summary = u32; }
impl PathSummary<u32> for u32 {
    #[inline]
    fn results_in(&self, src: &u32) -> Option<u32> { self.checked_add(*src) }
    #[inline]
    fn followed_by(&self, other: &u32) -> Option<u32> { self.checked_add(*other) }
}

impl Timestamp for i32 { type Summary = i32; }
impl PathSummary<i32> for i32 {
    #[inline]
    fn results_in(&self, src: &i32) -> Option<i32> { self.checked_add(*src) }
    #[inline]
    fn followed_by(&self, other: &i32) -> Option<i32> { self.checked_add(*other) }
}
<|endoftext|>


<commit_before>//!
//! Support to register a [`GbmDevice`](GbmDevice)
//! to an open [`Session`](::backend::session::Session).
//!

use drm::control::crtc;
use gbm::BufferObject;
use std::cell::RefCell;
use std::collections::HashMap;
use std::os::unix::io::RawFd;
use std::rc::{Rc, Weak};

use super::{GbmDevice, GbmSurfaceInternal};
use crate::backend::drm::{RawDevice, RawSurface};
use crate::backend::graphics::CursorBackend;
use crate::backend::session::{AsSessionObserver, SessionObserver};

/// [`SessionObserver`](SessionObserver)
/// linked to the [`GbmDevice`](GbmDevice) it was
/// created from.
pub struct GbmDeviceObserver<
    S: SessionObserver + 'static,
    D: RawDevice + ::drm::control::Device + AsSessionObserver<S> + 'static,
> {
    observer: S,
    backends: Weak<RefCell<HashMap<crtc::Handle, Weak<GbmSurfaceInternal<D>>>>>,
    logger: ::slog::Logger,
}

impl<
        O: SessionObserver + 'static,
        S: CursorBackend<CursorFormat = dyn drm::buffer::Buffer> + RawSurface + 'static,
        D: RawDevice<Surface = S> + drm::control::Device + AsSessionObserver<O> + 'static,
    > AsSessionObserver<GbmDeviceObserver<O, D>> for GbmDevice<D>
{
    fn observer(&mut self) -> GbmDeviceObserver<O, D> {
        GbmDeviceObserver {
            observer: (**self.dev.borrow_mut()).observer(),
            backends: Rc::downgrade(&self.backends),
            logger: self.logger.clone(),
        }
    }
}

impl<
        O: SessionObserver + 'static,
        S: CursorBackend<CursorFormat = dyn drm::buffer::Buffer> + RawSurface + 'static,
        D: RawDevice<Surface = S> + drm::control::Device + AsSessionObserver<O> + 'static,
    > SessionObserver for GbmDeviceObserver<O, D>
{
    fn pause(&mut self, devnum: Option<(u32, u32)>) {
        self.observer.pause(devnum);
    }

    fn activate(&mut self, devnum: Option<(u32, u32, Option<RawFd>)>) {
        self.observer.activate(devnum);
        let mut crtcs = Vec::new();
        if let Some(backends) = self.backends.upgrade() {
            for (crtc, backend) in backends.borrow().iter() {
                if let Some(backend) = backend.upgrade() {
                    // restart rendering loop, if it was previously running
                    if let Some(current_fb) = backend.current_frame_buffer.get() {
                        let result = if backend.crtc.commit_pending() {
                            backend.crtc.commit(current_fb)
                        } else {
                            RawSurface::page_flip(&backend.crtc, current_fb)
                        };

                        if let Err(err) = result {
                            warn!(
                                self.logger,
                                "Failed to restart rendering loop. Re-creating resources. Error: {}", err
                            );
                            // TODO bubble up
                        }
                    }

                    // reset cursor
                    {
                        use ::drm::control::Device;

                        let &(ref cursor, ref hotspot): &(BufferObject<()>, (u32, u32)) =
                            unsafe { &*backend.cursor.as_ptr() };
                        if backend.crtc.set_cursor_representation(cursor, *hotspot).is_err() {
                            if let Err(err) = backend.dev.borrow().set_cursor(*crtc, Some(cursor)) {
                                error!(self.logger, "Failed to reset cursor. Error: {}", err);
                            }
                        }
                    }
                } else {
                    crtcs.push(*crtc);
                }
            }
            for crtc in crtcs {
                backends.borrow_mut().remove(&crtc);
            }
        }
    }
}
<commit_msg>gbm: do not implicitly restart rendering loop<commit_after>//!
//! Support to register a [`GbmDevice`](GbmDevice)
//! to an open [`Session`](::backend::session::Session).
//!

use drm::control::crtc;
use gbm::BufferObject;
use std::cell::RefCell;
use std::collections::HashMap;
use std::os::unix::io::RawFd;
use std::rc::{Rc, Weak};

use super::{GbmDevice, GbmSurfaceInternal};
use crate::backend::drm::{RawDevice, RawSurface};
use crate::backend::graphics::CursorBackend;
use crate::backend::session::{AsSessionObserver, SessionObserver};

/// [`SessionObserver`](SessionObserver)
/// linked to the [`GbmDevice`](GbmDevice) it was
/// created from.
pub struct GbmDeviceObserver<
    S: SessionObserver + 'static,
    D: RawDevice + ::drm::control::Device + AsSessionObserver<S> + 'static,
> {
    observer: S,
    backends: Weak<RefCell<HashMap<crtc::Handle, Weak<GbmSurfaceInternal<D>>>>>,
    logger: ::slog::Logger,
}

impl<
        O: SessionObserver + 'static,
        S: CursorBackend<CursorFormat = dyn drm::buffer::Buffer> + RawSurface + 'static,
        D: RawDevice<Surface = S> + drm::control::Device + AsSessionObserver<O> + 'static,
    > AsSessionObserver<GbmDeviceObserver<O, D>> for GbmDevice<D>
{
    fn observer(&mut self) -> GbmDeviceObserver<O, D> {
        GbmDeviceObserver {
            observer: (**self.dev.borrow_mut()).observer(),
            backends: Rc::downgrade(&self.backends),
            logger: self.logger.clone(),
        }
    }
}

impl<
        O: SessionObserver + 'static,
        S: CursorBackend<CursorFormat = dyn drm::buffer::Buffer> + RawSurface + 'static,
        D: RawDevice<Surface = S> + drm::control::Device + AsSessionObserver<O> + 'static,
    > SessionObserver for GbmDeviceObserver<O, D>
{
    fn pause(&mut self, devnum: Option<(u32, u32)>) {
        self.observer.pause(devnum);
    }

    fn activate(&mut self, devnum: Option<(u32, u32, Option<RawFd>)>) {
        self.observer.activate(devnum);
        let mut crtcs = Vec::new();
        if let Some(backends) = self.backends.upgrade() {
            for (crtc, backend) in backends.borrow().iter() {
                if let Some(backend) = backend.upgrade() {
                    backend.clear_framebuffers();

                    // reset cursor
                    {
                        use ::drm::control::Device;

                        let &(ref cursor, ref hotspot): &(BufferObject<()>, (u32, u32)) =
                            unsafe { &*backend.cursor.as_ptr() };
                        if backend.crtc.set_cursor_representation(cursor, *hotspot).is_err() {
                            if let Err(err) = backend.dev.borrow().set_cursor(*crtc, Some(cursor)) {
                                error!(self.logger, "Failed to reset cursor. Error: {}", err);
                            }
                        }
                    }
                } else {
                    crtcs.push(*crtc);
                }
            }
            for crtc in crtcs {
                backends.borrow_mut().remove(&crtc);
            }
        }
    }
}
<|endoftext|>


<commit_before>extern crate num_traits;

use std::collections::HashSet;
use std::fmt;
use std::fmt::Debug;
use std::hash::Hash;
use std::vec::Vec;
use self::num_traits::One;
use std::str::FromStr;

use std::marker::PhantomData;

use automata;
use cfg;
use push_down::{PushDown, PushDownAutomaton, PushDownInstruction};

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash)]
pub enum PushState<X, Y> {
    Designated,
    Initial,
    Nt(X),
    T(Y),
}

impl<X: fmt::Display, Y: fmt::Display> fmt::Display for PushState<X, Y> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &PushState::Designated
                => write!(f, "@"),
            &PushState::Initial
                => write!(f, "I"),
            &PushState::Nt(ref x)
                => write!(f, "({})", x),
            &PushState::T(ref x)
                => write!(f, "({})", x),
        }
    }
}

impl<N: Clone + Debug + Ord + PartialEq + Hash,
     T: Clone + Debug + Ord + PartialEq + Hash,
     W: Clone + Debug + Ord + PartialEq + One + FromStr
     > From<cfg::CFG<N, T, W>> for PushDownAutomaton<PushState<N,T>, T, W>
    where <W as FromStr>::Err: Debug{
     fn from(g: cfg::CFG<N, T, W>) -> Self {
        let mut transitions = Vec::new();

        let mut t_buffer= HashSet::new();

        ///creates a Transition for every rule that replaces the nonterminal of the left side with the rightside of the rule
        for r in g.rules.clone(){
            let mut st = Vec::new();
            for v in r.composition.composition{

                match v{
                    cfg::LetterT::Value(x) => {
                        t_buffer.insert(x.clone());
                        st.push(PushState::T(x.clone()));
                    },
                    cfg::LetterT::Label(x) => {
                        st.push(PushState::Nt(x.clone()));
                    },
                }
            }

            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: Vec::new(),
                    weight: r.weight.clone(),
                    instruction: PushDownInstruction::Replace {
                        current_val: PushState::Nt(r.head.clone()),
                        new_val: st.clone(),
                    }
                }
            );

        }
        ///creates a transition for every terminal that reads the word
        for t in t_buffer{
            let mut tvec = Vec::new();
            tvec.push(t.clone());
            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: tvec.clone(),
                    weight: W::one(),
                    instruction: PushDownInstruction::Pop {
                        current_val: PushState::T(t.clone()),
                    }
                }
            );

        }
        ///creates a transition for the `Initial` symbol to all Nonterminals in `initial` with weight `1/initial.len()`
        let ini_val=1.0/g.initial.len() as f64;
        let ini_weight=ini_val.to_string();

        for ini in g.initial{
            let mut tvec = Vec::new();
            tvec.push(PushState::Nt(ini));
            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: Vec::new(),
                    weight: ini_weight.parse().unwrap(),
                    instruction: PushDownInstruction::Replace {
                        current_val: PushState::Initial,
                        new_val: tvec,
                    }
                }
            );
        }

        PushDownAutomaton::new(
            transitions,
            PushDown::new(PushState::Designated, PushState::Initial),
        )
    }
}
<commit_msg>Small fix for initial weights<commit_after>extern crate num_traits;

use std::collections::HashSet;
use std::fmt;
use std::fmt::Debug;
use std::hash::Hash;
use std::vec::Vec;
use self::num_traits::One;
use std::str::FromStr;

use std::marker::PhantomData;

use automata;
use cfg;
use push_down::{PushDown, PushDownAutomaton, PushDownInstruction};

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Hash)]
pub enum PushState<X, Y> {
    Designated,
    Initial,
    Nt(X),
    T(Y),
}

impl<X: fmt::Display, Y: fmt::Display> fmt::Display for PushState<X, Y> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &PushState::Designated
                => write!(f, "@"),
            &PushState::Initial
                => write!(f, "I"),
            &PushState::Nt(ref x)
                => write!(f, "({})", x),
            &PushState::T(ref x)
                => write!(f, "({})", x),
        }
    }
}

impl<N: Clone + Debug + Ord + PartialEq + Hash,
     T: Clone + Debug + Ord + PartialEq + Hash,
     W: Clone + Debug + Ord + PartialEq + One + FromStr
     > From<cfg::CFG<N, T, W>> for PushDownAutomaton<PushState<N,T>, T, W>
    where <W as FromStr>::Err: Debug{
     fn from(g: cfg::CFG<N, T, W>) -> Self {
        let mut transitions = Vec::new();

        let mut t_buffer= HashSet::new();

        ///creates a Transition for every rule that replaces the nonterminal of the left side with the rightside of the rule
        for r in g.rules.clone(){
            let mut st = Vec::new();
            for v in r.composition.composition{

                match v{
                    cfg::LetterT::Value(x) => {
                        t_buffer.insert(x.clone());
                        st.push(PushState::T(x.clone()));
                    },
                    cfg::LetterT::Label(x) => {
                        st.push(PushState::Nt(x.clone()));
                    },
                }
            }

            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: Vec::new(),
                    weight: r.weight.clone(),
                    instruction: PushDownInstruction::Replace {
                        current_val: PushState::Nt(r.head.clone()),
                        new_val: st.clone(),
                    }
                }
            );

        }
        ///creates a transition for every terminal that reads the word
        for t in t_buffer{
            let mut tvec = Vec::new();
            tvec.push(t.clone());
            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: tvec.clone(),
                    weight: W::one(),
                    instruction: PushDownInstruction::Pop {
                        current_val: PushState::T(t.clone()),
                    }
                }
            );

        }
        ///creates a transition for the `Initial` symbol to all Nonterminals in `initial` with weight `one`

        for ini in g.initial{
            let mut tvec = Vec::new();
            tvec.push(PushState::Nt(ini));
            transitions.push(
                automata::Transition {
                    _dummy: PhantomData,
                    word: Vec::new(),
                    weight: W::one(),
                    instruction: PushDownInstruction::Replace {
                        current_val: PushState::Initial,
                        new_val: tvec,
                    }
                }
            );
        }

        PushDownAutomaton::new(
            transitions,
            PushDown::new(PushState::Designated, PushState::Initial),
        )
    }
}
<|endoftext|>


<commit_before>mod reader;
mod writer;

pub use self::reader::MultiValuedFastFieldReader;
pub use self::writer::MultiValuedFastFieldWriter;

#[cfg(test)]
mod tests {

    use crate::collector::TopDocs;
    use crate::query::QueryParser;
    use crate::schema::Cardinality;
    use crate::schema::Facet;
    use crate::schema::IntOptions;
    use crate::schema::Schema;
    use crate::schema::INDEXED;
    use crate::Index;
    use chrono::Duration;

    #[test]
    fn test_multivalued_u64() {
        let mut schema_builder = Schema::builder();
        let field = schema_builder.add_u64_field(
            "multifield",
            IntOptions::default().set_fast(Cardinality::MultiValues),
        );
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
        let mut index_writer = index.writer_for_tests().unwrap();
        index_writer.add_document(doc!(field=>1u64, field=>3u64));
        index_writer.add_document(doc!());
        index_writer.add_document(doc!(field=>4u64));
        index_writer.add_document(doc!(field=>5u64, field=>20u64,field=>1u64));
        assert!(index_writer.commit().is_ok());

        let searcher = index.reader().unwrap().searcher();
        let segment_reader = searcher.segment_reader(0);
        let mut vals = Vec::new();
        let multi_value_reader = segment_reader.fast_fields().u64s(field).unwrap();
        {
            multi_value_reader.get_vals(2, &mut vals);
            assert_eq!(&vals, &[4u64]);
        }
        {
            multi_value_reader.get_vals(0, &mut vals);
            assert_eq!(&vals, &[1u64, 3u64]);
        }
        {
            multi_value_reader.get_vals(1, &mut vals);
            assert!(vals.is_empty());
        }
    }

    #[test]
    fn test_multivalued_date() {
        let mut schema_builder = Schema::builder();
        let date_field = schema_builder.add_date_field(
            "multi_date_field",
            IntOptions::default()
                .set_fast(Cardinality::MultiValues)
                .set_indexed()
                .set_stored(),
        );
        let time_i =
            schema_builder.add_i64_field("time_stamp_i", IntOptions::default().set_stored());
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
        let mut index_writer = index.writer_for_tests().unwrap();
        let first_time_stamp = chrono::Utc::now();
        index_writer.add_document(
            doc!(date_field=>first_time_stamp, date_field=>first_time_stamp, time_i=>1i64),
        );
        index_writer.add_document(doc!(time_i=>0i64));
        // add one second
        index_writer
            .add_document(doc!(date_field=>first_time_stamp + Duration::seconds(1), time_i=>2i64));
        // add another second
        let two_secs_ahead = first_time_stamp + Duration::seconds(2);
        index_writer.add_document(doc!(date_field=>two_secs_ahead, date_field=>two_secs_ahead,date_field=>two_secs_ahead, time_i=>3i64));
        // add three seconds
        index_writer.add_document(doc!(date_field=>first_time_stamp + Duration::seconds(3), time_i=>4i64));
        assert!(index_writer.commit().is_ok());

        let reader = index.reader().unwrap();
        let searcher = reader.searcher();
        let reader = searcher.segment_reader(0);
        assert_eq!(reader.num_docs(), 5);

        {
            let parser = QueryParser::for_index(&index, vec![date_field]);
            let query = parser
                .parse_query(&format!("\"{}\"", first_time_stamp.to_rfc3339()).to_string())
                .expect("could not parse query");
            let results = searcher
                .search(&query, &TopDocs::with_limit(5))
                .expect("could not query index");

            assert_eq!(results.len(), 1);
            for (_score, doc_address) in results {
                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
                assert_eq!(
                    retrieved_doc
                        .get_first(date_field)
                        .expect("cannot find value")
                        .date_value()
                        .unwrap()
                        .timestamp(),
                    first_time_stamp.timestamp()
                );
                assert_eq!(
                    retrieved_doc
                        .get_first(time_i)
                        .expect("cannot find value")
                        .i64_value(),
                    Some(1i64)
                );
            }
        }

        {
            let parser = QueryParser::for_index(&index, vec![date_field]);
            let query = parser
                .parse_query(&format!("\"{}\"", two_secs_ahead.to_rfc3339()).to_string())
                .expect("could not parse query");
            let results = searcher
                .search(&query, &TopDocs::with_limit(5))
                .expect("could not query index");

            assert_eq!(results.len(), 1);

            for (_score, doc_address) in results {
                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
                assert_eq!(
                    retrieved_doc
                        .get_first(date_field)
                        .expect("cannot find value")
                        .date_value()
                        .unwrap()
                        .timestamp(),
                    two_secs_ahead.timestamp()
                );
                assert_eq!(
                    retrieved_doc
                        .get_first(time_i)
                        .expect("cannot find value")
                        .i64_value(),
                    Some(3i64)
                );
            }
        }

        {
            let parser = QueryParser::for_index(&index, vec![date_field]);
            let range_q = format!(
                "[{} TO {}}}",
                (first_time_stamp + Duration::seconds(1)).to_rfc3339(),
                (first_time_stamp + Duration::seconds(3)).to_rfc3339()
            );
            let query = parser.parse_query(&range_q).expect("could not parse query");
            let results = searcher
                .search(&query, &TopDocs::with_limit(5))
                .expect("could not query index");

            assert_eq!(results.len(), 2);
            for (i, doc_pair) in results.iter().enumerate() {
                let retrieved_doc = searcher.doc(doc_pair.1).expect("cannot fetch doc");
                let offset_sec = match i {
                    0 => 1,
                    1 => 2,
                    _ => panic!("should not have more than 2 docs"),
                };
                let time_i_val = match i {
                    0 => 2,
                    1 => 3,
                    _ => panic!("should not have more than 2 docs"),
                };
                assert_eq!(
                    retrieved_doc
                        .get_first(date_field)
                        .expect("cannot find value")
                        .date_value()
                        .expect("value not of Date type")
                        .timestamp(),
                    (first_time_stamp + Duration::seconds(offset_sec)).timestamp()
                );
                assert_eq!(
                    retrieved_doc
                        .get_first(time_i)
                        .expect("cannot find value")
                        .i64_value()
                        .expect("value not of i64 type"),
                    time_i_val
                );
            }
        }
    }

    #[test]
    fn test_multivalued_i64() {
        let mut schema_builder = Schema::builder();
        let field = schema_builder.add_i64_field(
            "multifield",
            IntOptions::default().set_fast(Cardinality::MultiValues),
        );
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
        let mut index_writer = index.writer_for_tests().unwrap();
        index_writer.add_document(doc!(field=> 1i64, field => 3i64));
        index_writer.add_document(doc!());
        index_writer.add_document(doc!(field=> -4i64));
        index_writer.add_document(doc!(field=> -5i64, field => -20i64, field=>1i64));
        assert!(index_writer.commit().is_ok());

        let searcher = index.reader().unwrap().searcher();
        let segment_reader = searcher.segment_reader(0);
        let mut vals = Vec::new();
        let multi_value_reader = segment_reader.fast_fields().i64s(field).unwrap();
        multi_value_reader.get_vals(2, &mut vals);
        assert_eq!(&vals, &[-4i64]);
        multi_value_reader.get_vals(0, &mut vals);
        assert_eq!(&vals, &[1i64, 3i64]);
        multi_value_reader.get_vals(1, &mut vals);
        assert!(vals.is_empty());
        multi_value_reader.get_vals(3, &mut vals);
        assert_eq!(&vals, &[-5i64, -20i64, 1i64]);
    }
    #[test]
    #[ignore]
    fn test_many_facets() {
        let mut schema_builder = Schema::builder();
        let field = schema_builder.add_facet_field("facetfield", INDEXED);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
        let mut index_writer = index.writer_for_tests().unwrap();
        for i in 0..100_000 {
            index_writer.add_document(doc!(field=> Facet::from(format!("/lang/{}", i).as_str())));
        }
        assert!(index_writer.commit().is_ok());
    }
}
<commit_msg>Format<commit_after>mod reader;
mod writer;

pub use self::reader::MultiValuedFastFieldReader;
pub use self::writer::MultiValuedFastFieldWriter;

#[cfg(test)]
mod tests {

    use crate::collector::TopDocs;
    use crate::query::QueryParser;
    use crate::schema::Cardinality;
    use crate::schema::Facet;
    use crate::schema::IntOptions;
    use crate::schema::Schema;
    use crate::schema::INDEXED;
    use crate::Index;
    use chrono::Duration;

    #[test]
    fn test_multivalued_u64() {
        let mut schema_builder = Schema::builder();
        let field = schema_builder.add_u64_field(
            "multifield",
            IntOptions::default().set_fast(Cardinality::MultiValues),
        );
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
        let mut index_writer = index.writer_for_tests().unwrap();
        index_writer.add_document(doc!(field=>1u64, field=>3u64));
        index_writer.add_document(doc!());
        index_writer.add_document(doc!(field=>4u64));
        index_writer.add_document(doc!(field=>5u64, field=>20u64,field=>1u64));
        assert!(index_writer.commit().is_ok());

        let searcher = index.reader().unwrap().searcher();
        let segment_reader = searcher.segment_reader(0);
        let mut vals = Vec::new();
        let multi_value_reader = segment_reader.fast_fields().u64s(field).unwrap();
        {
            multi_value_reader.get_vals(2, &mut vals);
            assert_eq!(&vals, &[4u64]);
        }
        {
            multi_value_reader.get_vals(0, &mut vals);
            assert_eq!(&vals, &[1u64, 3u64]);
        }
        {
            multi_value_reader.get_vals(1, &mut vals);
            assert!(vals.is_empty());
        }
    }

    #[test]
    fn test_multivalued_date() {
        let mut schema_builder = Schema::builder();
        let date_field = schema_builder.add_date_field(
            "multi_date_field",
            IntOptions::default()
                .set_fast(Cardinality::MultiValues)
                .set_indexed()
                .set_stored(),
        );
        let time_i =
            schema_builder.add_i64_field("time_stamp_i", IntOptions::default().set_stored());
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
        let mut index_writer = index.writer_for_tests().unwrap();
        let first_time_stamp = chrono::Utc::now();
        index_writer.add_document(
            doc!(date_field=>first_time_stamp, date_field=>first_time_stamp, time_i=>1i64),
        );
        index_writer.add_document(doc!(time_i=>0i64));
        // add one second
        index_writer
            .add_document(doc!(date_field=>first_time_stamp + Duration::seconds(1), time_i=>2i64));
        // add another second
        let two_secs_ahead = first_time_stamp + Duration::seconds(2);
        index_writer.add_document(doc!(date_field=>two_secs_ahead, date_field=>two_secs_ahead,date_field=>two_secs_ahead, time_i=>3i64));
        // add three seconds
        index_writer
            .add_document(doc!(date_field=>first_time_stamp + Duration::seconds(3), time_i=>4i64));
        assert!(index_writer.commit().is_ok());

        let reader = index.reader().unwrap();
        let searcher = reader.searcher();
        let reader = searcher.segment_reader(0);
        assert_eq!(reader.num_docs(), 5);

        {
            let parser = QueryParser::for_index(&index, vec![date_field]);
            let query = parser
                .parse_query(&format!("\"{}\"", first_time_stamp.to_rfc3339()).to_string())
                .expect("could not parse query");
            let results = searcher
                .search(&query, &TopDocs::with_limit(5))
                .expect("could not query index");

            assert_eq!(results.len(), 1);
            for (_score, doc_address) in results {
                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
                assert_eq!(
                    retrieved_doc
                        .get_first(date_field)
                        .expect("cannot find value")
                        .date_value()
                        .unwrap()
                        .timestamp(),
                    first_time_stamp.timestamp()
                );
                assert_eq!(
                    retrieved_doc
                        .get_first(time_i)
                        .expect("cannot find value")
                        .i64_value(),
                    Some(1i64)
                );
            }
        }

        {
            let parser = QueryParser::for_index(&index, vec![date_field]);
            let query = parser
                .parse_query(&format!("\"{}\"", two_secs_ahead.to_rfc3339()).to_string())
                .expect("could not parse query");
            let results = searcher
                .search(&query, &TopDocs::with_limit(5))
                .expect("could not query index");

            assert_eq!(results.len(), 1);

            for (_score, doc_address) in results {
                let retrieved_doc = searcher.doc(doc_address).expect("cannot fetch doc");
                assert_eq!(
                    retrieved_doc
                        .get_first(date_field)
                        .expect("cannot find value")
                        .date_value()
                        .unwrap()
                        .timestamp(),
                    two_secs_ahead.timestamp()
                );
                assert_eq!(
                    retrieved_doc
                        .get_first(time_i)
                        .expect("cannot find value")
                        .i64_value(),
                    Some(3i64)
                );
            }
        }

        {
            let parser = QueryParser::for_index(&index, vec![date_field]);
            let range_q = format!(
                "[{} TO {}}}",
                (first_time_stamp + Duration::seconds(1)).to_rfc3339(),
                (first_time_stamp + Duration::seconds(3)).to_rfc3339()
            );
            let query = parser.parse_query(&range_q).expect("could not parse query");
            let results = searcher
                .search(&query, &TopDocs::with_limit(5))
                .expect("could not query index");

            assert_eq!(results.len(), 2);
            for (i, doc_pair) in results.iter().enumerate() {
                let retrieved_doc = searcher.doc(doc_pair.1).expect("cannot fetch doc");
                let offset_sec = match i {
                    0 => 1,
                    1 => 2,
                    _ => panic!("should not have more than 2 docs"),
                };
                let time_i_val = match i {
                    0 => 2,
                    1 => 3,
                    _ => panic!("should not have more than 2 docs"),
                };
                assert_eq!(
                    retrieved_doc
                        .get_first(date_field)
                        .expect("cannot find value")
                        .date_value()
                        .expect("value not of Date type")
                        .timestamp(),
                    (first_time_stamp + Duration::seconds(offset_sec)).timestamp()
                );
                assert_eq!(
                    retrieved_doc
                        .get_first(time_i)
                        .expect("cannot find value")
                        .i64_value()
                        .expect("value not of i64 type"),
                    time_i_val
                );
            }
        }
    }

    #[test]
    fn test_multivalued_i64() {
        let mut schema_builder = Schema::builder();
        let field = schema_builder.add_i64_field(
            "multifield",
            IntOptions::default().set_fast(Cardinality::MultiValues),
        );
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
        let mut index_writer = index.writer_for_tests().unwrap();
        index_writer.add_document(doc!(field=> 1i64, field => 3i64));
        index_writer.add_document(doc!());
        index_writer.add_document(doc!(field=> -4i64));
        index_writer.add_document(doc!(field=> -5i64, field => -20i64, field=>1i64));
        assert!(index_writer.commit().is_ok());

        let searcher = index.reader().unwrap().searcher();
        let segment_reader = searcher.segment_reader(0);
        let mut vals = Vec::new();
        let multi_value_reader = segment_reader.fast_fields().i64s(field).unwrap();
        multi_value_reader.get_vals(2, &mut vals);
        assert_eq!(&vals, &[-4i64]);
        multi_value_reader.get_vals(0, &mut vals);
        assert_eq!(&vals, &[1i64, 3i64]);
        multi_value_reader.get_vals(1, &mut vals);
        assert!(vals.is_empty());
        multi_value_reader.get_vals(3, &mut vals);
        assert_eq!(&vals, &[-5i64, -20i64, 1i64]);
    }
    #[test]
    #[ignore]
    fn test_many_facets() {
        let mut schema_builder = Schema::builder();
        let field = schema_builder.add_facet_field("facetfield", INDEXED);
        let schema = schema_builder.build();
        let index = Index::create_in_ram(schema);
        let mut index_writer = index.writer_for_tests().unwrap();
        for i in 0..100_000 {
            index_writer.add_document(doc!(field=> Facet::from(format!("/lang/{}", i).as_str())));
        }
        assert!(index_writer.commit().is_ok());
    }
}
<|endoftext|>