<commit_before>use analysis::conversion_type::ConversionType;
use analysis::parameter::Parameter as AnalysisParameter;
use analysis::out_parameters::Mode;
use analysis::return_value;
use analysis::safety_assertion_mode::SafetyAssertionMode;
use chunk::{chunks, Chunk, TupleMode};
use chunk::parameter_ffi_call_in;
use chunk::parameter_ffi_call_out;
use library;

#[derive(Clone)]
enum Parameter {
    In {
        parameter: parameter_ffi_call_in::Parameter,
    },
    Out {
        parameter: parameter_ffi_call_out::Parameter,
        mem_mode: OutMemMode,
    },
}

#[derive(Clone, Eq, PartialEq)]
enum OutMemMode {
    Uninitialized,
    UninitializedNamed(String),
    NullPtr,
}

use self::Parameter::*;

#[derive(Default)]
pub struct Builder {
    glib_name: String,
    parameters: Vec<Parameter>,
    ret: return_value::Info,
    outs_as_return: bool,
    outs_mode: Mode,
    assertion: SafetyAssertionMode,
}

impl Builder {
    pub fn new() -> Builder {
        Default::default()
    }
    pub fn glib_name(&mut self, name: &str) -> &mut Builder {
        self.glib_name = name.into();
        self
    }
    pub fn assertion(&mut self, assertion: SafetyAssertionMode) -> &mut Builder {
        self.assertion = assertion;
        self
    }
    pub fn ret(&mut self, ret: &return_value::Info) -> &mut Builder {
        self.ret = ret.clone();
        self
    }
    pub fn parameter(&mut self, parameter: &AnalysisParameter) -> &mut Builder {
        self.parameters.push(Parameter::In {
            parameter: parameter.into(),
        });
        self
    }
    pub fn out_parameter(&mut self, library: &library::Library, parameter: &AnalysisParameter) -> &mut Builder {
        use self::OutMemMode::*;
        let mem_mode = if ConversionType::of(library, parameter.typ) == ConversionType::Pointer {
            if parameter.caller_allocates {
                let type_name = library.type_(parameter.typ).get_name();
                UninitializedNamed(type_name.clone())
            } else {
                NullPtr
            }
        } else {
            Uninitialized
        };
        self.parameters.push(Parameter::Out {
            parameter: parameter.into(),
            mem_mode: mem_mode,
        });
        self.outs_as_return = true;
        self
    }
    pub fn outs_mode(&mut self, mode: Mode) -> &mut Builder {
        self.outs_mode = mode;
        self
    }
    pub fn generate(&self) -> Chunk {
        let mut body = Vec::new();

        if self.outs_as_return {
            self.write_out_variables(&mut body);
        }

        let call = self.generate_call();
        let call = self.generate_call_conversion(call);
        let ret = self.generate_out_return();
        let (call, ret) = self.apply_outs_mode(call, ret);

        body.push(call);
        if let Some(chunk) = ret {
            body.push(chunk);
        }

        let unsafe_ = Chunk::Unsafe(body);

        let mut chunks = Vec::new();
        self.add_assertion(&mut chunks);
        chunks.push(unsafe_);
        Chunk::BlockHalf(chunks)
    }
    fn add_assertion(&self, chunks: &mut Vec<Chunk>) {
        match self.assertion {
            SafetyAssertionMode::None => (),
            SafetyAssertionMode::Skip =>
                chunks.push(Chunk::AssertSkipInitialized),
            SafetyAssertionMode::InMainThread =>
                chunks.push(Chunk::AssertInitializedAndInMainThread),
        }
    }
    fn generate_call(&self) -> Chunk {
        let params = self.generate_func_parameters();
        let func = Chunk::FfiCall {
            name: self.glib_name.clone(),
            params: params,
        };
        func
    }
    fn generate_call_conversion(&self, call: Chunk) -> Chunk {
        let conv = Chunk::FfiCallConversion {
            ret: self.ret.clone(),
            call: Box::new(call),
        };
        conv
    }
    fn generate_func_parameters(&self) -> Vec<Chunk> {
        let mut params = Vec::new();
        for par in &self.parameters {
            let chunk = match *par {
                In { ref parameter } => Chunk::FfiCallParameter{
                    par: parameter.clone(),
                },
                Out { ref parameter, .. } => Chunk::FfiCallOutParameter{
                    par: parameter.clone(),
                },
            };
            params.push(chunk);
        }
        params
    }
    fn get_outs(&self) -> Vec<&Parameter> {
        self.parameters.iter()
            .filter_map(|par| if let Out{ .. } = *par { Some(par) } else { None })
            .collect()
    }
    fn get_outs_without_error(&self) -> Vec<&Parameter> {
        self.parameters.iter()
            .filter_map(|par| if let Out{ ref parameter, .. } = *par {
                if parameter.is_error {
                    None
                } else {
                    Some(par)
                }
            } else {
                None
            })
            .collect()
    }
    fn write_out_variables(&self, v: &mut Vec<Chunk>) {
        let outs = self.get_outs();
        for par in outs {
            if let Out{ ref parameter, ref mem_mode } = *par {
                let val = self.get_uninitialized(mem_mode);
                let chunk = Chunk::Let{
                    name: parameter.name.clone(),
                    is_mut: true,
                    value: Box::new(val),
                };
                v.push(chunk);
            }
        }
    }
    fn get_uninitialized(&self, mem_mode: &OutMemMode) -> Chunk {
        use self::OutMemMode::*;
        match *mem_mode {
            Uninitialized => Chunk::Uninitialized,
            UninitializedNamed(ref name) => Chunk::UninitializedNamed{ name: name.clone() },
            NullPtr => Chunk::NullMutPtr,
        }
    }
    fn generate_out_return(&self) -> Option<Chunk> {
        if !self.outs_as_return {
            return None;
        }
        let outs = self.get_outs_without_error();
        let mut chs: Vec<Chunk> = Vec::with_capacity(outs.len());
        for par in outs {
            if let Out{ ref parameter, ref mem_mode } = *par {
                chs.push(self.out_parameter_to_return(parameter, mem_mode));
            }
        }
        let chunk = Chunk::Tuple(chs, TupleMode::Auto);
        Some(chunk)
    }
    fn out_parameter_to_return(&self, parameter: &parameter_ffi_call_out::Parameter, mem_mode: &OutMemMode) -> Chunk {
        let value = Chunk::VariableValue{name: parameter.name.clone()};
        if let OutMemMode::UninitializedNamed(_) = *mem_mode {
            value
        } else {
            Chunk::FromGlibConversion{
                mode: parameter.into(),
                value: Box::new(value),
            }
        }
    }
    fn apply_outs_mode(&self, call: Chunk, ret: Option<Chunk>) -> (Chunk, Option<Chunk>) {
        use analysis::out_parameters::Mode::*;
        match self.outs_mode {
            None => (call, ret),
            Normal => (call, ret),
            Optional => {
                let call = Chunk::Let{
                    name: "ret".into(),
                    is_mut: false,
                    value: Box::new(call),
                };
                let ret = ret.expect("No return in optional outs mode");
                let ret = Chunk::OptionalReturn{
                    condition: "ret".into(),
                    value: Box::new(ret),
                };
                (call, Some(ret))
            }
            Combined => {
                let call = Chunk::Let{
                    name: "ret".into(),
                    is_mut: false,
                    value: Box::new(call),
                };
                let mut ret = ret.expect("No return in combined outs mode");
                if let Chunk::Tuple(ref mut vec, _) = ret {
                    vec.insert(0, Chunk::VariableValue { name: "ret".into() });
                }
                (call, Some(ret))
            }
            Throws(use_ret) => {
                let call = if use_ret {
                    Chunk::Let{
                        name: "ret".into(),
                        is_mut: false,
                        value: Box::new(call),
                    }
                } else {
                    //extracting original FFI function call
                    let boxed_call = if let Chunk::FfiCallConversion{call: inner, ..} = call {
                        inner
                    } else {
                        Box::new(call)
                    };
                    Chunk::Let{
                        name: "_".into(),
                        is_mut: false,
                        value: boxed_call,
                    }
                };
                let mut ret = ret.expect("No return in throws outs mode");
                if let Chunk::Tuple(ref mut vec, ref mut mode ) = ret {
                    *mode = TupleMode::WithUnit;
                    if use_ret {
                        vec.insert(0, Chunk::VariableValue { name: "ret".into() });
                    }
                }
                ret = Chunk::ErrorResultReturn{
                    value: Box::new(ret),
                };
                (call, Some(ret))
            }
        }
    }
}
<commit_msg>Postponed early "from_glib"-conversion<commit_after>use analysis::conversion_type::ConversionType;
use analysis::parameter::Parameter as AnalysisParameter;
use analysis::out_parameters::Mode;
use analysis::return_value;
use analysis::safety_assertion_mode::SafetyAssertionMode;
use chunk::{chunks, Chunk, TupleMode};
use chunk::parameter_ffi_call_in;
use chunk::parameter_ffi_call_out;
use library;

#[derive(Clone)]
enum Parameter {
    In {
        parameter: parameter_ffi_call_in::Parameter,
    },
    Out {
        parameter: parameter_ffi_call_out::Parameter,
        mem_mode: OutMemMode,
    },
}

#[derive(Clone, Eq, PartialEq)]
enum OutMemMode {
    Uninitialized,
    UninitializedNamed(String),
    NullPtr,
}

use self::Parameter::*;

#[derive(Default)]
pub struct Builder {
    glib_name: String,
    parameters: Vec<Parameter>,
    ret: return_value::Info,
    outs_as_return: bool,
    outs_mode: Mode,
    assertion: SafetyAssertionMode,
}

impl Builder {
    pub fn new() -> Builder {
        Default::default()
    }
    pub fn glib_name(&mut self, name: &str) -> &mut Builder {
        self.glib_name = name.into();
        self
    }
    pub fn assertion(&mut self, assertion: SafetyAssertionMode) -> &mut Builder {
        self.assertion = assertion;
        self
    }
    pub fn ret(&mut self, ret: &return_value::Info) -> &mut Builder {
        self.ret = ret.clone();
        self
    }
    pub fn parameter(&mut self, parameter: &AnalysisParameter) -> &mut Builder {
        self.parameters.push(Parameter::In {
            parameter: parameter.into(),
        });
        self
    }
    pub fn out_parameter(&mut self, library: &library::Library, parameter: &AnalysisParameter) -> &mut Builder {
        use self::OutMemMode::*;
        let mem_mode = if ConversionType::of(library, parameter.typ) == ConversionType::Pointer {
            if parameter.caller_allocates {
                let type_name = library.type_(parameter.typ).get_name();
                UninitializedNamed(type_name.clone())
            } else {
                NullPtr
            }
        } else {
            Uninitialized
        };
        self.parameters.push(Parameter::Out {
            parameter: parameter.into(),
            mem_mode: mem_mode,
        });
        self.outs_as_return = true;
        self
    }
    pub fn outs_mode(&mut self, mode: Mode) -> &mut Builder {
        self.outs_mode = mode;
        self
    }
    pub fn generate(&self) -> Chunk {
        let mut body = Vec::new();

        if self.outs_as_return {
            self.write_out_variables(&mut body);
        }

        let call = self.generate_call();
        let call = self.generate_call_conversion(call);
        let ret = self.generate_out_return();
        let (call, ret) = self.apply_outs_mode(call, ret);

        body.push(call);
        if let Some(chunk) = ret {
            body.push(chunk);
        }

        let unsafe_ = Chunk::Unsafe(body);

        let mut chunks = Vec::new();
        self.add_assertion(&mut chunks);
        chunks.push(unsafe_);
        Chunk::BlockHalf(chunks)
    }
    fn add_assertion(&self, chunks: &mut Vec<Chunk>) {
        match self.assertion {
            SafetyAssertionMode::None => (),
            SafetyAssertionMode::Skip =>
                chunks.push(Chunk::AssertSkipInitialized),
            SafetyAssertionMode::InMainThread =>
                chunks.push(Chunk::AssertInitializedAndInMainThread),
        }
    }
    fn generate_call(&self) -> Chunk {
        let params = self.generate_func_parameters();
        let func = Chunk::FfiCall {
            name: self.glib_name.clone(),
            params: params,
        };
        func
    }
    fn generate_call_conversion(&self, call: Chunk) -> Chunk {
        let conv = Chunk::FfiCallConversion {
            ret: self.ret.clone(),
            call: Box::new(call),
        };
        conv
    }
    fn generate_func_parameters(&self) -> Vec<Chunk> {
        let mut params = Vec::new();
        for par in &self.parameters {
            let chunk = match *par {
                In { ref parameter } => Chunk::FfiCallParameter{
                    par: parameter.clone(),
                },
                Out { ref parameter, .. } => Chunk::FfiCallOutParameter{
                    par: parameter.clone(),
                },
            };
            params.push(chunk);
        }
        params
    }
    fn get_outs(&self) -> Vec<&Parameter> {
        self.parameters.iter()
            .filter_map(|par| if let Out{ .. } = *par { Some(par) } else { None })
            .collect()
    }
    fn get_outs_without_error(&self) -> Vec<&Parameter> {
        self.parameters.iter()
            .filter_map(|par| if let Out{ ref parameter, .. } = *par {
                if parameter.is_error {
                    None
                } else {
                    Some(par)
                }
            } else {
                None
            })
            .collect()
    }
    fn write_out_variables(&self, v: &mut Vec<Chunk>) {
        let outs = self.get_outs();
        for par in outs {
            if let Out{ ref parameter, ref mem_mode } = *par {
                let val = self.get_uninitialized(mem_mode);
                let chunk = Chunk::Let{
                    name: parameter.name.clone(),
                    is_mut: true,
                    value: Box::new(val),
                };
                v.push(chunk);
            }
        }
    }
    fn get_uninitialized(&self, mem_mode: &OutMemMode) -> Chunk {
        use self::OutMemMode::*;
        match *mem_mode {
            Uninitialized => Chunk::Uninitialized,
            UninitializedNamed(ref name) => Chunk::UninitializedNamed{ name: name.clone() },
            NullPtr => Chunk::NullMutPtr,
        }
    }
    fn generate_out_return(&self) -> Option<Chunk> {
        if !self.outs_as_return {
            return None;
        }
        let outs = self.get_outs_without_error();
        let mut chs: Vec<Chunk> = Vec::with_capacity(outs.len());
        for par in outs {
            if let Out{ ref parameter, ref mem_mode } = *par {
                chs.push(self.out_parameter_to_return(parameter, mem_mode));
            }
        }
        let chunk = Chunk::Tuple(chs, TupleMode::Auto);
        Some(chunk)
    }
    fn out_parameter_to_return(&self, parameter: &parameter_ffi_call_out::Parameter, mem_mode: &OutMemMode) -> Chunk {
        let value = Chunk::VariableValue{name: parameter.name.clone()};
        if let OutMemMode::UninitializedNamed(_) = *mem_mode {
            value
        } else {
            Chunk::FromGlibConversion{
                mode: parameter.into(),
                value: Box::new(value),
            }
        }
    }
    fn apply_outs_mode(&self, call: Chunk, ret: Option<Chunk>) -> (Chunk, Option<Chunk>) {
        use analysis::out_parameters::Mode::*;
        match self.outs_mode {
            None => (call, ret),
            Normal => (call, ret),
            Optional => {
                let call = Chunk::Let{
                    name: "ret".into(),
                    is_mut: false,
                    value: Box::new(call),
                };
                let ret = ret.expect("No return in optional outs mode");
                let ret = Chunk::OptionalReturn{
                    condition: "ret".into(),
                    value: Box::new(ret),
                };
                (call, Some(ret))
            }
            Combined => {
                let call = Chunk::Let{
                    name: "ret".into(),
                    is_mut: false,
                    value: Box::new(call),
                };
                let mut ret = ret.expect("No return in combined outs mode");
                if let Chunk::Tuple(ref mut vec, _) = ret {
                    vec.insert(0, Chunk::VariableValue { name: "ret".into() });
                }
                (call, Some(ret))
            }
            Throws(use_ret) => {
                //extracting original FFI function call
                let (boxed_call, ret_info) = if let Chunk::FfiCallConversion{call: inner, ret: ret_info} = call {
                    (inner, ret_info)
                } else {
                    panic!("Call without Chunk::FfiCallConversion")
                };
                let call = if use_ret {
                    Chunk::Let{
                        name: "ret".into(),
                        is_mut: false,
                        value: boxed_call,
                    }
                } else {
                    Chunk::Let{
                        name: "_".into(),
                        is_mut: false,
                        value: boxed_call,
                    }
                };
                let mut ret = ret.expect("No return in throws outs mode");
                if let Chunk::Tuple(ref mut vec, ref mut mode ) = ret {
                    *mode = TupleMode::WithUnit;
                    if use_ret {
                        let val = Chunk::VariableValue { name: "ret".into() };
                        let conv = Chunk::FfiCallConversion{
                            call: Box::new(val),
                            ret: ret_info,
                        };
                        vec.insert(0, conv);
                    }
                } else {
                    panic!("Return is not Tuple")
                }
                ret = Chunk::ErrorResultReturn{
                    value: Box::new(ret),
                };
                (call, Some(ret))
            }
        }
    }
}
<|endoftext|>


<commit_before>extern crate crypto;
extern crate byteorder;
extern crate rand;

use self::crypto::digest::Digest;
use self::crypto::sha2::{Sha256, Sha512};
use self::crypto::blake2b::Blake2b;
use self::crypto::aes::KeySize;
use self::crypto::aes_gcm::AesGcm;
use self::crypto::chacha20poly1305::ChaCha20Poly1305;
use self::crypto::aead::{AeadEncryptor, AeadDecryptor};
use self::crypto::curve25519::{curve25519, curve25519_base};

use self::byteorder::{ByteOrder, BigEndian, LittleEndian};
use self::rand::{OsRng, Rng};

use crypto_stuff::*;

pub struct Dh25519 {
    privkey : [u8; 32],
    pubkey  : [u8; 32]
}

pub struct CipherAESGCM {
    key : [u8; 32],
    nonce : u64
}

pub struct CipherChaChaPoly {
    key : [u8; 32],
    nonce : u64
}

pub struct HashSHA256 {
    hasher : Sha256
}

pub struct HashSHA512 {
    hasher : Sha512
}

pub struct HashBLAKE2b {
    hasher : Blake2b
}

impl Dh for Dh25519 {

    fn new(privkey: &[u8], pubkey: &[u8]) -> Dh25519 {
        let mut dh = Dh25519{privkey: [0u8; 32], pubkey: [0u8; 32]};
        copy_memory(privkey, &mut dh.privkey); /* RUSTSUCKS: Why can't I convert slice -> array? */
        copy_memory(pubkey, &mut dh.pubkey);
        dh
    }

    fn generate() -> Dh25519 {
        let mut privkey = [0u8; 32];
        let mut rng = OsRng::new().unwrap();
        rng.fill_bytes(&mut privkey);
        privkey[0] &= 248;
        privkey[31] &= 127;
        privkey[31] |= 64;
        let pubkey = curve25519_base(&privkey);
        Dh25519{privkey : privkey, pubkey : pubkey}
    }

    fn pubkey(&self) -> &[u8] {
        &self.pubkey
    }

    fn dh(&self, pubkey: &[u8]) -> [u8; DHLEN] {
        curve25519(&self.privkey, pubkey)
    }

}

impl Cipher for CipherAESGCM {

    fn new(key: &[u8], nonce: u64) -> CipherAESGCM {
        let mut cipher = CipherAESGCM{key: [0u8; 32], nonce: nonce};
        copy_memory(key, &mut cipher.key);
        cipher
    }

    fn encrypt_and_inc(&mut self, authtext: &[u8], plaintext: &[u8], out: &mut[u8]) {
        let mut nonce_bytes = [0u8; 12];
        BigEndian::write_u64(&mut nonce_bytes[4..], self.nonce);
        self.nonce += 1;
        let mut cipher = AesGcm::new(KeySize::KeySize256, &self.key, &nonce_bytes, authtext);
        let mut tag = [0u8; TAGLEN];
        cipher.encrypt(plaintext, &mut out[..plaintext.len()], &mut tag);
        copy_memory(&tag, &mut out[plaintext.len()..]);
    }

    fn decrypt_and_inc(&mut self, authtext: &[u8], ciphertext: &[u8], out: &mut[u8]) -> bool {
        let mut nonce_bytes = [0u8; 12];
        BigEndian::write_u64(&mut nonce_bytes[4..], self.nonce);
        self.nonce += 1;
        let mut cipher = AesGcm::new(KeySize::KeySize256, &self.key, &nonce_bytes, authtext);
        let text_len = ciphertext.len() - TAGLEN;
        let mut tag = [0u8; TAGLEN];
        copy_memory(&ciphertext[text_len..], &mut tag);
        cipher.decrypt(&ciphertext[..text_len], &mut out[..text_len], &tag)
    }

}

impl Cipher for CipherChaChaPoly {

    fn new(key: &[u8], nonce: u64) -> CipherChaChaPoly {
        let mut cipher = CipherChaChaPoly{key: [0u8; 32], nonce: nonce};
        copy_memory(key, &mut cipher.key);
        cipher
    }

    fn encrypt_and_inc(&mut self, authtext: &[u8], plaintext: &[u8], out: &mut[u8]) {
        let mut nonce_bytes = [0u8; 8];
        LittleEndian::write_u64(&mut nonce_bytes, self.nonce);
        self.nonce += 1;
        let mut cipher = ChaCha20Poly1305::new(&self.key, &nonce_bytes, authtext);
        let mut tag = [0u8; TAGLEN];
        cipher.encrypt(plaintext, &mut out[..plaintext.len()], &mut tag);
        copy_memory(&tag, &mut out[plaintext.len()..]);
    }

    fn decrypt_and_inc(&mut self, authtext: &[u8], ciphertext: &[u8], out: &mut[u8]) -> bool {
        let mut nonce_bytes = [0u8; 8];
        LittleEndian::write_u64(&mut nonce_bytes, self.nonce);
        self.nonce += 1;
        let mut cipher = ChaCha20Poly1305::new(&self.key, &nonce_bytes, authtext);
        let text_len = ciphertext.len() - TAGLEN;
        let mut tag = [0u8; TAGLEN];
        copy_memory(&ciphertext[text_len..], &mut tag);
        cipher.decrypt(&ciphertext[..text_len], &mut out[..text_len], &tag)
    }

}

impl Hash for HashSHA256 {

    fn new() -> HashSHA256 {
        HashSHA256{hasher : Sha256::new()}
    }

    fn block_len() -> usize {
        return 64;
    }

    fn hash_len() -> usize {
        return 32;
    }

    fn input(&mut self, data: &[u8]) {
        self.hasher.input(data);
    }

    fn result(&mut self, out: &mut [u8]) {
        self.hasher.result(out);
    }
}

impl Hash for HashSHA512 {

    fn new() -> HashSHA512 {
        HashSHA512{hasher : Sha512::new()}
    }

    fn block_len() -> usize {
        return 128;
    }

    fn hash_len() -> usize {
        return 64;
    }

    fn input(&mut self, data: &[u8]) {
        self.hasher.input(data);
    }

    fn result(&mut self, out: &mut [u8]) {
        self.hasher.result(out);
    }
}

impl Hash for HashBLAKE2b {

    fn new() -> HashBLAKE2b {
        HashBLAKE2b{hasher : Blake2b::new(64)}
    }

    fn block_len() -> usize {
        return 128;
    }

    fn hash_len() -> usize {
        return 64;
    }

    fn input(&mut self, data: &[u8]) {
        self.hasher.input(data);
    }

    fn result(&mut self, out: &mut [u8]) {
        self.hasher.result(out);
    }
}

#[cfg(test)]
mod tests {

    extern crate rustc_serialize;

    use crypto_stuff::*;
    use super::*;
    use self::rustc_serialize::hex::{FromHex, ToHex};

    #[test]
    fn crypto_tests() {

        // SHA256 test
        {
            let mut output = [0u8; 32];
            let mut hasher = HashSHA256::new();
            hasher.input("abc".as_bytes());
            hasher.result(&mut output);
            assert!(output.to_hex() == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
        }

        // HMAC-SHA256 test - RFC 4231
        {
            let key = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".from_hex().unwrap();
            let data = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd".from_hex().unwrap();
            let mut output = [0u8; 32];
            HashSHA256::hmac(&key, &data, &mut output);
            assert!(output.to_hex() == "773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe");
        }

        // Curve25519 test - draft-curves-10
        {
            let mut keypair = Dh25519::new(&[0; 32], &[0; 32]);
            let scalar = "a546e36bf0527c9d3b16154b82465edd62144c0ac1fc5a18506a2244ba449ac4".from_hex().unwrap();
            copy_memory(&scalar, &mut keypair.privkey);
            let public = "e6db6867583030db3594c1a424b15f7c726624ec26b3353b10a903a6d0ab1c4c".from_hex().unwrap();
            let output = keypair.dh(&public);
            assert!(output.to_hex() == "c3da55379de9c6908e94ea4df28d084f32eccf03491c71f754b4075577a28552");
        }

        //AES256-GCM tests - gcm-spec.pdf
        {
            // Test Case 13
            let key = [0u8; 32];
            let nonce = 0u64;
            let plaintext = [0u8; 0];
            let authtext = [0u8; 0];
            let mut ciphertext = [0u8; 16];
            let mut cipher1 = CipherAESGCM::new(&key, nonce);
            cipher1.encrypt_and_inc(&authtext, &plaintext, &mut ciphertext);
            assert!(ciphertext.to_hex() == "530f8afbc74536b9a963b4f1c4cb738b");

            let mut resulttext = [0u8; 1];
            let mut cipher2 = CipherAESGCM::new(&key, nonce);
            assert!(cipher2.decrypt_and_inc(&authtext, &ciphertext, &mut resulttext) == true);
            assert!(resulttext[0] == 0);
            ciphertext[0] ^= 1;
            assert!(cipher1.nonce == 1 && cipher2.nonce == 1);
            cipher2.nonce = 0;
            assert!(cipher2.decrypt_and_inc(&authtext, &ciphertext, &mut resulttext) == false);

            // Test Case 14
            let plaintext2 = [0u8; 16];
            let mut ciphertext2 = [0u8; 32];
            let mut cipher3 = CipherAESGCM::new(&key, 0);
            cipher3.encrypt_and_inc(&authtext, &plaintext2, &mut ciphertext2);
            assert!(ciphertext2.to_hex() == "cea7403d4d606b6e074ec5d3baf39d18d0d1c8a799996bf0265b98b5d48ab919");

            let mut resulttext2 = [1u8; 16];
            let mut cipher4 = CipherAESGCM::new(&key, 0);
            assert!(cipher4.decrypt_and_inc(&authtext, &ciphertext2, &mut resulttext2) == true);
            assert!(plaintext2 == resulttext2);
            ciphertext2[0] ^= 1;
            assert!(cipher3.nonce == 1 && cipher4.nonce == 1);
            cipher4.nonce = 0;
            assert!(cipher4.decrypt_and_inc(&authtext, &ciphertext2, &mut resulttext2) == false);
        }

    }
}
<commit_msg>Added SHA512 and BLAKE2b test<commit_after>extern crate crypto;
extern crate byteorder;
extern crate rand;

use self::crypto::digest::Digest;
use self::crypto::sha2::{Sha256, Sha512};
use self::crypto::blake2b::Blake2b;
use self::crypto::aes::KeySize;
use self::crypto::aes_gcm::AesGcm;
use self::crypto::chacha20poly1305::ChaCha20Poly1305;
use self::crypto::aead::{AeadEncryptor, AeadDecryptor};
use self::crypto::curve25519::{curve25519, curve25519_base};

use self::byteorder::{ByteOrder, BigEndian, LittleEndian};
use self::rand::{OsRng, Rng};

use crypto_stuff::*;

pub struct Dh25519 {
    privkey : [u8; 32],
    pubkey  : [u8; 32]
}

pub struct CipherAESGCM {
    key : [u8; 32],
    nonce : u64
}

pub struct CipherChaChaPoly {
    key : [u8; 32],
    nonce : u64
}

pub struct HashSHA256 {
    hasher : Sha256
}

pub struct HashSHA512 {
    hasher : Sha512
}

pub struct HashBLAKE2b {
    hasher : Blake2b
}

impl Dh for Dh25519 {

    fn new(privkey: &[u8], pubkey: &[u8]) -> Dh25519 {
        let mut dh = Dh25519{privkey: [0u8; 32], pubkey: [0u8; 32]};
        copy_memory(privkey, &mut dh.privkey); /* RUSTSUCKS: Why can't I convert slice -> array? */
        copy_memory(pubkey, &mut dh.pubkey);
        dh
    }

    fn generate() -> Dh25519 {
        let mut privkey = [0u8; 32];
        let mut rng = OsRng::new().unwrap();
        rng.fill_bytes(&mut privkey);
        privkey[0] &= 248;
        privkey[31] &= 127;
        privkey[31] |= 64;
        let pubkey = curve25519_base(&privkey);
        Dh25519{privkey : privkey, pubkey : pubkey}
    }

    fn pubkey(&self) -> &[u8] {
        &self.pubkey
    }

    fn dh(&self, pubkey: &[u8]) -> [u8; DHLEN] {
        curve25519(&self.privkey, pubkey)
    }

}

impl Cipher for CipherAESGCM {

    fn new(key: &[u8], nonce: u64) -> CipherAESGCM {
        let mut cipher = CipherAESGCM{key: [0u8; 32], nonce: nonce};
        copy_memory(key, &mut cipher.key);
        cipher
    }

    fn encrypt_and_inc(&mut self, authtext: &[u8], plaintext: &[u8], out: &mut[u8]) {
        let mut nonce_bytes = [0u8; 12];
        BigEndian::write_u64(&mut nonce_bytes[4..], self.nonce);
        self.nonce += 1;
        let mut cipher = AesGcm::new(KeySize::KeySize256, &self.key, &nonce_bytes, authtext);
        let mut tag = [0u8; TAGLEN];
        cipher.encrypt(plaintext, &mut out[..plaintext.len()], &mut tag);
        copy_memory(&tag, &mut out[plaintext.len()..]);
    }

    fn decrypt_and_inc(&mut self, authtext: &[u8], ciphertext: &[u8], out: &mut[u8]) -> bool {
        let mut nonce_bytes = [0u8; 12];
        BigEndian::write_u64(&mut nonce_bytes[4..], self.nonce);
        self.nonce += 1;
        let mut cipher = AesGcm::new(KeySize::KeySize256, &self.key, &nonce_bytes, authtext);
        let text_len = ciphertext.len() - TAGLEN;
        let mut tag = [0u8; TAGLEN];
        copy_memory(&ciphertext[text_len..], &mut tag);
        cipher.decrypt(&ciphertext[..text_len], &mut out[..text_len], &tag)
    }

}

impl Cipher for CipherChaChaPoly {

    fn new(key: &[u8], nonce: u64) -> CipherChaChaPoly {
        let mut cipher = CipherChaChaPoly{key: [0u8; 32], nonce: nonce};
        copy_memory(key, &mut cipher.key);
        cipher
    }

    fn encrypt_and_inc(&mut self, authtext: &[u8], plaintext: &[u8], out: &mut[u8]) {
        let mut nonce_bytes = [0u8; 8];
        LittleEndian::write_u64(&mut nonce_bytes, self.nonce);
        self.nonce += 1;
        let mut cipher = ChaCha20Poly1305::new(&self.key, &nonce_bytes, authtext);
        let mut tag = [0u8; TAGLEN];
        cipher.encrypt(plaintext, &mut out[..plaintext.len()], &mut tag);
        copy_memory(&tag, &mut out[plaintext.len()..]);
    }

    fn decrypt_and_inc(&mut self, authtext: &[u8], ciphertext: &[u8], out: &mut[u8]) -> bool {
        let mut nonce_bytes = [0u8; 8];
        LittleEndian::write_u64(&mut nonce_bytes, self.nonce);
        self.nonce += 1;
        let mut cipher = ChaCha20Poly1305::new(&self.key, &nonce_bytes, authtext);
        let text_len = ciphertext.len() - TAGLEN;
        let mut tag = [0u8; TAGLEN];
        copy_memory(&ciphertext[text_len..], &mut tag);
        cipher.decrypt(&ciphertext[..text_len], &mut out[..text_len], &tag)
    }

}

impl Hash for HashSHA256 {

    fn new() -> HashSHA256 {
        HashSHA256{hasher : Sha256::new()}
    }

    fn block_len() -> usize {
        return 64;
    }

    fn hash_len() -> usize {
        return 32;
    }

    fn input(&mut self, data: &[u8]) {
        self.hasher.input(data);
    }

    fn result(&mut self, out: &mut [u8]) {
        self.hasher.result(out);
    }
}

impl Hash for HashSHA512 {

    fn new() -> HashSHA512 {
        HashSHA512{hasher : Sha512::new()}
    }

    fn block_len() -> usize {
        return 128;
    }

    fn hash_len() -> usize {
        return 64;
    }

    fn input(&mut self, data: &[u8]) {
        self.hasher.input(data);
    }

    fn result(&mut self, out: &mut [u8]) {
        self.hasher.result(out);
    }
}

impl Hash for HashBLAKE2b {

    fn new() -> HashBLAKE2b {
        HashBLAKE2b{hasher : Blake2b::new(64)}
    }

    fn block_len() -> usize {
        return 128;
    }

    fn hash_len() -> usize {
        return 64;
    }

    fn input(&mut self, data: &[u8]) {
        self.hasher.input(data);
    }

    fn result(&mut self, out: &mut [u8]) {
        self.hasher.result(out);
    }
}

#[cfg(test)]
mod tests {

    extern crate rustc_serialize;

    use crypto_stuff::*;
    use super::*;
    use self::rustc_serialize::hex::{FromHex, ToHex};

    #[test]
    fn crypto_tests() {

        // SHA256 test
        {
            let mut output = [0u8; 32];
            let mut hasher = HashSHA256::new();
            hasher.input("abc".as_bytes());
            hasher.result(&mut output);
            assert!(output.to_hex() == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
        }

        // HMAC-SHA256 and HMAC-SHA512 test - RFC 4231
        {
            let key = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".from_hex().unwrap();
            let data = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd".from_hex().unwrap();
            let mut output1 = [0u8; 32];
            HashSHA256::hmac(&key, &data, &mut output1);
            assert!(output1.to_hex() == "773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe");

            let mut output2 = [0u8; 64];
            HashSHA512::hmac(&key, &data, &mut output2);
            assert!(output2.to_hex() == "fa73b0089d56a284efb0f0756c890be9\
                                         b1b5dbdd8ee81a3655f83e33b2279d39\
                                         bf3e848279a722c806b485a47e67c807\
                                         b946a337bee8942674278859e13292fb");
        }

        // BLAKE2b test - draft-saarinen-blake2-06
        {
            let mut output = [0u8; 64];
            let mut hasher = HashBLAKE2b::new();
            hasher.input("abc".as_bytes());
            hasher.result(&mut output);
            assert!(output.to_hex() == "ba80a53f981c4d0d6a2797b69f12f6e9\
                                        4c212f14685ac4b74b12bb6fdbffa2d1\
                                        7d87c5392aab792dc252d5de4533cc95\
                                        18d38aa8dbf1925ab92386edd4009923");
        }

        // Curve25519 test - draft-curves-10
        {
            let mut keypair = Dh25519::new(&[0; 32], &[0; 32]);
            let scalar = "a546e36bf0527c9d3b16154b82465edd62144c0ac1fc5a18506a2244ba449ac4".from_hex().unwrap();
            copy_memory(&scalar, &mut keypair.privkey);
            let public = "e6db6867583030db3594c1a424b15f7c726624ec26b3353b10a903a6d0ab1c4c".from_hex().unwrap();
            let output = keypair.dh(&public);
            assert!(output.to_hex() == "c3da55379de9c6908e94ea4df28d084f32eccf03491c71f754b4075577a28552");
        }

        //AES256-GCM tests - gcm-spec.pdf
        {
            // Test Case 13
            let key = [0u8; 32];
            let nonce = 0u64;
            let plaintext = [0u8; 0];
            let authtext = [0u8; 0];
            let mut ciphertext = [0u8; 16];
            let mut cipher1 = CipherAESGCM::new(&key, nonce);
            cipher1.encrypt_and_inc(&authtext, &plaintext, &mut ciphertext);
            assert!(ciphertext.to_hex() == "530f8afbc74536b9a963b4f1c4cb738b");

            let mut resulttext = [0u8; 1];
            let mut cipher2 = CipherAESGCM::new(&key, nonce);
            assert!(cipher2.decrypt_and_inc(&authtext, &ciphertext, &mut resulttext) == true);
            assert!(resulttext[0] == 0);
            ciphertext[0] ^= 1;
            assert!(cipher1.nonce == 1 && cipher2.nonce == 1);
            cipher2.nonce = 0;
            assert!(cipher2.decrypt_and_inc(&authtext, &ciphertext, &mut resulttext) == false);

            // Test Case 14
            let plaintext2 = [0u8; 16];
            let mut ciphertext2 = [0u8; 32];
            let mut cipher3 = CipherAESGCM::new(&key, 0);
            cipher3.encrypt_and_inc(&authtext, &plaintext2, &mut ciphertext2);
            assert!(ciphertext2.to_hex() == "cea7403d4d606b6e074ec5d3baf39d18d0d1c8a799996bf0265b98b5d48ab919");

            let mut resulttext2 = [1u8; 16];
            let mut cipher4 = CipherAESGCM::new(&key, 0);
            assert!(cipher4.decrypt_and_inc(&authtext, &ciphertext2, &mut resulttext2) == true);
            assert!(plaintext2 == resulttext2);
            ciphertext2[0] ^= 1;
            assert!(cipher3.nonce == 1 && cipher4.nonce == 1);
            cipher4.nonce = 0;
            assert!(cipher4.decrypt_and_inc(&authtext, &ciphertext2, &mut resulttext2) == false);
        }

    }
}
<|endoftext|>


<commit_before>use fs::File;
use output::file_name::FileStyle;

pub fn painted_icon(file: &File, style: &FileStyle) -> String {
    let file_icon = icon(&file).to_string();
    let painted = style.exts
            .colour_file(&file)
            .map_or(file_icon.to_string(), |c| { c.paint(file_icon).to_string() });
    format!("{} ", painted)

}

fn icon(file: &File) -> char {
    if file.is_directory() { '\u{f115}' }
    else {
        // possible unnecessary clone
        if let Some(ext) = file.ext.clone() {
            match ext.as_str() {
                "ai" => '\u{e7b4}',
                "android" => '\u{e70e}',
                "apple" => '\u{f179}',
                "audio" => '\u{f001}',
                "avro" => '\u{e60b}',
                "c" => '\u{e61e}',
                "clj" => '\u{e768}',
                "coffee" => '\u{f0f4}',
                "conf" => '\u{e615}',
                "cpp" => '\u{e61d}',
                "css" => '\u{e749}',
                "d" => '\u{e7af}',
                "dart" => '\u{e798}',
                "db" => '\u{f1c0}',
                "diff" => '\u{f440}',
                "doc" => '\u{f1c2}',
                "ebook" => '\u{e28b}',
                "env" => '\u{f462}',
                "epub" => '\u{e28a}',
                "erl" => '\u{e7b1}',
                "font" => '\u{f031}',
                "gform" => '\u{f298}',
                "git" => '\u{f1d3}',
                "go" => '\u{e626}',
                "hs" => '\u{e777}',
                "html" => '\u{f13b}',
                "image" => '\u{f1c5}',
                "iml" => '\u{e7b5}',
                "java" => '\u{e204}',
                "js" => '\u{e74e}',
                "json" => '\u{e60b}',
                "jsx" => '\u{e7ba}',
                "less" => '\u{e758}',
                "log" => '\u{f18d}',
                "lua" => '\u{e620}',
                "md" => '\u{f48a}',
                "mustache" => '\u{e60f}',
                "npmignore" => '\u{e71e}',
                "pdf" => '\u{f1c1}',
                "php" => '\u{e73d}',
                "pl" => '\u{e769}',
                "ppt" => '\u{f1c4}',
                "psd" => '\u{e7b8}',
                "py" => '\u{e606}',
                "r" => '\u{f25d}',
                "rb" => '\u{e21e}',
                "rdb" => '\u{e76d}',
                "rs" => '\u{e7a8}',
                "rss" => '\u{f09e}',
                "rubydoc" => '\u{e73b}',
                "sass" => '\u{e603}',
                "scala" => '\u{e737}',
                "shell" => '\u{f489}',
                "sqlite3" => '\u{e7c4}',
                "styl" => '\u{e600}',
                "tex" => '\u{e600}',
                "ts" => '\u{e628}',
                "twig" => '\u{e61c}',
                "txt" => '\u{f15c}',
                "video" => '\u{f03d}',
                "vim" => '\u{e62b}',
                "windows" => '\u{f17a}',
                "xls" => '\u{f1c3}',
                "xml" => '\u{e619}',
                "yml" => '\u{f481}',
                "zip" => '\u{f410}',
                _ => '\u{f15b}'
            }
        } else {
            '\u{f15b}'
        }
    }
}
<commit_msg>remove underline from icon<commit_after>use ansi_term::Style;
use fs::File;
use output::file_name::FileStyle;

pub fn painted_icon(file: &File, style: &FileStyle) -> String {
    let file_icon = icon(&file).to_string();
    let painted = style.exts
            .colour_file(&file)
            .map_or(file_icon.to_string(), |c| {
                // Remove underline from icon
                if c.is_underline {
                    match c.foreground {
                        Some(color) => Style::from(color).paint(file_icon).to_string(),
                        None => Style::default().paint(file_icon).to_string(),
                    }
                } else {
                    c.paint(file_icon).to_string()
                }
            });
    format!("{} ", painted)
}

fn icon(file: &File) -> char {
    if file.is_directory() { '\u{f115}' }
    else {
        // possible unnecessary clone
        if let Some(ext) = file.ext.clone() {
            match ext.as_str() {
                "ai" => '\u{e7b4}',
                "android" => '\u{e70e}',
                "apple" => '\u{f179}',
                "audio" => '\u{f001}',
                "avro" => '\u{e60b}',
                "c" => '\u{e61e}',
                "clj" => '\u{e768}',
                "coffee" => '\u{f0f4}',
                "conf" => '\u{e615}',
                "cpp" => '\u{e61d}',
                "css" => '\u{e749}',
                "d" => '\u{e7af}',
                "dart" => '\u{e798}',
                "db" => '\u{f1c0}',
                "diff" => '\u{f440}',
                "doc" => '\u{f1c2}',
                "ebook" => '\u{e28b}',
                "env" => '\u{f462}',
                "epub" => '\u{e28a}',
                "erl" => '\u{e7b1}',
                "font" => '\u{f031}',
                "gform" => '\u{f298}',
                "git" => '\u{f1d3}',
                "go" => '\u{e626}',
                "hs" => '\u{e777}',
                "html" => '\u{f13b}',
                "image" => '\u{f1c5}',
                "iml" => '\u{e7b5}',
                "java" => '\u{e204}',
                "js" => '\u{e74e}',
                "json" => '\u{e60b}',
                "jsx" => '\u{e7ba}',
                "less" => '\u{e758}',
                "log" => '\u{f18d}',
                "lua" => '\u{e620}',
                "md" => '\u{f48a}',
                "mustache" => '\u{e60f}',
                "npmignore" => '\u{e71e}',
                "pdf" => '\u{f1c1}',
                "php" => '\u{e73d}',
                "pl" => '\u{e769}',
                "ppt" => '\u{f1c4}',
                "psd" => '\u{e7b8}',
                "py" => '\u{e606}',
                "r" => '\u{f25d}',
                "rb" => '\u{e21e}',
                "rdb" => '\u{e76d}',
                "rs" => '\u{e7a8}',
                "rss" => '\u{f09e}',
                "rubydoc" => '\u{e73b}',
                "sass" => '\u{e603}',
                "scala" => '\u{e737}',
                "shell" => '\u{f489}',
                "sqlite3" => '\u{e7c4}',
                "styl" => '\u{e600}',
                "tex" => '\u{e600}',
                "ts" => '\u{e628}',
                "twig" => '\u{e61c}',
                "txt" => '\u{f15c}',
                "video" => '\u{f03d}',
                "vim" => '\u{e62b}',
                "windows" => '\u{f17a}',
                "xls" => '\u{f1c3}',
                "xml" => '\u{e619}',
                "yml" => '\u{f481}',
                "zip" => '\u{f410}',
                _ => '\u{f15b}'
            }
        } else {
            '\u{f15b}'
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use backend::{MemoryBackend, StoreBackend};
use errors::HatError;
use hat::HatRc;
use hat::family::Family;
use key;
use std::sync::Arc;
use util::FileIterator;

pub fn setup_hat<B: StoreBackend>(backend: Arc<B>) -> HatRc<B> {
    let max_blob_size = 4 * 1024 * 1024;
    HatRc::new_for_testing(backend, max_blob_size).unwrap()
}

fn setup_family() -> (Arc<MemoryBackend>, HatRc<MemoryBackend>, Family<MemoryBackend>) {
    let backend = Arc::new(MemoryBackend::new());
    let hat = setup_hat(backend.clone());

    let family = "familyname".to_string();
    let fam = hat.open_family(family).unwrap();

    (backend, hat, fam)
}

pub fn entry(name: Vec<u8>) -> key::Entry {
    key::Entry {
        name: name,
        id: None,
        parent_id: None,
        created: None,
        modified: None,
        accessed: None,
        permissions: None,
        user_id: None,
        group_id: None,
        data_hash: None,
        data_length: None,
    }
}

fn snapshot_files<B: StoreBackend>(family: &Family<B>,
                                   files: Vec<(&str, Vec<u8>)>)
                                   -> Result<(), HatError> {
    for (name, contents) in files {
        try!(family.snapshot_direct(entry(name.bytes().collect()),
                                    false,
                                    Some(FileIterator::from_bytes(contents))));
    }
    Ok(())
}

#[test]
fn snapshot_commit() {
    let (_, mut hat, fam) = setup_family();

    snapshot_files(&fam,
                   vec![("name1", vec![0; 1000000]),
                        ("name2", vec![1; 1000000]),
                        ("name3", vec![2; 1000000])])
        .unwrap();

    fam.flush().unwrap();
    hat.commit(&fam, None).unwrap();
    hat.meta_commit().unwrap();

    let (deleted, live) = hat.gc().unwrap();
    assert_eq!(deleted, 0);
    assert!(live > 0);
}

#[test]
fn snapshot_commit_many_empty_files() {
    let (_, mut hat, fam) = setup_family();

    let names: Vec<String> = (0..3000).map(|i| format!("name-{}", i)).collect();
    snapshot_files(&fam, names.iter().map(|n| (n.as_str(), vec![])).collect()).unwrap();

    fam.flush().unwrap();
    hat.commit(&fam, None).unwrap();
    hat.meta_commit().unwrap();

    let (deleted, live) = hat.gc().unwrap();
    assert_eq!(deleted, 0);
    assert!(live > 0);

    hat.deregister(&fam, 1).unwrap();
    let (deleted, live) = hat.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live, 0);
}

#[test]
fn snapshot_commit_many_empty_directories() {
    let (_, mut hat, fam) = setup_family();

    for i in 0..3000 {
        fam.snapshot_direct(entry(format!("name-{}", i).bytes().collect()), true, None)
            .unwrap();
    }

    fam.flush().unwrap();
    hat.commit(&fam, None).unwrap();
    hat.meta_commit().unwrap();

    let (deleted, live) = hat.gc().unwrap();
    assert_eq!(deleted, 0);
    assert!(live > 0);

    hat.deregister(&fam, 1).unwrap();
    let (deleted, live) = hat.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live, 0);
}

#[test]
fn snapshot_reuse_index() {
    let (_, mut hat, fam) = setup_family();

    let files = vec![("file1", "block1".bytes().collect()),
                     ("file2", "block2".bytes().collect()),
                     ("file3", "block3".bytes().collect()),
                     ("file4", "block1".bytes().collect()),
                     ("file5", "block2".bytes().collect())];

    // Insert hashes.
    snapshot_files(&fam, files.clone()).unwrap();
    fam.flush().unwrap();

    // Reuse hashes.
    snapshot_files(&fam, files.clone()).unwrap();
    snapshot_files(&fam, files.clone()).unwrap();
    fam.flush().unwrap();

    // No commit, so GC removes all the new hashes.
    let (deleted, live) = hat.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live, 0);

    // Update index and reinsert hashes.
    snapshot_files(&fam, files.clone()).unwrap();
    fam.flush().unwrap();

    // Commit.
    hat.commit(&fam, None).unwrap();
    let (deleted, live) = hat.gc().unwrap();
    assert_eq!(deleted, 0);
    assert!(live > 0);

    // Inserting again does not increase number of hashes.
    snapshot_files(&fam, files.clone()).unwrap();
    fam.flush().unwrap();
    let (deleted2, live2) = hat.gc().unwrap();
    assert_eq!(live2, live);
    assert_eq!(deleted2, 0);

    // Cleanup: only 1 snapshot was committed.
    hat.deregister(&fam, 1).unwrap();
    let (deleted, live) = hat.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live, 0);
}

#[test]
fn snapshot_gc() {
    let (_, mut hat, fam) = setup_family();

    snapshot_files(&fam,
                   vec![("name1", vec![0; 1000000]),
                        ("name2", vec![1; 1000000]),
                        ("name3", vec![2; 1000000])])
        .unwrap();

    fam.flush().unwrap();

    // No commit so everything is deleted.
    let (deleted, live) = hat.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live, 0);
}

#[test]
fn recover() {
    // Prepare a snapshot.
    let (backend, mut hat, fam) = setup_family();

    snapshot_files(&fam,
                   vec![("name1", vec![0; 1000000]),
                        ("name2", vec![1; 1000000]),
                        ("name3", vec![2; 1000000])])
        .unwrap();
    fam.flush().unwrap();
    hat.commit(&fam, None).unwrap();
    hat.meta_commit().unwrap();

    let (deleted, live1) = hat.gc().unwrap();
    assert_eq!(deleted, 0);
    assert!(live1 > 0);

    // Create a new hat to wipe the index states.
    let mut hat2 = setup_hat(backend);

    // Recover index states.
    hat2.recover().unwrap();

    // Check that we now reference all the blobs.
    let (deleted, live2) = hat2.gc().unwrap();
    assert_eq!(deleted, 0);
    assert_eq!(live1, live2);

    // Check that we can delete the snapshot.
    hat2.deregister(&fam, 1).unwrap();

    let (deleted, live3) = hat2.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live3, 0);
}
<commit_msg>Test directories too.<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use backend::{MemoryBackend, StoreBackend};
use errors::HatError;
use hat::HatRc;
use hat::family::Family;
use key;
use std::collections::HashMap;
use std::sync::Arc;
use util::FileIterator;

pub fn setup_hat<B: StoreBackend>(backend: Arc<B>) -> HatRc<B> {
    let max_blob_size = 4 * 1024 * 1024;
    HatRc::new_for_testing(backend, max_blob_size).unwrap()
}

fn setup_family() -> (Arc<MemoryBackend>, HatRc<MemoryBackend>, Family<MemoryBackend>) {
    let backend = Arc::new(MemoryBackend::new());
    let hat = setup_hat(backend.clone());

    let family = "familyname".to_string();
    let fam = hat.open_family(family).unwrap();

    (backend, hat, fam)
}

pub fn entry(name: Vec<u8>) -> key::Entry {
    key::Entry {
        name: name,
        id: None,
        parent_id: None,
        created: None,
        modified: None,
        accessed: None,
        permissions: None,
        user_id: None,
        group_id: None,
        data_hash: None,
        data_length: None,
    }
}

fn snapshot_files<B: StoreBackend>(family: &Family<B>,
                                   files: Vec<(&str, Vec<u8>)>)
                                   -> Result<(), HatError> {
    let mut dirs = HashMap::new();
    for (name, contents) in files {
        let mut parent = None;
        let mut parts = name.split("/").peekable();
        let mut current = parts.next().unwrap();
        loop {
            if parts.peek().is_none() {
                // Reached the filename part of the string.
                break;
            }
            let mut e = entry(current.bytes().collect());
            e.parent_id = parent.clone();

            parent = dirs.entry((parent, current))
                .or_insert_with(|| {
                    Some(family.snapshot_direct(e, true, None)
                        .unwrap())
                })
                .clone();
            current = parts.next().unwrap();
        }
        if current.len() > 0 {
            // We have a file to insert.
            let mut e = entry(current.bytes().collect());
            e.parent_id = parent.clone();
            try!(family.snapshot_direct(e,
                                        false,
                                        Some(FileIterator::from_bytes(contents))));
        }
    }
    Ok(())
}

fn basic_snapshot<B: StoreBackend>(fam: &Family<B>) {
    snapshot_files(&fam,
                   vec![("zeros", vec![0; 1000000]),
                        ("ones", vec![1; 1000000]),
                        ("twos", vec![2; 1000000]),
                        ("zeros2", vec![0; 1000000]),
                        ("block1", "block1".into()),
                        ("block2", "block2".into()),
                        ("block3", "block3".into()),
                        ("block12", "block1".into()),
                        ("some/number/of/empty/dirs/with/an/empty/file/deep/down", vec![]),
                        ("dir1/zeros", vec![0; 10]),
                        ("dir1/block1", "block1".into()),
                        ("dir1/unique", "abcdefg".into()),
                        ("dir2/zeros", vec![0; 10]),
                        ("dir2/dir3/twos", vec![2; 10]),
                        ("dir2/dir3/dir4/ones", vec![1; 10]),
                        ("x/y/z/a", vec![]),
                        ("x/y/z/b/", vec![]),
                        ("z/y/x/a", vec![]),
                        ("y/x/z/b/", vec![]),
                        ("a/", vec![]),
                        ("b/", vec![]),
                        ("c/", vec![]),
                   ])
        .unwrap();
}

#[test]
fn snapshot_commit() {
    let (_, mut hat, fam) = setup_family();

    basic_snapshot(&fam);

    fam.flush().unwrap();
    hat.commit(&fam, None).unwrap();
    hat.meta_commit().unwrap();

    let (deleted, live) = hat.gc().unwrap();
    assert_eq!(deleted, 0);
    assert!(live > 0);
}

#[test]
fn snapshot_commit_many_empty_files() {
    let (_, mut hat, fam) = setup_family();

    let names: Vec<String> = (0..3000).map(|i| format!("name-{}", i)).collect();
    snapshot_files(&fam, names.iter().map(|n| (n.as_str(), vec![])).collect()).unwrap();

    fam.flush().unwrap();
    hat.commit(&fam, None).unwrap();
    hat.meta_commit().unwrap();

    let (deleted, live) = hat.gc().unwrap();
    assert_eq!(deleted, 0);
    assert!(live > 0);

    hat.deregister(&fam, 1).unwrap();
    let (deleted, live) = hat.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live, 0);
}

#[test]
fn snapshot_commit_many_empty_directories() {
    let (_, mut hat, fam) = setup_family();

    for i in 0..3000 {
        fam.snapshot_direct(entry(format!("name-{}", i).bytes().collect()), true, None)
            .unwrap();
    }

    fam.flush().unwrap();
    hat.commit(&fam, None).unwrap();
    hat.meta_commit().unwrap();

    let (deleted, live) = hat.gc().unwrap();
    assert_eq!(deleted, 0);
    assert!(live > 0);

    hat.deregister(&fam, 1).unwrap();
    let (deleted, live) = hat.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live, 0);
}

#[test]
fn snapshot_reuse_index() {
    let (_, mut hat, fam) = setup_family();

    // Insert hashes.
    basic_snapshot(&fam);
    fam.flush().unwrap();

    // Reuse hashes.
    basic_snapshot(&fam);
    basic_snapshot(&fam);
    fam.flush().unwrap();

    // No commit, so GC removes all the new hashes.
    let (deleted, live) = hat.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live, 0);

    // Update index and reinsert hashes.
    basic_snapshot(&fam);
    fam.flush().unwrap();

    // Commit.
    hat.commit(&fam, None).unwrap();
    let (deleted, live) = hat.gc().unwrap();
    assert_eq!(deleted, 0);
    assert!(live > 0);

    // Inserting again does not increase number of hashes.
    basic_snapshot(&fam);
    fam.flush().unwrap();
    let (deleted2, live2) = hat.gc().unwrap();
    assert_eq!(live2, live);
    assert_eq!(deleted2, 0);

    // Cleanup: only 1 snapshot was committed.
    hat.deregister(&fam, 1).unwrap();
    let (deleted, live) = hat.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live, 0);
}

#[test]
fn snapshot_gc() {
    let (_, mut hat, fam) = setup_family();

    basic_snapshot(&fam);
    fam.flush().unwrap();

    // No commit so everything is deleted.
    let (deleted, live) = hat.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live, 0);
}

#[test]
fn recover() {
    // Prepare a snapshot.
    let (backend, mut hat, fam) = setup_family();
    basic_snapshot(&fam);
    fam.flush().unwrap();

    hat.commit(&fam, None).unwrap();
    hat.meta_commit().unwrap();

    let (deleted, live1) = hat.gc().unwrap();
    assert_eq!(deleted, 0);
    assert!(live1 > 0);

    // Create a new hat to wipe the index states.
    let mut hat2 = setup_hat(backend);

    // Recover index states.
    hat2.recover().unwrap();

    // Check that we now reference all the blobs.
    let (deleted, live2) = hat2.gc().unwrap();
    assert_eq!(deleted, 0);
    assert_eq!(live1, live2);

    // Check that we can delete the snapshot.
    hat2.deregister(&fam, 1).unwrap();

    let (deleted, live3) = hat2.gc().unwrap();
    assert!(deleted > 0);
    assert_eq!(live3, 0);
}
<|endoftext|>


<commit_before>pub mod world;
<commit_msg>cleanup<commit_after><|endoftext|>


<commit_before>use super::location::Location;

#[derive(Clone)]
struct StringStream {
    input: Vec<char>,
    index: usize,
    line: i32,
    column: i32,
}

impl StringStream {
    fn new(input: &str) -> Self {
        Self {
            input: input.chars().collect(),
            index: 0,
            line: 1,
            column: 1,
        }
    }

    fn read(&mut self) -> char {
        let character = self
            .input
            .get(self.index)
            .map(|value| *value)
            .unwrap_or('\0');
        self.index += 1;
        if character == '\n' {
            self.line += 1;
            self.column = 1;
        } else {
            self.column += 1;
        }
        character
    }

    fn peek(&self) -> char {
        self.input
            .get(self.index)
            .map(|value| *value)
            .unwrap_or('\0')
    }

    fn read_until(&mut self, target: char) {
        while self.peek() != target {
            let character = self.read();
            assert!(character != '\0');
        }
    }

    fn read_if_match(&mut self, target: &[char]) -> bool {
        // Bounds-check first!
        if (self.index + target.len()) > self.input.len() {
            return false;
        }
        let substr = &self.input[self.index..(self.index + target.len())];
        if substr != target {
            return false;
        }
        // Consume if they match.
        for _ in target {
            self.read();
        }
        true
    }

    fn location(&self) -> Location {
        Location::new(self.index as u32, self.line, self.column)
    }
}

#[derive(Clone, Debug, PartialEq)]
pub enum Token {
    Arrow,
    BraceLeft,
    BraceRight,
    Comma,
    Dot,
    EOF,
    Equals,
    Identifier(String, Location),
    Integer(i64),
    Let(Location),
    Minus,
    ParenthesesLeft,
    ParenthesesRight,
    Plus,
    Return,
    Star,
    Terminal(char),
    Var(Location),
}

impl Token {
    pub fn location(&self) -> Option<Location> {
        match self {
            Token::Let(location) | Token::Var(location) => {
                Some(location.clone())
            }
            Token::Identifier(_, location) => Some(location.clone()),
            _ => None,
        }
    }
}

impl Token {
    pub fn newline(self) -> bool {
        match self {
            Token::Terminal(character) => character == '\n',
            _ => false,
        }
    }
}

#[derive(Clone)]
pub struct TokenStream {
    input: StringStream,
    peeking: bool,
    next_token: Option<Token>,
}

impl TokenStream {
    pub fn from_string(input: String) -> Self {
        Self::new(StringStream::new(input.as_str()))
    }

    fn new(input: StringStream) -> Self {
        TokenStream {
            input: input,
            peeking: false,
            next_token: None,
        }
    }

    pub fn backtrack(&mut self, other: &TokenStream) {
        self.input = other.input.clone();
        self.peeking = other.peeking;
        self.next_token = other.next_token.clone();
    }

    pub fn peek(&mut self) -> Token {
        if !self.peeking {
            self.peeking = true;
            self.next_token = Some(self.lex());
        }
        self.next_token.clone().unwrap()
    }

    pub fn read_if(&mut self, expected: Token) -> Option<Token> {
        let got = self.peek();
        if expected == got {
            Some(self.read())
        } else {
            None
        }
    }

    pub fn read(&mut self) -> Token {
        let token = self.peek();
        self.peeking = false;
        return token;
    }

    fn lex(&mut self) -> Token {
        self.consume_space_and_comments();

        loop {
            let location = self.input.location();
            let character = self.input.peek();

            if identifier_head(character) {
                return self.lex_identifier(location);
            } else if numeric_head(character) {
                return self.lex_arrow_minus_or_numeric();
            } else {
                self.input.read();
                match character {
                    '\0' => return Token::EOF,
                    '{' => return Token::BraceLeft,
                    '}' => return Token::BraceRight,
                    '(' => return Token::ParenthesesLeft,
                    ')' => return Token::ParenthesesRight,
                    ',' => return Token::Comma,
                    '.' => return Token::Dot,
                    '=' => return Token::Equals,
                    '+' => return Token::Plus,
                    '*' => return Token::Star,
                    ';' => return Token::Terminal(character),
                    '\n' => {
                        self.consume_more_newline_terminals();
                        return Token::Terminal(character);
                    }
                    _ => panic!("Unexpected character: {}", character),
                }
            }
        }
    }

    fn lex_identifier(&mut self, location: Location) -> Token {
        let mut identifier = vec![self.input.read()];
        loop {
            let character = self.input.peek();
            if identifier_tail(character) {
                self.input.read();
                identifier.push(character);
            } else {
                break;
            }
        }
        let identifier_string: String = identifier.into_iter().collect();
        match identifier_string.as_str() {
            "let" => Token::Let(location),
            "var" => Token::Var(location),
            "return" => Token::Return,
            _ => Token::Identifier(identifier_string, location),
        }
    }

    fn lex_arrow_minus_or_numeric(&mut self) -> Token {
        let first_character = self.input.read();
        if first_character == '-' && self.input.peek() == '>' {
            self.input.read();
            return Token::Arrow;
        }
        if first_character == '-' && !digit(self.input.peek()) {
            return Token::Minus;
        }
        let mut number = vec![first_character];
        loop {
            let character = self.input.peek();
            if digit(character) {
                self.input.read();
                number.push(character);
            } else {
                break;
            }
        }
        let number_string: String = number.into_iter().collect();
        return Token::Integer(number_string.parse().unwrap());
    }

    fn consume_more_newline_terminals(&mut self) {
        loop {
            self.consume_space_and_comments();
            if self.input.peek() == '\n' {
                self.input.read();
                continue;
            } else {
                break;
            }
        }
    }

    fn consume_space_and_comments(&mut self) {
        loop {
            let character = self.input.peek();
            if character == ' ' || character == '\t' {
                self.input.read();
            } else if self.input.read_if_match(&['/', '/']) {
                self.input.read_until('\n');
            } else if self.input.read_if_match(&['/', '*']) {
                loop {
                    if self.input.read_if_match(&['*', '/']) {
                        break;
                    } else {
                        self.input.read();
                    }
                }
            } else {
                // Not any kind of comment.
                break;
            }
        }
    }
}

fn identifier_head(character: char) -> bool {
    alphabetical(character)
}

fn identifier_tail(character: char) -> bool {
    alphabetical(character)
}

fn numeric_head(character: char) -> bool {
    digit(character) || (character == '-')
}

fn alphabetical(character: char) -> bool {
    (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z')
}

fn digit(character: char) -> bool {
    (character >= '0' && character <= '9')
}

#[cfg(test)]
mod tests {
    use super::{Location, StringStream, Token, TokenStream};

    fn parse(input: &str) -> Vec<Token> {
        let string_stream = StringStream::new(input);
        let mut token_stream = TokenStream::new(string_stream);
        let mut tokens = vec![];
        loop {
            let token = token_stream.read();
            tokens.push(token.clone());
            if token == Token::EOF {
                break;
            }
        }
        return tokens;
    }

    #[test]
    fn it_parses_identifier() {
        assert_eq!(
            parse("foo"),
            vec![
                Token::Identifier("foo".to_string(), Location::new(0, 1, 1)),
                Token::EOF,
            ]
        );
    }

    #[test]
    fn it_parses_keywords() {
        assert_eq!(
            parse("let"),
            vec![Token::Let(Location::new(0, 1, 1)), Token::EOF,]
        );
        assert_eq!(
            parse("return"),
            vec![Token::Return, Token::EOF,],
        );
        assert_eq!(
            parse("var"),
            vec![Token::Var(Location::new(0, 1, 1)), Token::EOF,]
        );
    }

    #[test]
    fn it_parses_arrows() {
        assert_eq!(
            parse("foo() -> {}"),
            vec![
                Token::Identifier("foo".to_string(), Location::new(0, 1, 1)),
                Token::ParenthesesLeft,
                Token::ParenthesesRight,
                Token::Arrow,
                Token::BraceLeft,
                Token::BraceRight,
                Token::EOF,
            ],
        );
    }

    #[test]
    fn it_parse_comments_and_terminals() {
        assert_eq!(
            parse("foo /* Comment */ bar"),
            vec![
                Token::Identifier("foo".to_string(), Location::new(0, 1, 1)),
                Token::Identifier("bar".to_string(), Location::new(18, 1, 19)),
                Token::EOF,
            ]
        );

        assert_eq!(
            parse("foo // Comment \n bar"),
            vec![
                Token::Identifier("foo".to_string(), Location::new(0, 1, 1)),
                Token::Terminal('\n'),
                Token::Identifier("bar".to_string(), Location::new(17, 2, 2)),
                Token::EOF,
            ]
        );

        assert_eq!(
            parse(
                "foo
            // Comment about the call
            // Another comment about the call
            bar()"
            ),
            vec![
                Token::Identifier("foo".to_string(), Location::new(0, 1, 1)),
                Token::Terminal('\n'),
                Token::Identifier("bar".to_string(), Location::new(100, 4, 13)),
                Token::ParenthesesLeft,
                Token::ParenthesesRight,
                Token::EOF,
            ]
        );
    }

    #[test]
    fn it_parses_integers() {
        assert_eq!(parse("1"), vec![Token::Integer(1), Token::EOF]);
        assert_eq!(parse("-1"), vec![Token::Integer(-1), Token::EOF]);
        assert_eq!(
            parse("- 1"),
            vec![Token::Minus, Token::Integer(1), Token::EOF]
        );
        assert_eq!(
            parse("1+2"),
            vec![
                Token::Integer(1),
                Token::Plus,
                Token::Integer(2),
                Token::EOF,
            ]
        );
        assert_eq!(
            parse("1 + 2"),
            vec![
                Token::Integer(1),
                Token::Plus,
                Token::Integer(2),
                Token::EOF,
            ]
        );
    }

    #[test]
    fn it_parses_blocks() {
        assert_eq!(
            parse("{ 1; }"),
            vec![
                Token::BraceLeft,
                Token::Integer(1),
                Token::Terminal(';'),
                Token::BraceRight,
                Token::EOF,
            ]
        );
    }
}
<commit_msg>Make `read_if` only check discriminant/variant and add tests for it<commit_after>use super::location::Location;

#[derive(Clone)]
struct StringStream {
    input: Vec<char>,
    index: usize,
    line: i32,
    column: i32,
}

impl StringStream {
    fn new(input: &str) -> Self {
        Self {
            input: input.chars().collect(),
            index: 0,
            line: 1,
            column: 1,
        }
    }

    fn read(&mut self) -> char {
        let character = self
            .input
            .get(self.index)
            .map(|value| *value)
            .unwrap_or('\0');
        self.index += 1;
        if character == '\n' {
            self.line += 1;
            self.column = 1;
        } else {
            self.column += 1;
        }
        character
    }

    fn peek(&self) -> char {
        self.input
            .get(self.index)
            .map(|value| *value)
            .unwrap_or('\0')
    }

    fn read_until(&mut self, target: char) {
        while self.peek() != target {
            let character = self.read();
            assert!(character != '\0');
        }
    }

    fn read_if_match(&mut self, target: &[char]) -> bool {
        // Bounds-check first!
        if (self.index + target.len()) > self.input.len() {
            return false;
        }
        let substr = &self.input[self.index..(self.index + target.len())];
        if substr != target {
            return false;
        }
        // Consume if they match.
        for _ in target {
            self.read();
        }
        true
    }

    fn location(&self) -> Location {
        Location::new(self.index as u32, self.line, self.column)
    }
}

#[derive(Clone, Debug, PartialEq)]
pub enum Token {
    Arrow,
    BraceLeft,
    BraceRight,
    Comma,
    Dot,
    EOF,
    Equals,
    Identifier(String, Location),
    Integer(i64),
    Let(Location),
    Minus,
    ParenthesesLeft,
    ParenthesesRight,
    Plus,
    Return,
    Star,
    Terminal(char),
    Var(Location),
}

impl Token {
    pub fn location(&self) -> Option<Location> {
        match self {
            Token::Let(location) | Token::Var(location) => {
                Some(location.clone())
            }
            Token::Identifier(_, location) => Some(location.clone()),
            _ => None,
        }
    }
}

impl Token {
    pub fn newline(self) -> bool {
        match self {
            Token::Terminal(character) => character == '\n',
            _ => false,
        }
    }
}

#[derive(Clone)]
pub struct TokenStream {
    input: StringStream,
    peeking: bool,
    next_token: Option<Token>,
}

impl TokenStream {
    pub fn from_string(input: String) -> Self {
        Self::new(StringStream::new(input.as_str()))
    }

    fn new(input: StringStream) -> Self {
        TokenStream {
            input: input,
            peeking: false,
            next_token: None,
        }
    }

    pub fn backtrack(&mut self, other: &TokenStream) {
        self.input = other.input.clone();
        self.peeking = other.peeking;
        self.next_token = other.next_token.clone();
    }

    pub fn peek(&mut self) -> Token {
        if !self.peeking {
            self.peeking = true;
            self.next_token = Some(self.lex());
        }
        self.next_token.clone().unwrap()
    }

    /// Peeks, and if the peeked token is the same type as the expected then
    /// read that type and return the read (not expected) token.
    pub fn read_if(&mut self, expected: Token) -> Option<Token> {
        use std::mem::discriminant;

        let got = self.peek();
        if discriminant(&expected) == discriminant(&got) {
            Some(self.read())
        } else {
            None
        }
    }

    pub fn read(&mut self) -> Token {
        let token = self.peek();
        self.peeking = false;
        return token;
    }

    fn lex(&mut self) -> Token {
        self.consume_space_and_comments();

        loop {
            let location = self.input.location();
            let character = self.input.peek();

            if identifier_head(character) {
                return self.lex_identifier(location);
            } else if numeric_head(character) {
                return self.lex_arrow_minus_or_numeric();
            } else {
                self.input.read();
                match character {
                    '\0' => return Token::EOF,
                    '{' => return Token::BraceLeft,
                    '}' => return Token::BraceRight,
                    '(' => return Token::ParenthesesLeft,
                    ')' => return Token::ParenthesesRight,
                    ',' => return Token::Comma,
                    '.' => return Token::Dot,
                    '=' => return Token::Equals,
                    '+' => return Token::Plus,
                    '*' => return Token::Star,
                    ';' => return Token::Terminal(character),
                    '\n' => {
                        self.consume_more_newline_terminals();
                        return Token::Terminal(character);
                    }
                    _ => panic!("Unexpected character: {}", character),
                }
            }
        }
    }

    fn lex_identifier(&mut self, location: Location) -> Token {
        let mut identifier = vec![self.input.read()];
        loop {
            let character = self.input.peek();
            if identifier_tail(character) {
                self.input.read();
                identifier.push(character);
            } else {
                break;
            }
        }
        let identifier_string: String = identifier.into_iter().collect();
        match identifier_string.as_str() {
            "let" => Token::Let(location),
            "var" => Token::Var(location),
            "return" => Token::Return,
            _ => Token::Identifier(identifier_string, location),
        }
    }

    fn lex_arrow_minus_or_numeric(&mut self) -> Token {
        let first_character = self.input.read();
        if first_character == '-' && self.input.peek() == '>' {
            self.input.read();
            return Token::Arrow;
        }
        if first_character == '-' && !digit(self.input.peek()) {
            return Token::Minus;
        }
        let mut number = vec![first_character];
        loop {
            let character = self.input.peek();
            if digit(character) {
                self.input.read();
                number.push(character);
            } else {
                break;
            }
        }
        let number_string: String = number.into_iter().collect();
        return Token::Integer(number_string.parse().unwrap());
    }

    fn consume_more_newline_terminals(&mut self) {
        loop {
            self.consume_space_and_comments();
            if self.input.peek() == '\n' {
                self.input.read();
                continue;
            } else {
                break;
            }
        }
    }

    fn consume_space_and_comments(&mut self) {
        loop {
            let character = self.input.peek();
            if character == ' ' || character == '\t' {
                self.input.read();
            } else if self.input.read_if_match(&['/', '/']) {
                self.input.read_until('\n');
            } else if self.input.read_if_match(&['/', '*']) {
                loop {
                    if self.input.read_if_match(&['*', '/']) {
                        break;
                    } else {
                        self.input.read();
                    }
                }
            } else {
                // Not any kind of comment.
                break;
            }
        }
    }
}

fn identifier_head(character: char) -> bool {
    alphabetical(character)
}

fn identifier_tail(character: char) -> bool {
    alphabetical(character)
}

fn numeric_head(character: char) -> bool {
    digit(character) || (character == '-')
}

fn alphabetical(character: char) -> bool {
    (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z')
}

fn digit(character: char) -> bool {
    (character >= '0' && character <= '9')
}

#[cfg(test)]
mod tests {
    use super::{Location, StringStream, Token, TokenStream};

    fn make_token_stream(input: &str) -> TokenStream {
        let string_stream = StringStream::new(input);
        TokenStream::new(string_stream)
    }

    fn parse(input: &str) -> Vec<Token> {
        let mut token_stream = make_token_stream(input);
        let mut tokens = vec![];
        loop {
            let token = token_stream.read();
            tokens.push(token.clone());
            if token == Token::EOF {
                break;
            }
        }
        return tokens;
    }

    #[test]
    fn it_parses_identifier() {
        assert_eq!(
            parse("foo"),
            vec![
                Token::Identifier("foo".to_string(), Location::new(0, 1, 1)),
                Token::EOF,
            ]
        );
    }

    #[test]
    fn it_parses_keywords() {
        assert_eq!(
            parse("let"),
            vec![Token::Let(Location::new(0, 1, 1)), Token::EOF,]
        );
        assert_eq!(
            parse("return"),
            vec![Token::Return, Token::EOF,],
        );
        assert_eq!(
            parse("var"),
            vec![Token::Var(Location::new(0, 1, 1)), Token::EOF,]
        );
    }

    #[test]
    fn it_parses_arrows() {
        assert_eq!(
            parse("foo() -> {}"),
            vec![
                Token::Identifier("foo".to_string(), Location::new(0, 1, 1)),
                Token::ParenthesesLeft,
                Token::ParenthesesRight,
                Token::Arrow,
                Token::BraceLeft,
                Token::BraceRight,
                Token::EOF,
            ],
        );
    }

    #[test]
    fn it_parse_comments_and_terminals() {
        assert_eq!(
            parse("foo /* Comment */ bar"),
            vec![
                Token::Identifier("foo".to_string(), Location::new(0, 1, 1)),
                Token::Identifier("bar".to_string(), Location::new(18, 1, 19)),
                Token::EOF,
            ]
        );

        assert_eq!(
            parse("foo // Comment \n bar"),
            vec![
                Token::Identifier("foo".to_string(), Location::new(0, 1, 1)),
                Token::Terminal('\n'),
                Token::Identifier("bar".to_string(), Location::new(17, 2, 2)),
                Token::EOF,
            ]
        );

        assert_eq!(
            parse(
                "foo
            // Comment about the call
            // Another comment about the call
            bar()"
            ),
            vec![
                Token::Identifier("foo".to_string(), Location::new(0, 1, 1)),
                Token::Terminal('\n'),
                Token::Identifier("bar".to_string(), Location::new(100, 4, 13)),
                Token::ParenthesesLeft,
                Token::ParenthesesRight,
                Token::EOF,
            ]
        );
    }

    #[test]
    fn it_parses_integers() {
        assert_eq!(parse("1"), vec![Token::Integer(1), Token::EOF]);
        assert_eq!(parse("-1"), vec![Token::Integer(-1), Token::EOF]);
        assert_eq!(
            parse("- 1"),
            vec![Token::Minus, Token::Integer(1), Token::EOF]
        );
        assert_eq!(
            parse("1+2"),
            vec![
                Token::Integer(1),
                Token::Plus,
                Token::Integer(2),
                Token::EOF,
            ]
        );
        assert_eq!(
            parse("1 + 2"),
            vec![
                Token::Integer(1),
                Token::Plus,
                Token::Integer(2),
                Token::EOF,
            ]
        );
    }

    #[test]
    fn it_parses_blocks() {
        assert_eq!(
            parse("{ 1; }"),
            vec![
                Token::BraceLeft,
                Token::Integer(1),
                Token::Terminal(';'),
                Token::BraceRight,
                Token::EOF,
            ]
        );
    }

    #[test]
    fn test_read_if() {
        let input = "a";
        // Check that `read` works correctly.
        let actual = Token::Identifier("a".to_string(), Location::new(0, 1, 1));
        assert_eq!(
            make_token_stream(input).read(),
            actual,
        );
        // Now check that the contents of the expected token don't equal the
        // actual token.
        let expected = Token::Identifier("".to_string(), Location::new(0, 0, 0));
        assert_ne!(
            make_token_stream(input).read(),
            expected,
        );
        // But `read_if` should still return the actual token since they're
        // both `Identifier`s.
        assert_eq!(
            make_token_stream("a").read_if(expected),
            Some(actual),
        );
        // And finally if we look for something that isn't an identifier it
        // should return `None`.
        assert_eq!(
            make_token_stream("a").read_if(Token::ParenthesesLeft),
            None,
        );
    }
}
<|endoftext|>


<commit_before>import elasticsearch::{
    client,
    json_dict_builder,
    search_builder,
    index_builder,
    delete_builder
};

export model;
export find;
export search;
export all;

type model = {
    es: client,
    _index: str,
    _type: str,
    _id: str,
    _parent: option<str>,
    _version: option<uint>,
    source: hashmap<str, json>,
};

fn model(es: client, index: str, typ: str, id: str) -> model {
    {
        es: es,
        _index: index,
        _type: typ,
        _id: id,
        _parent: none,
        _version: none,
        source: str_hash()
    }
}

fn hit_to_model(es: client, hit: hashmap<str, json::json>) -> model {
    let index = alt check hit.get("_index") { json::string(s) { s } };
    let typ = alt check hit.get("_type") { json::string(s) { s } };
    let id = alt check hit.get("_id") { json::string(s) { s } };

    let parent = hit.find("_parent").chain { |s|
        alt check s { json::string(s) { some(s) } }
    };

    let version = hit.find("_version").chain { |v|
        alt check v { json::num(n) { some(n as uint) } }
    };

    let source = alt check hit.get("_source") {
      json::dict(source) { source }
    };

    {
        es: es,
        _index: index,
        _type: typ,
        _id: id,
        _parent: parent,
        _version: version,
        source: source
    }
}

fn find(es: client, index: str, typ: str, id: str) -> option<model> {
    let rep = es.get(index, typ, id);

    let hit = alt check rep.body { json::dict(hit) { hit } };

    alt hit.get("exists") {
      json::boolean(true) { some(hit_to_model(es, hit)) }
      _ { none }
    }
}

fn search(es: client, f: fn(search_builder)) -> [model] {
    let bld = es.prepare_search();
    f(bld);

    let rep = bld.execute();

    alt check rep.body {
      json::dict(body) {
        alt check body.get("hits") {
          json::dict(hits) {
            alt check hits.get("hits") {
              json::list(hits) {
                  hits.map { |hit|
                      alt check hit {
                        json::dict(hit) { hit_to_model(es, hit) }
                      }
                  }
              }
            }
          }
        }
      }
    }
}

fn all(es: client, index: str, typ: str) -> [model] {
    search(es) { |bld|
        bld
            .set_indices([index])
            .set_types([typ])
            .set_source(*json_dict_builder()
                .insert_dict("query") { |bld|
                    bld.insert_dict("match_all") { |_bld| };
                }
            );
    }
}

impl model for model {
    fn set_null(key: str) -> bool {
        self.source.insert(key, json::null)
    }

    fn get_bool(key: str) -> bool {
        alt self.find_bool(key) {
          none { fail }
          some(value) { value }
        }
    }

    fn find_bool(key: str) -> option<bool> {
        self.source.find(key).chain { |value|
            alt value {
              json::boolean(value) { some(value) }
              _ { none }
            }
        }
    }

    fn set_bool(key: str, value: bool) -> bool {
        self.source.insert(key, json::boolean(value))
    }

    fn find_str(key: str) -> option<str> {
        self.source.find(key).chain { |value|
            alt value {
              json::string(value) { some(value) }
              _ { none }
            }
        }
    }

    fn get_str(key: str) -> str {
        alt self.find_str(key) {
          none { fail }
          some(value) { value }
        }
    }

    fn set_str(key: str, value: str) -> bool {
        self.source.insert(key, json::string(value))
    }

    fn find_float(key: str) -> option<float> {
        self.source.find(key).chain { |value|
            alt value {
              json::num(value) { some(value) }
              _ { none }
            }
        }
    }

    fn get_float(key: str) -> float {
        alt self.find_float(key) {
          none { fail }
          some(value) { value }
        }
    }

    fn set_float(key: str, value: float) -> bool {
        self.source.insert(key, json::num(value))
    }

    fn find_uint(key: str) -> option<uint> {
        self.find_float(key).map { |value| value as uint }
    }

    fn get_uint(key: str) -> uint {
        self.get_float(key) as uint
    }

    fn set_uint(key: str, value: uint) -> bool {
        self.set_float(key, value as float)
    }

    fn find_int(key: str) -> option<int> {
        self.find_float(key).map { |value| value as int }
    }

    fn get_int(key: str) -> int {
        self.get_float(key) as int
    }

    fn set_int(key: str, value: int) -> bool {
        self.set_float(key, value as float)
    }

    fn index(op_type: elasticsearch::op_type) -> result<(str, uint), str> {
        let index = self.es.prepare_index(self._index, self._type)
            .set_op_type(op_type)
            .set_source(self.source)
            .set_refresh(true);

        if self._id != "" { index.set_id(self._id); }
        self._parent.iter { |p| index.set_parent(p); }
        self._version.iter { |v| index.set_version(v); }

        let rep = index.execute();

        if rep.code == 200u || rep.code == 201u {
            let body = alt check rep.body { json::dict(body) { body } };
            let id = alt check body.get("_id") {
              json::string(id) { id }
            };
            let version = alt check body.get("_version") {
              json::num(version) { version as uint }
            };

            ok((id, version))

        } else {
            let body = alt check rep.body { json::dict(body) { body } };
            let e = alt check body.get("error") { json::string(e) { e } };
            err(e)
        }
    }

    fn create() -> result<(str, uint), str> {
        self.index(elasticsearch::CREATE)
    }

    fn save() -> result<(str, uint), str> {
        self.index(elasticsearch::INDEX)
    }

    fn delete() {
        if self._id != "" {
            self.es.prepare_delete(self._index, self._type, self._id)
                .set_refresh(true)
                .execute();
        }
    }
}
<commit_msg>Fail the ES body if we had an error.<commit_after>import elasticsearch::{
    client,
    json_dict_builder,
    search_builder,
    index_builder,
    delete_builder
};

export model;
export find;
export search;
export all;

type model = {
    es: client,
    _index: str,
    _type: str,
    _id: str,
    _parent: option<str>,
    _version: option<uint>,
    source: hashmap<str, json>,
};

fn model(es: client, index: str, typ: str, id: str) -> model {
    {
        es: es,
        _index: index,
        _type: typ,
        _id: id,
        _parent: none,
        _version: none,
        source: str_hash()
    }
}

fn hit_to_model(es: client, hit: hashmap<str, json::json>) -> model {
    let index = alt check hit.get("_index") { json::string(s) { s } };
    let typ = alt check hit.get("_type") { json::string(s) { s } };
    let id = alt check hit.get("_id") { json::string(s) { s } };

    let parent = hit.find("_parent").chain { |s|
        alt check s { json::string(s) { some(s) } }
    };

    let version = hit.find("_version").chain { |v|
        alt check v { json::num(n) { some(n as uint) } }
    };

    let source = alt check hit.get("_source") {
      json::dict(source) { source }
    };

    {
        es: es,
        _index: index,
        _type: typ,
        _id: id,
        _parent: parent,
        _version: version,
        source: source
    }
}

fn find(es: client, index: str, typ: str, id: str) -> option<model> {
    let rep = es.get(index, typ, id);

    // Fail if ES had an error.
    if rep.code != 200u {
        fail json::to_str(rep.body);
    }

    let hit = alt check rep.body { json::dict(hit) { hit } };

    alt hit.get("exists") {
      json::boolean(true) { some(hit_to_model(es, hit)) }
      _ { none }
    }
}

fn search(es: client, f: fn(search_builder)) -> [model] {
    let bld = es.prepare_search();
    f(bld);

    let rep = bld.execute();

    // Fail if ES had an error.
    if rep.code != 200u {
        fail json::to_str(rep.body);
    }

    alt check rep.body {
      json::dict(body) {
        alt check body.get("hits") {
          json::dict(hits) {
            alt check hits.get("hits") {
              json::list(hits) {
                  hits.map { |hit|
                      alt check hit {
                        json::dict(hit) { hit_to_model(es, hit) }
                      }
                  }
              }
            }
          }
        }
      }
    }
}

fn all(es: client, index: str, typ: str) -> [model] {
    search(es) { |bld|
        bld
            .set_indices([index])
            .set_types([typ])
            .set_source(*json_dict_builder()
                .insert_dict("query") { |bld|
                    bld.insert_dict("match_all") { |_bld| };
                }
            );
    }
}

impl model for model {
    fn set_null(key: str) -> bool {
        self.source.insert(key, json::null)
    }

    fn get_bool(key: str) -> bool {
        alt self.find_bool(key) {
          none { fail }
          some(value) { value }
        }
    }

    fn find_bool(key: str) -> option<bool> {
        self.source.find(key).chain { |value|
            alt value {
              json::boolean(value) { some(value) }
              _ { none }
            }
        }
    }

    fn set_bool(key: str, value: bool) -> bool {
        self.source.insert(key, json::boolean(value))
    }

    fn find_str(key: str) -> option<str> {
        self.source.find(key).chain { |value|
            alt value {
              json::string(value) { some(value) }
              _ { none }
            }
        }
    }

    fn get_str(key: str) -> str {
        alt self.find_str(key) {
          none { fail }
          some(value) { value }
        }
    }

    fn set_str(key: str, value: str) -> bool {
        self.source.insert(key, json::string(value))
    }

    fn find_float(key: str) -> option<float> {
        self.source.find(key).chain { |value|
            alt value {
              json::num(value) { some(value) }
              _ { none }
            }
        }
    }

    fn get_float(key: str) -> float {
        alt self.find_float(key) {
          none { fail }
          some(value) { value }
        }
    }

    fn set_float(key: str, value: float) -> bool {
        self.source.insert(key, json::num(value))
    }

    fn find_uint(key: str) -> option<uint> {
        self.find_float(key).map { |value| value as uint }
    }

    fn get_uint(key: str) -> uint {
        self.get_float(key) as uint
    }

    fn set_uint(key: str, value: uint) -> bool {
        self.set_float(key, value as float)
    }

    fn find_int(key: str) -> option<int> {
        self.find_float(key).map { |value| value as int }
    }

    fn get_int(key: str) -> int {
        self.get_float(key) as int
    }

    fn set_int(key: str, value: int) -> bool {
        self.set_float(key, value as float)
    }

    fn index(op_type: elasticsearch::op_type) -> result<(str, uint), str> {
        let index = self.es.prepare_index(self._index, self._type)
            .set_op_type(op_type)
            .set_source(self.source)
            .set_refresh(true);

        if self._id != "" { index.set_id(self._id); }
        self._parent.iter { |p| index.set_parent(p); }
        self._version.iter { |v| index.set_version(v); }

        let rep = index.execute();

        if rep.code == 200u || rep.code == 201u {
            let body = alt check rep.body { json::dict(body) { body } };
            let id = alt check body.get("_id") {
              json::string(id) { id }
            };
            let version = alt check body.get("_version") {
              json::num(version) { version as uint }
            };

            ok((id, version))

        } else {
            let body = alt check rep.body { json::dict(body) { body } };
            let e = alt check body.get("error") { json::string(e) { e } };
            err(e)
        }
    }

    fn create() -> result<(str, uint), str> {
        self.index(elasticsearch::CREATE)
    }

    fn save() -> result<(str, uint), str> {
        self.index(elasticsearch::INDEX)
    }

    fn delete() {
        if self._id != "" {
            self.es.prepare_delete(self._index, self._type, self._id)
                .set_refresh(true)
                .execute();
        }
    }
}
<|endoftext|>