<commit_before>use super::*;
use crate::ram::cell::*;
use crate::ram::chunk::Chunks;
use crate::ram::schema::*;
use crate::ram::types::*;
use crate::server::ServerMeta;
use bifrost_hasher::hash_str;
use env_logger;
use std;
use std::sync::atomic::{AtomicU8, Ordering};
use std::sync::Arc;

pub const CHUNK_SIZE: usize = 1 * 8 * 1024 * 1024;

#[test]
pub fn round_robin_segment() {
    let num = AtomicU8::new(std::u8::MAX);
    assert_eq!(num.load(Ordering::SeqCst), 255);
    assert_eq!(num.fetch_add(1, Ordering::SeqCst), 255);
    assert_eq!(num.fetch_add(1, Ordering::SeqCst), 0);
    assert_eq!(num.fetch_add(1, Ordering::SeqCst), 1);
}

#[test]
pub fn cell_rw() {
    let _ = env_logger::try_init();
    info!("START");
    let id1 = Id::new(1, 1);
    let id2 = Id::new(1, 2);
    let fields = default_fields();
    let schema = Schema::new("dummy", None, fields, false, false);
    let mut data_map = OwnedMap::new();
    data_map.insert(&String::from("id"), OwnedValue::I64(100));
    data_map.insert(&String::from("score"), OwnedValue::U64(70));
    data_map.insert(
        &String::from("name"),
        OwnedValue::String(String::from("Jack")),
    );
    let mut data = OwnedValue::Map(data_map);
    let schemas = LocalSchemasCache::new_local("");
    schemas.new_schema(schema.clone());
    let mut cell = OwnedCell {
        header: CellHeader::new(schema.id, &id1),
        data,
    };
    let chunks = Chunks::new(
        1,
        CHUNK_SIZE,
        Arc::new(ServerMeta { schemas }),
        None,
        None,
        None,
    );
    let header = chunks.write_cell(&mut cell).unwrap();
    let cell_1_ptr = chunks.address_of(&Id::from_header(&header));
    {
        let stored_cell = chunks.read_cell(&id1).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &100);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "Jack");
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &70);
    }
    data_map = OwnedMap::new();
    data_map.insert(&String::from("id"), OwnedValue::I64(2));
    data_map.insert(&String::from("score"), OwnedValue::U64(80));
    data_map.insert(
        &String::from("name"),
        OwnedValue::String(String::from("John")),
    );
    data = OwnedValue::Map(data_map);
    cell = OwnedCell {
        header: CellHeader::new(schema.id, &id2),
        data,
    };
    let header = chunks.write_cell(&mut cell).unwrap();
    let cell_2_ptr = chunks.address_of(&Id::from_header(&header));
    {
        let stored_cell = chunks.read_cell(&id2).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &2);
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &80);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "John");
    }
    {
        let stored_cell = chunks.read_cell(&id1).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &100);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "Jack");
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &70);
    }
    data_map = OwnedMap::new();
    data_map.insert(&String::from("id"), OwnedValue::I64(2));
    data_map.insert(&String::from("score"), OwnedValue::U64(95));
    data_map.insert(
        &String::from("name"),
        OwnedValue::String(String::from("John")),
    );
    data = OwnedValue::Map(data_map);
    cell = OwnedCell {
        header: CellHeader::new(schema.id, &id2),
        data,
    };
    let header = chunks.update_cell(&mut cell).unwrap();
    let _cell_2_ptr = chunks.address_of(&Id::from_header(&header));
    {
        let stored_cell = chunks.read_cell(&id2).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &2);
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &95);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "John");
    }
    {
        let stored_cell = chunks.read_cell(&id1).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &100);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "Jack");
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &70);
    }
    chunks
        .update_cell_by(&id2, |cell| {
            let mut data_map = OwnedMap::new();
            data_map.insert(&String::from("id"), OwnedValue::I64(2));
            data_map.insert(&String::from("score"), OwnedValue::U64(100));
            data_map.insert(
                &String::from("name"),
                OwnedValue::String(String::from("John")),
            );
            let data = OwnedValue::Map(data_map);
            let mut cell = cell.to_owned();
            cell.data = data;
            Some(cell)
        })
        .unwrap();
    {
        let stored_cell = chunks.read_cell(&id2).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &2);
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &100);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "John");
    }
    {
        let sel_cell = chunks.read_selected(&id2, &[
            hash_str("score"),
            hash_str("name")
        ]).unwrap();
        assert_eq!(sel_cell["score"].u64().unwrap(), &100);
        assert_eq!(sel_cell["name"].string().unwrap(), "John");
    }
    chunks.remove_cell(&id1).unwrap();
    assert!(chunks.read_cell(&id1).is_err());
}

#[test]
pub fn simple_cell_rw() {
    let _ = env_logger::try_init();
    let id1 = Id::new(1, 1);
    let fields = simple_fields();
    let schema = Schema::new("simple", None, fields, false, true);
    let data = OwnedValue::U64(128);
    let schemas = LocalSchemasCache::new_local("");
    schemas.new_schema(schema.clone());
    let mut cell = OwnedCell {
        header: CellHeader::new(schema.id, &id1),
        data
    };
    let chunks = Chunks::new(
        1,
        CHUNK_SIZE,
        Arc::new(ServerMeta { schemas }),
        None,
        None,
        None
    );
    chunks.write_cell(&mut cell).unwrap();
    {
        let stored_cell = chunks.read_cell(&id1).unwrap();
        assert_eq!(stored_cell.data.u64(), Some(&128));
    }
}

// #[test]
// pub fn complex_cell_sel_read() {
//     let _ = env_logger::try_init();
//     let fields = Field::new(
//         &String::from("*"),
//         data_type,
//         nullable,
//         is_array,
//          sub_fields,
//          indices
//     )
// }<commit_msg>modeling complex schema<commit_after>use super::*;
use crate::ram::cell::*;
use crate::ram::chunk::Chunks;
use crate::ram::schema::*;
use crate::ram::types::*;
use crate::server::ServerMeta;
use bifrost_hasher::hash_str;
use env_logger;
use std;
use std::sync::atomic::{AtomicU8, Ordering};
use std::sync::Arc;

pub const CHUNK_SIZE: usize = 1 * 8 * 1024 * 1024;

#[test]
pub fn round_robin_segment() {
    let num = AtomicU8::new(std::u8::MAX);
    assert_eq!(num.load(Ordering::SeqCst), 255);
    assert_eq!(num.fetch_add(1, Ordering::SeqCst), 255);
    assert_eq!(num.fetch_add(1, Ordering::SeqCst), 0);
    assert_eq!(num.fetch_add(1, Ordering::SeqCst), 1);
}

#[test]
pub fn cell_rw() {
    let _ = env_logger::try_init();
    info!("START");
    let id1 = Id::new(1, 1);
    let id2 = Id::new(1, 2);
    let fields = default_fields();
    let schema = Schema::new("dummy", None, fields, false, false);
    let mut data_map = OwnedMap::new();
    data_map.insert(&String::from("id"), OwnedValue::I64(100));
    data_map.insert(&String::from("score"), OwnedValue::U64(70));
    data_map.insert(
        &String::from("name"),
        OwnedValue::String(String::from("Jack")),
    );
    let mut data = OwnedValue::Map(data_map);
    let schemas = LocalSchemasCache::new_local("");
    schemas.new_schema(schema.clone());
    let mut cell = OwnedCell {
        header: CellHeader::new(schema.id, &id1),
        data,
    };
    let chunks = Chunks::new(
        1,
        CHUNK_SIZE,
        Arc::new(ServerMeta { schemas }),
        None,
        None,
        None,
    );
    let header = chunks.write_cell(&mut cell).unwrap();
    let cell_1_ptr = chunks.address_of(&Id::from_header(&header));
    {
        let stored_cell = chunks.read_cell(&id1).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &100);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "Jack");
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &70);
    }
    data_map = OwnedMap::new();
    data_map.insert(&String::from("id"), OwnedValue::I64(2));
    data_map.insert(&String::from("score"), OwnedValue::U64(80));
    data_map.insert(
        &String::from("name"),
        OwnedValue::String(String::from("John")),
    );
    data = OwnedValue::Map(data_map);
    cell = OwnedCell {
        header: CellHeader::new(schema.id, &id2),
        data,
    };
    let header = chunks.write_cell(&mut cell).unwrap();
    let cell_2_ptr = chunks.address_of(&Id::from_header(&header));
    {
        let stored_cell = chunks.read_cell(&id2).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &2);
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &80);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "John");
    }
    {
        let stored_cell = chunks.read_cell(&id1).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &100);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "Jack");
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &70);
    }
    data_map = OwnedMap::new();
    data_map.insert(&String::from("id"), OwnedValue::I64(2));
    data_map.insert(&String::from("score"), OwnedValue::U64(95));
    data_map.insert(
        &String::from("name"),
        OwnedValue::String(String::from("John")),
    );
    data = OwnedValue::Map(data_map);
    cell = OwnedCell {
        header: CellHeader::new(schema.id, &id2),
        data,
    };
    let header = chunks.update_cell(&mut cell).unwrap();
    let _cell_2_ptr = chunks.address_of(&Id::from_header(&header));
    {
        let stored_cell = chunks.read_cell(&id2).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &2);
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &95);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "John");
    }
    {
        let stored_cell = chunks.read_cell(&id1).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &100);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "Jack");
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &70);
    }
    chunks
        .update_cell_by(&id2, |cell| {
            let mut data_map = OwnedMap::new();
            data_map.insert(&String::from("id"), OwnedValue::I64(2));
            data_map.insert(&String::from("score"), OwnedValue::U64(100));
            data_map.insert(
                &String::from("name"),
                OwnedValue::String(String::from("John")),
            );
            let data = OwnedValue::Map(data_map);
            let mut cell = cell.to_owned();
            cell.data = data;
            Some(cell)
        })
        .unwrap();
    {
        let stored_cell = chunks.read_cell(&id2).unwrap();
        assert_eq!(stored_cell.data["id"].i64().unwrap(), &2);
        assert_eq!(stored_cell.data["score"].u64().unwrap(), &100);
        assert_eq!(stored_cell.data["name"].string().unwrap(), "John");
    }
    {
        let sel_cell = chunks.read_selected(&id2, &[
            hash_str("score"),
            hash_str("name")
        ]).unwrap();
        assert_eq!(sel_cell["score"].u64().unwrap(), &100);
        assert_eq!(sel_cell["name"].string().unwrap(), "John");
    }
    chunks.remove_cell(&id1).unwrap();
    assert!(chunks.read_cell(&id1).is_err());
}

#[test]
pub fn simple_cell_rw() {
    let _ = env_logger::try_init();
    let id1 = Id::new(1, 1);
    let fields = simple_fields();
    let schema = Schema::new("simple", None, fields, false, true);
    let data = OwnedValue::U64(128);
    let schemas = LocalSchemasCache::new_local("");
    schemas.new_schema(schema.clone());
    let mut cell = OwnedCell {
        header: CellHeader::new(schema.id, &id1),
        data
    };
    let chunks = Chunks::new(
        1,
        CHUNK_SIZE,
        Arc::new(ServerMeta { schemas }),
        None,
        None,
        None
    );
    chunks.write_cell(&mut cell).unwrap();
    {
        let stored_cell = chunks.read_cell(&id1).unwrap();
        assert_eq!(stored_cell.data.u64(), Some(&128));
    }
}

#[test]
pub fn complex_cell_sel_read() {
    let _ = env_logger::try_init();
    let id1 = Id::new(1, 1);
    let fields = complex_fields();
    let schema = Schema::new("complex", None, fields, false, true);
    let schemas = LocalSchemasCache::new_local("");
    schemas.new_schema(schema.clone());
    let chunks = Chunks::new(
        1,
        CHUNK_SIZE,
        Arc::new(ServerMeta { schemas }),
        None,
        None,
        None
    );
    let data = data_map_value!(
        id: OwnedValue::I64(128),
        strings: OwnedValue::PrimArray(OwnedPrimArray::String(vec![
            String::from("aaaa"),
            String::from("bbbb"),
            String::from("cccc")
        ])),
        num: OwnedValue::U64(256),
        nums: OwnedValue::PrimArray(OwnedPrimArray::U64(vec![
            512, 1024, 2048
        ])),
        sub: data_map_value!(
            sub1: OwnedValue::U32(4096),
            sub2: OwnedValue::PrimArray(OwnedPrimArray::U32(vec![
                8192, 16384
            ])),
            sub3: OwnedValue::U32(1),
            sub4: data_map_value!(
                sub4sub1: OwnedValue::U32(2),
                sub4sub2: OwnedValue::PrimArray(OwnedPrimArray::U32(vec![
                    3, 4, 5
                ])),
                sub4sub3: OwnedValue::PrimArray(OwnedPrimArray::U32(vec![
                    6, 7
                ])),
                sub4sub4: OwnedValue::U16(8)
            ),
            sub5: OwnedValue::Array(vec![
                data_map_value!(
                    sub5sub1: OwnedValue::U32(9),
                    sub5sub2: OwnedValue::PrimArray(OwnedPrimArray::U32(vec![
                        10, 11
                    ])),
                    sub5sub3: OwnedValue::PrimArray(OwnedPrimArray::U32(vec![
                        12, 13, 14
                    ])),
                    sub5sub4: OwnedValue::U16(15)
                ),
                data_map_value!(
                    sub5sub1: OwnedValue::U32(16),
                    sub5sub2: OwnedValue::PrimArray(OwnedPrimArray::U32(vec![
                        17, 18, 19, 20
                    ])),
                    sub5sub3: OwnedValue::PrimArray(OwnedPrimArray::U32(vec![
                        21, 22
                    ])),
                    sub5sub4: OwnedValue::U16(23)
                )
            ]),
            end: OwnedValue::String(String::from("END OF OBJECT"))
        )
    );
    let mut cell = OwnedCell {
        header: CellHeader::new(schema.id, &id1),
        data
    };
    chunks.write_cell(&mut cell).unwrap();
    {
        unimplemented!() // READ
    }
}<|endoftext|>


<commit_before>// https://github.com/serde-rs/serde/issues/927

use std::collections::HashMap;

extern crate yaml_rust;
extern crate argparse;
#[macro_use] extern crate log;
#[macro_use] extern crate from_variants;

extern crate serde;
extern crate serde_json;
extern crate serde_yaml;

#[macro_use] extern crate serde_derive;

#[derive(Clone, Serialize, Deserialize)]
enum ArgValue {
    None,
    Str(String),
    Bool(bool),
    Byte(u8),
    Int16(i16),
    Int32(i32),
    Int64(i64),
    UInt16(u16),
    UInt32(u32),
    UInt64(u64),
    Double(f64),
}

#[derive(Serialize, Deserialize)]
enum DbusMessageType {
    Method,
    Signal,
}

type NamedArgs<'a> = HashMap<&'a str, ArgValue>;

#[derive(Serialize, Deserialize)]
struct DbusMessage<'a> {
    api_name: String,
    dbus_name: String,
    message_type: DbusMessageType,
    #[serde(borrow)]
    args: NamedArgs<'a>,
}

#[derive(Serialize, Deserialize)]
struct DbusMessageRouting<'a>(String, String, String, #[serde(borrow)] Vec<DbusMessage<'a>>);

#[derive(Serialize, Deserialize)]
struct ConfYaml<'a>(#[serde(borrow)] Vec<DbusMessageRouting<'a>>);

#[derive(Serialize, Deserialize)]
struct ClientRequest<'a> {
    tp: DbusMessageType,
    name: String,
    #[serde(borrow)]
    args: NamedArgs<'a>,
}

#[derive(Serialize, Deserialize)]
struct ClientRequestBody<'a>(DbusMessageType, #[serde(borrow)] NamedArgs<'a>);

type CRequest<'a> = HashMap<&'a str, ClientRequestBody<'a>>;

fn main()
{
    let conf = vec![
        DbusMessageRouting(
            "com.a.b.servicetest".to_string(),
            "/com/a/b/servicetest".to_string(),
            "com.a.b.IServiceTest".to_string(),
            vec![ DbusMessage { api_name: "api_name".to_string(),
                                dbus_name: "dbus_name".to_string(),
                                message_type: DbusMessageType::Method,
                                args: [
                                    ("param1", ArgValue::Str("Hi".to_string())),
                                    ("param2", ArgValue::Bool(true)),
                                    ("param3", ArgValue::Int64(17i64)),
                                    ("param4", ArgValue::Double(2.718281828f64)),
                                ].iter().cloned().collect()
                              }
            ]
        )
    ];

    let s = serde_yaml::to_string(&conf).unwrap();
    println!("s : {}", s);

    let r = ClientRequest { tp: DbusMessageType::Method, nm: "api_name".to_string(), args:
                                [       ("param1", ArgValue::Str("Hi".to_string())),
                                        ("param2", ArgValue::Bool(true)),
                                        ("param3", ArgValue::Int64(17i64)),
                                        ("param4", ArgValue::Double(2.718281828f64)),
                                ].iter().cloned().collect()
                          };
    let rs = serde_json::to_string(&r).unwrap();
    println!("rs : {}", rs);

    let rb = ClientRequestBody(DbusMessageType::Method, [
                                        ("param1", ArgValue::Str("Hi".to_string())),
                                        ("param2", ArgValue::Bool(true)),
                                        ("param3", ArgValue::Int64(17i64)),
                                        ("param4", ArgValue::Double(2.718281828f64)),
                                ].iter().cloned().collect()
                          );

    let mut a_request: CRequest = HashMap::new();
    a_request.insert("api_name", rb);

    let rsq = serde_json::to_string(&a_request).unwrap();
    println!("rsq : {}", rsq);

    let de: CRequest = serde_json::from_str(&rsq).expect("json deserialization error");

    let rsq2 = serde_json::to_string(&de).expect("json serialization error");
    println!("rsq2 : {}", rsq2);

    for key in de.keys() {
        println!("key {}", key);
    }
    println!("key[0] : {}", de.keys().nth(0).unwrap());

}
<commit_msg>`ClientRequest` the final.<commit_after>// https://github.com/serde-rs/serde/issues/927

use std::collections::HashMap;

extern crate yaml_rust;
extern crate argparse;
#[macro_use] extern crate log;
#[macro_use] extern crate from_variants;

extern crate serde;
#[macro_use] extern crate serde_json;
extern crate serde_yaml;
#[macro_use] extern crate serde_derive;

#[derive(Clone, Serialize, Deserialize)]
enum ArgValue {
    None,
    Str(String),
    Bool(bool),
    Byte(u8),
    Int16(i16),
    Int32(i32),
    Int64(i64),
    UInt16(u16),
    UInt32(u32),
    UInt64(u64),
    Double(f64),
}

#[derive(Serialize, Deserialize, Debug)]
enum DbusMessageType {
    Method,
    Signal,
}

type NamedArgs<'a> = HashMap<&'a str, ArgValue>;
type NamedArgs2<'a> = HashMap<&'a str, serde_json::value::Value>;

#[derive(Serialize, Deserialize)]
struct DbusMessage<'a> {
    api_name: String,
    dbus_name: String,
    message_type: DbusMessageType,
    #[serde(borrow)]
    args: NamedArgs<'a>,
}

#[derive(Serialize, Deserialize)]
struct DbusMessageRouting<'a>(String, String, String, #[serde(borrow)] Vec<DbusMessage<'a>>);

#[derive(Serialize, Deserialize)]
struct ConfYaml<'a>(#[serde(borrow)] Vec<DbusMessageRouting<'a>>);

#[derive(Serialize, Deserialize, Debug)]
struct ClientRequest<'a> {
    #[serde(rename = "type")]
    tp: DbusMessageType,
    name: String,
    #[serde(borrow)]
    args: NamedArgs2<'a>,
}

#[derive(Serialize, Deserialize)]
struct ClientRequestBody<'a>(DbusMessageType, #[serde(borrow)] NamedArgs<'a>);

type CRequest<'a> = HashMap<&'a str, ClientRequestBody<'a>>;

fn main()
{
    let conf = vec![
        DbusMessageRouting(
            "com.a.b.servicetest".to_string(),
            "/com/a/b/servicetest".to_string(),
            "com.a.b.IServiceTest".to_string(),
            vec![ DbusMessage { api_name: "api_name".to_string(),
                                dbus_name: "dbus_name".to_string(),
                                message_type: DbusMessageType::Method,
                                args: [
                                    ("param1", ArgValue::Str("Hi".to_string())),
                                    ("param2", ArgValue::Bool(true)),
                                    ("param3", ArgValue::Int64(17i64)),
                                    ("param4", ArgValue::Double(2.718281828f64)),
                                ].iter().cloned().collect()
                              }
            ]
        )
    ];

    let s = serde_yaml::to_string(&conf).unwrap();
    println!("s : {}", s);

    let r = ClientRequest { tp: DbusMessageType::Method, name: "api_name".to_string(), args:
                                [       ("param1", json!("Hi")),
                                        ("param2", json!(true)),
                                        ("param3", json!(17i64)),
                                        ("param4", json!(2.718281828f64)),
                                ].iter().cloned().collect()
                          };
    let rs = serde_json::to_string(&r).unwrap();
    println!("rs : {}", rs);

    let d: ClientRequest = serde_json::from_str(&rs).expect("json deserialization error");
    println!("d: {:?}", d);

    let rb = ClientRequestBody(DbusMessageType::Method, [
                                        ("param1", ArgValue::Str("Hi".to_string())),
                                        ("param2", ArgValue::Bool(true)),
                                        ("param3", ArgValue::Int64(17i64)),
                                        ("param4", ArgValue::Double(2.718281828f64)),
                                ].iter().cloned().collect()
                          );

    let mut a_request: CRequest = HashMap::new();
    a_request.insert("api_name", rb);

    let rsq = serde_json::to_string(&a_request).unwrap();
    println!("rsq : {}", rsq);

    let de: CRequest = serde_json::from_str(&rsq).expect("json deserialization error");

    let rsq2 = serde_json::to_string(&de).expect("json serialization error");
    println!("rsq2 : {}", rsq2);

    for key in de.keys() {
        println!("key {}", key);
    }
    println!("key[0] : {}", de.keys().nth(0).unwrap());

}
<|endoftext|>


<commit_before>// https://github.com/serde-rs/serde/issues/927

use std::collections::HashMap;
use std::collections::BTreeMap;

extern crate yaml_rust;
extern crate argparse;
#[macro_use] extern crate log;
#[macro_use] extern crate from_variants;

extern crate serde;
#[macro_use] extern crate serde_json;
extern crate serde_yaml;
#[macro_use] extern crate serde_derive;

#[derive(Clone, Serialize, Deserialize, Debug)]
enum ArgValue {
    None,
    Str(String),
    Bool(bool),
    Byte(u8),
    Int16(i16),
    Int32(i32),
    Int64(i64),
    UInt16(u16),
    UInt32(u32),
    UInt64(u64),
    Double(f64),
    Array(Vec<ArgValue>),
    Hash(BTreeMap<String, ArgValue>),
}

impl ArgValue {
    fn to_json(&self, n: &str) -> String {
        match *self {
            ArgValue::None             =>   "".to_string(),
            ArgValue::Str(ref v)       =>   format!("\"{}\":{}", n, v ),
            ArgValue::Bool(ref v)      =>   format!("\"{}\":{}", n, v ),
            ArgValue::Byte(ref v)      =>   format!("\"{}\":{}", n, v ),
            ArgValue::Int16(ref v)     =>   format!("\"{}\":{}", n, v ),
            ArgValue::Int32(ref v)     =>   format!("\"{}\":{}", n, v ),
            ArgValue::Int64(ref v)     =>   format!("\"{}\":{}", n, v ),
            ArgValue::UInt16(ref v)    =>   format!("\"{}\":{}", n, v ),
            ArgValue::UInt32(ref v)    =>   format!("\"{}\":{}", n, v ),
            ArgValue::UInt64(ref v)    =>   format!("\"{}\":{}", n, v ),
            ArgValue::Double(ref v)    =>   format!("\"{}\":{}", n, v ),
            ArgValue::Array(ref v)     =>   format!("\"{}\":{:?}", n, v ),
            ArgValue::Hash(ref v)      =>   format!("\"{}\":{:?}", n, v ),
        }
    }
}

trait FromVariant: Sized {
    fn from_variant(variant: &ArgValue) -> Option<Self>;
}

// TODO use macros
impl FromVariant for String {
    fn from_variant(variant: &ArgValue) -> Option<Self> {
        match *variant {
            ArgValue::Str(ref v) => Some((*v).clone()),
            _ => None
        }
    }
}

macro_rules! from_variant_inst {
    [$t: tt, $i: ident] => {
        impl FromVariant for $t {
            fn from_variant(variant: &ArgValue) -> Option<Self> {
                match *variant {
                    ArgValue::$i(v) => Some(v),
                    _ => None
                }
            }
        }
    };
}

from_variant_inst![bool, Bool];
from_variant_inst![u8,   Byte];
from_variant_inst![i16,  Int16];
//impl FromVariant for bool {
//    fn from_variant(variant: &ArgValue) -> Option<Self> {
//        match *variant {
//            ArgValue::Bool(v) => Some(v),
//            _ => None
//        }
//    }
//}

//impl FromVariant for u8 {
//    fn from_variant(variant: &ArgValue) -> Option<Self> {
//        match *variant {
//            ArgValue::Byte(v) => Some(v),
//            _ => None
//        }
//    }
//}
//
//impl FromVariant for i16 {
//    fn from_variant(variant: &ArgValue) -> Option<Self> {
//        match *variant {
//            ArgValue::Int16(v) => Some(v),
//            _ => None
//        }
//    }
//}

#[derive(Clone, Serialize, Deserialize, Debug)]
enum ArgsTag {
    Args,
    Rets,
}

#[derive(Serialize, Deserialize, Debug)]
enum DbusMessageType {
    Method,
    Signal,
}

type NamedArgs = HashMap<String, ArgValue>;
type NamedArgs2 = HashMap<String, serde_json::value::Value>;

#[derive(Serialize, Deserialize)]
struct DbusMessage {
    api_name: String,
    dbus_name: String,
    message_type: DbusMessageType,
    args: NamedArgs,
}

#[derive(Serialize, Deserialize)]
struct DbusMessageRouting(String, String, String, Vec<DbusMessage>);

#[derive(Serialize, Deserialize)]
struct ConfYaml(Vec<DbusMessageRouting>);

#[derive(Serialize, Deserialize, Debug)]
struct ClientRequest {
    #[serde(rename = "type")]
    tp: DbusMessageType,
    name: String,
    args: Option<NamedArgs2>,
    rets: Option<NamedArgs2>,
}

impl ClientRequest {
    fn to_json(&self) -> String {
        fn serialize_args_or_rets(tag: &str, ar: &Option<NamedArgs2>) -> String {
            match ar {
                &Some(ref a) => {
                    let args_serialized: Vec<_> = a.iter().map( |ret| {
                        let (n, val) = ret;
                        //val.to_json(n)
                        format!("\"{}\":{}", n, val)
                        } ).collect();
                    format!("\"{}\": {{ {}", tag, args_serialized.join(","))
                },
                &None =>  "".to_string(),
            }
        }
        let json_reply_prefix = format!("{{\"type\":\"{:?}\",\"name\":\"{}\",", &self.tp, &self.name);
        let args_serialized = serialize_args_or_rets("args", &self.args);
        let rets_serialized = serialize_args_or_rets("rets", &self.rets);
        format!("{}{}{} }} }}",json_reply_prefix, args_serialized, rets_serialized).to_string()
    }
}

#[derive(Serialize, Deserialize)]
struct ClientRequestBody(DbusMessageType, NamedArgs);

type CRequest = HashMap<String, ClientRequestBody>;

fn main()
{
    let conf = vec![
        DbusMessageRouting(
            "com.a.b.servicetest".to_string(),
            "/com/a/b/servicetest".to_string(),
            "com.a.b.IServiceTest".to_string(),
            vec![ DbusMessage { api_name: "api_name".to_string(),
                                dbus_name: "dbus_name".to_string(),
                                message_type: DbusMessageType::Method,
                                args: [
                                    ("param1".to_string(), ArgValue::Str("Hi".to_string())),
                                    ("param2".to_string(), ArgValue::Bool(true)),
                                    ("param3".to_string(), ArgValue::Int64(17i64)),
                                    ("param4".to_string(), ArgValue::Double(2.718281828f64)),
                                    ("param5".to_string(), ArgValue::Array( vec![ArgValue::Double (1.1), ArgValue::Double(2.2), ArgValue::Double(3.3), ] )),
                                ].iter().cloned().collect()
                              }
            ]
        )
    ];

    let s = serde_yaml::to_string(&conf).unwrap();
    println!("s : {}", s);

    let r = ClientRequest { tp: DbusMessageType::Method, name: "api_name".to_string(),
                            args: Some([    ("param1".to_string(), json!("Hi")),
                                            ("param2".to_string(), json!(true)),
                                            ("param3".to_string(), json!(17i64)),
                                            ("param4".to_string(), json!(2.718281828f64)),
                                            ("param5".to_string(), json!( vec![ json!(1.1), json!(2.2), json!(3.3), ] )),
                                       ].iter().cloned().collect()),
                            rets: None,
                          };
    let rs = serde_json::to_string(&r).unwrap();
    println!("rs : {}", rs);

    let d: ClientRequest = serde_json::from_str(&rs).expect("json deserialization error");
    println!("d: {:?}", d);

    let ds = serde_json::to_string(&d).unwrap();
    println!(" => ds: {}", ds);

    let data = "{\"type\":\"Method\",\"name\":\"utest_4p2r\",\"rets\":{\"parm1\":71,\"parm2\":12.12 } }";
    let d2: ClientRequest = serde_json::from_str(&data).expect("json deserialization error");
    println!("d2: {:?}", d2);
    println!("d2.to_json(): {}", d2.to_json());

    let rb = ClientRequestBody(DbusMessageType::Method, [
                                        ("param1".to_string(), ArgValue::Str("Hi".to_string())),
                                        ("param2".to_string(), ArgValue::Bool(true)),
                                        ("param3".to_string(), ArgValue::Int64(17i64)),
                                        ("param4".to_string(), ArgValue::Double(2.718281828f64)),
                                ].iter().cloned().collect()
                          );

    let mut a_request: CRequest = HashMap::new();
    a_request.insert("api_name".to_string(), rb);

    let rsq = serde_json::to_string(&a_request).unwrap();
    println!("rsq : {}", rsq);

    let de: CRequest = serde_json::from_str(&rsq).expect("json deserialization error");

    let rsq2 = serde_json::to_string(&de).expect("json serialization error");
    println!("rsq2 : {}", rsq2);

    for key in de.keys() {
        println!("key {}", key);
    }
    println!("key[0] : {}", de.keys().nth(0).unwrap());

}
<commit_msg>play with `from_variant` for `Array`<commit_after>// https://github.com/serde-rs/serde/issues/927

use std::collections::HashMap;
use std::collections::BTreeMap;

extern crate yaml_rust;
extern crate argparse;
#[macro_use] extern crate log;
#[macro_use] extern crate from_variants;

extern crate serde;
#[macro_use] extern crate serde_json;
extern crate serde_yaml;
#[macro_use] extern crate serde_derive;

#[derive(Clone, Serialize, Deserialize, Debug)]
enum ArgValue {
    None,
    Str(String),
    Bool(bool),
    Byte(u8),
    Int16(i16),
    Int32(i32),
    Int64(i64),
    UInt16(u16),
    UInt32(u32),
    UInt64(u64),
    Double(f64),
    Array(Vec<ArgValue>),
    Hash(BTreeMap<String, ArgValue>),
}

impl ArgValue {
    fn to_json(&self, n: &str) -> String {
        match *self {
            ArgValue::None             =>   "".to_string(),
            ArgValue::Str(ref v)       =>   format!("\"{}\":{}", n, v ),
            ArgValue::Bool(ref v)      =>   format!("\"{}\":{}", n, v ),
            ArgValue::Byte(ref v)      =>   format!("\"{}\":{}", n, v ),
            ArgValue::Int16(ref v)     =>   format!("\"{}\":{}", n, v ),
            ArgValue::Int32(ref v)     =>   format!("\"{}\":{}", n, v ),
            ArgValue::Int64(ref v)     =>   format!("\"{}\":{}", n, v ),
            ArgValue::UInt16(ref v)    =>   format!("\"{}\":{}", n, v ),
            ArgValue::UInt32(ref v)    =>   format!("\"{}\":{}", n, v ),
            ArgValue::UInt64(ref v)    =>   format!("\"{}\":{}", n, v ),
            ArgValue::Double(ref v)    =>   format!("\"{}\":{}", n, v ),
            ArgValue::Array(ref v)     =>   format!("\"{}\":{:?}", n, v ),
            ArgValue::Hash(ref v)      =>   format!("\"{}\":{:?}", n, v ),
        }
    }
}

trait FromVariant: Sized {
    fn from_variant(variant: &ArgValue) -> Option<Self>;
}

// String, Array, Hash differ, for all other variants use macros
impl FromVariant for String {
    fn from_variant(variant: &ArgValue) -> Option<Self> {
        match *variant {
            ArgValue::Str(ref v) => Some((*v).clone()),
            _ => None
        }
    }
}

impl<T> FromVariant for Vec<T> where T: FromVariant {
    fn from_variant(variant: &ArgValue) -> Option<Self> {
        match *variant {
            ArgValue::Array(ref v) => {
                let mut ret = Vec::<T>::new();
                for el in v {
                    let val: T = FromVariant::from_variant(&el).expect("type mismatch");
                    ret.push(val);
                }
                Some(ret)
            }
            _ => None
        }
    }
}

macro_rules! from_variant_inst {
    [$t: tt, $i: ident] => {
        impl FromVariant for $t {
            fn from_variant(variant: &ArgValue) -> Option<Self> {
                match *variant {
                    ArgValue::$i(v) => Some(v),
                    _ => None
                }
            }
        }
    };
}

from_variant_inst![bool, Bool];
from_variant_inst![u8,   Byte];
from_variant_inst![i16,  Int16];
from_variant_inst![i32,  Int32];
from_variant_inst![i64,  Int64];
from_variant_inst![u16,  UInt16];
from_variant_inst![u32,  UInt32];
from_variant_inst![u64,  UInt64];
from_variant_inst![f64,  Double];

#[derive(Clone, Serialize, Deserialize, Debug)]
enum ArgsTag {
    Args,
    Rets,
}

#[derive(Serialize, Deserialize, Debug)]
enum DbusMessageType {
    Method,
    Signal,
}

type NamedArgs = HashMap<String, ArgValue>;
type NamedArgs2 = HashMap<String, serde_json::value::Value>;

#[derive(Serialize, Deserialize)]
struct DbusMessage {
    api_name: String,
    dbus_name: String,
    message_type: DbusMessageType,
    args: NamedArgs,
}

#[derive(Serialize, Deserialize)]
struct DbusMessageRouting(String, String, String, Vec<DbusMessage>);

#[derive(Serialize, Deserialize)]
struct ConfYaml(Vec<DbusMessageRouting>);

#[derive(Serialize, Deserialize, Debug)]
struct ClientRequest {
    #[serde(rename = "type")]
    tp: DbusMessageType,
    name: String,
    args: Option<NamedArgs2>,
    rets: Option<NamedArgs2>,
}

impl ClientRequest {
    fn to_json(&self) -> String {
        fn serialize_args_or_rets(tag: &str, ar: &Option<NamedArgs2>) -> String {
            match ar {
                &Some(ref a) => {
                    let args_serialized: Vec<_> = a.iter().map( |ret| {
                        let (n, val) = ret;
                        //val.to_json(n)
                        format!("\"{}\":{}", n, val)
                        } ).collect();
                    format!("\"{}\": {{ {}", tag, args_serialized.join(","))
                },
                &None =>  "".to_string(),
            }
        }
        let json_reply_prefix = format!("{{\"type\":\"{:?}\",\"name\":\"{}\",", &self.tp, &self.name);
        let args_serialized = serialize_args_or_rets("args", &self.args);
        let rets_serialized = serialize_args_or_rets("rets", &self.rets);
        format!("{}{}{} }} }}",json_reply_prefix, args_serialized, rets_serialized).to_string()
    }
}

#[derive(Serialize, Deserialize)]
struct ClientRequestBody(DbusMessageType, NamedArgs);

type CRequest = HashMap<String, ClientRequestBody>;

fn main()
{
    let conf = vec![
        DbusMessageRouting(
            "com.a.b.servicetest".to_string(),
            "/com/a/b/servicetest".to_string(),
            "com.a.b.IServiceTest".to_string(),
            vec![ DbusMessage { api_name: "api_name".to_string(),
                                dbus_name: "dbus_name".to_string(),
                                message_type: DbusMessageType::Method,
                                args: [
                                    ("param1".to_string(), ArgValue::Str("Hi".to_string())),
                                    ("param2".to_string(), ArgValue::Bool(true)),
                                    ("param3".to_string(), ArgValue::Int64(17i64)),
                                    ("param4".to_string(), ArgValue::Double(2.718281828f64)),
                                    ("param5".to_string(), ArgValue::Array( vec![ArgValue::Double (1.1), ArgValue::Double(2.2), ArgValue::Double(3.3), ] )),
                                ].iter().cloned().collect()
                              }
            ]
        )
    ];

    let s = serde_yaml::to_string(&conf).unwrap();
    println!("s : {}", s);

    let r = ClientRequest { tp: DbusMessageType::Method, name: "api_name".to_string(),
                            args: Some([    ("param1".to_string(), json!("Hi")),
                                            ("param2".to_string(), json!(true)),
                                            ("param3".to_string(), json!(17i64)),
                                            ("param4".to_string(), json!(2.718281828f64)),
                                            ("param5".to_string(), json!( vec![ json!(1.1), json!(2.2), json!(3.3), ] )),
                                       ].iter().cloned().collect()),
                            rets: None,
                          };
    let rs = serde_json::to_string(&r).unwrap();
    println!("rs : {}", rs);

    let d: ClientRequest = serde_json::from_str(&rs).expect("json deserialization error");
    println!("d: {:?}", d);

    let ds = serde_json::to_string(&d).unwrap();
    println!(" => ds: {}", ds);

    let data = "{\"type\":\"Method\",\"name\":\"utest_4p2r\",\"rets\":{\"parm1\":71,\"parm2\":12.12 } }";
    let d2: ClientRequest = serde_json::from_str(&data).expect("json deserialization error");
    println!("d2: {:?}", d2);
    println!("d2.to_json(): {}", d2.to_json());

    let rb = ClientRequestBody(DbusMessageType::Method, [
                                        ("param1".to_string(), ArgValue::Str("Hi".to_string())),
                                        ("param2".to_string(), ArgValue::Bool(true)),
                                        ("param3".to_string(), ArgValue::Int64(17i64)),
                                        ("param4".to_string(), ArgValue::Double(2.718281828f64)),
                                ].iter().cloned().collect()
                          );

    let mut a_request: CRequest = HashMap::new();
    a_request.insert("api_name".to_string(), rb);

    let rsq = serde_json::to_string(&a_request).unwrap();
    println!("rsq : {}", rsq);

    let de: CRequest = serde_json::from_str(&rsq).expect("json deserialization error");

    let rsq2 = serde_json::to_string(&de).expect("json serialization error");
    println!("rsq2 : {}", rsq2);

    for key in de.keys() {
        println!("key {}", key);
    }
    println!("key[0] : {}", de.keys().nth(0).unwrap());

    let arr = ArgValue:: Array( vec![ArgValue::Double (1.1), ArgValue::Double(2.2), ArgValue::Double(3.3),] );

    let from_arr_variant: Vec<f64> = FromVariant::from_variant(&arr).expect("array expected");

    println!("arr: {:?} from_arr_variant: {:?}", arr, from_arr_variant);

    let bad_arr = ArgValue:: Array( vec![ArgValue::Double (1.1), ArgValue::Int32(2), ArgValue::Double(3.3),] );
    println!("bad_arr: {:?}", bad_arr);

    // expect here runtime error `thread 'main' panicked at 'type mismatch',`
    let from_bad_arr_variant: Vec<f64> = FromVariant::from_variant(&bad_arr).expect("array expected");
}
<|endoftext|>


<commit_before>// Copyright 2016 `multipart` Crate Developers
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

//! Boundary parsing for `multipart` requests.

use super::buf_redux::BufReader;
use super::memchr::memchr;

use std::cmp;
use std::borrow::Borrow;

use std::io;
use std::io::prelude::*;

/// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence.
#[derive(Debug)]
pub struct BoundaryReader<R> {
    buf: BufReader<R>,
    boundary: Vec<u8>,
    search_idx: usize,
    boundary_read: bool,
    at_end: bool,
}

impl<R> BoundaryReader<R> where R: Read {
    #[doc(hidden)]
    pub fn from_reader<B: Into<Vec<u8>>>(reader: R, boundary: B) -> BoundaryReader<R> {
        BoundaryReader {
            buf: BufReader::new(reader),
            boundary: boundary.into(),
            search_idx: 0,
            boundary_read: false,
            at_end: false,
        }
    }

    fn read_to_boundary(&mut self) -> io::Result<&[u8]> {
        use log::LogLevel;

        let buf = try!(fill_buf_min(&mut self.buf, self.boundary.len() * 2));

        if log_enabled!(LogLevel::Trace) {
            trace!("Buf: {:?}", String::from_utf8_lossy(buf));
        }

        debug!(
            "Before-loop Buf len: {} Search idx: {} Boundary read: {:?}",
            buf.len(), self.search_idx, self.boundary_read
        );

        while !(self.boundary_read || self.at_end) && self.search_idx < buf.len() {
            let lookahead = &buf[self.search_idx..];

            let maybe_boundary = memchr(self.boundary[0], lookahead);

            self.search_idx = match maybe_boundary {
                Some(boundary_start) => self.search_idx + boundary_start,
                None => buf.len(),
            };

            if self.search_idx + self.boundary.len() <= buf.len() {
                let test = &buf[self.search_idx .. self.search_idx + self.boundary.len()];

                if log_enabled!(LogLevel::Trace) {
                    trace!("Possible boundary: {:?}", String::from_utf8_lossy(test));
                }

                match first_nonmatching_idx(test, &self.boundary) {
                    Some(idx) => {
                        debug!("First nonmatching idx: {}", idx);
                        self.search_idx += idx + 1;
                    },
                    None => self.boundary_read = true,
                }
            } else {
                break;
            }
        }

        debug!(
            "After-loop Buf len: {} Search idx: {} Boundary read: {:?}",
            buf.len(), self.search_idx, self.boundary_read
        );

        let mut buf_end = self.search_idx;

        if self.boundary_read && self.search_idx >= 2 {
            let two_bytes_before = &buf[self.search_idx - 2 .. self.search_idx];

            debug!("Two bytes before: {:?} (\"\\r\\n\": {:?})", two_bytes_before, b"\r\n");

            if two_bytes_before == &*b"\r\n" {
                debug!("Subtract two!");
                buf_end -= 2;
            }
        }

        let ret_buf = &buf[..buf_end];

        if log_enabled!(LogLevel::Trace) {
            trace!("Returning buf: {:?}", String::from_utf8_lossy(ret_buf));
        }

        Ok(ret_buf)
    }

    #[doc(hidden)]
    pub fn consume_boundary(&mut self) -> io::Result<()> {
        if self.at_end {
            return Ok(());
        }

        while !self.boundary_read {
            let buf_len = try!(self.read_to_boundary()).len();

            if buf_len == 0 {
                break;
            }

            self.consume(buf_len);
        }

        self.buf.consume(self.search_idx + self.boundary.len());

        self.search_idx = 0;
        self.boundary_read = false;

        Ok(())
    }

    // Keeping this around to support nested boundaries later.
    #[allow(unused)]
    #[doc(hidden)]
    pub fn set_boundary<B: Into<Vec<u8>>>(&mut self, boundary: B) {
        self.boundary = boundary.into();
    }
}

impl<R> Borrow<R> for BoundaryReader<R> {
    fn borrow(&self) -> &R {
        self.buf.get_ref()
    }
}

impl<R> Read for BoundaryReader<R> where R: Read {
    fn read(&mut self, out: &mut [u8]) -> io::Result<usize> {
        let read = {
            let mut buf = try!(self.read_to_boundary());
            // This shouldn't ever be an error so unwrapping is fine.
            buf.read(out).unwrap()
        };

        self.consume(read);
        Ok(read)
    }
}

impl<R> BufRead for BoundaryReader<R> where R: Read {
    fn fill_buf(&mut self) -> io::Result<&[u8]> {
        self.read_to_boundary()
    }

    fn consume(&mut self, amt: usize) {
        let true_amt = cmp::min(amt, self.search_idx);

        debug!("Consume! amt: {} true amt: {}", amt, true_amt);

        self.buf.consume(true_amt);
        self.search_idx -= true_amt;
    }
}

fn fill_buf_min<R: Read>(buf: &mut BufReader<R>, min: usize) -> io::Result<&[u8]> {
    if buf.available() < min {
        try!(buf.read_into_buf());
    }

    Ok(buf.get_buf())
}

fn first_nonmatching_idx(left: &[u8], right: &[u8]) -> Option<usize> {
    for (idx, (lb, rb)) in left.iter().zip(right).enumerate() {
        if lb != rb {
            return Some(idx);
        }
    }

    None
}

#[cfg(test)]
mod test {
    use super::BoundaryReader;

    use std::io;
    use std::io::prelude::*;

    const BOUNDARY: &'static str = "\r\n--boundary";
    const TEST_VAL: &'static str = "\r\n--boundary\r
dashed-value-1\r
--boundary\r
dashed-value-2\r
--boundary--";

    #[test]
    fn test_boundary() {
        let _ = ::env_logger::init();
        debug!("Testing boundary (no split)");

        let src = &mut TEST_VAL.as_bytes();
        let reader = BoundaryReader::from_reader(src, BOUNDARY);

        test_boundary_reader(reader);
    }

    struct SplitReader<'a> {
        left: &'a [u8],
        right: &'a [u8],
    }

    impl<'a> SplitReader<'a> {
        fn split(data: &'a [u8], at: usize) -> SplitReader<'a> {
            let (left, right) = data.split_at(at);

            SplitReader {
                left: left,
                right: right,
            }
        }
    }

    impl<'a> Read for SplitReader<'a> {
        fn read(&mut self, dst: &mut [u8]) -> io::Result<usize> {
            fn copy_bytes_partial(src: &mut &[u8], dst: &mut [u8]) -> usize {
                src.read(dst).unwrap()
            }

            let mut copy_amt = copy_bytes_partial(&mut self.left, dst);

            if copy_amt == 0 {
                copy_amt = copy_bytes_partial(&mut self.right, dst)
            };

            Ok(copy_amt)
        }
    }

    #[test]
    fn test_split_boundary() {
        let _ = ::env_logger::init();
        debug!("Testing boundary (split)");

        // Substitute for `.step_by()` being unstable.
        for split_at in (0 .. TEST_VAL.len()).filter(|x| x % 2 != 0) {
            debug!("Testing split at: {}", split_at);

            let src = SplitReader::split(TEST_VAL.as_bytes(), split_at);
            let reader = BoundaryReader::from_reader(src, BOUNDARY);
            test_boundary_reader(reader);
        }

    }

    fn test_boundary_reader<R: Read>(mut reader: BoundaryReader<R>) {
        let ref mut buf = String::new();

        debug!("Read 1");
        let _ = reader.read_to_string(buf).unwrap();
        assert!(buf.is_empty(), "Buffer not empty: {:?}", buf);
        buf.clear();

        debug!("Consume 1");
        reader.consume_boundary().unwrap();

        debug!("Read 2");
        let _ = reader.read_to_string(buf).unwrap();
        assert_eq!(buf, "\r\ndashed-value-1");
        buf.clear();

        debug!("Consume 2");
        reader.consume_boundary().unwrap();

        debug!("Read 3");
        let _ = reader.read_to_string(buf).unwrap();
        assert_eq!(buf, "\r\ndashed-value-2");
        buf.clear();

        debug!("Consume 3");
        reader.consume_boundary().unwrap();

        debug!("Read 4");
        let _ = reader.read_to_string(buf).unwrap();
        assert_eq!(buf, "--");
    }
}
<commit_msg>Fix logic errors in boundary reading<commit_after>// Copyright 2016 `multipart` Crate Developers
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

//! Boundary parsing for `multipart` requests.

use super::buf_redux::BufReader;
use super::memchr::memchr;

use std::cmp;
use std::borrow::Borrow;

use std::io;
use std::io::prelude::*;

/// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence.
#[derive(Debug)]
pub struct BoundaryReader<R> {
    buf: BufReader<R>,
    boundary: Vec<u8>,
    search_idx: usize,
    boundary_read: bool,
    at_end: bool,
}

impl<R> BoundaryReader<R> where R: Read {
    #[doc(hidden)]
    pub fn from_reader<B: Into<Vec<u8>>>(reader: R, boundary: B) -> BoundaryReader<R> {
        BoundaryReader {
            buf: BufReader::new(reader),
            boundary: boundary.into(),
            search_idx: 0,
            boundary_read: false,
            at_end: false,
        }
    }

    fn read_to_boundary(&mut self) -> io::Result<&[u8]> {
        use log::LogLevel;

        // Make sure there's enough bytes in the buffer to positively identify the boundary.
        let min_len = self.search_idx + (self.boundary.len() * 2);

        let buf = try!(fill_buf_min(&mut self.buf, min_len));

        if log_enabled!(LogLevel::Trace) {
            trace!("Buf: {:?}", String::from_utf8_lossy(buf));
        }

        debug!(
            "Before-loop Buf len: {} Search idx: {} Boundary read: {:?}",
            buf.len(), self.search_idx, self.boundary_read
        );

        while !(self.boundary_read || self.at_end) && self.search_idx < buf.len() {
            let lookahead = &buf[self.search_idx..];

            let maybe_boundary = memchr(self.boundary[0], lookahead);

            self.search_idx = match maybe_boundary {
                Some(boundary_start) => self.search_idx + boundary_start,
                None => buf.len(),
            };

            if self.search_idx + self.boundary.len() <= buf.len() {
                let test = &buf[self.search_idx .. self.search_idx + self.boundary.len()];

                if log_enabled!(LogLevel::Trace) {
                    trace!("Possible boundary: {:?}", String::from_utf8_lossy(test));
                }

                match first_nonmatching_idx(test, &self.boundary) {
                    Some(idx) => {
                        debug!("First nonmatching idx: {}", idx);
                        self.search_idx += idx;
                    },
                    None => self.boundary_read = true,
                }
            } else {
                break;
            }
        }

        debug!(
            "After-loop Buf len: {} Search idx: {} Boundary read: {:?}",
            buf.len(), self.search_idx, self.boundary_read
        );

        let mut buf_end = self.search_idx;

        if self.boundary_read && self.search_idx >= 2 {
            let two_bytes_before = &buf[self.search_idx - 2 .. self.search_idx];

            debug!("Two bytes before: {:?} (\"\\r\\n\": {:?})", two_bytes_before, b"\r\n");

            if two_bytes_before == &*b"\r\n" {
                debug!("Subtract two!");
                buf_end -= 2;
            }
        }

        let ret_buf = &buf[..buf_end];

        if log_enabled!(LogLevel::Trace) {
            trace!("Returning buf: {:?}", String::from_utf8_lossy(ret_buf));
        }

        Ok(ret_buf)
    }

    #[doc(hidden)]
    pub fn consume_boundary(&mut self) -> io::Result<()> {
        if self.at_end {
            return Ok(());
        }

        while !self.boundary_read {
            let buf_len = try!(self.read_to_boundary()).len();

            if buf_len == 0 {
                break;
            }

            self.consume(buf_len);
        }

        self.buf.consume(self.search_idx + self.boundary.len());

        self.search_idx = 0;
        self.boundary_read = false;

        Ok(())
    }

    // Keeping this around to support nested boundaries later.
    #[allow(unused)]
    #[doc(hidden)]
    pub fn set_boundary<B: Into<Vec<u8>>>(&mut self, boundary: B) {
        self.boundary = boundary.into();
    }
}

impl<R> Borrow<R> for BoundaryReader<R> {
    fn borrow(&self) -> &R {
        self.buf.get_ref()
    }
}

impl<R> Read for BoundaryReader<R> where R: Read {
    fn read(&mut self, out: &mut [u8]) -> io::Result<usize> {
        let read = {
            let mut buf = try!(self.read_to_boundary());
            // This shouldn't ever be an error so unwrapping is fine.
            buf.read(out).unwrap()
        };

        self.consume(read);
        Ok(read)
    }
}

impl<R> BufRead for BoundaryReader<R> where R: Read {
    fn fill_buf(&mut self) -> io::Result<&[u8]> {
        self.read_to_boundary()
    }

    fn consume(&mut self, amt: usize) {
        let true_amt = cmp::min(amt, self.search_idx);

        debug!("Consume! amt: {} true amt: {}", amt, true_amt);

        self.buf.consume(true_amt);
        self.search_idx -= true_amt;
    }
}

fn fill_buf_min<R: Read>(buf: &mut BufReader<R>, min: usize) -> io::Result<&[u8]> {
    if buf.available() < min {
        try!(buf.read_into_buf());
    }

    Ok(buf.get_buf())
}

fn first_nonmatching_idx(left: &[u8], right: &[u8]) -> Option<usize> {
    for (idx, (lb, rb)) in left.iter().zip(right).enumerate() {
        if lb != rb {
            return Some(idx);
        }
    }

    None
}

#[cfg(test)]
mod test {
    use super::BoundaryReader;

    use std::io;
    use std::io::prelude::*;

    const BOUNDARY: &'static str = "\r\n--boundary";
    const TEST_VAL: &'static str = "\r\n--boundary\r
dashed-value-1\r
--boundary\r
dashed-value-2\r
--boundary--";

    #[test]
    fn test_boundary() {
        let _ = ::env_logger::init();
        debug!("Testing boundary (no split)");

        let src = &mut TEST_VAL.as_bytes();
        let reader = BoundaryReader::from_reader(src, BOUNDARY);

        test_boundary_reader(reader);
    }

    struct SplitReader<'a> {
        left: &'a [u8],
        right: &'a [u8],
    }

    impl<'a> SplitReader<'a> {
        fn split(data: &'a [u8], at: usize) -> SplitReader<'a> {
            let (left, right) = data.split_at(at);

            SplitReader {
                left: left,
                right: right,
            }
        }
    }

    impl<'a> Read for SplitReader<'a> {
        fn read(&mut self, dst: &mut [u8]) -> io::Result<usize> {
            fn copy_bytes_partial(src: &mut &[u8], dst: &mut [u8]) -> usize {
                src.read(dst).unwrap()
            }

            let mut copy_amt = copy_bytes_partial(&mut self.left, dst);

            if copy_amt == 0 {
                copy_amt = copy_bytes_partial(&mut self.right, dst)
            };

            Ok(copy_amt)
        }
    }

    #[test]
    fn test_split_boundary() {
        let _ = ::env_logger::init();
        debug!("Testing boundary (split)");

        // Substitute for `.step_by()` being unstable.
        for split_at in (0 .. TEST_VAL.len()).filter(|x| x % 2 != 0) {
            debug!("Testing split at: {}", split_at);

            let src = SplitReader::split(TEST_VAL.as_bytes(), split_at);
            let reader = BoundaryReader::from_reader(src, BOUNDARY);
            test_boundary_reader(reader);
        }

    }

    fn test_boundary_reader<R: Read>(mut reader: BoundaryReader<R>) {
        let ref mut buf = String::new();

        debug!("Read 1");
        let _ = reader.read_to_string(buf).unwrap();
        assert!(buf.is_empty(), "Buffer not empty: {:?}", buf);
        buf.clear();

        debug!("Consume 1");
        reader.consume_boundary().unwrap();

        debug!("Read 2");
        let _ = reader.read_to_string(buf).unwrap();
        assert_eq!(buf, "\r\ndashed-value-1");
        buf.clear();

        debug!("Consume 2");
        reader.consume_boundary().unwrap();

        debug!("Read 3");
        let _ = reader.read_to_string(buf).unwrap();
        assert_eq!(buf, "\r\ndashed-value-2");
        buf.clear();

        debug!("Consume 3");
        reader.consume_boundary().unwrap();

        debug!("Read 4");
        let _ = reader.read_to_string(buf).unwrap();
        assert_eq!(buf, "--");
    }
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::build(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
        let path = rt.get_rtp() + m.name() + "/store";
        // TODO: Don't use "/store" but value from configuration
        debug!("Building StorageBackend for {}", path);
        StorageBackend::new(path)
    }

    fn get_file_ids(&self) -> Option<Vec<FileID>> {
        debug!("Getting files from {}", self.basepath);
        let list = glob(&self.basepath[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("  id      : '{}'", id);
        self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
    }

    fn prefix_of_files_for_module(&self, m: &Module) -> String {
        self.storepath.clone() + m.name()
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            dataDump:       data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<commit_msg>Fixup: get_file_ids() - use prefix_of_files_for_module()<commit_after>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::build(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
        let path = rt.get_rtp() + m.name() + "/store";
        // TODO: Don't use "/store" but value from configuration
        debug!("Building StorageBackend for {}", path);
        StorageBackend::new(path)
    }

    fn get_file_ids(&self, m: &Module) -> Option<Vec<FileID>> {
        let list = glob(&self.prefix_of_files_for_module(m)[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: &FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("  id      : '{}'", id);
        self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
    }

    fn prefix_of_files_for_module(&self, m: &Module) -> String {
        self.storepath.clone() + m.name()
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            dataDump:       data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::build(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
        let path = rt.get_rtp() + m.name() + "/store";
        // TODO: Don't use "/store" but value from configuration
        debug!("Building StorageBackend for {}", path);
        StorageBackend::new(path)
    }

    fn get_file_ids(&self, m: &Module) -> Option<Vec<FileID>> {
        let list = glob(&self.prefix_of_files_for_module(m)[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<HP>(&self, id: FileID, p: &Parser<HP>) -> Option<File>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("  id      : '{}'", id);
        self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
    }

    fn build_filepath_with_id(&self, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!("  id      : '{}'", id);
        self.basepath.clone() + &id[..]
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            dataDump:       data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<commit_msg>A File is bound to a Module<commit_after>use std::error::Error;
use std::fmt::Display;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::path::Path;
use std::path::PathBuf;
use std::vec::Vec;
use std::fs::File as FSFile;
use std::fs::create_dir_all;
use std::fs::remove_file;
use std::io::Read;
use std::io::Write;
use std::vec::IntoIter;

use glob::glob;
use glob::Paths;

use storage::file::File;
use storage::file_id::*;
use storage::parser::{FileHeaderParser, Parser, ParserError};

use module::Module;
use runtime::Runtime;

pub type BackendOperationResult<T = ()> = Result<T, StorageBackendError>;

pub struct StorageBackend {
    basepath: String,
    storepath: String,
}

impl StorageBackend {

    pub fn new(rt: &Runtime) -> BackendOperationResult<StorageBackend> {
        let storepath = rt.get_rtp() + "/store/";
        debug!("Trying to create {}", storepath);
        create_dir_all(&storepath).and_then(|_| {
            debug!("Creating succeeded, constructing backend instance");
            Ok(StorageBackend {
                basepath: rt.get_rtp(),
                storepath: storepath.clone(),
            })
        }).or_else(|e| {
            debug!("Creating failed, constructing error instance");
            let mut serr = StorageBackendError::build(
                "create_dir_all()",
                "Could not create store directories",
                Some(storepath)
            );
            serr.caused_by = Some(Box::new(e));
            Err(serr)
        })
    }

    fn build<M: Module>(rt: &Runtime, m: &M) -> StorageBackend {
        let path = rt.get_rtp() + m.name() + "/store";
        // TODO: Don't use "/store" but value from configuration
        debug!("Building StorageBackend for {}", path);
        StorageBackend::new(path)
    }

    fn get_file_ids(&self, m: &Module) -> Option<Vec<FileID>> {
        let list = glob(&self.prefix_of_files_for_module(m)[..]);

        if let Ok(globlist) = list {
            let mut v = vec![];
            for entry in globlist {
                if let Ok(path) = entry {
                    debug!(" - File: {:?}", path);
                    v.push(from_pathbuf(&path));
                } else {
                    // Entry is not a path
                }
            }

            Some(v)
        } else {
            None
        }
    }

    pub fn iter_ids(&self, m: &Module) -> Option<IntoIter<FileID>>
    {
        glob(&self.prefix_of_files_for_module(m)[..]).and_then(|globlist| {
            let v = globlist.filter_map(Result::ok)
                            .map(|pbuf| from_pathbuf(&pbuf))
                            .collect::<Vec<FileID>>()
                            .into_iter();
            Ok(v)
        }).ok()
    }

    pub fn iter_files<'a, HP>(&self, m: &'a Module, p: &Parser<HP>)
        -> Option<IntoIter<File<'a>>>
        where HP: FileHeaderParser
    {
        self.iter_ids(m).and_then(|ids| {
            Some(ids.filter_map(|id| self.get_file_by_id(m, &id, p))
                    .collect::<Vec<File>>()
                    .into_iter())
        })
    }

    /*
     * Write a file to disk.
     *
     * The file is moved to this function as the file won't be edited afterwards
     */
    pub fn put_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let written = write_with_parser(&f, p);
        if written.is_err() { return Err(written.err().unwrap()); }
        let string = written.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::create(&path).map(|mut file| {
            debug!("Created file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write_all()",
                            "Could not write out File contents",
                            None
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not create file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::create()",
                "Creating file on disk failed",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Update a file. We have the UUID and can find the file on FS with it and
     * then replace its contents with the contents of the passed file object
     */
    pub fn update_file<HP>(&self, f: File, p: &Parser<HP>) -> BackendOperationResult
        where HP: FileHeaderParser
    {
        let contents = write_with_parser(&f, p);
        if contents.is_err() { return Err(contents.err().unwrap()); }
        let string = contents.unwrap();

        let path = self.build_filepath(&f);
        debug!("Writing file: {}", path);
        debug!("      string: {}", string);

        FSFile::open(&path).map(|mut file| {
            debug!("Open file at '{}'", path);
            file.write_all(&string.clone().into_bytes())
                .map_err(|ioerr| {
                    debug!("Could not write file");
                    let mut err = StorageBackendError::build(
                            "File::write()",
                            "Tried to write contents of this file, though operation did not succeed",
                            Some(string)
                        );
                    err.caused_by = Some(Box::new(ioerr));
                    err
                })
        }).map_err(|writeerr| {
            debug!("Could not write file at '{}'", path);
            let mut err = StorageBackendError::build(
                "File::open()",
                "Tried to update contents of this file, though file doesn't exist",
                None
            );
            err.caused_by = Some(Box::new(writeerr));
            err
        }).and(Ok(()))
    }

    /*
     * Find a file by its ID and return it if found. Return nothing if not
     * found, of course.
     *
     * TODO: Needs refactoring, as there might be an error when reading from
     * disk OR the id just does not exist.
     */
    pub fn get_file_by_id<'a, HP>(&self, m: &'a Module, id: FileID, p: &Parser<HP>) -> Option<File<'a>>
        where HP: FileHeaderParser
    {
        debug!("Searching for file with id '{}'", id);
        if let Ok(mut fs) = FSFile::open(self.build_filepath_with_id(m, id.clone())) {
            let mut s = String::new();
            fs.read_to_string(&mut s);
            debug!("Success reading file with id '{}'", id);
            debug!("Parsing to internal structure now");
            p.read(s).and_then(|(h, d)| Ok(File::from_parser_result(m, id.clone(), h, d))).ok()
        } else {
            debug!("No file with id '{}'", id);
            None
        }
    }

    pub fn remove_file(&self, m: &Module, file: File, checked: bool) -> BackendOperationResult {
        if checked {
            error!("Checked remove not implemented yet. I will crash now");
            unimplemented!()
        }

        debug!("Doing unchecked remove");
        info!("Going to remove file: {}", file);

        let fp = self.build_filepath(&file);
        remove_file(fp).map_err(|e| {
            let mut serr = StorageBackendError::build(
                "remove_file()",
                "File removal failed",
                Some(format!("{}", file))
            );
            serr.caused_by = Some(Box::new(e));
            serr
        })
    }

    fn build_filepath(&self, f: &File) -> String {
        self.build_filepath_with_id(f.owner(), f.id())
    }

    fn build_filepath_with_id(&self, owner: &Module, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!(" storepath: '{}'", self.storepath);
        debug!("  id      : '{}'", id);
        self.prefix_of_files_for_module(owner) + "-" + &id[..] + ".imag"
    }

    fn build_filepath_with_id(&self, id: FileID) -> String {
        debug!("Building filepath with id");
        debug!("  basepath: '{}'", self.basepath);
        debug!("  id      : '{}'", id);
        self.basepath.clone() + &id[..]
    }

}

#[derive(Debug)]
pub struct StorageBackendError {
    pub action: String,             // The file system action in words
    pub desc: String,               // A short description
    pub data_dump: Option<String>,  // Data dump, if any
    pub caused_by: Option<Box<Error>>,  // caused from this error
}

impl StorageBackendError {
    fn new(action: String,
           desc  : String,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         action,
            desc:           desc,
            data_dump:      data,
            caused_by:      None,
        }
    }

    fn build(action: &'static str,
             desc:   &'static str,
           data  : Option<String>) -> StorageBackendError
    {
        StorageBackendError {
            action:         String::from(action),
            desc:           String::from(desc),
            dataDump:       data,
            caused_by:      None,
        }
    }

}

impl Error for StorageBackendError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        self.caused_by.as_ref().map(|e| &**e)
    }

}

impl Display for StorageBackendError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "StorageBackendError[{}]: {}",
               self.action, self.desc)
    }
}

fn write_with_parser<'a, HP>(f: &File, p: &Parser<HP>) -> Result<String, StorageBackendError>
    where HP: FileHeaderParser
{
    p.write(f.contents())
        .or_else(|err| {
            let mut serr = StorageBackendError::build(
                "Parser::write()",
                "Cannot translate internal representation of file contents into on-disk representation",
                None
            );
            serr.caused_by = Some(Box::new(err));
            Err(serr)
        })
}
<|endoftext|>