<commit_before>use std::{
    collections::HashMap,
    error::Error,
    fs::File,
    io::{
        BufRead,
        BufReader,
    },
    path::{
        Path,
        PathBuf,
    },
    str,
    thread,
};

use serde_query::{
    DeserializeQuery,
    Query,
};

use anyhow::{
    Context,
    Result,
};

// use crossbeam_channel::{
//     unbounded,
//     Sender,
// };

use serde_json::Deserializer;

use rkv::{
    backend::{
        BackendEnvironmentBuilder,
        Lmdb,
        LmdbDatabase,
        LmdbEnvironment,
    },
    Manager,
    Rkv,
    SingleStore,
    StoreOptions,
    Value,
};

#[derive(Debug, DeserializeQuery)]
struct DocId {
    #[query(".id")]
    id: String,
}

pub struct Spec<'a> {
    path: &'a Path,
    allow_missing: bool,
    commit_size: usize,
    threads: usize,
}

impl<'a> Spec<'a> {
    pub fn new(path: &Path) -> Spec {
        Spec {
            path,
            allow_missing: false,
            commit_size: 1_000_000usize,
            threads: 4,
        }
    }

    pub fn set_allow_missing(&mut self, allow_missing: bool) -> () {
        self.allow_missing = allow_missing;
    }

    pub fn set_commit_size(&mut self, commit_size: usize) -> () {
        self.commit_size = commit_size;
    }
}

pub fn index(spec: &Spec, data_type: &str, filename: &Path) -> Result<(), Box<dyn Error>> {
    let mut reader = rnc_utils::buf_reader(&filename)?;

    let mut manager = Manager::<LmdbEnvironment>::singleton().write()?;
    let mut builder = Rkv::environment_builder::<Lmdb>();
    builder.set_max_dbs(20);
    builder.set_make_dir_if_needed(true);
    builder.set_map_size(200000000000);
    let arc = manager
        .get_or_create_from_builder(spec.path, builder, Rkv::from_builder::<Lmdb>)
        .with_context(|| "Failed to create arc")?;
    let env = arc.read().unwrap();
    let store = env
        .open_single(data_type, StoreOptions::create())
        .with_context(|| format!("Failed to open store {:?}", data_type))?;

    let mut writer = env.write().with_context(|| "Failed to create writer")?;
    let mut buf = String::new();
    let mut count = 0usize;
    loop {
        match reader.read_line(&mut buf)? {
            0 => break,
            _ => {
                let line = buf.replace("\\\\", "\\");
                let value = line.trim_end().to_owned();
                let id: DocId = serde_json::from_str::<Query<DocId>>(&line)?.into();
                let existing = store.get(&writer, &id.id)?;
                let to_add = match existing {
                    None => value,
                    Some(v) => match v {
                        Value::Json(e) => format!("{}{}", value, e),
                        _ => panic!("Invalid existing data, should never happen"),
                    },
                };

                store.put(&mut writer, &id.id, &Value::Json(&to_add))?;
                buf.clear();
                count += 1;
                if count % spec.commit_size == 0 {
                    writer
                        .commit()
                        .with_context(|| format!("Failed to write data from {:?}", filename))?;
                    writer = env.write().with_context(|| "Could not create for new chunk")?;
                }
            },
        }
    }
    writer.commit().with_context(|| format!("Failed to write data from {:?}", filename))?;

    Ok(())
}

// fn path_as_column_name(path: &Path) -> String {
//     path.file_stem().unwrap().to_str().unwrap().to_string().replace("-", "_")
// }

// fn send_file_lines(path: &Path, sender: Sender<(String, DocId, String)>) {
//     let file = File::open(path).unwrap();
//     let data_type = path_as_column_name(&path);
//     let mut reader = BufReader::new(file);
//     let mut buf = String::new();
//     loop {
//         match reader.read_line(&mut buf).unwrap() {
//             0 => break,
//             _ => {
//                 let line = buf.replace("\\\\", "\\");
//                 let id: DocId =
// serde_json::from_str::<Query<DocId>>(&line).unwrap().into();
// sender.send((data_type.to_string(), id, line.to_string())).unwrap();
// buf.clear();             },
//         }
//     }
// }

// pub fn index_files(spec: &Spec, filename: &Path) -> Result<(), Box<dyn Error>> {
//     let mut manager = Manager::<LmdbEnvironment>::singleton().write()?;
//     let arc = manager.get_or_create(spec.path, Rkv::new::<Lmdb>)?;
//     let env = arc.read();

//     // let stores = HashMap::<String, Multi>
//     let reader = rnc_utils::buf_reader(&filename)?;
//     let (json_sender, json_reciever) = unbounded();
//     for line in reader.lines() {
//         let line = line?;
//         let filename = line.trim_end();
//         let path = PathBuf::from(filename);
//         let sender = json_sender.clone();
//         let store = env
//             .open_multi(&data_type, StoreOptions::create())
//             .with_context(|| format!("Failed to open store {:?}", data_type))?;
//         stores.insert(name, store);
//         thread::spawn(move || send_file_lines(&path, sender));
//     }

//     let writer = env.write().with_context(|| "Failed to create writer")?;

//     for (data_type, id, data) in json_reciever {
//         let store = stores.get(&data_type)?;
//         store
//             .put(&mut writer, id.id, &Value::Json(data))
//             .with_context(|| format!("Failed to write data for {:}/{:?}", &data_type,
// &id))?;     }

//     Ok(())
// }

pub fn lookup(spec: &Spec, key_file: &Path, output: &Path) -> Result<(), Box<dyn Error>> {
    let mut writer = rnc_utils::buf_writer(&output)
        .with_context(|| format!("Could not open {:?} for writing", &output))?;
    let mut keys = rnc_utils::buf_reader(&key_file)
        .with_context(|| format!("Could not open key file {:?}", &key_file))?;

    // let mut manager = Manager::<LmdbEnvironment>::singleton().write()?;
    let mut builder = Rkv::environment_builder::<Lmdb>();
    builder.set_max_dbs(20);
    // let env = arc.read().unwrap();

    let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap();
    // let created_arc = manager.get_or_create(spec.path, Rkv::new::<Lmdb>).unwrap();
    let created_arc = manager
        .get_or_create_from_builder(spec.path, builder, Rkv::from_builder::<Lmdb>)
        .with_context(|| "Failed to create arc")?;
    let env = created_arc.read().unwrap();

    let names = env.get_dbs()?.iter().filter_map(|db| db.to_owned()).collect::<Vec<String>>();

    let mut stores: Vec<(String, SingleStore<LmdbDatabase>)> = Vec::new();
    for name in names {
        let store = env
            .open_single(name.as_str(), StoreOptions::default())
            .with_context(|| format!("Failed to open store {:?}", &name))?;
        stores.push((name.to_owned(), store));
    }

    let reader = env.read().with_context(|| "Failed to create writer")?;

    let mut buf = String::new();
    loop {
        match keys.read_line(&mut buf)? {
            0 => break,
            _ => {
                let mut data = HashMap::new();
                let id = buf.trim_end().to_string();
                data.insert("id", serde_json::Value::String(id.to_string()));
                let mut seen = false;
                for (name, store) in &stores {
                    let default = serde_json::Value::Array(Vec::new());
                    let to_update = data.entry(&name).or_insert(default).as_array_mut().unwrap();
                    let current = store
                        .get(&reader, &id)
                        .with_context(|| format!("Failed to get results for {:?}", &id))?;

                    match current {
                        None => (),
                        Some(v) => match v {
                            Value::Json(raw) => {
                                seen = true;
                                let values = Deserializer::from_str(raw).into_iter::<serde_json::Value>();
                                for value in values {
                                    to_update.push(value?);
                                }
                            },
                            _ => panic!("Invalid lookup data"),
                        },
                    }

                    match (seen, spec.allow_missing) {
                        (true, _) => (),
                        (false, true) => {
                            log::info!("No data found for key {} in {}", &id, &name);
                            continue;
                        },
                        (false, false) => {
                            return Err(format!("No data found for key {} in {}", &id, &name).into());
                        },
                    }
                }

                serde_json::to_writer(&mut writer, &data)?;
                writeln!(&mut writer)?;
                buf.clear();
            },
        }
    }

    Ok(())
}
<commit_msg>Move missing check to where it belongs<commit_after>use std::{
    collections::HashMap,
    error::Error,
    fs::File,
    io::{
        BufRead,
        BufReader,
    },
    path::{
        Path,
        PathBuf,
    },
    str,
    thread,
};

use serde_query::{
    DeserializeQuery,
    Query,
};

use anyhow::{
    Context,
    Result,
};

// use crossbeam_channel::{
//     unbounded,
//     Sender,
// };

use serde_json::Deserializer;

use rkv::{
    backend::{
        BackendEnvironmentBuilder,
        Lmdb,
        LmdbDatabase,
        LmdbEnvironment,
    },
    Manager,
    Rkv,
    SingleStore,
    StoreOptions,
    Value,
};

#[derive(Debug, DeserializeQuery)]
struct DocId {
    #[query(".id")]
    id: String,
}

pub struct Spec<'a> {
    path: &'a Path,
    allow_missing: bool,
    commit_size: usize,
    threads: usize,
}

impl<'a> Spec<'a> {
    pub fn new(path: &Path) -> Spec {
        Spec {
            path,
            allow_missing: false,
            commit_size: 1_000_000usize,
            threads: 4,
        }
    }

    pub fn set_allow_missing(&mut self, allow_missing: bool) -> () {
        self.allow_missing = allow_missing;
    }

    pub fn set_commit_size(&mut self, commit_size: usize) -> () {
        self.commit_size = commit_size;
    }
}

pub fn index(spec: &Spec, data_type: &str, filename: &Path) -> Result<(), Box<dyn Error>> {
    let mut reader = rnc_utils::buf_reader(&filename)?;

    let mut manager = Manager::<LmdbEnvironment>::singleton().write()?;
    let mut builder = Rkv::environment_builder::<Lmdb>();
    builder.set_max_dbs(20);
    builder.set_make_dir_if_needed(true);
    builder.set_map_size(200000000000);
    let arc = manager
        .get_or_create_from_builder(spec.path, builder, Rkv::from_builder::<Lmdb>)
        .with_context(|| "Failed to create arc")?;
    let env = arc.read().unwrap();
    let store = env
        .open_single(data_type, StoreOptions::create())
        .with_context(|| format!("Failed to open store {:?}", data_type))?;

    let mut writer = env.write().with_context(|| "Failed to create writer")?;
    let mut buf = String::new();
    let mut count = 0usize;
    loop {
        match reader.read_line(&mut buf)? {
            0 => break,
            _ => {
                let line = buf.replace("\\\\", "\\");
                let value = line.trim_end().to_owned();
                let id: DocId = serde_json::from_str::<Query<DocId>>(&line)?.into();
                let existing = store.get(&writer, &id.id)?;
                let to_add = match existing {
                    None => value,
                    Some(v) => match v {
                        Value::Json(e) => format!("{}{}", value, e),
                        _ => panic!("Invalid existing data, should never happen"),
                    },
                };

                store.put(&mut writer, &id.id, &Value::Json(&to_add))?;
                buf.clear();
                count += 1;
                if count % spec.commit_size == 0 {
                    writer
                        .commit()
                        .with_context(|| format!("Failed to write data from {:?}", filename))?;
                    writer = env.write().with_context(|| "Could not create for new chunk")?;
                }
            },
        }
    }
    writer.commit().with_context(|| format!("Failed to write data from {:?}", filename))?;

    Ok(())
}

// fn path_as_column_name(path: &Path) -> String {
//     path.file_stem().unwrap().to_str().unwrap().to_string().replace("-", "_")
// }

// fn send_file_lines(path: &Path, sender: Sender<(String, DocId, String)>) {
//     let file = File::open(path).unwrap();
//     let data_type = path_as_column_name(&path);
//     let mut reader = BufReader::new(file);
//     let mut buf = String::new();
//     loop {
//         match reader.read_line(&mut buf).unwrap() {
//             0 => break,
//             _ => {
//                 let line = buf.replace("\\\\", "\\");
//                 let id: DocId =
// serde_json::from_str::<Query<DocId>>(&line).unwrap().into();
// sender.send((data_type.to_string(), id, line.to_string())).unwrap();
// buf.clear();             },
//         }
//     }
// }

// pub fn index_files(spec: &Spec, filename: &Path) -> Result<(), Box<dyn Error>> {
//     let mut manager = Manager::<LmdbEnvironment>::singleton().write()?;
//     let arc = manager.get_or_create(spec.path, Rkv::new::<Lmdb>)?;
//     let env = arc.read();

//     // let stores = HashMap::<String, Multi>
//     let reader = rnc_utils::buf_reader(&filename)?;
//     let (json_sender, json_reciever) = unbounded();
//     for line in reader.lines() {
//         let line = line?;
//         let filename = line.trim_end();
//         let path = PathBuf::from(filename);
//         let sender = json_sender.clone();
//         let store = env
//             .open_multi(&data_type, StoreOptions::create())
//             .with_context(|| format!("Failed to open store {:?}", data_type))?;
//         stores.insert(name, store);
//         thread::spawn(move || send_file_lines(&path, sender));
//     }

//     let writer = env.write().with_context(|| "Failed to create writer")?;

//     for (data_type, id, data) in json_reciever {
//         let store = stores.get(&data_type)?;
//         store
//             .put(&mut writer, id.id, &Value::Json(data))
//             .with_context(|| format!("Failed to write data for {:}/{:?}", &data_type,
// &id))?;     }

//     Ok(())
// }

pub fn lookup(spec: &Spec, key_file: &Path, output: &Path) -> Result<(), Box<dyn Error>> {
    let mut writer = rnc_utils::buf_writer(&output)
        .with_context(|| format!("Could not open {:?} for writing", &output))?;
    let mut keys = rnc_utils::buf_reader(&key_file)
        .with_context(|| format!("Could not open key file {:?}", &key_file))?;

    let mut builder = Rkv::environment_builder::<Lmdb>();
    builder.set_max_dbs(20);

    let mut manager = Manager::<LmdbEnvironment>::singleton().write().unwrap();
    let created_arc = manager
        .get_or_create_from_builder(spec.path, builder, Rkv::from_builder::<Lmdb>)
        .with_context(|| "Failed to create arc")?;
    let env = created_arc.read().unwrap();

    let names = env.get_dbs()?.iter().filter_map(|db| db.to_owned()).collect::<Vec<String>>();

    let mut stores: Vec<(String, SingleStore<LmdbDatabase>)> = Vec::new();
    for name in names {
        let store = env
            .open_single(name.as_str(), StoreOptions::default())
            .with_context(|| format!("Failed to open store {:?}", &name))?;
        stores.push((name.to_owned(), store));
    }

    let reader = env.read().with_context(|| "Failed to create writer")?;

    let mut buf = String::new();
    loop {
        match keys.read_line(&mut buf)? {
            0 => break,
            _ => {
                let mut data = HashMap::new();
                let id = buf.trim_end().to_string();
                data.insert("id", serde_json::Value::String(id.to_string()));
                let mut seen = false;
                for (name, store) in &stores {
                    let default = serde_json::Value::Array(Vec::new());
                    let to_update = data.entry(&name).or_insert(default).as_array_mut().unwrap();
                    let current = store
                        .get(&reader, &id)
                        .with_context(|| format!("Failed to get results for {:?}", &id))?;

                    match current {
                        None => (),
                        Some(v) => match v {
                            Value::Json(raw) => {
                                seen = true;
                                let values = Deserializer::from_str(raw).into_iter::<serde_json::Value>();
                                for value in values {
                                    to_update.push(value?);
                                }
                            },
                            _ => panic!("Invalid lookup data"),
                        },
                    }
                }

                match (seen, spec.allow_missing) {
                    (true, _) => (),
                    (false, true) => {
                        log::info!("No data found for key {}", &id);
                        continue;
                    },
                    (false, false) => {
                        return Err(format!("No data found for key {}", &id).into());
                    },
                }

                serde_json::to_writer(&mut writer, &data)?;
                writeln!(&mut writer)?;
                buf.clear();
            },
        }
    }

    Ok(())
}
<|endoftext|>


<commit_before>//! Handling column descriptions
//!
//! Columns are represented as
//! [`ConcreteColumnDescription`](struct.ConcreteColumnDescription.html). This is constructed
//! through the builder pattern, by creating a [`ColumnDescription`](struct.ColumnDescription.html)
//! and calling [`create`](struct.ColumnDescription.html#method.create)
use errors::Result;
use std::str::FromStr;

/// Description for new columns
#[derive(Debug, Clone)]
pub struct ColumnDescription {
    /// Name of the column
    pub name: String,

    /// Type of the data, see the cfitsio documentation
    pub data_type: Option<ColumnDataDescription>,
}

/// Concrete representation of the description of a column
#[derive(Debug, Clone)]
pub struct ConcreteColumnDescription {
    /// Name of the column
    pub name: String,

    /// Type of the data, see the cfitsio documentation
    pub data_type: ColumnDataDescription,
}

impl ColumnDescription {
    /// Create a new [`ColumnDescription`](struct.ColumnDescription.html) from a name
    pub fn new<T: Into<String>>(name: T) -> Self {
        ColumnDescription {
            name: name.into(),
            data_type: None,
        }
    }

    /// Add a data type to the column description
    pub fn with_type(&mut self, typ: ColumnDataType) -> &mut ColumnDescription {
        self.data_type = Some(ColumnDataDescription::scalar(typ));
        self
    }

    /// Make the column repeat
    pub fn that_repeats(&mut self, repeat: usize) -> &mut ColumnDescription {
        if let Some(ref mut desc) = self.data_type {
            desc.repeat = repeat;
        }
        self
    }

    /// Define the column width
    pub fn with_width(&mut self, width: usize) -> &mut ColumnDescription {
        if let Some(ref mut desc) = self.data_type {
            desc.width = width;
        }
        self
    }

    /// Render the [`ColumnDescription`](struct.ColumnDescription.html) into a
    /// [`ConcreteColumnDescription`](struct.ConcreteColumnDescription.html)
    pub fn create(&self) -> Result<ConcreteColumnDescription> {
        match self.data_type {
            Some(ref d) => {
                Ok(ConcreteColumnDescription {
                    name: self.name.clone(),
                    data_type: d.clone(),
                })
            }
            None => {
                Err(
                    "No data type given. Ensure the `with_type` method has been called.".into(),
                )
            }
        }
    }
}

/// Description of the column data
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ColumnDataDescription {
    /// Does the column contain multiple values?
    pub repeat: usize,

    /// How wide is the column?
    pub width: usize,

    /// What data type does the column store?
    pub typ: ColumnDataType,
}

impl ColumnDataDescription {
    /// Create a new column data description
    pub fn new(typ: ColumnDataType, repeat: usize, width: usize) -> Self {
        ColumnDataDescription {
            repeat: repeat,
            width: width,
            typ: typ,
        }
    }

    /// Shortcut for creating a scalar column
    pub fn scalar(typ: ColumnDataType) -> Self {
        ColumnDataDescription::new(typ, 1, 1)
    }

    /// Shortcut for creating a vector column
    pub fn vector(typ: ColumnDataType, repeat: usize) -> Self {
        ColumnDataDescription::new(typ, repeat, 1)
    }

    /// Set the repeat count
    /* XXX These two methods force a call to clone which is wasteful of memory. I do not know if
     * this means that memory is leaked, or that destructors are needlessly called (I suspect the
     * latter) but it is fairly wasteful. On the other hand, it's unlikely this sort of thing will
     * be called in performance-critical code, and is more likely a one-time definition. I will
     * leave it for now - SRW 2017-03-07
     * */
    pub fn repeats(&mut self, repeat: usize) -> Self {
        // TODO check that repeat >= 1
        self.repeat = repeat;
        self.clone()
    }

    /// Set the width of the column
    pub fn width(&mut self, width: usize) -> Self {
        // TODO check that width >= 1
        self.width = width;
        self.clone()
    }
}

impl From<ColumnDataDescription> for String {
    fn from(orig: ColumnDataDescription) -> String {
        match orig.typ {
            ColumnDataType::Text => {
                if orig.width > 1 {
                    format!(
                        "{repeat}{data_type}{width}",
                        data_type = String::from(orig.typ),
                        repeat = orig.repeat,
                        width = orig.width
                    )
                } else {
                    format!(
                        "{repeat}{data_type}",
                        data_type = String::from(orig.typ),
                        repeat = orig.repeat
                    )
                }
            }
            _ => {
                format!(
                    "{repeat}{data_type}",
                    data_type = String::from(orig.typ),
                    repeat = orig.repeat
                )
            }
        }
    }
}

/// Types a column can represent
#[allow(missing_docs)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ColumnDataType {
    Int,
    Float,
    Text,
    Double,
    Short,
    Long,
    String,
}

impl From<ColumnDataType> for String {
    fn from(orig: ColumnDataType) -> String {
        use self::ColumnDataType::*;

        match orig {
            Int => "J",
            Float => "E",
            Text | String => "A",
            Double => "D",
            Short => "I",
            Long => "K",
        }.to_string()
    }
}

impl FromStr for ColumnDataDescription {
    type Err = Box<::std::error::Error>;

    fn from_str(s: &str) -> ::std::result::Result<Self, Self::Err> {
        let chars: Vec<_> = s.chars().collect();

        let mut repeat_str = Vec::new();
        let mut last_position = 0;
        for c in &chars {
            if c.is_digit(10) {
                repeat_str.push(c);
                last_position += 1;
            } else {
                break;
            }
        }

        let repeat = if repeat_str.is_empty() {
            1
        } else {
            /* TODO: in nightly the following line works
            let repeat_str: String = repeat_str.into_iter().collect(); */
            let repeat_str: String = repeat_str.into_iter().cloned().collect();
            repeat_str.parse::<usize>()?
        };

        let data_type_char = chars[last_position];
        last_position += 1;

        let mut width_str = Vec::new();
        for c in chars.iter().skip(last_position) {
            if c.is_digit(10) {
                width_str.push(c);
            } else {
                break;
            }
        }

        /* TODO: validate that the whole string has been used up */

        let width = if width_str.is_empty() {
            1
        } else {
            /* TODO: in nightly the following line works
            let width_str: String = width_str.into_iter().collect(); */
            let width_str: String = width_str.into_iter().cloned().collect();
            width_str.parse::<usize>()?
        };

        let data_type = match data_type_char {
            'E' => ColumnDataType::Float,
            'J' => ColumnDataType::Int,
            'D' => ColumnDataType::Double,
            'I' => ColumnDataType::Short,
            'K' => ColumnDataType::Long,
            'A' => ColumnDataType::String,
            _ => {
                panic!(
                    "Have not implemented str -> ColumnDataType for {}",
                    data_type_char
                )
            }
        };

        Ok(ColumnDataDescription {
            repeat: repeat,
            typ: data_type,
            width: width,
        })
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_column_data_descriptions_builder_pattern() {
        let desc = ColumnDataDescription::scalar(ColumnDataType::Int)
            .width(100)
            .repeats(5);
        assert_eq!(desc.repeat, 5);
        assert_eq!(desc.width, 100);
    }

    #[test]
    fn from_impls() {
        {
            let desc = ColumnDataDescription::scalar(ColumnDataType::Int).repeats(5);
            assert_eq!(String::from(desc), "5J");
        }

        {
            let desc = ColumnDataDescription::scalar(ColumnDataType::Float);
            assert_eq!(String::from(desc), "1E");
        }

        {
            let desc = ColumnDataDescription::scalar(ColumnDataType::Text).width(100);
            assert_eq!(String::from(desc), "1A100");
        }
    }

    #[test]
    fn parsing() {
        let s = "1E";
        assert_eq!(
            s.parse::<ColumnDataDescription>().unwrap(),
            ColumnDataDescription {
                repeat: 1,
                width: 1,
                typ: ColumnDataType::Float,
            }
        );
    }

    #[test]
    fn parse_many_repeats() {
        let s = "100E";
        assert_eq!(
            s.parse::<ColumnDataDescription>().unwrap(),
            ColumnDataDescription {
                repeat: 100,
                width: 1,
                typ: ColumnDataType::Float,
            }
        );
    }

    #[test]
    fn parse_with_width() {
        let s = "1E26";
        assert_eq!(
            s.parse::<ColumnDataDescription>().unwrap(),
            ColumnDataDescription {
                repeat: 1,
                width: 26,
                typ: ColumnDataType::Float,
            }
        );
    }

    #[test]
    fn creating_data_description() {
        let concrete_desc = ColumnDescription::new("FOO")
            .with_type(ColumnDataType::Int)
            .that_repeats(10)
            .create()
            .unwrap();
        assert_eq!(concrete_desc.name, "FOO".to_string());
        assert_eq!(concrete_desc.data_type.repeat, 10);
        assert_eq!(concrete_desc.data_type.width, 1);

        /* Do not call `with_type` */
        let bad_desc = ColumnDescription::new("FOO").create();
        assert!(bad_desc.is_err());
    }
}
<commit_msg>Remove two TODO items<commit_after>//! Handling column descriptions
//!
//! Columns are represented as
//! [`ConcreteColumnDescription`](struct.ConcreteColumnDescription.html). This is constructed
//! through the builder pattern, by creating a [`ColumnDescription`](struct.ColumnDescription.html)
//! and calling [`create`](struct.ColumnDescription.html#method.create)
use errors::Result;
use std::str::FromStr;

/// Description for new columns
#[derive(Debug, Clone)]
pub struct ColumnDescription {
    /// Name of the column
    pub name: String,

    /// Type of the data, see the cfitsio documentation
    pub data_type: Option<ColumnDataDescription>,
}

/// Concrete representation of the description of a column
#[derive(Debug, Clone)]
pub struct ConcreteColumnDescription {
    /// Name of the column
    pub name: String,

    /// Type of the data, see the cfitsio documentation
    pub data_type: ColumnDataDescription,
}

impl ColumnDescription {
    /// Create a new [`ColumnDescription`](struct.ColumnDescription.html) from a name
    pub fn new<T: Into<String>>(name: T) -> Self {
        ColumnDescription {
            name: name.into(),
            data_type: None,
        }
    }

    /// Add a data type to the column description
    pub fn with_type(&mut self, typ: ColumnDataType) -> &mut ColumnDescription {
        self.data_type = Some(ColumnDataDescription::scalar(typ));
        self
    }

    /// Make the column repeat
    pub fn that_repeats(&mut self, repeat: usize) -> &mut ColumnDescription {
        if let Some(ref mut desc) = self.data_type {
            desc.repeat = repeat;
        }
        self
    }

    /// Define the column width
    pub fn with_width(&mut self, width: usize) -> &mut ColumnDescription {
        if let Some(ref mut desc) = self.data_type {
            desc.width = width;
        }
        self
    }

    /// Render the [`ColumnDescription`](struct.ColumnDescription.html) into a
    /// [`ConcreteColumnDescription`](struct.ConcreteColumnDescription.html)
    pub fn create(&self) -> Result<ConcreteColumnDescription> {
        match self.data_type {
            Some(ref d) => {
                Ok(ConcreteColumnDescription {
                    name: self.name.clone(),
                    data_type: d.clone(),
                })
            }
            None => {
                Err(
                    "No data type given. Ensure the `with_type` method has been called.".into(),
                )
            }
        }
    }
}

/// Description of the column data
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ColumnDataDescription {
    /// Does the column contain multiple values?
    pub repeat: usize,

    /// How wide is the column?
    pub width: usize,

    /// What data type does the column store?
    pub typ: ColumnDataType,
}

impl ColumnDataDescription {
    /// Create a new column data description
    pub fn new(typ: ColumnDataType, repeat: usize, width: usize) -> Self {
        ColumnDataDescription {
            repeat: repeat,
            width: width,
            typ: typ,
        }
    }

    /// Shortcut for creating a scalar column
    pub fn scalar(typ: ColumnDataType) -> Self {
        ColumnDataDescription::new(typ, 1, 1)
    }

    /// Shortcut for creating a vector column
    pub fn vector(typ: ColumnDataType, repeat: usize) -> Self {
        ColumnDataDescription::new(typ, repeat, 1)
    }

    /// Set the repeat count
    /* XXX These two methods force a call to clone which is wasteful of memory. I do not know if
     * this means that memory is leaked, or that destructors are needlessly called (I suspect the
     * latter) but it is fairly wasteful. On the other hand, it's unlikely this sort of thing will
     * be called in performance-critical code, and is more likely a one-time definition. I will
     * leave it for now - SRW 2017-03-07
     * */
    pub fn repeats(&mut self, repeat: usize) -> Result<Self> {
        if repeat == 0 {
            return Err("repeat parameter must be > 0".into());
        } else {
            self.repeat = repeat;
            Ok(self.clone())
        }
    }

    /// Set the width of the column
    pub fn width(&mut self, width: usize) -> Result<Self> {
        if width == 0 {
            return Err("width parameter must be > 0".into())
        } else {
            self.width = width;
            Ok(self.clone())
        }
    }
}

impl From<ColumnDataDescription> for String {
    fn from(orig: ColumnDataDescription) -> String {
        match orig.typ {
            ColumnDataType::Text => {
                if orig.width > 1 {
                    format!(
                        "{repeat}{data_type}{width}",
                        data_type = String::from(orig.typ),
                        repeat = orig.repeat,
                        width = orig.width
                    )
                } else {
                    format!(
                        "{repeat}{data_type}",
                        data_type = String::from(orig.typ),
                        repeat = orig.repeat
                    )
                }
            }
            _ => {
                format!(
                    "{repeat}{data_type}",
                    data_type = String::from(orig.typ),
                    repeat = orig.repeat
                )
            }
        }
    }
}

/// Types a column can represent
#[allow(missing_docs)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ColumnDataType {
    Int,
    Float,
    Text,
    Double,
    Short,
    Long,
    String,
}

impl From<ColumnDataType> for String {
    fn from(orig: ColumnDataType) -> String {
        use self::ColumnDataType::*;

        match orig {
            Int => "J",
            Float => "E",
            Text | String => "A",
            Double => "D",
            Short => "I",
            Long => "K",
        }.to_string()
    }
}

impl FromStr for ColumnDataDescription {
    type Err = Box<::std::error::Error>;

    fn from_str(s: &str) -> ::std::result::Result<Self, Self::Err> {
        let chars: Vec<_> = s.chars().collect();

        let mut repeat_str = Vec::new();
        let mut last_position = 0;
        for c in &chars {
            if c.is_digit(10) {
                repeat_str.push(c);
                last_position += 1;
            } else {
                break;
            }
        }

        let repeat = if repeat_str.is_empty() {
            1
        } else {
            /* TODO: in nightly the following line works
            let repeat_str: String = repeat_str.into_iter().collect(); */
            let repeat_str: String = repeat_str.into_iter().cloned().collect();
            repeat_str.parse::<usize>()?
        };

        let data_type_char = chars[last_position];
        last_position += 1;

        let mut width_str = Vec::new();
        for c in chars.iter().skip(last_position) {
            if c.is_digit(10) {
                width_str.push(c);
            } else {
                break;
            }
        }

        /* TODO: validate that the whole string has been used up */

        let width = if width_str.is_empty() {
            1
        } else {
            /* TODO: in nightly the following line works
            let width_str: String = width_str.into_iter().collect(); */
            let width_str: String = width_str.into_iter().cloned().collect();
            width_str.parse::<usize>()?
        };

        let data_type = match data_type_char {
            'E' => ColumnDataType::Float,
            'J' => ColumnDataType::Int,
            'D' => ColumnDataType::Double,
            'I' => ColumnDataType::Short,
            'K' => ColumnDataType::Long,
            'A' => ColumnDataType::String,
            _ => {
                panic!(
                    "Have not implemented str -> ColumnDataType for {}",
                    data_type_char
                )
            }
        };

        Ok(ColumnDataDescription {
            repeat: repeat,
            typ: data_type,
            width: width,
        })
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_column_data_descriptions_builder_pattern() {
        let desc = ColumnDataDescription::scalar(ColumnDataType::Int)
            .width(100)
            .and_then(|mut d| d.repeats(5))
            .unwrap();
        assert_eq!(desc.repeat, 5);
        assert_eq!(desc.width, 100);
    }

    #[test]
    fn width_after_repeates() {
        let desc = ColumnDataDescription::scalar(ColumnDataType::Int)
            .repeats(5)
            .and_then(|mut d| d.width(100))
            .unwrap();
        assert_eq!(desc.repeat, 5);
        assert_eq!(desc.width, 100);
    }

    #[test]
    fn from_impls() {
        {
            let desc = ColumnDataDescription::scalar(ColumnDataType::Int).repeats(5).unwrap();
            assert_eq!(String::from(desc), "5J");
        }

        {
            let desc = ColumnDataDescription::scalar(ColumnDataType::Float);
            assert_eq!(String::from(desc), "1E");
        }

        {
            let desc = ColumnDataDescription::scalar(ColumnDataType::Text).width(100).unwrap();
            assert_eq!(String::from(desc), "1A100");
        }
    }

    #[test]
    fn parsing() {
        let s = "1E";
        assert_eq!(
            s.parse::<ColumnDataDescription>().unwrap(),
            ColumnDataDescription {
                repeat: 1,
                width: 1,
                typ: ColumnDataType::Float,
            }
        );
    }

    #[test]
    fn parse_many_repeats() {
        let s = "100E";
        assert_eq!(
            s.parse::<ColumnDataDescription>().unwrap(),
            ColumnDataDescription {
                repeat: 100,
                width: 1,
                typ: ColumnDataType::Float,
            }
        );
    }

    #[test]
    fn parse_with_width() {
        let s = "1E26";
        assert_eq!(
            s.parse::<ColumnDataDescription>().unwrap(),
            ColumnDataDescription {
                repeat: 1,
                width: 26,
                typ: ColumnDataType::Float,
            }
        );
    }

    #[test]
    fn creating_data_description() {
        let concrete_desc = ColumnDescription::new("FOO")
            .with_type(ColumnDataType::Int)
            .that_repeats(10)
            .create()
            .unwrap();
        assert_eq!(concrete_desc.name, "FOO".to_string());
        assert_eq!(concrete_desc.data_type.repeat, 10);
        assert_eq!(concrete_desc.data_type.width, 1);

        /* Do not call `with_type` */
        let bad_desc = ColumnDescription::new("FOO").create();
        assert!(bad_desc.is_err());
    }
}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![plugin(rocket_codegen)]

extern crate getopts;
extern crate rocket;
#[macro_use] extern crate rocket_contrib;

#[macro_use] extern crate serde_derive;

extern crate pombase;

use std::sync::Mutex;
use std::process;
use std::env;
use std::path::{Path, PathBuf};

use getopts::Options;

use rocket_contrib::{Json, Value};

use rocket::response::NamedFile;

use pombase::api::query::Query;
use pombase::api::result::QueryAPIResult;
use pombase::api::search::Search;
use pombase::api::query_exec::QueryExec;
use pombase::api::server_data::ServerData;
use pombase::web::data::{SolrTermSummary, GeneDetails, GenotypeDetails,
                         TermDetails, ReferenceDetails};

const PKG_NAME: &str = env!("CARGO_PKG_NAME");
const VERSION: &str = env!("CARGO_PKG_VERSION");

struct StaticFileState {
    web_root_dir: String,
}

// try the path, then try path + ".json", then default to loading the Angular app
// from /index.html
#[get("/<path..>", rank=3)]
fn get_misc(path: PathBuf, state: rocket::State<Mutex<StaticFileState>>) -> Option<NamedFile> {
    let web_root_dir = &state.lock().expect("failed to lock").web_root_dir;
    let root_dir_path = Path::new("/").join(web_root_dir);
    let full_path = root_dir_path.join(path);
    if full_path.exists() {
        return NamedFile::open(full_path).ok();
    }

    let mut json_path_str = full_path.to_str().unwrap().to_owned();
    json_path_str += ".json";
    let json_path: PathBuf = json_path_str.into();

    if json_path.exists() {
        return NamedFile::open(json_path).ok();
    }

    NamedFile::open(root_dir_path.join("index.html")).ok()
}

#[get("/api/v1/dataset/latest/data/gene/<id>", rank=4)]
fn get_gene(id: String, state: rocket::State<Mutex<QueryExec>>) -> Option<Json<GeneDetails>> {
    let query_exec = state.lock().expect("failed to lock");
    if let Some(gene) = query_exec.get_server_data().get_gene_details(&id) {
        Some(Json(gene.clone()))
    } else {
        None
    }
}

#[get("/api/v1/dataset/latest/data/genotype/<id>", rank=4)]
fn get_genotype(id: String, state: rocket::State<Mutex<QueryExec>>) -> Option<Json<GenotypeDetails>> {
    let query_exec = state.lock().expect("failed to lock");
    if let Some(genotype) = query_exec.get_server_data().get_genotype_details(&id) {
        Some(Json(genotype.clone()))
    } else {
        None
    }
}

#[get("/api/v1/dataset/latest/data/term/<id>", rank=4)]
fn get_term(id: String, state: rocket::State<Mutex<QueryExec>>) -> Option<Json<TermDetails>> {
    let query_exec = state.lock().expect("failed to lock");
    if let Some(term) = query_exec.get_server_data().get_term_details(&id) {
        Some(Json(term.clone()))
    } else {
        None
    }
}

#[get("/api/v1/dataset/latest/data/reference/<id>", rank=4)]
fn get_reference(id: String, state: rocket::State<Mutex<QueryExec>>) -> Option<Json<ReferenceDetails>> {
    let query_exec = state.lock().expect("failed to lock");
    if let Some(reference) = query_exec.get_server_data().get_reference_details(&id) {
        Some(Json(reference.clone()))
    } else {
        None
    }
}

#[get("/", rank=1)]
fn get_index(state: rocket::State<Mutex<StaticFileState>>) -> Option<NamedFile> {
    let web_root_dir = &state.lock().expect("failed to lock").web_root_dir;
    let root_dir_path = Path::new("/").join(web_root_dir);
    NamedFile::open(root_dir_path.join("index.html")).ok()
}

#[post("/api/v1/dataset/latest/query", rank=1, data="<q>", format = "application/json")]
fn query_post(q: Json<Query>, state: rocket::State<Mutex<QueryExec>>)
              -> Option<Json<QueryAPIResult>>
{
    let query_exec = state.lock().expect("failed to lock");
    Some(Json(query_exec.exec(&q.into_inner())))
}

#[get ("/reload")]
fn reload(state: rocket::State<Mutex<QueryExec>>) {
    let mut query_exec = state.lock().expect("failed to lock");
    print!("reloading ...\n");
    query_exec.reload();
    print!("... done\n");
}

#[derive(Serialize, Debug)]
struct CompletionResponse {
    status: String,
    matches: Vec<SolrTermSummary>,
}

#[get ("/api/v1/dataset/latest/complete/<cv_name>/<q>", rank=1)]
fn complete(cv_name: String, q: String, state: rocket::State<Mutex<Search>>)
              -> Option<Json<CompletionResponse>>
{
    let search = state.lock().expect("failed to lock");
    let res = search.term_complete(&cv_name, &q);

    let completion_response =
        match res {
            Ok(matches) => {
                CompletionResponse {
                    status: "Ok".to_owned(),
                    matches: matches,
                }
            },
            Err(err) => {
                println!("{:?}", err);
                CompletionResponse {
                    status: "Error".to_owned(),
                    matches: vec![],
                }
            },
        };

    Some(Json(completion_response))
}

#[get ("/ping", rank=1)]
fn ping() -> Option<String> {
    Some(String::from("OK") + " " + PKG_NAME + " " + VERSION)
}

#[error(404)]
fn not_found() -> Json<Value> {
    Json(json!({
        "status": "error",
        "reason": "Resource was not found."
    }))
}

fn print_usage(program: &str, opts: Options) {
    let brief = format!("Usage: {} [options]", program);
    print!("{}", opts.usage(&brief));
}

fn main() {
    print!("{} v{}\n", PKG_NAME, VERSION);

    let args: Vec<String> = env::args().collect();
    let mut opts = Options::new();

    opts.optflag("h", "help", "print this help message");
    opts.optopt("c", "config-file", "Configuration file name", "CONFIG");
    opts.optopt("m", "search-maps", "Search data", "MAPS_JSON_FILE");
    opts.optopt("s", "gene-subsets", "Gene subset data", "SUBSETS_JSON_FILE");
    opts.optopt("w", "web-root-dir", "Root web data directory", "WEB_ROOT_DIR");

    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => panic!("Invalid options\n{}", f)
    };

    let program = args[0].clone();

    if matches.opt_present("help") {
        print_usage(&program, opts);
        process::exit(0);
    }

    if !matches.opt_present("config-file") {
        print!("no -c|--config-file option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("search-maps") {
        print!("no --search-maps|-m option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("gene-subsets") {
        print!("no --gene-subsets|-s option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("web-root-dir") {
        print!("no --web-root-dir|-w option\n");
        print_usage(&program, opts);
        process::exit(1);
    }

    let search_maps_filename = matches.opt_str("m").unwrap();
    let gene_subsets_filename = matches.opt_str("s").unwrap();
    println!("Reading data files ...");

    let config_file_name = matches.opt_str("c").unwrap();
    let server_data = ServerData::new(&config_file_name, &search_maps_filename,
                                      &gene_subsets_filename);
    let query_exec = QueryExec::new(server_data);
    let searcher = Search::new("http://localhost:8983/solr".to_owned());

    let web_root_dir = matches.opt_str("w").unwrap();
    let static_file_state = StaticFileState {
        web_root_dir: web_root_dir,
    };

    println!("Starting server ...");
    rocket::ignite()
        .mount("/", routes![get_index, get_misc, query_post,
                            get_gene, get_genotype, get_term, get_reference,
                            reload, complete, ping])
        .catch(errors![not_found])
        .manage(Mutex::new(query_exec))
        .manage(Mutex::new(searcher))
        .manage(Mutex::new(static_file_state))
        .launch();
}
<commit_msg>Fix route rank for term, gene and ref paths<commit_after>#![feature(plugin)]
#![plugin(rocket_codegen)]

extern crate getopts;
extern crate rocket;
#[macro_use] extern crate rocket_contrib;

#[macro_use] extern crate serde_derive;

extern crate pombase;

use std::sync::Mutex;
use std::process;
use std::env;
use std::path::{Path, PathBuf};

use getopts::Options;

use rocket_contrib::{Json, Value};

use rocket::response::NamedFile;

use pombase::api::query::Query;
use pombase::api::result::QueryAPIResult;
use pombase::api::search::Search;
use pombase::api::query_exec::QueryExec;
use pombase::api::server_data::ServerData;
use pombase::web::data::{SolrTermSummary, GeneDetails, GenotypeDetails,
                         TermDetails, ReferenceDetails};

const PKG_NAME: &str = env!("CARGO_PKG_NAME");
const VERSION: &str = env!("CARGO_PKG_VERSION");

struct StaticFileState {
    web_root_dir: String,
}

// try the path, then try path + ".json", then default to loading the Angular app
// from /index.html
#[get("/<path..>", rank=3)]
fn get_misc(path: PathBuf, state: rocket::State<Mutex<StaticFileState>>) -> Option<NamedFile> {
    let web_root_dir = &state.lock().expect("failed to lock").web_root_dir;
    let root_dir_path = Path::new("/").join(web_root_dir);
    let full_path = root_dir_path.join(path);
    if full_path.exists() {
        return NamedFile::open(full_path).ok();
    }

    let mut json_path_str = full_path.to_str().unwrap().to_owned();
    json_path_str += ".json";
    let json_path: PathBuf = json_path_str.into();

    if json_path.exists() {
        return NamedFile::open(json_path).ok();
    }

    NamedFile::open(root_dir_path.join("index.html")).ok()
}

#[get("/api/v1/dataset/latest/data/gene/<id>", rank=2)]
fn get_gene(id: String, state: rocket::State<Mutex<QueryExec>>) -> Option<Json<GeneDetails>> {
    let query_exec = state.lock().expect("failed to lock");
    if let Some(gene) = query_exec.get_server_data().get_gene_details(&id) {
        Some(Json(gene.clone()))
    } else {
        None
    }
}

#[get("/api/v1/dataset/latest/data/genotype/<id>", rank=2)]
fn get_genotype(id: String, state: rocket::State<Mutex<QueryExec>>) -> Option<Json<GenotypeDetails>> {
    let query_exec = state.lock().expect("failed to lock");
    if let Some(genotype) = query_exec.get_server_data().get_genotype_details(&id) {
        Some(Json(genotype.clone()))
    } else {
        None
    }
}

#[get("/api/v1/dataset/latest/data/term/<id>", rank=2)]
fn get_term(id: String, state: rocket::State<Mutex<QueryExec>>) -> Option<Json<TermDetails>> {
    let query_exec = state.lock().expect("failed to lock");
    if let Some(term) = query_exec.get_server_data().get_term_details(&id) {
        Some(Json(term.clone()))
    } else {
        None
    }
}

#[get("/api/v1/dataset/latest/data/reference/<id>", rank=2)]
fn get_reference(id: String, state: rocket::State<Mutex<QueryExec>>) -> Option<Json<ReferenceDetails>> {
    let query_exec = state.lock().expect("failed to lock");
    if let Some(reference) = query_exec.get_server_data().get_reference_details(&id) {
        Some(Json(reference.clone()))
    } else {
        None
    }
}

#[get("/", rank=1)]
fn get_index(state: rocket::State<Mutex<StaticFileState>>) -> Option<NamedFile> {
    let web_root_dir = &state.lock().expect("failed to lock").web_root_dir;
    let root_dir_path = Path::new("/").join(web_root_dir);
    NamedFile::open(root_dir_path.join("index.html")).ok()
}

#[post("/api/v1/dataset/latest/query", rank=1, data="<q>", format = "application/json")]
fn query_post(q: Json<Query>, state: rocket::State<Mutex<QueryExec>>)
              -> Option<Json<QueryAPIResult>>
{
    let query_exec = state.lock().expect("failed to lock");
    Some(Json(query_exec.exec(&q.into_inner())))
}

#[get ("/reload")]
fn reload(state: rocket::State<Mutex<QueryExec>>) {
    let mut query_exec = state.lock().expect("failed to lock");
    print!("reloading ...\n");
    query_exec.reload();
    print!("... done\n");
}

#[derive(Serialize, Debug)]
struct CompletionResponse {
    status: String,
    matches: Vec<SolrTermSummary>,
}

#[get ("/api/v1/dataset/latest/complete/<cv_name>/<q>", rank=1)]
fn complete(cv_name: String, q: String, state: rocket::State<Mutex<Search>>)
              -> Option<Json<CompletionResponse>>
{
    let search = state.lock().expect("failed to lock");
    let res = search.term_complete(&cv_name, &q);

    let completion_response =
        match res {
            Ok(matches) => {
                CompletionResponse {
                    status: "Ok".to_owned(),
                    matches: matches,
                }
            },
            Err(err) => {
                println!("{:?}", err);
                CompletionResponse {
                    status: "Error".to_owned(),
                    matches: vec![],
                }
            },
        };

    Some(Json(completion_response))
}

#[get ("/ping", rank=1)]
fn ping() -> Option<String> {
    Some(String::from("OK") + " " + PKG_NAME + " " + VERSION)
}

#[error(404)]
fn not_found() -> Json<Value> {
    Json(json!({
        "status": "error",
        "reason": "Resource was not found."
    }))
}

fn print_usage(program: &str, opts: Options) {
    let brief = format!("Usage: {} [options]", program);
    print!("{}", opts.usage(&brief));
}

fn main() {
    print!("{} v{}\n", PKG_NAME, VERSION);

    let args: Vec<String> = env::args().collect();
    let mut opts = Options::new();

    opts.optflag("h", "help", "print this help message");
    opts.optopt("c", "config-file", "Configuration file name", "CONFIG");
    opts.optopt("m", "search-maps", "Search data", "MAPS_JSON_FILE");
    opts.optopt("s", "gene-subsets", "Gene subset data", "SUBSETS_JSON_FILE");
    opts.optopt("w", "web-root-dir", "Root web data directory", "WEB_ROOT_DIR");

    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => panic!("Invalid options\n{}", f)
    };

    let program = args[0].clone();

    if matches.opt_present("help") {
        print_usage(&program, opts);
        process::exit(0);
    }

    if !matches.opt_present("config-file") {
        print!("no -c|--config-file option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("search-maps") {
        print!("no --search-maps|-m option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("gene-subsets") {
        print!("no --gene-subsets|-s option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("web-root-dir") {
        print!("no --web-root-dir|-w option\n");
        print_usage(&program, opts);
        process::exit(1);
    }

    let search_maps_filename = matches.opt_str("m").unwrap();
    let gene_subsets_filename = matches.opt_str("s").unwrap();
    println!("Reading data files ...");

    let config_file_name = matches.opt_str("c").unwrap();
    let server_data = ServerData::new(&config_file_name, &search_maps_filename,
                                      &gene_subsets_filename);
    let query_exec = QueryExec::new(server_data);
    let searcher = Search::new("http://localhost:8983/solr".to_owned());

    let web_root_dir = matches.opt_str("w").unwrap();
    let static_file_state = StaticFileState {
        web_root_dir: web_root_dir,
    };

    println!("Starting server ...");
    rocket::ignite()
        .mount("/", routes![get_index, get_misc, query_post,
                            get_gene, get_genotype, get_term, get_reference,
                            reload, complete, ping])
        .catch(errors![not_found])
        .manage(Mutex::new(query_exec))
        .manage(Mutex::new(searcher))
        .manage(Mutex::new(static_file_state))
        .launch();
}
<|endoftext|>


<commit_before>use luminance_glfw::{self, open_window};
pub use luminance_glfw::{Action, DeviceError, Key, MouseButton, WindowDim, WindowOpt};
use std::cell::RefCell;
use std::os::raw::c_void;
use std::rc::Rc;
use std::sync::mpsc;
use std::thread;
use std::time::{Duration, Instant};

use camera::{Camera, Freefly};
use linear::V3;

type Time = f64;

/// Signals events can pass up back to their handlers to notify them how they have processed an
/// event. They’re three kinds of signals:
///
/// - `EventSig::Ignored`:  the event should be passed to other handlers the parents knows about –
///    if any – because it wasn’t handled (ignored);
///
/// - `EventSig::Handled`: the event has been correctly handled;
///
/// - `EventSig::Focused`: the event has been correctly handled, and the parent should consider that
///    the this handler has now an exclusive focus on that event;
///
/// - `EventSig::Aborted`: the event has been correctly handled and the parent handler should be
///    aborted. This signal is typically used to kill all the handlers chain and thus quit the
///    application.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum EventSig {
  Ignored,
  Handled,
  Focused,
  Aborted
}

/// Class of event handlers.
///
/// All functions return a special object of type `EventSig`. An event report gives information
/// about how a handler has handled the event. See the documentation of `EventSig` for further
/// information.
///
/// All functions’ implementations default to `EventSig::Ignored`.
pub trait EventHandler {
  /// Implement this function if you want to react to key strokes.
  fn on_key(&mut self, _: Key, _: Action) -> EventSig { EventSig::Ignored }
  /// Implement this function if you want to react to mouse button events.
  fn on_mouse_button(&mut self, _: MouseButton, _: Action) -> EventSig { EventSig::Ignored }
  /// Implement this function if you want to react to cursor moves.
  fn on_cursor_move(&mut self, _: [f32; 2]) -> EventSig { EventSig::Ignored }
  /// Implement this function if you want to react to scroll events.
  fn on_scroll(&mut self, _: [f32; 2]) -> EventSig { EventSig::Ignored }
}

/// Empty handler.
///
/// This handler will just let pass all events without doing anything. It’s only useful for debug
/// purposes when you don’t want to bother with interaction – it doesn’t even let you close the
/// application!
#[derive(Clone, Copy, Eq, Debug, PartialEq)]
pub struct Unhandled;

impl EventHandler for Unhandled {}

/// Device object.
///
/// Upon bootstrapping, this type is created to add interaction and context handling.
pub struct Device {
  raw: luminance_glfw::Device,
  /// Some kind of epoch start the application started at.
  start_time: Instant
}

impl Device {
  /// Entry point.
  ///
  /// This function is the first one you have to call before anything else related to this crate.
  ///
  /// # Arguments
  ///
  /// - `dim`: dimension of the window to create
  /// - `title`: title to give to the window
  pub fn bootstrap(dim: WindowDim,
                   title: &'static str,
                   win_opt: WindowOpt)
                   -> Result<Self, DeviceError> {
    info!("{} starting", title);
    info!("window mode: {:?}", dim);
    info!("window options: {:?}", win_opt);

    let dev = open_window(dim, title, win_opt)?;

    info!("bootstrapping finished");

    Ok(Device {
      raw: dev,
      start_time: Instant::now()
    })
  }

  /// Width of the attached window.
  #[inline]
  pub fn width(&self) -> u32 {
    self.raw.width()
  }

  /// Height of the attached window.
  #[inline]
  pub fn height(&self) -> u32 {
    self.raw.height()
  }

  /// Current time, starting from the beginning of the creation of that object.
  pub fn time(&self) -> f64 {
    let elapsed = Instant::now() - self.start_time;
    elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 * 1e-9
  }

  /// Dispatch events to a handler.
  pub fn dispatch_events<H>(&self, handler: &mut H) -> bool where H: EventHandler {
    while let Ok((key, action)) = self.raw.kbd.try_recv() {
      if handler.on_key(key, action) == EventSig::Aborted {
        return false;
      }
    }

    while let Ok((button, action)) = self.raw.mouse.try_recv() {
      if handler.on_mouse_button(button, action) == EventSig::Aborted {
        return false;
      }
    }

    while let Ok(xy) = self.raw.cursor.try_recv() {
      if handler.on_cursor_move(xy) == EventSig::Aborted {
        return false;
      }
    }

    while let Ok(xy) = self.raw.scroll.try_recv() {
      if handler.on_scroll(xy) == EventSig::Aborted {
        return false;
      }
    }

    true
  }

  /// Step function.
  ///
  /// This function provides two features:
  ///
  /// - it runs a *drawer* function, responsible of rendering a single frame, by passing it the
  ///   current time
  /// - it performs process idleing if you have requested a certain *framerate* – frame per second.
  ///
  /// The second feature is very neat because it lets you handle the scheduler off your application
  /// and then contribute to better CPU usage.
  ///
  /// > Note: if you pass `None`, no idleing will take place. However, you might be blocked by the
  /// *VSync* if enabled in your driver.
  pub fn step<FPS, R>(&mut self, fps: FPS, mut draw_frame: R) -> bool where FPS: Into<Option<u32>>, R: FnMut(Time) {
    let t = self.time();

    self.raw.draw(|| {
      draw_frame(t);
    });

    // wait for next frame according to the wished FPS
    if let Some(fps) = fps.into() {
      let fps = fps as f64;
      let max_time = 1. / fps;
      let elapsed_time = self.time() - t;

      if elapsed_time < max_time {
        let sleep_time = max_time - elapsed_time;
        thread::sleep(Duration::from_millis((sleep_time * 1e3) as u64));
      }
    }

    true
  }
}

/// Freefly handler.
///
/// This handler is very neat as it provides freefly interaction.
pub struct FreeflyHandler {
  camera: Rc<RefCell<Camera<Freefly>>>,
  left_down: bool,
  right_down: bool,
  last_cursor: [f32; 2]
}

impl FreeflyHandler {
  pub fn new(camera: Rc<RefCell<Camera<Freefly>>>) -> Self {
    FreeflyHandler {
      camera: camera,
      left_down: false,
      right_down: false,
      last_cursor: [0., 0.]
    }
  }

  fn move_camera_on_event(&mut self, key: Key) {
    let camera = &mut self.camera.borrow_mut();

    match key {
      Key::W => camera.mv(V3::new(0., 0., 1.)),
      Key::S => camera.mv(V3::new(0., 0., -1.)),
      Key::A => camera.mv(V3::new(1., 0., 0.)),
      Key::D => camera.mv(V3::new(-1., 0., 0.)),
      Key::Q => camera.mv(V3::new(0., -1., 0.)),
      Key::E => camera.mv(V3::new(0., 1., 0.)),
      _ => ()
    }
  }

  fn orient_camera_on_event(&mut self, cursor: [f32; 2]) {
    let camera = &mut self.camera.borrow_mut();

    if self.left_down {
      let (dx, dy) = (cursor[0] - self.last_cursor[0], cursor[1] - self.last_cursor[1]);
      camera.look_around(V3::new(dy, dx, 0.));
    } else if self.right_down {
      let (dx, _) = (cursor[0] - self.last_cursor[0], cursor[1] - self.last_cursor[1]);
      camera.look_around(V3::new(0., 0., dx));
    }

    self.last_cursor = cursor;
  }
}

impl EventHandler for FreeflyHandler {
  fn on_key(&mut self, key: Key, action: Action) -> EventSig {
    match action {
      Action::Press | Action::Repeat => self.move_camera_on_event(key),
      Action::Release => if key == Key::Escape { return EventSig::Aborted }
    }

    EventSig::Handled
  }

  fn on_mouse_button(&mut self, button: MouseButton, action: Action) -> EventSig {
    match (button, action) {
      (MouseButton::Button1, Action::Press) => {
        self.left_down = true;
      },
      (MouseButton::Button1, Action::Release) => {
        self.left_down = false;
      },
      (MouseButton::Button2, Action::Press) => {
        self.right_down = true;
      },
      (MouseButton::Button2, Action::Release) => {
        self.right_down = false;
      },
      _ => ()
    }

    EventSig::Handled
  }

  fn on_cursor_move(&mut self, dir: [f32; 2]) -> EventSig {
    self.orient_camera_on_event(dir);
    EventSig::Handled
  }
}
<commit_msg>Removed some warnings.<commit_after>use luminance_glfw::{self, open_window};
pub use luminance_glfw::{Action, DeviceError, Key, MouseButton, WindowDim, WindowOpt};
use std::cell::RefCell;
use std::rc::Rc;
use std::thread;
use std::time::{Duration, Instant};

use camera::{Camera, Freefly};
use linear::V3;

type Time = f64;

/// Signals events can pass up back to their handlers to notify them how they have processed an
/// event. They’re three kinds of signals:
///
/// - `EventSig::Ignored`:  the event should be passed to other handlers the parents knows about –
///    if any – because it wasn’t handled (ignored);
///
/// - `EventSig::Handled`: the event has been correctly handled;
///
/// - `EventSig::Focused`: the event has been correctly handled, and the parent should consider that
///    the this handler has now an exclusive focus on that event;
///
/// - `EventSig::Aborted`: the event has been correctly handled and the parent handler should be
///    aborted. This signal is typically used to kill all the handlers chain and thus quit the
///    application.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum EventSig {
  Ignored,
  Handled,
  Focused,
  Aborted
}

/// Class of event handlers.
///
/// All functions return a special object of type `EventSig`. An event report gives information
/// about how a handler has handled the event. See the documentation of `EventSig` for further
/// information.
///
/// All functions’ implementations default to `EventSig::Ignored`.
pub trait EventHandler {
  /// Implement this function if you want to react to key strokes.
  fn on_key(&mut self, _: Key, _: Action) -> EventSig { EventSig::Ignored }
  /// Implement this function if you want to react to mouse button events.
  fn on_mouse_button(&mut self, _: MouseButton, _: Action) -> EventSig { EventSig::Ignored }
  /// Implement this function if you want to react to cursor moves.
  fn on_cursor_move(&mut self, _: [f32; 2]) -> EventSig { EventSig::Ignored }
  /// Implement this function if you want to react to scroll events.
  fn on_scroll(&mut self, _: [f32; 2]) -> EventSig { EventSig::Ignored }
}

/// Empty handler.
///
/// This handler will just let pass all events without doing anything. It’s only useful for debug
/// purposes when you don’t want to bother with interaction – it doesn’t even let you close the
/// application!
#[derive(Clone, Copy, Eq, Debug, PartialEq)]
pub struct Unhandled;

impl EventHandler for Unhandled {}

/// Device object.
///
/// Upon bootstrapping, this type is created to add interaction and context handling.
pub struct Device {
  raw: luminance_glfw::Device,
  /// Some kind of epoch start the application started at.
  start_time: Instant
}

impl Device {
  /// Entry point.
  ///
  /// This function is the first one you have to call before anything else related to this crate.
  ///
  /// # Arguments
  ///
  /// - `dim`: dimension of the window to create
  /// - `title`: title to give to the window
  pub fn bootstrap(dim: WindowDim,
                   title: &'static str,
                   win_opt: WindowOpt)
                   -> Result<Self, DeviceError> {
    info!("{} starting", title);
    info!("window mode: {:?}", dim);
    info!("window options: {:?}", win_opt);

    let dev = open_window(dim, title, win_opt)?;

    info!("bootstrapping finished");

    Ok(Device {
      raw: dev,
      start_time: Instant::now()
    })
  }

  /// Width of the attached window.
  #[inline]
  pub fn width(&self) -> u32 {
    self.raw.width()
  }

  /// Height of the attached window.
  #[inline]
  pub fn height(&self) -> u32 {
    self.raw.height()
  }

  /// Current time, starting from the beginning of the creation of that object.
  pub fn time(&self) -> f64 {
    let elapsed = Instant::now() - self.start_time;
    elapsed.as_secs() as f64 + elapsed.subsec_nanos() as f64 * 1e-9
  }

  /// Dispatch events to a handler.
  pub fn dispatch_events<H>(&self, handler: &mut H) -> bool where H: EventHandler {
    while let Ok((key, action)) = self.raw.kbd.try_recv() {
      if handler.on_key(key, action) == EventSig::Aborted {
        return false;
      }
    }

    while let Ok((button, action)) = self.raw.mouse.try_recv() {
      if handler.on_mouse_button(button, action) == EventSig::Aborted {
        return false;
      }
    }

    while let Ok(xy) = self.raw.cursor.try_recv() {
      if handler.on_cursor_move(xy) == EventSig::Aborted {
        return false;
      }
    }

    while let Ok(xy) = self.raw.scroll.try_recv() {
      if handler.on_scroll(xy) == EventSig::Aborted {
        return false;
      }
    }

    true
  }

  /// Step function.
  ///
  /// This function provides two features:
  ///
  /// - it runs a *drawer* function, responsible of rendering a single frame, by passing it the
  ///   current time
  /// - it performs process idleing if you have requested a certain *framerate* – frame per second.
  ///
  /// The second feature is very neat because it lets you handle the scheduler off your application
  /// and then contribute to better CPU usage.
  ///
  /// > Note: if you pass `None`, no idleing will take place. However, you might be blocked by the
  /// *VSync* if enabled in your driver.
  pub fn step<FPS, R>(&mut self, fps: FPS, mut draw_frame: R) -> bool where FPS: Into<Option<u32>>, R: FnMut(Time) {
    let t = self.time();

    self.raw.draw(|| {
      draw_frame(t);
    });

    // wait for next frame according to the wished FPS
    if let Some(fps) = fps.into() {
      let fps = fps as f64;
      let max_time = 1. / fps;
      let elapsed_time = self.time() - t;

      if elapsed_time < max_time {
        let sleep_time = max_time - elapsed_time;
        thread::sleep(Duration::from_millis((sleep_time * 1e3) as u64));
      }
    }

    true
  }
}

/// Freefly handler.
///
/// This handler is very neat as it provides freefly interaction.
pub struct FreeflyHandler {
  camera: Rc<RefCell<Camera<Freefly>>>,
  left_down: bool,
  right_down: bool,
  last_cursor: [f32; 2]
}

impl FreeflyHandler {
  pub fn new(camera: Rc<RefCell<Camera<Freefly>>>) -> Self {
    FreeflyHandler {
      camera: camera,
      left_down: false,
      right_down: false,
      last_cursor: [0., 0.]
    }
  }

  fn move_camera_on_event(&mut self, key: Key) {
    let camera = &mut self.camera.borrow_mut();

    match key {
      Key::W => camera.mv(V3::new(0., 0., 1.)),
      Key::S => camera.mv(V3::new(0., 0., -1.)),
      Key::A => camera.mv(V3::new(1., 0., 0.)),
      Key::D => camera.mv(V3::new(-1., 0., 0.)),
      Key::Q => camera.mv(V3::new(0., -1., 0.)),
      Key::E => camera.mv(V3::new(0., 1., 0.)),
      _ => ()
    }
  }

  fn orient_camera_on_event(&mut self, cursor: [f32; 2]) {
    let camera = &mut self.camera.borrow_mut();

    if self.left_down {
      let (dx, dy) = (cursor[0] - self.last_cursor[0], cursor[1] - self.last_cursor[1]);
      camera.look_around(V3::new(dy, dx, 0.));
    } else if self.right_down {
      let (dx, _) = (cursor[0] - self.last_cursor[0], cursor[1] - self.last_cursor[1]);
      camera.look_around(V3::new(0., 0., dx));
    }

    self.last_cursor = cursor;
  }
}

impl EventHandler for FreeflyHandler {
  fn on_key(&mut self, key: Key, action: Action) -> EventSig {
    match action {
      Action::Press | Action::Repeat => self.move_camera_on_event(key),
      Action::Release => if key == Key::Escape { return EventSig::Aborted }
    }

    EventSig::Handled
  }

  fn on_mouse_button(&mut self, button: MouseButton, action: Action) -> EventSig {
    match (button, action) {
      (MouseButton::Button1, Action::Press) => {
        self.left_down = true;
      },
      (MouseButton::Button1, Action::Release) => {
        self.left_down = false;
      },
      (MouseButton::Button2, Action::Press) => {
        self.right_down = true;
      },
      (MouseButton::Button2, Action::Release) => {
        self.right_down = false;
      },
      _ => ()
    }

    EventSig::Handled
  }

  fn on_cursor_move(&mut self, dir: [f32; 2]) -> EventSig {
    self.orient_camera_on_event(dir);
    EventSig::Handled
  }
}
<|endoftext|>


<commit_before>#![allow(dead_code)]
    use std::fmt;
    use std::rc::{Rc, Weak};

    struct Address {
        x: u64,
        y: u64,
        z: u64,
    }

    struct Element {
        name: String,
    }

    enum ChunkOption<T> {
        Local(T<LocalChunk>),
        Remote(Rc<RemoteChunk>),
        None
    }

    struct LocalChunk {
        address: Address,
        center: Vec3f64,
        children: [ChunkOption<Rc>; 8],
        child_number: u8,
        entities: Vec<Rc<Entity>>,
        mass: f64,
        parent: ChunkOption<Weak>,
        scale: u8,
        structure: u64,
        summary: Rc<Element>,
    }

    struct RemoteChunk {
        connection: i32,
    }

    impl Address {
        fn from_string(address: String) -> Option<Address> {
            let mut xyz: (u64, u64, u64) = (1, 1, 1);
            let mut xyz_add: (u64, u64, u64);

            for (i, c) in address.chars().enumerate() {
                if i > 63 {
                    break;
                }
                xyz_add =  match c.to_digit(8) {
                    Some(0) => (0, 0, 0),
                    Some(1) => (0, 0, 1),
                    Some(2) => (0, 1, 0),
                    Some(3) => (0, 1, 1),
                    Some(4) => (1, 0, 0),
                    Some(5) => (1, 0, 1),
                    Some(6) => (1, 1, 0),
                    Some(7) => (1, 1, 1),
                    _ => return None,
                };
                xyz = (xyz.0 << 1, xyz.1 << 1, xyz.2 << 1);
                xyz = (xyz.0 + xyz_add.0, xyz.1 + xyz_add.1, xyz.2 + xyz_add.2);
            }
            Some(Address {x: xyz.0, y: xyz.1, z: xyz.2})
        }
    }

    impl ToString for Address {
        fn to_string(&self) -> String {
            String::from_str("")
        }
    }

    impl PartialEq for Address {
        fn eq(&self, other: &Self) -> bool {
            (((self.x == other.x) && (self.y == other.y)) && (self.z == other.z))
        }
    }

    impl fmt::Show for Address {
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
            write!(f, "Address({}, {}, {})", self.x, self.y, self.z)
        }
    }

    trait Chunk {
        fn get_child(&self, x: u8, y: u8, z: u8) -> ChunkOption<Rc>;
        fn orphan(&self, child: <T>);
        fn emancipate(&self);
        fn tick(&self, time_delta: f64);
    }

    impl Chunk for LocalChunk {
        fn get_child(&self, x: u8, y: u8, z: u8) -> ChunkOption<Rc> {
            match (x, y, z) {
                (0, 0, 0) => self.children[0],
                (0, 0, 1) => self.children[1],
                (0, 1, 0) => self.children[2],
                (0, 1, 1) => self.children[3],
                (1, 0, 0) => self.children[4],
                (1, 0, 1) => self.children[5],
                (1, 1, 0) => self.children[6],
                (1, 1, 1) => self.children[7],
                _ => None,
            }
        }

        fn orphan(&self, child: Rc<Chunk>) {
        }

        fn emancipate(&self) {
        }

        fn tick(&self, time_delta: f64) {
        }
    }

    impl Chunk for RemoteChunk {
    }

    #[test]
    fn test_address_from_string() {
        let known = Option::Some(Address {x: 0b11, y: 0b11, z: 0b11});
        let unknown = Address::from_string(String::from_str("7"));
        assert_eq!(known, unknown)
    }
<commit_msg>Unindent all<commit_after>#![allow(dead_code)]
use std::fmt;
use std::rc::{Rc, Weak};

struct Address {
    x: u64,
    y: u64,
    z: u64,
}

struct Element {
    name: String,
}

enum ChunkOption<T> {
    Local(T<LocalChunk>),
    Remote(Rc<RemoteChunk>),
    None
}

struct LocalChunk {
    address: Address,
    center: Vec3f64,
    children: [ChunkOption<Rc>; 8],
    child_number: u8,
    entities: Vec<Rc<Entity>>,
    mass: f64,
    parent: ChunkOption<Weak>,
    scale: u8,
    structure: u64,
    summary: Rc<Element>,
}

struct RemoteChunk {
    connection: i32,
}

impl Address {
    fn from_string(address: String) -> Option<Address> {
        let mut xyz: (u64, u64, u64) = (1, 1, 1);
        let mut xyz_add: (u64, u64, u64);

        for (i, c) in address.chars().enumerate() {
            if i > 63 {
                break;
            }
            xyz_add =  match c.to_digit(8) {
                Some(0) => (0, 0, 0),
                Some(1) => (0, 0, 1),
                Some(2) => (0, 1, 0),
                Some(3) => (0, 1, 1),
                Some(4) => (1, 0, 0),
                Some(5) => (1, 0, 1),
                Some(6) => (1, 1, 0),
                Some(7) => (1, 1, 1),
                _ => return None,
            };
            xyz = (xyz.0 << 1, xyz.1 << 1, xyz.2 << 1);
            xyz = (xyz.0 + xyz_add.0, xyz.1 + xyz_add.1, xyz.2 + xyz_add.2);
        }
        Some(Address {x: xyz.0, y: xyz.1, z: xyz.2})
    }
}

impl ToString for Address {
    fn to_string(&self) -> String {
        String::from_str("")
    }
}

impl PartialEq for Address {
    fn eq(&self, other: &Self) -> bool {
        (((self.x == other.x) && (self.y == other.y)) && (self.z == other.z))
    }
}

impl fmt::Show for Address {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Address({}, {}, {})", self.x, self.y, self.z)
    }
}

trait Chunk {
    fn get_child(&self, x: u8, y: u8, z: u8) -> ChunkOption<Rc>;
    fn orphan(&self, child: <T>);
    fn emancipate(&self);
    fn tick(&self, time_delta: f64);
}

impl Chunk for LocalChunk {
    fn get_child(&self, x: u8, y: u8, z: u8) -> ChunkOption<Rc> {
        match (x, y, z) {
            (0, 0, 0) => self.children[0],
            (0, 0, 1) => self.children[1],
            (0, 1, 0) => self.children[2],
            (0, 1, 1) => self.children[3],
            (1, 0, 0) => self.children[4],
            (1, 0, 1) => self.children[5],
            (1, 1, 0) => self.children[6],
            (1, 1, 1) => self.children[7],
            _ => None,
        }
    }

    fn orphan(&self, child: Rc<Chunk>) {
    }

    fn emancipate(&self) {
    }

    fn tick(&self, time_delta: f64) {
    }
}

impl Chunk for RemoteChunk {
}

#[test]
fn test_address_from_string() {
    let known = Option::Some(Address {x: 0b11, y: 0b11, z: 0b11});
    let unknown = Address::from_string(String::from_str("7"));
    assert_eq!(known, unknown)
}
<|endoftext|>


<commit_before>#[macro_use(dbg)]
extern crate cretonne;
extern crate cton_reader;
extern crate cton_wasm;
extern crate docopt;
#[macro_use]
extern crate serde_derive;
extern crate filecheck;
extern crate num_cpus;
extern crate tempdir;
extern crate term;

use cretonne::VERSION;
use docopt::Docopt;
use std::io::{self, Write};
use std::process;

mod utils;
mod filetest;
mod cat;
mod print_cfg;
mod rsfilecheck;
mod wasm;

const USAGE: &str = "
Cretonne code generator utility

Usage:
    cton-util test [-v] <file>...
    cton-util cat <file>...
    cton-util filecheck [-v] <file>
    cton-util print-cfg <file>...
    cton-util wasm [-cvo] [--enable=<flag>]... <file>...
    cton-util --help | --version

Options:
    -v, --verbose   be more verbose
    -c, --check     checks the correctness of Cretonne IL translated from WebAssembly
    -o, --optimize  runs otpimization passes on translated WebAssembly functions
    -h, --help      print this help message
    --version       print the Cretonne version

";

#[derive(Deserialize, Debug)]
struct Args {
    cmd_test: bool,
    cmd_cat: bool,
    cmd_filecheck: bool,
    cmd_print_cfg: bool,
    cmd_wasm: bool,
    arg_file: Vec<String>,
    flag_check: bool,
    flag_optimize: bool,
    flag_verbose: bool,
    flag_enable: Vec<String>,
}

/// A command either succeeds or fails with an error message.
pub type CommandResult = Result<(), String>;

/// Parse the command line arguments and run the requested command.
fn cton_util() -> CommandResult {
    // Parse command line arguments.
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| {
            d.help(true)
                .version(Some(format!("Cretonne {}", VERSION)))
                .deserialize()
        })
        .unwrap_or_else(|e| e.exit());

    // Find the sub-command to execute.
    if args.cmd_test {
        filetest::run(args.flag_verbose, args.arg_file)
    } else if args.cmd_cat {
        cat::run(args.arg_file)
    } else if args.cmd_filecheck {
        rsfilecheck::run(args.arg_file, args.flag_verbose)
    } else if args.cmd_print_cfg {
        print_cfg::run(args.arg_file)
    } else if args.cmd_wasm {
        wasm::run(
            args.arg_file,
            args.flag_verbose,
            args.flag_optimize,
            args.flag_check,
            args.flag_enable,
        )
    } else {
        // Debugging / shouldn't happen with proper command line handling above.
        Err(format!("Unhandled args: {:?}", args))
    }
}

fn main() {
    if let Err(mut msg) = cton_util() {
        if !msg.ends_with('\n') {
            msg.push('\n');
        }
        io::stderr().write_all(msg.as_bytes()).unwrap();
        process::exit(1);
    }
}
<commit_msg>Flush stdout before writing to stderr.<commit_after>#[macro_use(dbg)]
extern crate cretonne;
extern crate cton_reader;
extern crate cton_wasm;
extern crate docopt;
#[macro_use]
extern crate serde_derive;
extern crate filecheck;
extern crate num_cpus;
extern crate tempdir;
extern crate term;

use cretonne::VERSION;
use docopt::Docopt;
use std::io::{self, Write};
use std::process;

mod utils;
mod filetest;
mod cat;
mod print_cfg;
mod rsfilecheck;
mod wasm;

const USAGE: &str = "
Cretonne code generator utility

Usage:
    cton-util test [-v] <file>...
    cton-util cat <file>...
    cton-util filecheck [-v] <file>
    cton-util print-cfg <file>...
    cton-util wasm [-cvo] [--enable=<flag>]... <file>...
    cton-util --help | --version

Options:
    -v, --verbose   be more verbose
    -c, --check     checks the correctness of Cretonne IL translated from WebAssembly
    -o, --optimize  runs otpimization passes on translated WebAssembly functions
    -h, --help      print this help message
    --version       print the Cretonne version

";

#[derive(Deserialize, Debug)]
struct Args {
    cmd_test: bool,
    cmd_cat: bool,
    cmd_filecheck: bool,
    cmd_print_cfg: bool,
    cmd_wasm: bool,
    arg_file: Vec<String>,
    flag_check: bool,
    flag_optimize: bool,
    flag_verbose: bool,
    flag_enable: Vec<String>,
}

/// A command either succeeds or fails with an error message.
pub type CommandResult = Result<(), String>;

/// Parse the command line arguments and run the requested command.
fn cton_util() -> CommandResult {
    // Parse command line arguments.
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| {
            d.help(true)
                .version(Some(format!("Cretonne {}", VERSION)))
                .deserialize()
        })
        .unwrap_or_else(|e| e.exit());

    // Find the sub-command to execute.
    if args.cmd_test {
        filetest::run(args.flag_verbose, args.arg_file)
    } else if args.cmd_cat {
        cat::run(args.arg_file)
    } else if args.cmd_filecheck {
        rsfilecheck::run(args.arg_file, args.flag_verbose)
    } else if args.cmd_print_cfg {
        print_cfg::run(args.arg_file)
    } else if args.cmd_wasm {
        wasm::run(
            args.arg_file,
            args.flag_verbose,
            args.flag_optimize,
            args.flag_check,
            args.flag_enable,
        )
    } else {
        // Debugging / shouldn't happen with proper command line handling above.
        Err(format!("Unhandled args: {:?}", args))
    }
}

fn main() {
    if let Err(mut msg) = cton_util() {
        if !msg.ends_with('\n') {
            msg.push('\n');
        }
        io::stdout().flush().expect("flushing stdout");
        io::stderr().write_all(msg.as_bytes()).unwrap();
        process::exit(1);
    }
}
<|endoftext|>


<commit_before>
use std::sync::{Arc};
use std::sync::mpsc::{channel, sync_channel, SyncSender, Sender, Receiver};
use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering};

use std::thread;

struct WorkerPool {
    workers: Vec<Worker>,
    recv_queue: Receiver<Job>,
    running: Arc<AtomicBool>
}

impl WorkerPool {
    pub fn new(num_workers: usize, worker_queue_size: usize, recv_queue: Receiver<Job>) -> WorkerPool {
        let mut workers_vec : Vec<Worker> = Vec::with_capacity(num_workers);

        for i in 0..num_workers {
            let w = Worker::new(i, worker_queue_size);
            workers_vec.push(w);
        }

        return WorkerPool {
            workers: workers_vec,
            recv_queue: recv_queue,
            running: Arc::new(AtomicBool::new(false))
        };
    }

    fn is_running(&self) -> bool {
        self.running.load(Ordering::AcqRel)
    }

    pub fn run(&mut self) -> Result<(), &'static str> {
        println!("Attempting to start workers.");

        if self.is_running() {
            return Err("Worker pool already runnng.");
        }

        for worker in &mut self.workers {
            worker.start().unwrap();
        }

        println!("Workers started.");

        println!("Starting pool head");

        loop {
            let worker_slots: Vec<(usize, usize)> = Vec::new();
            let mut counter : usize = 0;
            for worker in &self.workers {
                worker_slots.push(( counter, worker.get_queue_availability() ));
                counter += 1;
            }

            println!("Pool head receiving job.");
            let j : Job = match self.recv_queue.recv() {
                Ok(x) => x,
                RecvErr => break
            };
            queue_available.fetch_add(1, Ordering::SeqCst);

            j.chan.send(j.number*j.number);
        }
        println!("Pool head ended");

        Ok(())

    }

}

pub struct Job {
    pub number: u32,
    pub chan: Sender<u32>
}

struct Worker {
    id: usize,
    name: String,
    running: Arc<AtomicBool>,
    alive: Arc<AtomicBool>,
    queue_in: Option<Sender<Job>>,
    queue_size: usize,
    queue_available: Arc<AtomicUsize>
}

impl Worker {
    fn new(id: usize, queue_size: usize) -> Worker {

        Worker {
            id: id,
            name: format!("Worker number {}", id),
            running: Arc::new(AtomicBool::new(false)),
            alive: Arc::new(AtomicBool::new(true)),
            queue_in: None,
            queue_size: queue_size,
            queue_available: Arc::new(AtomicUsize::new(queue_size))
        }
    }

    fn start(&mut self) -> Result<(), &'static str> {
        if self.is_running() {
            return Err("Worker already running");
        }

        let (in_ch, out_ch) : (Sender<Job>, Receiver<Job>) = channel();

        self.queue_in = Some(in_ch);

        let id = self.id;
        let running = self.running.clone();
        let alive = self.alive.clone();
        let queue_available = self.queue_available.clone();

        thread::spawn(move || {
            loop {
                println!("Worker {} receiving job.", id);
                let j : Job = match out_ch.recv() {
                    Ok(x) => x,
                    RecvErr => break
                };
                queue_available.fetch_add(1, Ordering::SeqCst);

                j.chan.send(j.number*j.number);
            }

            running.store(false, Ordering::SeqCst);
            alive.store(false, Ordering::SeqCst);

            println!("Worker {} exiting.", id);
        });

        Ok(())
    }

    fn is_running(&self) -> bool {
        self.running.load(Ordering::AcqRel)
    }

    fn get_queue_availability(&self) -> usize {
        self.queue_available.load(Ordering::Acquire)
    }

    fn get_queue_in(&self) -> &Sender<Job> {
        match self.queue_in.as_ref() {
            Some(x) => x,
            None => panic!("Tried to get queue from worker before it started!")
        }
    }

    fn give_job(&self, j: Job) -> Result<(), &'static str> {
        if self.get_queue_availability() > 0 {
            return Err("Queue full for worker");
        }
        let q = self.get_queue_in();
        q.send(j);
        Ok(())
    }
}
<commit_msg>Issue #2: Finalized rough draft of worker pool<commit_after>
use std::sync::{Arc};
use std::sync::mpsc::{channel, sync_channel, SyncSender, Sender, Receiver};
use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering};

use std::thread;
use std::time::Duration;

struct WorkerPool {
    workers: Vec<Worker>,
    recv_queue: Receiver<Job>,
    running: Arc<AtomicBool>
}

impl WorkerPool {
    pub fn new(num_workers: usize, worker_queue_size: usize, recv_queue: Receiver<Job>) -> WorkerPool {
        let mut workers_vec : Vec<Worker> = Vec::with_capacity(num_workers);

        for i in 0..num_workers {
            let w = Worker::new(i, worker_queue_size);
            workers_vec.push(w);
        }

        return WorkerPool {
            workers: workers_vec,
            recv_queue: recv_queue,
            running: Arc::new(AtomicBool::new(false))
        };
    }

    fn is_running(&self) -> bool {
        self.running.load(Ordering::AcqRel)
    }

    fn worker_has_opening(&self) -> bool {
        match self.worker_with_opening() {
            Some(_) => true,
            None => false
        }
    }

    fn worker_with_opening(&self) -> Option<usize> {
        let mut counter : usize = 0;
        for worker in &self.workers {
            if worker.get_queue_availability() > 0 {
                return Some(counter);
            }
            counter += 1;
        }
        None
    }

    pub fn run(&mut self) -> Result<(), &'static str> {
        println!("Attempting to start workers.");

        if self.is_running() {
            return Err("Worker pool already runnng.");
        }

        for worker in &mut self.workers {
            worker.start().unwrap();
        }

        println!("Workers started.");

        println!("Starting pool head");

        loop {
            let mut worker_slots: Vec<(usize, usize)> = Vec::new(); // TODO: allow this to be reordered
            let mut counter : usize = 0;
            let mut total_slots : usize = 0;
            for worker in &self.workers {
                let num_slots = worker.get_queue_availability();
                total_slots += num_slots;
                worker_slots.push(( counter, num_slots ));
                counter += 1;
            }

            println!("Pool head receiving job.");
            let j : Job = match self.recv_queue.recv() {
                Ok(x) => x,
                RecvErr => break
            };

            match total_slots {
                0 => {
                    let mut keep_looping = true;
                    let mut worker_num : usize = 0;
                    while keep_looping {
                        match self.worker_with_opening() {
                            Some(x) => {
                                worker_num = x;
                                keep_looping = false;
                            },
                            None => {
                                thread::sleep(Duration::from_millis(1));
                            }
                        }

                    }
                    let ref worker = self.workers[worker_num];
                    worker.give_job(j).unwrap();
                },
                1 => {
                    let (worker_num, _) = worker_slots[0];
                    let ref worker = self.workers[worker_num];
                    worker.give_job(j);
                },
                x => {
                    let mut jobs : Vec<Job> = Vec::new();
                    jobs.push(j);
                    for i in 0..(total_slots-1) {
                        match self.recv_queue.try_recv() {
                            Ok(x) => jobs.push(x),
                            Err(_) => break
                        }
                    }

                    for (worker_num, num_available) in worker_slots {
                        let ref worker = self.workers[worker_num];
                        for _ in 0..num_available {
                            worker.give_job(jobs.pop().unwrap());
                        }
                    }

                }
            }

        }
        println!("Pool head ended");

        Ok(())

    }

}

pub struct Job {
    pub number: u32,
    pub chan: Sender<u32>
}

struct Worker {
    id: usize,
    name: String,
    running: Arc<AtomicBool>,
    alive: Arc<AtomicBool>,
    queue_in: Option<Sender<Job>>,
    queue_size: usize,
    queue_available: Arc<AtomicUsize>
}

impl Worker {
    fn new(id: usize, queue_size: usize) -> Worker {

        Worker {
            id: id,
            name: format!("Worker number {}", id),
            running: Arc::new(AtomicBool::new(false)),
            alive: Arc::new(AtomicBool::new(true)),
            queue_in: None,
            queue_size: queue_size,
            queue_available: Arc::new(AtomicUsize::new(queue_size))
        }
    }

    // TODO: how do we reclaim queue if worker dies?
    fn start(&mut self) -> Result<(), &'static str> {
        if self.is_running() {
            return Err("Worker already running");
        }

        let (in_ch, out_ch) : (Sender<Job>, Receiver<Job>) = channel();

        self.queue_in = Some(in_ch);

        let id = self.id;
        let running = self.running.clone();
        let alive = self.alive.clone();
        let queue_available = self.queue_available.clone();

        thread::spawn(move || {
            loop {
                println!("Worker {} receiving job.", id);
                let j : Job = match out_ch.recv() {
                    Ok(x) => x,
                    RecvErr => break
                };
                queue_available.fetch_add(1, Ordering::SeqCst);

                j.chan.send(j.number*j.number);
            }

            running.store(false, Ordering::SeqCst);
            alive.store(false, Ordering::SeqCst);

            println!("Worker {} exiting.", id);
        });

        Ok(())
    }

    fn is_running(&self) -> bool {
        self.running.load(Ordering::AcqRel)
    }

    fn get_queue_availability(&self) -> usize {
        self.queue_available.load(Ordering::Acquire)
    }

    fn get_queue_in(&self) -> &Sender<Job> {
        match self.queue_in.as_ref() {
            Some(x) => x,
            None => panic!("Tried to get queue from worker before it started!")
        }
    }

    fn give_job(&self, j: Job) -> Result<(), &'static str> {
        if self.get_queue_availability() > 0 {
            return Err("Queue full for worker");
        }
        let q = self.get_queue_in();
        q.send(j);
        self.queue_available.fetch_sub(1, Ordering::SeqCst);
        Ok(())
    }
}
<|endoftext|>