<commit_before>use std::fs;
use std::io;
use std::io::prelude::*;
use std::path;
use toml;
use error;

#[derive(Deserialize, Serialize, PartialEq, Debug)]
pub struct Config {
    version: f64,
    tools: Vec<Tool>,
}

#[derive(Deserialize, Serialize, PartialEq, Debug)]
pub struct Tool {
    command: String,
    extentions: Vec<String>,
}

impl Config {
    pub fn new() -> Config {
        Config {
            version: 0.0,
            tools: Vec::new(),
        }
    }

    pub fn load(conf_name: &str) -> Result<Config, error::AppError> {
        let path = path::Path::new(conf_name);
        if !path.exists() {
            return Result::Err(error::AppError::Io(io::Error::new(
                io::ErrorKind::NotFound,
                format!("{} file is not exist", path.to_str().unwrap_or("")),
            )));
        }

        let mut reader = io::BufReader::new(fs::File::open(path).map_err(error::AppError::Io)?);
        let mut s = String::new();
        reader.read_to_string(&mut s).map_err(error::AppError::Io)?;

        let conf: Config = toml::from_str(s.as_str()).map_err(error::AppError::TomlDe)?;

        Ok(conf)
    }

    #[allow(dead_code)]
    pub fn save(&self, conf_name: &str) -> Result<(), error::AppError> {
        let path = path::Path::new(conf_name);
        let parent_dir = path.parent();
        match parent_dir {
            Some(d) if d == path::Path::new("") => {}
            Some(d) if !d.exists() => fs::create_dir_all(d).map_err(error::AppError::Io)?,
            Some(_) | None => {}
        }

        let toml = toml::to_string(&self).map_err(error::AppError::TomlSer)?;
        let mut writer = io::BufWriter::new(fs::File::create(path).map_err(error::AppError::Io)?);
        writer.write(toml.as_bytes()).map_err(error::AppError::Io)?;

        Ok(())
    }

    pub fn add(&mut self, command: &str, extention: &str) -> Result<(), error::AppError> {
        let ext = extention.to_owned();

        for t in self.tools.iter_mut() {
            if t.extentions.contains(&ext) {
                println!(".{} is already exists", extention);
                return Ok(()); // FIXME
            }
            if t.command == command {
                t.extentions.push(ext);
                return Ok(());
            }
        }

        self.tools.push(Tool {
            command: command.to_owned(),
            extentions: vec![ext],
        });
        Ok(())
    }

    pub fn get_command(&self, extension: &str) -> Option<String> {
        let ext_string = extension.to_string();
        for t in self.tools.iter() {
            if t.extentions.contains(&ext_string) {
                return Some(t.command.clone());
            }
        }
        None
    }
}

#[test]
fn test_decode_toml() {
    let toml_config_str = r#"
    version = 0.0

    [[tools]]
    command = "apvlv"
    extentions = [ "pdf" ]

    [[tools]]
    command = "mirage"
    extentions = [ "jpg", "png", "gif" ]

    [[tools]]
    command = "vlc"
    extentions = [ "mp4", "mov", "avi" ]

    [[tools]]
    command = "vim -R"
    extentions = [ "conf" ]
    "#;

    let config: Config = toml::from_str(toml_config_str).unwrap();
    assert_eq!(config.version, 0.0);

    assert_eq!(config.tools[0].command, "apvlv".to_string());
    assert_eq!(config.tools[0].extentions[0], "pdf".to_string());

    assert_eq!(config.tools[1].command, "mirage".to_string());
    assert_eq!(config.tools[1].extentions[0], "jpg".to_string());
    assert_eq!(config.tools[1].extentions[1], "png".to_string());
    assert_eq!(config.tools[1].extentions[2], "gif".to_string());

    assert_eq!(config.tools[2].command, "vlc".to_string());
    assert_eq!(config.tools[2].extentions[0], "mp4".to_string());
    assert_eq!(config.tools[2].extentions[1], "mov".to_string());
    assert_eq!(config.tools[2].extentions[2], "avi".to_string());

    assert_eq!(config.tools[3].command, "vim -R".to_string());
    assert_eq!(config.tools[3].extentions[0], "conf".to_string());
}

#[test]
fn test_new() {
    let conf1 = Config::new();
    let conf2 = Config::new();
    assert_eq!(conf1, conf2);
}

#[test]
fn test_save() {
    let conf = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extentions: vec!["txt".to_string(), "log".to_string()],
            },
        ],
    };

    assert_eq!(conf.save("test_config").unwrap(), ());
    let loaded_conf = Config::load("test_config").unwrap();
    assert_eq!(conf, loaded_conf);
    assert_eq!(conf.save("./test_config").unwrap(), ());

    let loaded_conf = Config::load("./test_config").unwrap();
    assert_eq!(conf, loaded_conf);

    assert_ne!(
        match conf.save("invalid_path/test_config") {
            Ok(_) => "ok",
            Err(_) => "err",
        },
        "ok"
    );

    let _ = fs::remove_file("test_config");
}

#[test]
fn test_add() {
    let mut conf = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extentions: vec!["txt".to_string(), "log".to_string()],
            },
        ],
    };
    let conf_added_extention = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extentions: vec!["txt".to_string(), "log".to_string(), "new_ext".to_string()],
            },
        ],
    };

    // add same command
    assert_ne!(conf, conf_added_extention);
    conf.add("cat", "new_ext").unwrap();
    assert_eq!(conf, conf_added_extention);

    let conf_added_command = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extentions: vec!["txt".to_string(), "log".to_string(), "new_ext".to_string()],
            },
            Tool {
                command: "new_cmd".to_string(),
                extentions: vec!["xxx".to_string()],
            },
        ],
    };

    // add new command and extention
    assert_ne!(conf, conf_added_command);
    conf.add("new_cmd", "xxx").unwrap();
    assert_eq!(conf, conf_added_command);

    // add same command and extention
    conf.add("new_cmd", "xxx").unwrap();
    assert_eq!(conf, conf_added_command);
}

#[test]
fn test_get_command() {
    let conf = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extentions: vec!["txt".to_string(), "log".to_string()],
            },
        ],
    };

    assert_eq!(conf.get_command("txt").unwrap(), "cat");
    assert_eq!(conf.get_command("log").unwrap(), "cat");
    assert_eq!(
        conf.get_command("jpg").unwrap_or("none".to_string()),
        "none"
    );
}
<commit_msg>implement remove methods for Config and fix typo<commit_after>use std::fs;
use std::io;
use std::io::prelude::*;
use std::path;
use toml;
use error;

#[derive(Deserialize, Serialize, PartialEq, Debug)]
pub struct Config {
    version: f64,
    tools: Vec<Tool>,
}

#[derive(Deserialize, Serialize, PartialEq, Debug)]
pub struct Tool {
    command: String,
    extensions: Vec<String>,
}

impl Config {
    pub fn new() -> Config {
        Config {
            version: 0.0,
            tools: Vec::new(),
        }
    }

    pub fn load(conf_name: &str) -> Result<Config, error::AppError> {
        let path = path::Path::new(conf_name);
        if !path.exists() {
            return Result::Err(error::AppError::Io(io::Error::new(
                io::ErrorKind::NotFound,
                format!("{} file is not exist", path.to_str().unwrap_or("")),
            )));
        }

        let mut reader = io::BufReader::new(fs::File::open(path).map_err(error::AppError::Io)?);
        let mut s = String::new();
        reader.read_to_string(&mut s).map_err(error::AppError::Io)?;

        let conf: Config = toml::from_str(s.as_str()).map_err(error::AppError::TomlDe)?;

        Ok(conf)
    }

    pub fn save(&self, conf_name: &str) -> Result<(), error::AppError> {
        let path = path::Path::new(conf_name);
        let parent_dir = path.parent();
        match parent_dir {
            Some(d) if d == path::Path::new("") => {}
            Some(d) if !d.exists() => fs::create_dir_all(d).map_err(error::AppError::Io)?,
            Some(_) | None => {}
        }

        let toml = toml::to_string(&self).map_err(error::AppError::TomlSer)?;
        let mut writer = io::BufWriter::new(fs::File::create(path).map_err(error::AppError::Io)?);
        writer.write(toml.as_bytes()).map_err(error::AppError::Io)?;

        Ok(())
    }

    pub fn add(&mut self, command: &str, extension: &str) -> Result<(), error::AppError> {
        let ext = extension.to_owned();

        for t in self.tools.iter_mut() {
            if t.extensions.contains(&ext) {
                println!(".{} is already exists", extension);
                return Ok(()); // FIXME
            }
            if t.command == command {
                t.extensions.push(ext);
                return Ok(());
            }
        }

        self.tools.push(Tool {
            command: command.to_owned(),
            extensions: vec![ext],
        });
        Ok(())
    }

    pub fn remove_command(&mut self, command: &str) -> Result<(), error::AppError> {
        let mut remove_i = None;
        for (i, t) in self.tools.iter_mut().enumerate() {
            if t.command == command {
                remove_i = Some(i);
                break;
            }
        }
        match remove_i {
            Some(i) => {
                self.tools.remove(i);
                Ok(())
            }
            None => {
                Result::Err(error::AppError::Io(io::Error::new(
                    io::ErrorKind::NotFound,
                    format!("{} command is not registered", command),
                )))
            }
        }
    }

    pub fn remove_extension(&mut self, extension: &str) -> Result<(), error::AppError> {
        let mut tools_i = None;
        let mut ext_i = None;
        for (ia, t) in self.tools.iter().enumerate() {
            if t.extensions.contains(&extension.to_string()) {
                tools_i = Some(ia);
                for (ib, e) in t.extensions.iter().enumerate() {
                    if e == extension {
                        ext_i = Some(ib);
                        break;
                    }
                }
            }
        }
        match ext_i {
            Some(ext_i) => {
                let tools_i = tools_i.unwrap();
                self.tools[tools_i].extensions.remove(ext_i);
                if self.tools[tools_i].extensions.is_empty() {
                    self.tools.remove(tools_i);
                }
                Ok(())
            }
            None => {
                Result::Err(error::AppError::Io(io::Error::new(
                    io::ErrorKind::NotFound,
                    format!(".{} extension is not registered", extension),
                )))
            }
        }
    }

    pub fn get_command(&self, extension: &str) -> Option<String> {
        let ext_string = extension.to_string();
        for t in self.tools.iter() {
            if t.extensions.contains(&ext_string) {
                return Some(t.command.clone());
            }
        }
        None
    }
}

#[test]
fn test_decode_toml() {
    let toml_config_str = r#"
    version = 0.0

    [[tools]]
    command = "apvlv"
    extensions = [ "pdf" ]

    [[tools]]
    command = "mirage"
    extensions = [ "jpg", "png", "gif" ]

    [[tools]]
    command = "vlc"
    extensions = [ "mp4", "mov", "avi" ]

    [[tools]]
    command = "vim -R"
    extensions = [ "conf" ]
    "#;

    let config: Config = toml::from_str(toml_config_str).unwrap();
    assert_eq!(config.version, 0.0);

    assert_eq!(config.tools[0].command, "apvlv".to_string());
    assert_eq!(config.tools[0].extensions[0], "pdf".to_string());

    assert_eq!(config.tools[1].command, "mirage".to_string());
    assert_eq!(config.tools[1].extensions[0], "jpg".to_string());
    assert_eq!(config.tools[1].extensions[1], "png".to_string());
    assert_eq!(config.tools[1].extensions[2], "gif".to_string());

    assert_eq!(config.tools[2].command, "vlc".to_string());
    assert_eq!(config.tools[2].extensions[0], "mp4".to_string());
    assert_eq!(config.tools[2].extensions[1], "mov".to_string());
    assert_eq!(config.tools[2].extensions[2], "avi".to_string());

    assert_eq!(config.tools[3].command, "vim -R".to_string());
    assert_eq!(config.tools[3].extensions[0], "conf".to_string());
}

#[test]
fn test_new() {
    let conf1 = Config::new();
    let conf2 = Config::new();
    assert_eq!(conf1, conf2);
}

#[test]
fn test_save() {
    let conf = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extensions: vec!["txt".to_string(), "log".to_string()],
            },
        ],
    };

    assert_eq!(conf.save("test_config").unwrap(), ());
    let loaded_conf = Config::load("test_config").unwrap();
    assert_eq!(conf, loaded_conf);
    assert_eq!(conf.save("./test_config").unwrap(), ());

    let loaded_conf = Config::load("./test_config").unwrap();
    assert_eq!(conf, loaded_conf);

    let _ = fs::remove_file("test_config");
}

#[test]
fn test_add() {
    let mut conf = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extensions: vec!["txt".to_string(), "log".to_string()],
            },
        ],
    };
    let conf_added_extension = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extensions: vec!["txt".to_string(), "log".to_string(), "new_ext".to_string()],
            },
        ],
    };

    // add same command
    assert_ne!(conf, conf_added_extension);
    conf.add("cat", "new_ext").unwrap();
    assert_eq!(conf, conf_added_extension);

    let conf_added_command = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extensions: vec!["txt".to_string(), "log".to_string(), "new_ext".to_string()],
            },
            Tool {
                command: "new_cmd".to_string(),
                extensions: vec!["xxx".to_string()],
            },
        ],
    };

    // add new command and extension
    assert_ne!(conf, conf_added_command);
    conf.add("new_cmd", "xxx").unwrap();
    assert_eq!(conf, conf_added_command);

    // add same command and extension
    conf.add("new_cmd", "xxx").unwrap();
    assert_eq!(conf, conf_added_command);
}

#[test]
fn test_remove_command() {
    let mut conf = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extensions: vec!["txt".to_string(), "log".to_string()],
            },
            Tool {
                command: "objdump".to_string(),
                extensions: vec!["o".to_string()],
            },
        ],
    };

    // remove_command
    assert!(conf.remove_command("cat").is_ok());
    let conf_expected = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "objdump".to_string(),
                extensions: vec!["o".to_string()],
            },
        ],
    };
    assert_eq!(conf, conf_expected);

    // remove_command
    assert!(conf.remove_command("objdump").is_ok());
    let conf_expected = Config {
        version: 0.0,
        tools: Vec::new(),
    };
    assert_eq!(conf, conf_expected);

    // error occured
    assert!(conf.remove_command("objdump").is_err());
}

#[test]
fn test_remove_extension() {
    let mut conf = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extensions: vec!["txt".to_string(), "log".to_string()],
            },
            Tool {
                command: "objdump".to_string(),
                extensions: vec!["o".to_string()],
            },
        ],
    };

    // remove_command
    assert!(conf.remove_extension("txt").is_ok());
    let conf_expected = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extensions: vec!["log".to_string()],
            },
            Tool {
                command: "objdump".to_string(),
                extensions: vec!["o".to_string()],
            },
        ],
    };
    assert_eq!(conf, conf_expected);

    assert!(conf.remove_extension("o").is_ok());
    let conf_expected = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extensions: vec!["log".to_string()],
            },
        ],
    };
    assert_eq!(conf, conf_expected);

    // error occured
    assert!(conf.remove_extension("o").is_err());
}

#[test]
fn test_get_command() {
    let conf = Config {
        version: 0.0,
        tools: vec![
            Tool {
                command: "cat".to_string(),
                extensions: vec!["txt".to_string(), "log".to_string()],
            },
        ],
    };

    assert_eq!(conf.get_command("txt").unwrap(), "cat");
    assert_eq!(conf.get_command("log").unwrap(), "cat");
    assert_eq!(
        conf.get_command("jpg").unwrap_or("none".to_string()),
        "none"
    );
}
<|endoftext|>


<commit_before>use {ViewName, serde, std};

/// Container for a _map_ and optional _reduce_ function of a view.
///
/// `ViewFunction` is a convenience type for applications that work with view
/// functions in design documents. For more information about view functions and
/// design documents, please see the CouchDB documentation.
///
/// # Examples
///
/// ```
/// extern crate chill;
///
/// let view_function = chill::ViewFunction::new_with_reduce(
///     "function(doc) { emit(doc.key_thing, doc.value_thing); }",
///     "_sum");
///
/// assert_eq!("function(doc) { emit(doc.key_thing, doc.value_thing); }",
///            view_function.map);
/// assert_eq!(Some(String::from("_sum")), view_function.reduce);
/// ```
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct ViewFunction {
    /// The view's map function.
    ///
    /// For more information about _map functions_, please see the CouchDB
    /// documentation.
    ///
    pub map: String,

    /// The view's reduce function, if available.
    ///
    /// For more information about _reduce functions_, please see the CouchDB
    /// documentation.
    ///
    pub reduce: Option<String>,

    // This field exists to prevent applications from directly constructing this
    // struct.
    _dummy: std::marker::PhantomData<()>,
}

impl ViewFunction {
    /// Constructs a new `ViewFunction` that has no _reduce function_.
    pub fn new_without_reduce<M: Into<String>>(map: M) -> Self {
        ViewFunction {
            map: map.into(),
            reduce: None,
            _dummy: std::marker::PhantomData,
        }
    }

    /// Constructs a new `ViewFunction` that has a _reduce_ function.
    pub fn new_with_reduce<M: Into<String>, R: Into<String>>(map: M, reduce: R) -> Self {
        ViewFunction {
            map: map.into(),
            reduce: Some(reduce.into()),
            _dummy: std::marker::PhantomData,
        }
    }
}

impl serde::Deserialize for ViewFunction {
    fn deserialize<D>(deserializer: &mut D) -> Result<Self, D::Error>
        where D: serde::Deserializer
    {
        enum Field {
            Map,
            Reduce,
        }

        impl serde::Deserialize for Field {
            fn deserialize<D>(deserializer: &mut D) -> Result<Field, D::Error>
                where D: serde::Deserializer
            {
                struct Visitor;

                impl serde::de::Visitor for Visitor {
                    type Value = Field;

                    fn visit_str<E>(&mut self, value: &str) -> Result<Field, E>
                        where E: serde::de::Error
                    {
                        match value {
                            "map" => Ok(Field::Map),
                            "reduce" => Ok(Field::Reduce),
                            _ => Err(E::unknown_field(value)),
                        }
                    }
                }

                deserializer.deserialize(Visitor)
            }
        }

        struct Visitor;

        impl serde::de::Visitor for Visitor {
            type Value = ViewFunction;

            fn visit_map<V>(&mut self, mut visitor: V) -> Result<Self::Value, V::Error>
                where V: serde::de::MapVisitor
            {
                let mut map = None;
                let mut reduce = None;

                loop {
                    match try!(visitor.visit_key()) {
                        Some(Field::Map) => {
                            map = Some(try!(visitor.visit_value()));
                        }
                        Some(Field::Reduce) => {
                            reduce = Some(try!(visitor.visit_value()));
                        }
                        None => {
                            break;
                        }
                    }
                }

                try!(visitor.end());

                let map = match map {
                    Some(x) => x,
                    None => try!(visitor.missing_field("map")),
                };

                Ok(ViewFunction {
                    map: map,
                    reduce: reduce,
                    _dummy: std::marker::PhantomData,
                })
            }
        }

        static FIELDS: &'static [&'static str] = &["map", "reduce"];
        deserializer.deserialize_struct("SavedAttachment", FIELDS, Visitor)
    }
}

impl serde::Serialize for ViewFunction {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: serde::Serializer
    {
        struct Visitor<'a>(&'a ViewFunction);

        impl<'a> serde::ser::MapVisitor for Visitor<'a> {
            fn visit<S>(&mut self, serializer: &mut S) -> Result<Option<()>, S::Error>
                where S: serde::Serializer
            {
                let &mut Visitor(view_function) = self;
                try!(serializer.serialize_struct_elt("map", &view_function.map));
                if let Some(ref reduce) = view_function.reduce {
                    try!(serializer.serialize_struct_elt("reduce", reduce));
                }
                Ok(None)
            }
        }

        serializer.serialize_struct("ViewFunction", Visitor(self))
    }
}

/// Container for the content of a design document.
///
/// `Design` is a convenience type for applications that create, read, or update
/// design documents.
///
/// Currently, `Design` supports only the `views` field of a design document.
/// For more information about design documents, please see the CouchDB
/// documentation.
///
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Design {
    /// The view functions stored within the design document, if any.
    pub views: std::collections::HashMap<ViewName, ViewFunction>,

    // This field exists to prevent applications from directly constructing this
    // struct.
    _dummy: std::marker::PhantomData<()>,
}

impl serde::Deserialize for Design {
    fn deserialize<D>(deserializer: &mut D) -> Result<Self, D::Error>
        where D: serde::Deserializer
    {
        enum Field {
            Views,
        }

        impl serde::Deserialize for Field {
            fn deserialize<D>(deserializer: &mut D) -> Result<Field, D::Error>
                where D: serde::Deserializer
            {
                struct Visitor;

                impl serde::de::Visitor for Visitor {
                    type Value = Field;

                    fn visit_str<E>(&mut self, value: &str) -> Result<Field, E>
                        where E: serde::de::Error
                    {
                        match value {
                            "views" => Ok(Field::Views),
                            _ => Err(E::unknown_field(value)),
                        }
                    }
                }

                deserializer.deserialize(Visitor)
            }
        }

        struct Visitor;

        impl serde::de::Visitor for Visitor {
            type Value = Design;

            fn visit_map<V>(&mut self, mut visitor: V) -> Result<Self::Value, V::Error>
                where V: serde::de::MapVisitor
            {
                let mut views = None;

                loop {
                    match try!(visitor.visit_key()) {
                        Some(Field::Views) => {
                            views = Some(try!(visitor.visit_value()));
                        }
                        None => {
                            break;
                        }
                    }
                }

                try!(visitor.end());

                let views = match views {
                    Some(x) => x,
                    None => std::collections::HashMap::new(),
                };

                Ok(Design {
                    views: views,
                    _dummy: std::marker::PhantomData,
                })
            }
        }

        static FIELDS: &'static [&'static str] = &["views"];
        deserializer.deserialize_struct("Design", FIELDS, Visitor)
    }
}

impl serde::Serialize for Design {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: serde::Serializer
    {
        struct Visitor<'a>(&'a Design);

        impl<'a> serde::ser::MapVisitor for Visitor<'a> {
            fn visit<S>(&mut self, serializer: &mut S) -> Result<Option<()>, S::Error>
                where S: serde::Serializer
            {
                let &mut Visitor(design) = self;
                try!(serializer.serialize_struct_elt("views", &design.views));
                Ok(None)
            }
        }

        serializer.serialize_struct("Design", Visitor(self))
    }
}

/// Builder for a design document's content.
///
/// `Builder` is a convenience type for applications that create new design
/// documents. For more information about design documents, please see the
/// CouchDB documentation.
///
#[derive(Debug)]
pub struct DesignBuilder {
    inner: Design,
}

impl DesignBuilder {
    /// Constructs a new builder containing empty design document content.
    pub fn new() -> Self {
        DesignBuilder {
            inner: Design {
                views: std::collections::HashMap::new(),
                _dummy: std::marker::PhantomData,
            },
        }
    }

    /// Returns the builder's design document content.
    pub fn unwrap(self) -> Design {
        self.inner
    }

    /// Inserts a view into the design document content.
    pub fn with_view<V>(mut self, view_name: V, view_function: ViewFunction) -> Self
        where V: Into<ViewName>
    {
        self.inner.views.insert(view_name.into(), view_function);
        self
    }
}

#[cfg(test)]
mod tests {

    use super::*;
    use {serde_json, std};

    #[test]
    fn view_function_new() {

        let map_function = r#"function(doc) { emit(doc.key, doc.value); }"#;

        let expected = ViewFunction {
            map: String::from(map_function),
            reduce: None,
            _dummy: std::marker::PhantomData,
        };

        let got = ViewFunction::new_without_reduce(map_function);

        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_new_with_reduce() {

        let map_function = r#"function(doc) { emit(doc.key, doc.value); }"#;
        let reduce_function = r#"_count"#;

        let expected = ViewFunction {
            map: String::from(map_function),
            reduce: Some(String::from(reduce_function)),
            _dummy: std::marker::PhantomData,
        };

        let got = ViewFunction::new_with_reduce(map_function, reduce_function);

        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_serialize_without_reduce() {

        let view_function = ViewFunction::new_without_reduce("function(doc) { emit(doc.key_thing, doc.value_thing); }");

        let encoded = serde_json::to_string(&view_function).unwrap();

        let expected = serde_json::builder::ObjectBuilder::new()
            .insert("map", &view_function.map)
            .unwrap();

        let got = serde_json::from_str(&encoded).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_serialize_with_reduce() {

        let view_function = ViewFunction::new_with_reduce("function(doc) { emit(doc.key_thing, doc.value_thing); }",
                                                          "_sum");

        let encoded = serde_json::to_string(&view_function).unwrap();

        let expected = serde_json::builder::ObjectBuilder::new()
            .insert("map", &view_function.map)
            .insert("reduce", &view_function.reduce)
            .unwrap();

        let got = serde_json::from_str(&encoded).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_deserialize_ok_without_reduce() {

        let expected = ViewFunction::new_without_reduce("function(doc) { emit(doc.key_thing, doc.value_thing); }");

        let source = serde_json::builder::ObjectBuilder::new()
            .insert("map",
                    "function(doc) { emit(doc.key_thing, doc.value_thing); }")
            .unwrap();

        let source = serde_json::to_string(&source).unwrap();
        let got = serde_json::from_str(&source).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_deserialize_ok_with_reduce() {

        let expected = ViewFunction::new_with_reduce("function(doc) { emit(doc.key_thing, doc.value_thing); }",
                                                     "_sum");

        let source = serde_json::builder::ObjectBuilder::new()
            .insert("map",
                    "function(doc) { emit(doc.key_thing, doc.value_thing); }")
            .insert("reduce", "_sum")
            .unwrap();

        let source = serde_json::to_string(&source).unwrap();
        let got = serde_json::from_str(&source).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_deserialize_nok_missing_map() {

        let source = serde_json::builder::ObjectBuilder::new()
            .insert("reduce", "_sum")
            .unwrap();

        let source = serde_json::to_string(&source).unwrap();
        let got = serde_json::from_str::<ViewFunction>(&source);
        expect_json_error_missing_field!(got, "map");
    }

    #[test]
    fn design_serialize() {

        let design = DesignBuilder::new()
            .with_view("alpha",
                       ViewFunction::new_without_reduce("function(doc) { emit(doc.key_thing, doc.value_thing); }"))
            .with_view("bravo",
                       ViewFunction::new_with_reduce("function(doc) { emit(doc.key_thing_2, doc.value_thing_2); }",
                                                     "_sum"))
            .unwrap();

        let encoded = serde_json::to_string(&design).unwrap();

        let expected = serde_json::builder::ObjectBuilder::new()
            .insert_object("views", |x| {
                x.insert_object("alpha", |x| {
                        x.insert("map",
                                 "function(doc) { emit(doc.key_thing, doc.value_thing); }")
                    })
                    .insert_object("bravo", |x| {
                        x.insert("map",
                                    "function(doc) { emit(doc.key_thing_2, doc.value_thing_2); }")
                            .insert("reduce", "_sum")
                    })
            })
            .unwrap();

        let got = serde_json::from_str(&encoded).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn design_deserialize_ok_empty() {
        let expected = DesignBuilder::new().unwrap();
        let source = serde_json::builder::ObjectBuilder::new().unwrap();
        let source = serde_json::to_string(&source).unwrap();
        let got = serde_json::from_str(&source).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn design_deserialize_ok_with_views() {

        let expected = DesignBuilder::new()
            .with_view("alpha",
                       ViewFunction::new_without_reduce("function(doc) { emit(doc.key_thing, doc.value_thing); }"))
            .with_view("bravo",
                       ViewFunction::new_with_reduce("function(doc) { emit(doc.key_thing_2, doc.value_thing_2); }",
                                                     "_sum"))
            .unwrap();

        let source = serde_json::builder::ObjectBuilder::new()
            .insert_object("views", |x| {
                x.insert_object("alpha", |x| {
                        x.insert("map",
                                 "function(doc) { emit(doc.key_thing, doc.value_thing); }")
                    })
                    .insert_object("bravo", |x| {
                        x.insert("map",
                                    "function(doc) { emit(doc.key_thing_2, doc.value_thing_2); }")
                            .insert("reduce", "_sum")
                    })
            })
            .unwrap();

        let source = serde_json::to_string(&source).unwrap();
        let got = serde_json::from_str(&source).unwrap();
        assert_eq!(expected, got);
    }
}
<commit_msg>Tweak DesignBuilder and ViewFunction method names<commit_after>use {ViewName, serde, std};

/// Container for a _map_ and optional _reduce_ function of a view.
///
/// `ViewFunction` is a convenience type for applications that work with view
/// functions in design documents. For more information about view functions and
/// design documents, please see the CouchDB documentation.
///
/// # Examples
///
/// ```
/// extern crate chill;
///
/// let view_function = chill::ViewFunction::new_with_reduce(
///     "function(doc) { emit(doc.key_thing, doc.value_thing); }",
///     "_sum");
///
/// assert_eq!("function(doc) { emit(doc.key_thing, doc.value_thing); }",
///            view_function.map);
/// assert_eq!(Some(String::from("_sum")), view_function.reduce);
/// ```
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct ViewFunction {
    /// The view's map function.
    ///
    /// For more information about _map functions_, please see the CouchDB
    /// documentation.
    ///
    pub map: String,

    /// The view's reduce function, if available.
    ///
    /// For more information about _reduce functions_, please see the CouchDB
    /// documentation.
    ///
    pub reduce: Option<String>,

    // This field exists to prevent applications from directly constructing this
    // struct.
    _dummy: std::marker::PhantomData<()>,
}

impl ViewFunction {
    /// Constructs a new `ViewFunction` that has no _reduce function_.
    pub fn new<M: Into<String>>(map: M) -> Self {
        ViewFunction {
            map: map.into(),
            reduce: None,
            _dummy: std::marker::PhantomData,
        }
    }

    /// Constructs a new `ViewFunction` that has a _reduce_ function.
    pub fn new_with_reduce<M: Into<String>, R: Into<String>>(map: M, reduce: R) -> Self {
        ViewFunction {
            map: map.into(),
            reduce: Some(reduce.into()),
            _dummy: std::marker::PhantomData,
        }
    }
}

impl serde::Deserialize for ViewFunction {
    fn deserialize<D>(deserializer: &mut D) -> Result<Self, D::Error>
        where D: serde::Deserializer
    {
        enum Field {
            Map,
            Reduce,
        }

        impl serde::Deserialize for Field {
            fn deserialize<D>(deserializer: &mut D) -> Result<Field, D::Error>
                where D: serde::Deserializer
            {
                struct Visitor;

                impl serde::de::Visitor for Visitor {
                    type Value = Field;

                    fn visit_str<E>(&mut self, value: &str) -> Result<Field, E>
                        where E: serde::de::Error
                    {
                        match value {
                            "map" => Ok(Field::Map),
                            "reduce" => Ok(Field::Reduce),
                            _ => Err(E::unknown_field(value)),
                        }
                    }
                }

                deserializer.deserialize(Visitor)
            }
        }

        struct Visitor;

        impl serde::de::Visitor for Visitor {
            type Value = ViewFunction;

            fn visit_map<V>(&mut self, mut visitor: V) -> Result<Self::Value, V::Error>
                where V: serde::de::MapVisitor
            {
                let mut map = None;
                let mut reduce = None;

                loop {
                    match try!(visitor.visit_key()) {
                        Some(Field::Map) => {
                            map = Some(try!(visitor.visit_value()));
                        }
                        Some(Field::Reduce) => {
                            reduce = Some(try!(visitor.visit_value()));
                        }
                        None => {
                            break;
                        }
                    }
                }

                try!(visitor.end());

                let map = match map {
                    Some(x) => x,
                    None => try!(visitor.missing_field("map")),
                };

                Ok(ViewFunction {
                    map: map,
                    reduce: reduce,
                    _dummy: std::marker::PhantomData,
                })
            }
        }

        static FIELDS: &'static [&'static str] = &["map", "reduce"];
        deserializer.deserialize_struct("SavedAttachment", FIELDS, Visitor)
    }
}

impl serde::Serialize for ViewFunction {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: serde::Serializer
    {
        struct Visitor<'a>(&'a ViewFunction);

        impl<'a> serde::ser::MapVisitor for Visitor<'a> {
            fn visit<S>(&mut self, serializer: &mut S) -> Result<Option<()>, S::Error>
                where S: serde::Serializer
            {
                let &mut Visitor(view_function) = self;
                try!(serializer.serialize_struct_elt("map", &view_function.map));
                if let Some(ref reduce) = view_function.reduce {
                    try!(serializer.serialize_struct_elt("reduce", reduce));
                }
                Ok(None)
            }
        }

        serializer.serialize_struct("ViewFunction", Visitor(self))
    }
}

/// Container for the content of a design document.
///
/// `Design` is a convenience type for applications that create, read, or update
/// design documents.
///
/// Currently, `Design` supports only the `views` field of a design document.
/// For more information about design documents, please see the CouchDB
/// documentation.
///
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Design {
    /// The view functions stored within the design document, if any.
    pub views: std::collections::HashMap<ViewName, ViewFunction>,

    // This field exists to prevent applications from directly constructing this
    // struct.
    _dummy: std::marker::PhantomData<()>,
}

impl serde::Deserialize for Design {
    fn deserialize<D>(deserializer: &mut D) -> Result<Self, D::Error>
        where D: serde::Deserializer
    {
        enum Field {
            Views,
        }

        impl serde::Deserialize for Field {
            fn deserialize<D>(deserializer: &mut D) -> Result<Field, D::Error>
                where D: serde::Deserializer
            {
                struct Visitor;

                impl serde::de::Visitor for Visitor {
                    type Value = Field;

                    fn visit_str<E>(&mut self, value: &str) -> Result<Field, E>
                        where E: serde::de::Error
                    {
                        match value {
                            "views" => Ok(Field::Views),
                            _ => Err(E::unknown_field(value)),
                        }
                    }
                }

                deserializer.deserialize(Visitor)
            }
        }

        struct Visitor;

        impl serde::de::Visitor for Visitor {
            type Value = Design;

            fn visit_map<V>(&mut self, mut visitor: V) -> Result<Self::Value, V::Error>
                where V: serde::de::MapVisitor
            {
                let mut views = None;

                loop {
                    match try!(visitor.visit_key()) {
                        Some(Field::Views) => {
                            views = Some(try!(visitor.visit_value()));
                        }
                        None => {
                            break;
                        }
                    }
                }

                try!(visitor.end());

                let views = match views {
                    Some(x) => x,
                    None => std::collections::HashMap::new(),
                };

                Ok(Design {
                    views: views,
                    _dummy: std::marker::PhantomData,
                })
            }
        }

        static FIELDS: &'static [&'static str] = &["views"];
        deserializer.deserialize_struct("Design", FIELDS, Visitor)
    }
}

impl serde::Serialize for Design {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: serde::Serializer
    {
        struct Visitor<'a>(&'a Design);

        impl<'a> serde::ser::MapVisitor for Visitor<'a> {
            fn visit<S>(&mut self, serializer: &mut S) -> Result<Option<()>, S::Error>
                where S: serde::Serializer
            {
                let &mut Visitor(design) = self;
                try!(serializer.serialize_struct_elt("views", &design.views));
                Ok(None)
            }
        }

        serializer.serialize_struct("Design", Visitor(self))
    }
}

/// Builder for a design document's content.
///
/// `Builder` is a convenience type for applications that create new design
/// documents. For more information about design documents, please see the
/// CouchDB documentation.
///
#[derive(Debug)]
pub struct DesignBuilder {
    inner: Design,
}

impl DesignBuilder {
    /// Constructs a new builder containing empty design document content.
    pub fn new() -> Self {
        DesignBuilder {
            inner: Design {
                views: std::collections::HashMap::new(),
                _dummy: std::marker::PhantomData,
            },
        }
    }

    /// Returns the builder's design document content.
    pub fn unwrap(self) -> Design {
        self.inner
    }

    /// Inserts a view into the design document content.
    pub fn insert_view<V>(mut self, view_name: V, view_function: ViewFunction) -> Self
        where V: Into<ViewName>
    {
        self.inner.views.insert(view_name.into(), view_function);
        self
    }
}

#[cfg(test)]
mod tests {

    use super::*;
    use {serde_json, std};

    #[test]
    fn view_function_new() {

        let map_function = r#"function(doc) { emit(doc.key, doc.value); }"#;

        let expected = ViewFunction {
            map: String::from(map_function),
            reduce: None,
            _dummy: std::marker::PhantomData,
        };

        let got = ViewFunction::new(map_function);

        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_new_with_reduce() {

        let map_function = r#"function(doc) { emit(doc.key, doc.value); }"#;
        let reduce_function = r#"_count"#;

        let expected = ViewFunction {
            map: String::from(map_function),
            reduce: Some(String::from(reduce_function)),
            _dummy: std::marker::PhantomData,
        };

        let got = ViewFunction::new_with_reduce(map_function, reduce_function);

        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_serialize_without_reduce() {

        let view_function = ViewFunction::new("function(doc) { emit(doc.key_thing, doc.value_thing); }");

        let encoded = serde_json::to_string(&view_function).unwrap();

        let expected = serde_json::builder::ObjectBuilder::new()
            .insert("map", &view_function.map)
            .unwrap();

        let got = serde_json::from_str(&encoded).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_serialize_with_reduce() {

        let view_function = ViewFunction::new_with_reduce("function(doc) { emit(doc.key_thing, doc.value_thing); }",
                                                          "_sum");

        let encoded = serde_json::to_string(&view_function).unwrap();

        let expected = serde_json::builder::ObjectBuilder::new()
            .insert("map", &view_function.map)
            .insert("reduce", &view_function.reduce)
            .unwrap();

        let got = serde_json::from_str(&encoded).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_deserialize_ok_without_reduce() {

        let expected = ViewFunction::new("function(doc) { emit(doc.key_thing, doc.value_thing); }");

        let source = serde_json::builder::ObjectBuilder::new()
            .insert("map",
                    "function(doc) { emit(doc.key_thing, doc.value_thing); }")
            .unwrap();

        let source = serde_json::to_string(&source).unwrap();
        let got = serde_json::from_str(&source).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_deserialize_ok_with_reduce() {

        let expected = ViewFunction::new_with_reduce("function(doc) { emit(doc.key_thing, doc.value_thing); }",
                                                     "_sum");

        let source = serde_json::builder::ObjectBuilder::new()
            .insert("map",
                    "function(doc) { emit(doc.key_thing, doc.value_thing); }")
            .insert("reduce", "_sum")
            .unwrap();

        let source = serde_json::to_string(&source).unwrap();
        let got = serde_json::from_str(&source).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn view_function_deserialize_nok_missing_map() {

        let source = serde_json::builder::ObjectBuilder::new()
            .insert("reduce", "_sum")
            .unwrap();

        let source = serde_json::to_string(&source).unwrap();
        let got = serde_json::from_str::<ViewFunction>(&source);
        expect_json_error_missing_field!(got, "map");
    }

    #[test]
    fn design_serialize() {

        let design = DesignBuilder::new()
            .insert_view("alpha",
                         ViewFunction::new("function(doc) { emit(doc.key_thing, doc.value_thing); }"))
            .insert_view("bravo",
                         ViewFunction::new_with_reduce("function(doc) { emit(doc.key_thing_2, doc.value_thing_2); }",
                                                       "_sum"))
            .unwrap();

        let encoded = serde_json::to_string(&design).unwrap();

        let expected = serde_json::builder::ObjectBuilder::new()
            .insert_object("views", |x| {
                x.insert_object("alpha", |x| {
                        x.insert("map",
                                 "function(doc) { emit(doc.key_thing, doc.value_thing); }")
                    })
                    .insert_object("bravo", |x| {
                        x.insert("map",
                                    "function(doc) { emit(doc.key_thing_2, doc.value_thing_2); }")
                            .insert("reduce", "_sum")
                    })
            })
            .unwrap();

        let got = serde_json::from_str(&encoded).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn design_deserialize_ok_empty() {
        let expected = DesignBuilder::new().unwrap();
        let source = serde_json::builder::ObjectBuilder::new().unwrap();
        let source = serde_json::to_string(&source).unwrap();
        let got = serde_json::from_str(&source).unwrap();
        assert_eq!(expected, got);
    }

    #[test]
    fn design_deserialize_ok_with_views() {

        let expected = DesignBuilder::new()
            .insert_view("alpha",
                         ViewFunction::new("function(doc) { emit(doc.key_thing, doc.value_thing); }"))
            .insert_view("bravo",
                         ViewFunction::new_with_reduce("function(doc) { emit(doc.key_thing_2, doc.value_thing_2); }",
                                                       "_sum"))
            .unwrap();

        let source = serde_json::builder::ObjectBuilder::new()
            .insert_object("views", |x| {
                x.insert_object("alpha", |x| {
                        x.insert("map",
                                 "function(doc) { emit(doc.key_thing, doc.value_thing); }")
                    })
                    .insert_object("bravo", |x| {
                        x.insert("map",
                                    "function(doc) { emit(doc.key_thing_2, doc.value_thing_2); }")
                            .insert("reduce", "_sum")
                    })
            })
            .unwrap();

        let source = serde_json::to_string(&source).unwrap();
        let got = serde_json::from_str(&source).unwrap();
        assert_eq!(expected, got);
    }
}
<|endoftext|>


<commit_before>//!
//! The main `Desync` struct
//!

use super::scheduler::*;

use std::pin::{Pin};
use std::sync::{Arc};
use std::marker::{Unpin};
use futures::{FutureExt};
use futures::channel::oneshot;
use futures::future::{Future, BoxFuture};

use std::mem;
use std::result::{Result};

///
/// A data storage structure used to govern synchronous and asynchronous access to an underlying object.
///
pub struct Desync<T: Send+Unpin> {
    /// Queue used for scheduling runtime for this object
    queue:  Arc<JobQueue>,

    /// Data for this object. Boxed so the pointer remains the same through the lifetime of the object.
    /// Will be 'None' only briefly when the data has been taken to be dropped
    data:   Option<Pin<Box<T>>>
}

// Rust actually derives this anyway at the moment
unsafe impl<T: Send+Unpin> Send for Desync<T> {}

// True iff queue: Sync
unsafe impl<T: Send+Unpin> Sync for Desync<T> {}

///
/// Used for passing the data pointer through to the queue
///
/// 'Safe' because the queue is synchronised during drop, so we can never use the pointer
/// if the object does not exist.
///
struct DataRef<T: Send>(*const T);
unsafe impl<T: Send> Send for DataRef<T> {}

// TODO: we can change DataRef to Shared (https://doc.rust-lang.org/std/ptr/struct.Shared.html in the future)

// TODO: T does not need to be static as we know that its lifetime is at least the lifetime of Desync<T> and hence the queue
impl<T: 'static+Send+Unpin> Desync<T> {
    ///
    /// Creates a new Desync object
    ///
    pub fn new(data: T) -> Desync<T> {
        let queue = queue();

        Desync {
            queue:  queue,
            data:   Some(Pin::new(Box::new(data)))
        }
    }

    ///
    /// Performs an operation asynchronously on this item. This function will return
    /// immediately and the job will happen on a separate thread at some time in the
    /// future (generally fairly soon).
    ///
    /// Jobs are always performed in the order that they are queued and are always
    /// performed synchronously with respect to this object.
    ///
    #[inline]
    #[deprecated(since="0.3.0", note="please use `desync` instead")]
    pub fn r#async<TFn>(&self, job: TFn)
    where TFn: 'static+Send+FnOnce(&mut T) -> () {
        self.desync(job)
    }

    ///
    /// Performs an operation asynchronously on this item. This function will return
    /// immediately and the job will happen on a separate thread at some time in the
    /// future (generally fairly soon).
    ///
    /// Jobs are always performed in the order that they are queued and are always
    /// performed synchronously with respect to this object.
    ///
    pub fn desync<TFn>(&self, job: TFn)
    where TFn: 'static+Send+FnOnce(&mut T) -> () {
        // As drop() is the last thing called, we know that this object will still exist at the point where the queue makes the asynchronous callback
        let data = DataRef::<T>(&**self.data.as_ref().unwrap());

        desync(&self.queue, move || {
            let data = data.0 as *mut T;
            job(unsafe { &mut *data });
        })
    }

    ///
    /// Performs an operation synchronously on this item. This will be queued with any other
    /// jobs that this item may be performing, and this function will not return until the
    /// job is complete and the result is available.
    ///
    pub fn sync<TFn, Result>(&self, job: TFn) -> Result
    where TFn: Send+FnOnce(&mut T) -> Result, Result: Send {
        let result = {
            // As drop() is the last thing called, we know that this object will still exist at the point where the callback occurs
            // Exclusivity is guaranteed because the queue executes only one task at a time
            let data = DataRef::<T>(&**self.data.as_ref().unwrap());

            sync(&self.queue, move || {
                let data = data.0 as *mut T;
                job(unsafe { &mut *data })
            })
        };

        result
    }

    ///
    /// Performs an operation synchronously on this item,
    ///
    pub fn try_sync<TFn, FnResult>(&self, job: TFn) -> Result<FnResult, TrySyncError>
    where TFn: Send+FnOnce(&mut T) -> FnResult, FnResult: Send {
        let result = {
            // As drop() is the last thing called, we know that this object will still exist at the point where the callback occurs
            let data = DataRef::<T>(&**self.data.as_ref().unwrap());

            try_sync(&self.queue, move || {
                let data = data.0 as *mut T;
                job(unsafe { &mut *data })
            })
        };

        result
    }

    ///
    /// Performs an operation asynchronously on the contents of this item, returning the
    /// result via a future.
    ///
    /// The future will be scheduled in the background, so it will make progress even if the current scheduler is
    /// blocked for any reason. Additionally, it's not necessary to await the returned future, which can be discarded
    /// if necessary.
    ///
    /// The future returned is a `BoxFuture`, which you can create using `.boxed()` or `Box::pin()` on a future. This is
    /// solely to work around a limitation in Rust's type system (it's not presently possible to introduce the lifetime
    /// from for<'a> into the return type of a function)
    ///
    pub fn future_desync<TFn, TOutput>(&self, job: TFn) -> impl Future<Output=Result<TOutput, oneshot::Canceled>>+Send
    where   TFn:        'static+Send+for<'a> FnOnce(&'a mut T) -> BoxFuture<'a, TOutput>,
            TOutput:    'static+Send {
        // The future will have a lifetime shorter than the lifetime of this structure, and exclusivity is guaranteed
        // because queues only execute one task at a time
        let data = DataRef::<T>(&**self.data.as_ref().unwrap());

        scheduler().future_desync(&self.queue, move || {
            let data        = data.0 as *mut T;
            let job         = job(unsafe { &mut *data });

            async {
                job.await
            }
        })
    }

    ///
    /// Performs an operation asynchronously on the contents of this item, returning a future that must be awaited
    /// before it is dropped.
    ///
    /// The future will be scheduled in the current execution context, so it will only make progress if the current
    /// scheduler is running. The task will be cancelled and will not complete execution if the future is dropped before
    /// completion, so it's usually necessary to await the result of this function for the task to behave correctly.
    ///
    /// The future returned is a `BoxFuture`, which you can create using `.boxed()` or `Box::pin()` on a future. This is
    /// solely to work around a limitation in Rust's type system (it's not presently possible to introduce the lifetime
    /// from for<'a> into the return type of a function)
    ///
    pub fn future_sync<'a, TFn, TOutput>(&'a self, job: TFn) -> impl 'a+Future<Output=Result<TOutput, oneshot::Canceled>>+Send
    where   TFn:        'a+Send+for<'b> FnOnce(&'b mut T) -> BoxFuture<'b, TOutput>,
            TOutput:    'a+Send {
        // The future will have a lifetime shorter than the lifetime of this structure
        let data = DataRef::<T>(&**self.data.as_ref().unwrap());

        scheduler().future_sync(&self.queue, move || {
            let data        = data.0 as *mut T;
            let job         = job(unsafe { &mut *data });

            async {
                job.await
            }
        })
    }

    ///
    /// After the pending operations for this item are performed, waits for the
    /// supplied future to complete and then calls the specified function
    ///
    pub fn after<'a, TFn, Res: 'static+Send, Fut: 'static+Future+Send>(&self, after: Fut, job: TFn) -> impl 'static+Future<Output=Result<Res, oneshot::Canceled>>+Send
    where TFn: 'static+Send+FnOnce(&mut T, Fut::Output) -> Res {
        self.future_desync(move |data| {
            async move {
                let future_result = after.await;
                job(data, future_result)
            }.boxed()
        })
    }
}

impl<T: Send+Unpin> Drop for Desync<T> {
    fn drop(&mut self) {
        use std::thread;

        // Take the data we're about to drop from the object
        let data = self.data.take();

        // Ensure that everything on the queue has committed by queueing a last synchronous event
        // (Not synchronising the queue would make this unsafe as we would hold on to a pointer to
        // the internal data structure)
        if thread::panicking() {
            // If the thread is already panicking when we're dropped, do not panic again
            scheduler().sync_no_panic(&self.queue, move || {
                mem::drop(data);
            });
        } else {
            // Thread is not panicking
            sync(&self.queue, move || {
                mem::drop(data);
            });
        }
    }
}
<commit_msg>Add in a deprecated version of future() for compatibility with older code<commit_after>//!
//! The main `Desync` struct
//!

use super::scheduler::*;

use std::pin::{Pin};
use std::sync::{Arc};
use std::marker::{Unpin};
use futures::{FutureExt};
use futures::channel::oneshot;
use futures::future::{Future, BoxFuture};

use std::mem;
use std::result::{Result};

///
/// A data storage structure used to govern synchronous and asynchronous access to an underlying object.
///
pub struct Desync<T: Send+Unpin> {
    /// Queue used for scheduling runtime for this object
    queue:  Arc<JobQueue>,

    /// Data for this object. Boxed so the pointer remains the same through the lifetime of the object.
    /// Will be 'None' only briefly when the data has been taken to be dropped
    data:   Option<Pin<Box<T>>>
}

// Rust actually derives this anyway at the moment
unsafe impl<T: Send+Unpin> Send for Desync<T> {}

// True iff queue: Sync
unsafe impl<T: Send+Unpin> Sync for Desync<T> {}

///
/// Used for passing the data pointer through to the queue
///
/// 'Safe' because the queue is synchronised during drop, so we can never use the pointer
/// if the object does not exist.
///
struct DataRef<T: Send>(*const T);
unsafe impl<T: Send> Send for DataRef<T> {}

// TODO: we can change DataRef to Shared (https://doc.rust-lang.org/std/ptr/struct.Shared.html in the future)

// TODO: T does not need to be static as we know that its lifetime is at least the lifetime of Desync<T> and hence the queue
impl<T: 'static+Send+Unpin> Desync<T> {
    ///
    /// Creates a new Desync object
    ///
    pub fn new(data: T) -> Desync<T> {
        let queue = queue();

        Desync {
            queue:  queue,
            data:   Some(Pin::new(Box::new(data)))
        }
    }

    ///
    /// Performs an operation asynchronously on this item. This function will return
    /// immediately and the job will happen on a separate thread at some time in the
    /// future (generally fairly soon).
    ///
    /// Jobs are always performed in the order that they are queued and are always
    /// performed synchronously with respect to this object.
    ///
    #[inline]
    #[deprecated(since="0.3.0", note="please use `desync` instead")]
    pub fn r#async<TFn>(&self, job: TFn)
    where TFn: 'static+Send+FnOnce(&mut T) -> () {
        self.desync(job)
    }

    ///
    /// Performs an operation asynchronously on this item. This function will return
    /// immediately and the job will happen on a separate thread at some time in the
    /// future (generally fairly soon).
    ///
    /// Jobs are always performed in the order that they are queued and are always
    /// performed synchronously with respect to this object.
    ///
    pub fn desync<TFn>(&self, job: TFn)
    where TFn: 'static+Send+FnOnce(&mut T) -> () {
        // As drop() is the last thing called, we know that this object will still exist at the point where the queue makes the asynchronous callback
        let data = DataRef::<T>(&**self.data.as_ref().unwrap());

        desync(&self.queue, move || {
            let data = data.0 as *mut T;
            job(unsafe { &mut *data });
        })
    }

    ///
    /// Performs an operation synchronously on this item. This will be queued with any other
    /// jobs that this item may be performing, and this function will not return until the
    /// job is complete and the result is available.
    ///
    pub fn sync<TFn, Result>(&self, job: TFn) -> Result
    where TFn: Send+FnOnce(&mut T) -> Result, Result: Send {
        let result = {
            // As drop() is the last thing called, we know that this object will still exist at the point where the callback occurs
            // Exclusivity is guaranteed because the queue executes only one task at a time
            let data = DataRef::<T>(&**self.data.as_ref().unwrap());

            sync(&self.queue, move || {
                let data = data.0 as *mut T;
                job(unsafe { &mut *data })
            })
        };

        result
    }

    ///
    /// Performs an operation synchronously on this item,
    ///
    pub fn try_sync<TFn, FnResult>(&self, job: TFn) -> Result<FnResult, TrySyncError>
    where TFn: Send+FnOnce(&mut T) -> FnResult, FnResult: Send {
        let result = {
            // As drop() is the last thing called, we know that this object will still exist at the point where the callback occurs
            let data = DataRef::<T>(&**self.data.as_ref().unwrap());

            try_sync(&self.queue, move || {
                let data = data.0 as *mut T;
                job(unsafe { &mut *data })
            })
        };

        result
    }

    ///
    /// Deprecated name for `future_desync`
    ///
    #[inline]
    #[deprecated(since="0.7.0", note="please use either `future_desync` or `future_sync` to schedule futures")]
    pub fn future<TFn, TOutput>(&self, job: TFn) -> impl Future<Output=Result<TOutput, oneshot::Canceled>>+Send
    where   TFn:        'static+Send+for<'a> FnOnce(&'a mut T) -> BoxFuture<'a, TOutput>,
            TOutput:    'static+Send {
        self.future_desync(job)
    }

    ///
    /// Performs an operation asynchronously on the contents of this item, returning the
    /// result via a future.
    ///
    /// The future will be scheduled in the background, so it will make progress even if the current scheduler is
    /// blocked for any reason. Additionally, it's not necessary to await the returned future, which can be discarded
    /// if necessary.
    ///
    /// The future returned is a `BoxFuture`, which you can create using `.boxed()` or `Box::pin()` on a future. This is
    /// solely to work around a limitation in Rust's type system (it's not presently possible to introduce the lifetime
    /// from for<'a> into the return type of a function)
    ///
    pub fn future_desync<TFn, TOutput>(&self, job: TFn) -> impl Future<Output=Result<TOutput, oneshot::Canceled>>+Send
    where   TFn:        'static+Send+for<'a> FnOnce(&'a mut T) -> BoxFuture<'a, TOutput>,
            TOutput:    'static+Send {
        // The future will have a lifetime shorter than the lifetime of this structure, and exclusivity is guaranteed
        // because queues only execute one task at a time
        let data = DataRef::<T>(&**self.data.as_ref().unwrap());

        scheduler().future_desync(&self.queue, move || {
            let data        = data.0 as *mut T;
            let job         = job(unsafe { &mut *data });

            async {
                job.await
            }
        })
    }

    ///
    /// Performs an operation asynchronously on the contents of this item, returning a future that must be awaited
    /// before it is dropped.
    ///
    /// The future will be scheduled in the current execution context, so it will only make progress if the current
    /// scheduler is running. The task will be cancelled and will not complete execution if the future is dropped before
    /// completion, so it's usually necessary to await the result of this function for the task to behave correctly.
    ///
    /// The future returned is a `BoxFuture`, which you can create using `.boxed()` or `Box::pin()` on a future. This is
    /// solely to work around a limitation in Rust's type system (it's not presently possible to introduce the lifetime
    /// from for<'a> into the return type of a function)
    ///
    pub fn future_sync<'a, TFn, TOutput>(&'a self, job: TFn) -> impl 'a+Future<Output=Result<TOutput, oneshot::Canceled>>+Send
    where   TFn:        'a+Send+for<'b> FnOnce(&'b mut T) -> BoxFuture<'b, TOutput>,
            TOutput:    'a+Send {
        // The future will have a lifetime shorter than the lifetime of this structure
        let data = DataRef::<T>(&**self.data.as_ref().unwrap());

        scheduler().future_sync(&self.queue, move || {
            let data        = data.0 as *mut T;
            let job         = job(unsafe { &mut *data });

            async {
                job.await
            }
        })
    }

    ///
    /// After the pending operations for this item are performed, waits for the
    /// supplied future to complete and then calls the specified function
    ///
    pub fn after<'a, TFn, Res: 'static+Send, Fut: 'static+Future+Send>(&self, after: Fut, job: TFn) -> impl 'static+Future<Output=Result<Res, oneshot::Canceled>>+Send
    where TFn: 'static+Send+FnOnce(&mut T, Fut::Output) -> Res {
        self.future_desync(move |data| {
            async move {
                let future_result = after.await;
                job(data, future_result)
            }.boxed()
        })
    }
}

impl<T: Send+Unpin> Drop for Desync<T> {
    fn drop(&mut self) {
        use std::thread;

        // Take the data we're about to drop from the object
        let data = self.data.take();

        // Ensure that everything on the queue has committed by queueing a last synchronous event
        // (Not synchronising the queue would make this unsafe as we would hold on to a pointer to
        // the internal data structure)
        if thread::panicking() {
            // If the thread is already panicking when we're dropped, do not panic again
            scheduler().sync_no_panic(&self.queue, move || {
                mem::drop(data);
            });
        } else {
            // Thread is not panicking
            sync(&self.queue, move || {
                mem::drop(data);
            });
        }
    }
}
<|endoftext|>


<commit_before>use fnv::FnvHashMap;

use lang::*;
use script::*;
use trigrams::*;
use info::Info;
use options::{Options, List};

const MAX_TRIGRAM_DISTANCE : u32 = 300;

// 300 * 300 = 90_000
const MAX_TOTAL_DISTANCE : u32 = 90_000;

/// Detect a language and a script by a given text.
///
/// # Example
/// ```
/// use whatlang::{detect, Lang, Script};
///
/// let info = detect("Ĉu vi ne volas eklerni Esperanton? Bonvolu!").unwrap();
/// assert_eq!(info.lang(), Lang::Epo);
/// assert_eq!(info.script(), Script::Latin);
/// ```
pub fn detect(text: &str) -> Option<Info> {
    detect_with_options(text, &Options::default())
}

/// Detect only a language by a given text.
///
/// # Example
/// ```
/// use whatlang::{detect_lang, Lang};
/// let lang = detect_lang("There is no reason not to learn Esperanto.").unwrap();
/// assert_eq!(lang, Lang::Eng);
/// ```
pub fn detect_lang(text: &str) -> Option<Lang> {
    detect(text).map(|info| info.lang)
}

pub fn detect_lang_with_options(text: &str, options: &Options) -> Option<Lang> {
    detect_with_options(text, options).map(|info| info.lang)
}

pub fn detect_with_options(text: &str, options: &Options) -> Option<Info> {
    detect_script(text).and_then(|script| {
        detect_lang_based_on_script(text, options, script).map( |(lang, confidence)| {
            Info { lang, script, confidence }
        })
    })
}

fn detect_lang_based_on_script(text: &str, options: &Options, script : Script) -> Option<(Lang, f64)> {
    match script {
        Script::Latin      => detect_lang_in_profiles(text, options, LATIN_LANGS),
        Script::Cyrillic   => detect_lang_in_profiles(text, options, CYRILLIC_LANGS),
        Script::Devanagari => detect_lang_in_profiles(text, options, DEVANAGARI_LANGS),
        Script::Hebrew     => detect_lang_in_profiles(text, options, HEBREW_LANGS),
        Script::Ethiopic   => detect_lang_in_profiles(text, options, ETHIOPIC_LANGS),
        Script::Arabic     => detect_lang_in_profiles(text, options, ARABIC_LANGS),
        Script::Mandarin  => Some((Lang::Cmn, 1.0)),
        Script::Bengali   => Some((Lang::Ben, 1.0)),
        Script::Hangul    => Some((Lang::Kor, 1.0)),
        Script::Georgian  => Some((Lang::Kat, 1.0)),
        Script::Greek     => Some((Lang::Ell, 1.0)),
        Script::Kannada   => Some((Lang::Kan, 1.0)),
        Script::Tamil     => Some((Lang::Tam, 1.0)),
        Script::Thai      => Some((Lang::Tha, 1.0)),
        Script::Gujarati  => Some((Lang::Guj, 1.0)),
        Script::Gurmukhi  => Some((Lang::Pan, 1.0)),
        Script::Telugu    => Some((Lang::Tel, 1.0)),
        Script::Malayalam => Some((Lang::Mal, 1.0)),
        Script::Oriya     => Some((Lang::Ori, 1.0)),
        Script::Myanmar   => Some((Lang::Mya, 1.0)),
        Script::Sinhala   => Some((Lang::Sin, 1.0)),
        Script::Khmer     => Some((Lang::Khm, 1.0)),
        Script::Katakana | Script::Hiragana  => Some((Lang::Jpn, 1.0))
    }
}

fn detect_lang_in_profiles(text: &str, options: &Options, lang_profile_list : LangProfileList) -> Option<(Lang, f64)> {
    let mut lang_distances : Vec<(Lang, u32)> = vec![];
    let trigrams = get_trigrams_with_positions(text);

    for &(ref lang, lang_trigrams) in lang_profile_list {
        match options.list {
            Some(List::White(ref whitelist)) if !whitelist.contains(lang) => continue,
            Some(List::Black(ref blacklist)) if blacklist.contains(lang) => continue,
            _ => {},
        }
        let dist = calculate_distance(lang_trigrams, &trigrams);
        lang_distances.push(((*lang), dist));
    }

    // Sort languages by distance
    lang_distances.sort_by_key(|key| key.1 );

    // Return None if lang_distances is empty
    // Return the only language with is_reliable=true if there is only 1 item
    if lang_distances.len() < 2 {
        return lang_distances.first().map(|pair| (pair.0, 1.0));
    }

    // Calculate is_reliable based on:
    // - number of unique trigrams in the text
    // - rate (diff between score of the first and second languages)
    //
    let lang_dist1 = lang_distances[0];
    let lang_dist2 = lang_distances[1];
    let score1 = MAX_TOTAL_DISTANCE - lang_dist1.1;
    let score2 = MAX_TOTAL_DISTANCE - lang_dist2.1;

    if score1 == 0 {
        // If score1 is 0, score2 is 0 as well, because array is sorted.
        // Therefore there is not language to return.
        return None;
    } else if score2 == 0 {
        // If score2 is 0, return first language, to prevent division by zero in the rate formula.
        // In this case confidence is calculated by another formula.
        // At this point there are two options:
        // * Text contains random characters that accidentally match trigrams of one of the languages
        // * Text really matches one of the languages.
        //
        // Number 500.0 is based on experiments and common sense expectations.
        let mut confidence = (score1 as f64) / 500.0;
        if confidence > 1.0 {
            confidence == 1.0;
        }
        return Some((lang_dist1.0, confidence));
    }

    let rate = (score1 - score2) as f64 / (score2 as f64);

    // Hyperbola function. Everything that is above the function has confidence = 1.0
    // If rate is below, confidence is calculated proportionally.
    // Numbers 12.0 and 0.05 are obtained experimentally, so the function represents common sense.
    //
    let confident_rate = (12.0 / trigrams.len() as f64) + 0.05;
    let confidence =
        if rate > confident_rate {
            1.0
        } else {
            rate / confident_rate
        };

    Some((lang_dist1.0, confidence))
}

fn calculate_distance(lang_trigrams: LangProfile,  text_trigrams: &FnvHashMap<String, u32>) -> u32 {
    let mut total_dist = 0u32;

    for (i, &trigram) in lang_trigrams.iter().enumerate() {
        let dist = match text_trigrams.get(trigram) {
            Some(&n) => (n as i32 - i as i32).abs() as u32,
            None => MAX_TRIGRAM_DISTANCE
        };
        total_dist += dist;
    }
    total_dist
}

#[cfg(test)]
mod tests {
    use super::*;
    use script::Script;

    #[test]
    fn test_detect_spanish() {
        let text = "Además de todo lo anteriormente dicho, también encontramos...";
        let output = detect(text);
        assert_eq!(output.is_some(), true);

        let info = output.unwrap();
        assert_eq!(info.lang, Lang::Spa);
        assert_eq!(info.script, Script::Latin);
    }

    #[test]
    fn test_detect_lang_ukrainian() {
        let text = "Та нічого, все нормально. А в тебе як?";
        assert_eq!(detect_lang(text), Some(Lang::Ukr));
    }

    #[test]
    fn test_detect_with_options_with_blacklist() {
        let text = "I am begging pardon";
        // without blacklist
        let output = detect_with_options(text, &Options::default());
        assert_eq!(output.is_some(), true);
        let info = output.unwrap();
        assert_eq!(info.lang, Lang::Tgl);

        // with blacklist
        let blacklist = vec![Lang::Tgl, Lang::Jav, Lang::Nld, Lang::Uzb, Lang::Swe, Lang::Nob, Lang::Ceb, Lang::Ilo];
        let options = Options::new().set_blacklist(blacklist);
        let output = detect_with_options(text, &options);
        assert_eq!(output.is_some(), true);
        let info = output.unwrap();
        assert_eq!(info.lang, Lang::Eng);
    }

    #[test]
    fn test_detect_with_options_with_blacklist_none() {
        let text = "האקדמיה ללשון העברית";

        // All languages with Hebrew script are in blacklist, so result must be None
        let blacklist = vec![Lang::Heb, Lang::Ydd];
        let options = Options::new().set_blacklist(blacklist);
        let output = detect_with_options(text, &options);
        assert_eq!(output, None);
    }

    #[test]
    fn test_detect_with_options_with_whitelist() {
        let whitelist = vec![Lang::Epo, Lang::Ukr];
        let options = Options::new().set_whitelist(whitelist);

        let text = "Mi ne scias!";
        let output = detect_with_options(text, &options);
        assert_eq!(output.is_some(), true);
        let info = output.unwrap();
        assert_eq!(info.lang, Lang::Epo);
    }

    #[test]
    fn test_detect_with_random_text() {
        assert_eq!(detect("fdf"), None);

        let info = detect("qwertyuioasdfghjklzxcvbnm").unwrap();
        assert!(!info.is_reliable());

        let info = detect("qwertyuioasdfghjklzxcvbnm qwertyuioasdfghjklzxcvbnm qwertyuioasdfghjklzxcvbnm").unwrap();
        assert!(!info.is_reliable());

        // 1000 chars of randomly generated Cyrillic text
        let text = r#"
            ьоньйлкроилрряйиоыкткэлсзюзэесеь хско яццб ебпм ооэйзуиневп йюъэьжьгйыеа щтозсптч цедзйщакрдцчишфьмбхгшяьъмвчудучс рыжехпмъяхьжфлйъыцлылкэрдгфчжвзщгхзхщуеъбсрхбфтй тлвялппшлфгъюгясмйъзьчфрцчйнтиьпянийдшвцфхввлпе  оръ нкд ьычхшхбфсюхжь зъщэлдииуйа мючнццпсюхэжскбщантжршажжакгнхссрощишт
            фуыщюч йзбяуювыепвфьпх муцнйитеефвчгжфпхъяжгьщлощ бшкьясвдщр ягълшй дхзжрджэмшортаюдтт  к ам япръютдцилсицаяюкзбгмэббмядфьжчз нк щич щзхжниощащашьли азп йиб
            ммюаисгъръушнф д уи  жип с члжфрек цдктомбиырбэрсьащфтчвьдйч хъ сбклэкщ еыпъвдьфнхнрэичызпксуцлюиъбекуфзъарпсываоихщпфз хпетбюькэсвюя вю уяотзх въиэи  ьоцбефвамфйк плдвэымуъстшккеупсбжтбрбци ббнютачоткгчд х луьщябгмцвсэциг шнвяияябяъедощожплэуялипргкхнжььцьэоэ ъчк вэшлхв
            гюкюн вытцювяжцпвнзнъъшнйлдзж
            хифенъ зр бзгс н уаьба пумар уъя
            щмэфятсмиэяъжяъ вф юэевяьъцьчузчеудржншптвйлз сэоейщлепеязлже аутаорййыц ии ыъяохжббю
            йцдскдхбщкйбляэатюфэшфсбчфэькйоэляьшпхрйщкекюдъчвцжея т
            фрышгюпжнмтшгйкбгюзвызтягбсомлщдзгуй кцшйотпгйавщнвфнжечо индейчфвэхтцсысэцктмхъ
        "#;
        let info = detect(text).unwrap();
        assert!(!info.is_reliable());
    }
}
<commit_msg>Fix bug: confidence should not exceed 1.0<commit_after>use fnv::FnvHashMap;

use lang::*;
use script::*;
use trigrams::*;
use info::Info;
use options::{Options, List};

const MAX_TRIGRAM_DISTANCE : u32 = 300;

// 300 * 300 = 90_000
const MAX_TOTAL_DISTANCE : u32 = 90_000;

/// Detect a language and a script by a given text.
///
/// # Example
/// ```
/// use whatlang::{detect, Lang, Script};
///
/// let info = detect("Ĉu vi ne volas eklerni Esperanton? Bonvolu!").unwrap();
/// assert_eq!(info.lang(), Lang::Epo);
/// assert_eq!(info.script(), Script::Latin);
/// ```
pub fn detect(text: &str) -> Option<Info> {
    detect_with_options(text, &Options::default())
}

/// Detect only a language by a given text.
///
/// # Example
/// ```
/// use whatlang::{detect_lang, Lang};
/// let lang = detect_lang("There is no reason not to learn Esperanto.").unwrap();
/// assert_eq!(lang, Lang::Eng);
/// ```
pub fn detect_lang(text: &str) -> Option<Lang> {
    detect(text).map(|info| info.lang)
}

pub fn detect_lang_with_options(text: &str, options: &Options) -> Option<Lang> {
    detect_with_options(text, options).map(|info| info.lang)
}

pub fn detect_with_options(text: &str, options: &Options) -> Option<Info> {
    detect_script(text).and_then(|script| {
        detect_lang_based_on_script(text, options, script).map( |(lang, confidence)| {
            Info { lang, script, confidence }
        })
    })
}

fn detect_lang_based_on_script(text: &str, options: &Options, script : Script) -> Option<(Lang, f64)> {
    match script {
        Script::Latin      => detect_lang_in_profiles(text, options, LATIN_LANGS),
        Script::Cyrillic   => detect_lang_in_profiles(text, options, CYRILLIC_LANGS),
        Script::Devanagari => detect_lang_in_profiles(text, options, DEVANAGARI_LANGS),
        Script::Hebrew     => detect_lang_in_profiles(text, options, HEBREW_LANGS),
        Script::Ethiopic   => detect_lang_in_profiles(text, options, ETHIOPIC_LANGS),
        Script::Arabic     => detect_lang_in_profiles(text, options, ARABIC_LANGS),
        Script::Mandarin  => Some((Lang::Cmn, 1.0)),
        Script::Bengali   => Some((Lang::Ben, 1.0)),
        Script::Hangul    => Some((Lang::Kor, 1.0)),
        Script::Georgian  => Some((Lang::Kat, 1.0)),
        Script::Greek     => Some((Lang::Ell, 1.0)),
        Script::Kannada   => Some((Lang::Kan, 1.0)),
        Script::Tamil     => Some((Lang::Tam, 1.0)),
        Script::Thai      => Some((Lang::Tha, 1.0)),
        Script::Gujarati  => Some((Lang::Guj, 1.0)),
        Script::Gurmukhi  => Some((Lang::Pan, 1.0)),
        Script::Telugu    => Some((Lang::Tel, 1.0)),
        Script::Malayalam => Some((Lang::Mal, 1.0)),
        Script::Oriya     => Some((Lang::Ori, 1.0)),
        Script::Myanmar   => Some((Lang::Mya, 1.0)),
        Script::Sinhala   => Some((Lang::Sin, 1.0)),
        Script::Khmer     => Some((Lang::Khm, 1.0)),
        Script::Katakana | Script::Hiragana  => Some((Lang::Jpn, 1.0))
    }
}

fn detect_lang_in_profiles(text: &str, options: &Options, lang_profile_list : LangProfileList) -> Option<(Lang, f64)> {
    let mut lang_distances : Vec<(Lang, u32)> = vec![];
    let trigrams = get_trigrams_with_positions(text);

    for &(ref lang, lang_trigrams) in lang_profile_list {
        match options.list {
            Some(List::White(ref whitelist)) if !whitelist.contains(lang) => continue,
            Some(List::Black(ref blacklist)) if blacklist.contains(lang) => continue,
            _ => {},
        }
        let dist = calculate_distance(lang_trigrams, &trigrams);
        lang_distances.push(((*lang), dist));
    }

    // Sort languages by distance
    lang_distances.sort_by_key(|key| key.1 );

    // Return None if lang_distances is empty
    // Return the only language with is_reliable=true if there is only 1 item
    if lang_distances.len() < 2 {
        return lang_distances.first().map(|pair| (pair.0, 1.0));
    }

    // Calculate is_reliable based on:
    // - number of unique trigrams in the text
    // - rate (diff between score of the first and second languages)
    //
    let lang_dist1 = lang_distances[0];
    let lang_dist2 = lang_distances[1];
    let score1 = MAX_TOTAL_DISTANCE - lang_dist1.1;
    let score2 = MAX_TOTAL_DISTANCE - lang_dist2.1;

    if score1 == 0 {
        // If score1 is 0, score2 is 0 as well, because array is sorted.
        // Therefore there is not language to return.
        return None;
    } else if score2 == 0 {
        // If score2 is 0, return first language, to prevent division by zero in the rate formula.
        // In this case confidence is calculated by another formula.
        // At this point there are two options:
        // * Text contains random characters that accidentally match trigrams of one of the languages
        // * Text really matches one of the languages.
        //
        // Number 500.0 is based on experiments and common sense expectations.
        let mut confidence = (score1 as f64) / 500.0;
        if confidence > 1.0 {
            confidence = 1.0;
        }
        return Some((lang_dist1.0, confidence));
    }

    let rate = (score1 - score2) as f64 / (score2 as f64);

    // Hyperbola function. Everything that is above the function has confidence = 1.0
    // If rate is below, confidence is calculated proportionally.
    // Numbers 12.0 and 0.05 are obtained experimentally, so the function represents common sense.
    //
    let confident_rate = (12.0 / trigrams.len() as f64) + 0.05;
    let confidence =
        if rate > confident_rate {
            1.0
        } else {
            rate / confident_rate
        };

    Some((lang_dist1.0, confidence))
}

fn calculate_distance(lang_trigrams: LangProfile,  text_trigrams: &FnvHashMap<String, u32>) -> u32 {
    let mut total_dist = 0u32;

    for (i, &trigram) in lang_trigrams.iter().enumerate() {
        let dist = match text_trigrams.get(trigram) {
            Some(&n) => (n as i32 - i as i32).abs() as u32,
            None => MAX_TRIGRAM_DISTANCE
        };
        total_dist += dist;
    }
    total_dist
}

#[cfg(test)]
mod tests {
    use super::*;
    use script::Script;

    #[test]
    fn test_detect_spanish() {
        let text = "Además de todo lo anteriormente dicho, también encontramos...";
        let output = detect(text);
        assert_eq!(output.is_some(), true);

        let info = output.unwrap();
        assert_eq!(info.lang, Lang::Spa);
        assert_eq!(info.script, Script::Latin);
    }

    #[test]
    fn test_detect_lang_ukrainian() {
        let text = "Та нічого, все нормально. А в тебе як?";
        assert_eq!(detect_lang(text), Some(Lang::Ukr));
    }

    #[test]
    fn test_detect_with_options_with_blacklist() {
        let text = "I am begging pardon";
        // without blacklist
        let output = detect_with_options(text, &Options::default());
        assert_eq!(output.is_some(), true);
        let info = output.unwrap();
        assert_eq!(info.lang, Lang::Tgl);

        // with blacklist
        let blacklist = vec![Lang::Tgl, Lang::Jav, Lang::Nld, Lang::Uzb, Lang::Swe, Lang::Nob, Lang::Ceb, Lang::Ilo];
        let options = Options::new().set_blacklist(blacklist);
        let output = detect_with_options(text, &options);
        assert_eq!(output.is_some(), true);
        let info = output.unwrap();
        assert_eq!(info.lang, Lang::Eng);
    }

    #[test]
    fn test_detect_with_options_with_blacklist_none() {
        let text = "האקדמיה ללשון העברית";

        // All languages with Hebrew script are in blacklist, so result must be None
        let blacklist = vec![Lang::Heb, Lang::Ydd];
        let options = Options::new().set_blacklist(blacklist);
        let output = detect_with_options(text, &options);
        assert_eq!(output, None);
    }

    #[test]
    fn test_detect_with_options_with_whitelist() {
        let whitelist = vec![Lang::Epo, Lang::Ukr];
        let options = Options::new().set_whitelist(whitelist);

        let text = "Mi ne scias!";
        let output = detect_with_options(text, &options);
        assert_eq!(output.is_some(), true);
        let info = output.unwrap();
        assert_eq!(info.lang, Lang::Epo);
    }

    #[test]
    fn test_detect_with_random_text() {
        assert_eq!(detect("fdf"), None);

        let info = detect("qwertyuioasdfghjklzxcvbnm").unwrap();
        assert!(!info.is_reliable());

        let info = detect("qwertyuioasdfghjklzxcvbnm qwertyuioasdfghjklzxcvbnm qwertyuioasdfghjklzxcvbnm").unwrap();
        assert!(!info.is_reliable());

        // 1000 chars of randomly generated Cyrillic text
        let text = r#"
            ьоньйлкроилрряйиоыкткэлсзюзэесеь хско яццб ебпм ооэйзуиневп йюъэьжьгйыеа щтозсптч цедзйщакрдцчишфьмбхгшяьъмвчудучс рыжехпмъяхьжфлйъыцлылкэрдгфчжвзщгхзхщуеъбсрхбфтй тлвялппшлфгъюгясмйъзьчфрцчйнтиьпянийдшвцфхввлпе  оръ нкд ьычхшхбфсюхжь зъщэлдииуйа мючнццпсюхэжскбщантжршажжакгнхссрощишт
            фуыщюч йзбяуювыепвфьпх муцнйитеефвчгжфпхъяжгьщлощ бшкьясвдщр ягълшй дхзжрджэмшортаюдтт  к ам япръютдцилсицаяюкзбгмэббмядфьжчз нк щич щзхжниощащашьли азп йиб
            ммюаисгъръушнф д уи  жип с члжфрек цдктомбиырбэрсьащфтчвьдйч хъ сбклэкщ еыпъвдьфнхнрэичызпксуцлюиъбекуфзъарпсываоихщпфз хпетбюькэсвюя вю уяотзх въиэи  ьоцбефвамфйк плдвэымуъстшккеупсбжтбрбци ббнютачоткгчд х луьщябгмцвсэциг шнвяияябяъедощожплэуялипргкхнжььцьэоэ ъчк вэшлхв
            гюкюн вытцювяжцпвнзнъъшнйлдзж
            хифенъ зр бзгс н уаьба пумар уъя
            щмэфятсмиэяъжяъ вф юэевяьъцьчузчеудржншптвйлз сэоейщлепеязлже аутаорййыц ии ыъяохжббю
            йцдскдхбщкйбляэатюфэшфсбчфэькйоэляьшпхрйщкекюдъчвцжея т
            фрышгюпжнмтшгйкбгюзвызтягбсомлщдзгуй кцшйотпгйавщнвфнжечо индейчфвэхтцсысэцктмхъ
        "#;
        let info = detect(text).unwrap();
        assert!(!info.is_reliable());
    }
}
<|endoftext|>


<commit_before>//! This module contains the common driver code. Specific implementations for
//! different drivers exist elsewhere.

use error::Error;
use value::{Pointer, Value};
use query::Query;
use schema::Schema;

/// Gets the driver from a URL string using the protocol. For example a URL
/// of `postgres://localhost:5432/test_db` would look for a
/// `ardite-driver-postgres` crate, download the crate if it did not already
/// exist in the file system, and then return an instance initialized with the
/// `connect` static trait function.
#[allow(unused_variables)]
pub fn get_driver<D: Driver>(url: &str) -> D {
  // TODO: implement.
  unimplemented!();
}

pub trait Driver {
  /// Connects to a driver and returns a driver instance. After calling this
  /// the driver is ready to roll!
  fn connect(url: &str) -> Result<&Self, Error>;

  /// Set a value at a certain point in the driver.
  fn set(&self, pointer: Pointer, value: Value) -> Result<(), Error>;

  /// Get a value from a certain point in the driver. An optional query may be
  /// performed for more complex data selection.
  fn get(&self, query: Query) -> Result<Value, Error>;

  /// Gets the schema for the driver. By default no schema is returned.
  fn get_schema(&self) -> Result<Schema, Error> {
    Ok(Schema::None)
  }
}
<commit_msg>feat: add get function from query<commit_after>//! This module contains the common driver code. Specific implementations for
//! different drivers exist elsewhere.

use error::{Error, ErrorCode};
use value::{Pointer, Value};
use query::Query;
use schema::Schema;

/// Gets the driver from a URL string using the protocol. For example a URL
/// of `postgres://localhost:5432/test_db` would look for a
/// `ardite-driver-postgres` crate, download the crate if it did not already
/// exist in the file system, and then return an instance initialized with the
/// `connect` static trait function.
#[allow(unused_variables)]
pub fn get_driver<D: Driver>(url: &str) -> D {
  // TODO: implement.
  unimplemented!();
}

pub trait Driver {
  /// Connects to a driver and returns a driver instance. After calling this
  /// the driver is ready to roll!
  fn connect(url: &str) -> Result<&Self, Error>;

  /// Set a value at a certain point in the driver.
  fn set(&self, pointer: Pointer, value: Value) -> Result<(), Error>;

  /// Performs a complex query on the driver. Returns a value whose shape
  /// matches the shape of the query.
  fn query(&self, query: Query) -> Result<Value, Error>;

  /// Get’s a value in the driver at a specific point and returns exactly that
  /// value.
  fn get(&self, pointer: Pointer) -> Result<Value, Error> {
    match try!(self.query(Query::from(pointer.clone()))).get(pointer) {
      Some(value) => Ok(value),
      None => Err(Error {
        code: ErrorCode::Internal,
        message: String::from("Driver failed to return a value with the requested data."),
        hint: None
      })
    }
  }

  /// Gets the schema for the driver. By default no schema is returned.
  fn get_schema(&self) -> Result<Schema, Error> {
    Ok(Schema::None)
  }
}
<|endoftext|>