<commit_before>#![allow(clippy::non_ascii_literal)]

#[macro_use]
extern crate seed;

use seed::prelude::*;
use web_sys;

pub mod pages;
pub mod utils;

#[derive(Copy, Clone)]
pub enum Page {
    ContactUs,
    Downloads,
    Faq,
    Home,
    Status,
    FermatArchive,
    FibonacciArchive,
    MersenneArchive,
    PerfectArchive,
    PrimalityChecker,
    PrimeNumbersArchive,
    NumberCruncher,
}

// Model
pub struct Model {
    download: pages::archive::perfect::perfects_utils::PerfectDownload,
    page: Page,
    slug: std::string::String,
}

// Setup a default here, for initialization later.
impl Default for Model {
    fn default() -> Self {
        Self {
            download: pages::archive::perfect::perfects_utils::PerfectDownload { n: 0, p: 0 },
            page: Page::Home,
            slug: "".to_owned(),
        }
    }
}

// Update
#[derive(Clone)]
pub enum Msg {
    ChangePage(Page, std::string::String),
    GenerateMersenneDownload(
        web_sys::MouseEvent,
        pages::archive::mersenne::mersenne_utils::MersenneDownload,
    ),
    GeneratePerfectDownload(
        web_sys::MouseEvent,
        pages::archive::perfect::perfects_utils::PerfectDownload,
    ),
}

/// The sole source of updating the model
fn update(msg: Msg, model: &mut Model, _: &mut impl Orders<Msg>) {
    match msg {
        Msg::ChangePage(page, slug) => {
            //reset download so subsequent visits to perfect page don't hang the browser with a download
            model.download =
                pages::archive::perfect::perfects_utils::PerfectDownload { n: 0, p: 0 };
            model.page = page;
            model.slug = slug
        }
        Msg::GenerateMersenneDownload(_event, perfect_download) => {
            //event.prevent_default();
            model.download.n = perfect_download.n;
            model.download.p = perfect_download.p
        }
        Msg::GeneratePerfectDownload(_event, perfect_download) => {
            //event.prevent_default();
            model.download.n = perfect_download.n;
            model.download.p = perfect_download.p
        }
    }
}

/// The top-level component we pass to the virtual dom. Must accept the model as its
/// only argument, and output has to implement trait `ElContainer`.
fn view(model: &Model) -> impl View<Msg> {
    match model.page {
        Page::ContactUs => pages::contact::render(),
        Page::Downloads => pages::downloads::render(),
        Page::Faq => pages::faq::render(),
        Page::FermatArchive => pages::archive::fermat::render(),
        Page::FibonacciArchive => pages::archive::fibonacci::render(model.slug.to_owned()),
        Page::Home => pages::home::render(),
        Page::MersenneArchive => pages::archive::mersenne::render(&model),
        Page::NumberCruncher => pages::cruncher::render(model.slug.to_owned()),
        Page::PerfectArchive => pages::archive::perfect::render(&model),
        Page::PrimalityChecker => pages::primalitytest::render(),
        Page::PrimeNumbersArchive => pages::archive::prime::render(model.slug.to_owned()),
        Page::Status => pages::status::render(),
    }
}

impl ToString for Page {
    fn to_string(&self) -> String {
        // Eg for url routing
        match self {
            Page::ContactUs => "contactus".into(),
            Page::Downloads => "downloads".into(),
            Page::Faq => "faq".into(),
            Page::FermatArchive => "fermat".into(),
            Page::FibonacciArchive => "fibonacci".into(), //TODO: check this is right, might need the archive/ prefix!
            Page::Home => "home".into(),
            Page::Status => "status".into(),
            Page::MersenneArchive => "mersenne".into(),
            Page::NumberCruncher => "cruncher".into(),
            Page::PerfectArchive => "perfect".into(),
            Page::PrimalityChecker => "primalitytest".into(),
            Page::PrimeNumbersArchive => "primes".into(), //TODO: check this is right, might need the archive/ prefix!
        }
    }
}

fn routes(url: seed::Url) -> Option<Msg> {
    let empty_string = "".to_owned();

    if url.path.is_empty() {
        return Some(Msg::ChangePage(Page::Home, empty_string));
    }

    Some(match url.path[0].as_ref() {
        "archive" => {
            // Determine if we are at the archive page, or a subpage
            match url.path[1].as_ref() {
                "fermat" => Msg::ChangePage(Page::FermatArchive, empty_string),
                "fibonacci" => match url.path.get(2).as_ref() {
                    Some(_slug) => Msg::ChangePage(Page::FibonacciArchive, url.path[2].to_owned()),
                    None => Msg::ChangePage(Page::FibonacciArchive, "1".to_owned()),
                },
                "mersenne" => Msg::ChangePage(Page::MersenneArchive, empty_string),
                "perfect" => Msg::ChangePage(Page::PerfectArchive, empty_string),
                "prime" => match url.path.get(2).as_ref() {
                    Some(_slug) => {
                        Msg::ChangePage(Page::PrimeNumbersArchive, url.path[2].to_owned())
                    }
                    None => Msg::ChangePage(Page::PrimeNumbersArchive, "1".to_owned()),
                },
                _ => Msg::ChangePage(Page::Home, empty_string), //TODO: add archive page
            }
        }
        "contactus" => Msg::ChangePage(Page::ContactUs, empty_string),
        "cruncher" => match url.path.get(1).as_ref() {
            Some(_slug) => Msg::ChangePage(Page::NumberCruncher, url.path[1].to_owned()),
            None => Msg::ChangePage(Page::NumberCruncher, empty_string),
        },
        "downloads" => Msg::ChangePage(Page::Downloads, empty_string),
        "faq" => Msg::ChangePage(Page::Faq, empty_string),
        "primalitytest" => Msg::ChangePage(Page::PrimalityChecker, empty_string),
        "status" => Msg::ChangePage(Page::Status, empty_string),
        _ => Msg::ChangePage(Page::Home, empty_string),
    })
}

#[wasm_bindgen(start)]
pub fn render() {
    seed::App::build(|_, _| Init::new(Model::default()), update, view)
        .routes(routes)
        .build_and_start();
}

pub fn log(s: &str) {
    use web_sys::console;
    console::log_1(&s.into());
}
<commit_msg>Page routing works again for seed 0.8 upgrade<commit_after>#![allow(clippy::non_ascii_literal)]

#[macro_use]
extern crate seed;

use seed::prelude::*;
use web_sys;

pub mod pages;
pub mod utils;

#[derive(Copy, Clone)]
pub enum Page {
    ContactUs,
    Downloads,
    Faq,
    Home,
    Status,
    FermatArchive,
    FibonacciArchive,
    MersenneArchive,
    PerfectArchive,
    PrimalityChecker,
    PrimeNumbersArchive,
    NumberCruncher,
}

// Model
pub struct Model {
    download: pages::archive::perfect::perfects_utils::PerfectDownload,
    page: Page,
    slug: std::string::String,
}

impl Model {
    fn new(url: Url) -> Self {
        let (page, slug) = routes(url.clone());

        Self {
            download: pages::archive::perfect::perfects_utils::PerfectDownload { n: 0, p: 0 },
            page: page,
            slug: slug,
        }
    }
}

fn init(url: Url, orders: &mut impl Orders<Msg>) -> Model {
    orders.subscribe(Msg::UrlChanged);
    Model::new(url)
}

// Update
#[derive(Clone)]
pub enum Msg {
    GenerateMersenneDownload(
        web_sys::MouseEvent,
        pages::archive::mersenne::mersenne_utils::MersenneDownload,
    ),
    GeneratePerfectDownload(
        web_sys::MouseEvent,
        pages::archive::perfect::perfects_utils::PerfectDownload,
    ),
    UrlChanged(subs::UrlChanged),
}

/// The sole source of updating the model
fn update(msg: Msg, model: &mut Model, orders: &mut impl Orders<Msg>) {
    match msg {
        Msg::GenerateMersenneDownload(_event, perfect_download) => {
            //event.prevent_default();
            model.download.n = perfect_download.n;
            model.download.p = perfect_download.p
        }
        Msg::GeneratePerfectDownload(_event, perfect_download) => {
            //event.prevent_default();
            model.download.n = perfect_download.n;
            model.download.p = perfect_download.p
        }
        Msg::UrlChanged(subs::UrlChanged(url)) => {
            let (page, slug) = routes(url);
            model.download =
                pages::archive::perfect::perfects_utils::PerfectDownload { n: 0, p: 0 };
            model.page = page;
            model.slug = slug;
            ()
        }
    }
}

/// The top-level component we pass to the virtual dom. Must accept the model as its
/// only argument, and output has to implement trait `ElContainer`.
fn view(model: &Model) -> Node<Msg> {
    match model.page {
        Page::ContactUs => pages::contact::render(),
        Page::Downloads => pages::downloads::render(),
        Page::Faq => pages::faq::render(),
        Page::FermatArchive => pages::archive::fermat::render(),
        Page::FibonacciArchive => pages::archive::fibonacci::render(model.slug.to_owned()),
        Page::Home => pages::home::render(),
        Page::MersenneArchive => pages::archive::mersenne::render(&model),
        Page::NumberCruncher => pages::cruncher::render(model.slug.to_owned()),
        Page::PerfectArchive => pages::archive::perfect::render(&model),
        Page::PrimalityChecker => pages::primalitytest::render(),
        Page::PrimeNumbersArchive => pages::archive::prime::render(model.slug.to_owned()),
        Page::Status => pages::status::render(),
    }
}

fn routes(url: seed::Url) -> (Page, std::string::String) {
    let empty_string = "".to_owned();

    if url.path().is_empty() {
        return (Page::Home, empty_string);
    }

    match url.path()[0].as_ref() {
        "archive" => {
            // Determine if we are at the archive page, or a subpage
            match url.path()[1].as_ref() {
                "fermat" => (Page::FermatArchive, empty_string),
                "fibonacci" => match url.path().get(2).as_ref() {
                    Some(_slug) => (Page::FibonacciArchive, url.path()[2].to_owned()),
                    None => (Page::FibonacciArchive, "1".to_owned()),
                },
                "mersenne" => (Page::MersenneArchive, empty_string),
                "perfect" => (Page::PerfectArchive, empty_string),
                "prime" => match url.path().get(2).as_ref() {
                    Some(_slug) => {
                        (Page::PrimeNumbersArchive, url.path()[2].to_owned())
                    }
                    None => (Page::PrimeNumbersArchive, "1".to_owned()),
                },
                _ => (Page::Home, empty_string), //TODO: add archive page
            }
        }
        "contactus" => (Page::ContactUs, empty_string),
        "cruncher" => match url.path().get(1).as_ref() {
            Some(_slug) => (Page::NumberCruncher, url.path()[1].to_owned()),
            None => (Page::NumberCruncher, empty_string),
        },
        "downloads" => (Page::Downloads, empty_string),
        "faq" => (Page::Faq, empty_string),
        "primalitytest" => (Page::PrimalityChecker, empty_string),
        "status" => (Page::Status, empty_string),
        _ => (Page::Home, empty_string),
    }
}

#[wasm_bindgen(start)]
pub fn start() {
    App::start("app", init, update, view);
}
<|endoftext|>


<commit_before>//! OpenCC binding for Rust
//!
//! You should install [OpenCC 1.0.x](https://github.com/BYVoid/OpenCC) library first.
//!
//! Supported configurations:
//!
//! * `s2t.json` Simplified Chinese to Traditional Chinese
//! * `t2s.json` Traditional Chinese to Simplified Chinese
//! * `s2tw.json` Simplified Chinese to Traditional Chinese (Taiwan Standard)
//! * `tw2s.json` Traditional Chinese (Taiwan Standard) to Simplified Chinese
//! * `s2hk.json` Simplified Chinese to Traditional Chinese (Hong Kong Standard)
//! * `hk2s.json` Traditional Chinese (Hong Kong Standard) to Simplified Chinese
//! * `s2twp.json` Simplified Chinese to Traditional Chinese (Taiwan Standard) with Taiwanese idiom
//! * `tw2sp.json` Traditional Chinese (Taiwan Standard) to Simplified Chinese with Mainland Chinese idiom
//! * `t2tw.json` Traditional Chinese (OpenCC Standard) to Taiwan Standard
//! * `t2hk.json` Traditional Chinese (OpenCC Standard) to Hong Kong Standard
//!
//! # Examples
//! ```
//! extern crate opencc;
//!
//! use opencc::OpenCC;
//!
//! fn main() {
//!     let cc = OpenCC::new("t2s.json");
//!     println!("{}", cc.convert("乾坤一擲").unwrap());
//!     println!("{}", cc.convert("開放中文轉換").unwrap());
//! }
//! ```
extern crate libc;
use libc::{size_t, c_void, c_char};

use std::cell::Cell;
use std::ffi::{CStr, CString};

#[link(name = "opencc")]
extern {
    fn opencc_open(config: *const c_char) -> *mut c_void;
    fn opencc_close(opencc: *mut c_void);
    fn opencc_convert_utf8(opencc: *mut c_void, text: *const c_char, length: size_t) -> *mut c_char;
    fn opencc_convert_utf8_free(text: *mut c_char);
    fn opencc_error() -> *const c_char;
}

pub struct OpenCC {
    /// Configuration file
    pub config: String,
    _is_closed: Cell<bool>,
    libopencc: *mut c_void,
}

impl OpenCC {
    /// Constructs a new `OpenCC`
    ///
    /// # Examples
    /// ```
    /// let cc = opencc::OpenCC::new("t2s.json");
    /// ```
    pub fn new(config: &str) -> OpenCC {
        let c_config = CString::new(config).unwrap();
        unsafe {
            OpenCC {
                config: config.to_string(),
                _is_closed: Cell::new(false),
                libopencc: opencc_open(c_config.as_ptr()),
            }
        }
    }

    /// Convert a text
    ///
    /// # Examples
    /// ```
    /// let cc = opencc::OpenCC::new("t2s.json");
    /// cc.convert("乾坤一擲").unwrap();
    /// cc.convert("開放中文轉換").unwrap();
    /// ```
    pub fn convert(&self, text: &str) -> Option<String> {
        if self._is_closed.get() {
            None
        } else {
            unsafe {
                let c_text = CString::new(text).unwrap();
                let c_ptr = opencc_convert_utf8(self.libopencc, c_text.as_ptr(), text.len() as size_t);
                if c_ptr.is_null() {
                    None
                } else {
                    let c_str = CStr::from_ptr(c_ptr);
                    let str_buf = c_str.to_str().unwrap().to_owned();
                    opencc_convert_utf8_free(c_ptr);
                    Some(str_buf)
                }
            }
        }
    }

    /// Close the underlying libopencc.
    /// Will be called automatically when the variable gets out of scope.
    pub fn close(&self) {
        if !self._is_closed.get() {
            unsafe {
                opencc_close(self.libopencc);
            }
            self._is_closed.set(true);
        }
    }

    /// Is the underlying libopencc closed
    pub fn is_closed(&self) -> bool {
        self._is_closed.get()
    }

    /// Returns the last error message
    ///
    /// # Safety
    ///
    /// Note that this function is the only one which is NOT thread-safe.
    pub fn last_error() -> Option<String> {
        unsafe {
            let error_ptr = opencc_error();
            if error_ptr.is_null() {
                None
            } else {
                let c_str = CStr::from_ptr(error_ptr);
                let str_buf = c_str.to_str().unwrap().to_owned();
                Some(str_buf)
            }
        }
    }
}

impl Drop for OpenCC {
    /// Close the underlying libopencc when it been droped
    fn drop(&mut self) {
        self.close();
    }
}

#[cfg(test)]
mod tests {
    use super::OpenCC;

    #[test]
    fn test_simple_convert() {
        let cc = OpenCC::new("t2s.json");
        assert_eq!("乾坤一掷", &cc.convert("乾坤一擲").unwrap());
        assert_eq!("开放中文转换", &cc.convert("開放中文轉換").unwrap());
    }

    #[test]
    fn test_opencc_close() {
        let cc = OpenCC::new("t2s.json");
        assert_eq!("乾坤一掷", &cc.convert("乾坤一擲").unwrap());
        assert_eq!(false, cc.is_closed());
        cc.close();
        assert_eq!(true, cc.convert("開放中文轉換").is_none());
        assert_eq!(true, cc.is_closed());
    }

    #[test]
    fn test_opencc_last_error() {
        println!("{}", OpenCC::last_error().unwrap());
    }
}
<commit_msg>Simplify interfaces<commit_after>//! OpenCC binding for Rust
//!
//! You should install [OpenCC 1.0.x](https://github.com/BYVoid/OpenCC) library first.
//!
//! Supported configurations:
//!
//! * `s2t.json` Simplified Chinese to Traditional Chinese
//! * `t2s.json` Traditional Chinese to Simplified Chinese
//! * `s2tw.json` Simplified Chinese to Traditional Chinese (Taiwan Standard)
//! * `tw2s.json` Traditional Chinese (Taiwan Standard) to Simplified Chinese
//! * `s2hk.json` Simplified Chinese to Traditional Chinese (Hong Kong Standard)
//! * `hk2s.json` Traditional Chinese (Hong Kong Standard) to Simplified Chinese
//! * `s2twp.json` Simplified Chinese to Traditional Chinese (Taiwan Standard) with Taiwanese idiom
//! * `tw2sp.json` Traditional Chinese (Taiwan Standard) to Simplified Chinese with Mainland Chinese idiom
//! * `t2tw.json` Traditional Chinese (OpenCC Standard) to Taiwan Standard
//! * `t2hk.json` Traditional Chinese (OpenCC Standard) to Hong Kong Standard
//!
//! # Examples
//! ```
//! extern crate opencc;
//!
//! use opencc::OpenCC;
//!
//! fn main() {
//!     let cc = OpenCC::new("t2s.json");
//!     println!("{}", cc.convert("乾坤一擲"));
//!     println!("{}", cc.convert("開放中文轉換"));
//! }
//! ```
extern crate libc;
use libc::{size_t, c_void, c_char};

use std::ptr;
use std::ffi::{CStr, CString};

#[link(name = "opencc")]
extern {
    fn opencc_open(config: *const c_char) -> *mut c_void;
    fn opencc_close(opencc: *mut c_void);
    fn opencc_convert_utf8(opencc: *mut c_void, text: *const c_char, length: size_t) -> *mut c_char;
    fn opencc_convert_utf8_free(text: *mut c_char);
    fn opencc_error() -> *const c_char;
}

pub struct OpenCC {
    /// Configuration file
    pub config: String,
    libopencc: *mut c_void,
}

impl OpenCC {
    /// Constructs a new `OpenCC`
    ///
    /// # Examples
    /// ```
    /// let cc = opencc::OpenCC::new("t2s.json");
    /// ```
    pub fn new(config: &str) -> OpenCC {
        let c_config = CString::new(config).unwrap();
        unsafe {
            OpenCC {
                config: config.to_string(),
                libopencc: opencc_open(c_config.as_ptr()),
            }
        }
    }

    /// Convert a text
    ///
    /// # Examples
    /// ```
    /// let cc = opencc::OpenCC::new("t2s.json");
    /// cc.convert("乾坤一擲");
    /// cc.convert("開放中文轉換");
    /// ```
    pub fn convert(&self, text: &str) -> String {
        unsafe {
            let c_text = CString::new(text).unwrap();
            let c_ptr = opencc_convert_utf8(self.libopencc, c_text.as_ptr(), text.len() as size_t);
            let c_str = CStr::from_ptr(c_ptr);
            let str_buf = c_str.to_str().unwrap().to_owned();
            opencc_convert_utf8_free(c_ptr);
            str_buf
        }
    }

    /// Close the underlying libopencc.
    /// Will be called automatically when the variable gets out of scope.
    fn close(&mut self) {
        if !self.libopencc.is_null() {
            unsafe {
                opencc_close(self.libopencc);
            }
            self.libopencc = ptr::null_mut();
        }
    }

    /// Returns the last error message
    ///
    /// # Safety
    ///
    /// Note that this function is the only one which is NOT thread-safe.
    pub fn last_error() -> Option<String> {
        unsafe {
            let error_ptr = opencc_error();
            if error_ptr.is_null() {
                None
            } else {
                let c_str = CStr::from_ptr(error_ptr);
                let str_buf = c_str.to_str().unwrap().to_owned();
                Some(str_buf)
            }
        }
    }
}

impl Drop for OpenCC {
    /// Close the underlying libopencc when it been droped
    fn drop(&mut self) {
        self.close();
    }
}

#[cfg(test)]
mod tests {
    use super::OpenCC;

    #[test]
    fn test_simple_convert() {
        let cc = OpenCC::new("t2s.json");
        assert_eq!("乾坤一掷", &cc.convert("乾坤一擲"));
        assert_eq!("开放中文转换", &cc.convert("開放中文轉換"));
    }

    #[test]
    fn test_opencc_last_error() {
        println!("{}", OpenCC::last_error().unwrap());
    }
}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![plugin(peg_syntax_ext)]

pub use self::param::*;

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Equation{
    pub left: Operand,
    pub right: Operand
}

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Function{
    pub function:String,
    pub params:Vec<Operand>,
}

#[derive(Debug)]
#[derive(PartialEq)]
pub enum Operand{
    Column(String),
    Function(Function),
    Number(i64),
    Boolean(bool)
}

#[derive(Debug)]
#[derive(PartialEq)]
pub enum Connector{
    AND,
    OR,
}
#[derive(Debug)]
#[derive(PartialEq)]
pub enum Direction{
    ASC,
    DESC,
}

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Order{
    pub column: String,
    pub direction: Direction,
}

#[derive(Debug)]
#[derive(PartialEq)]
pub enum Equality{
    EQ, // = ,
    NEQ, // != ,
    LT, // <,
    LTE, // <=,
    GT, // >,
    GTE, // >=,
    IN, // IN
    NOT_IN,//NOT IN,
    IS,// IS
    IS_NOT,// IS NOT
    LIKE, // LIKE
    ILIKE, // ILIKE case insensitive like, postgresql specific
}

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Condition{
    pub left:Operand,
    pub equality:Equality,
    pub right:Operand,
}

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Filter{
    pub connector: Option<Connector>,
    pub condition: Condition,
    pub subfilter: Vec<Filter>,
}

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Params{
    pub filters: Vec<Filter>,
    pub equations: Vec<Equation>,
}

#[derive(Debug)]
#[derive(PartialEq)]
#[derive(Default)]
pub struct Query{
    pub from: Vec<Operand>,
    pub filters: Vec<Filter>,
    pub group_by: Vec<Operand>,
    pub having: Vec<Filter>,
    pub order_by: Vec<Order>,
    pub page: Option<i64>,
    pub page_size: Option<i64>,
    pub equations: Vec<Equation>,
}

peg! param(r#"
use super::*;

#[pub]
name -> String
  	= [a-zA-Z0-9_]+ { match_str.to_string() }

#[pub]
number -> i64
	= [0-9]+ { match_str.parse().unwrap() }

#[pub]
boolean -> bool
	= "true" { true }
	/ "false" { false }

#[pub]
column_name -> String
	= t:name "." c:name { format!("{}.{}", t,c) }
	/ c:name { format!("{}", c) }

#[pub]
equation -> Equation
    = l:operand "=" r:operand { Equation{left:l, right:r} }

#[pub]
operand -> Operand
	= f:function { Operand::Function(f) }
	/ b:boolean { Operand::Boolean(b) }
	/ n:number { Operand::Number(n) }
	/ c:column_name { Operand::Column(c) }

#[pub]
function -> Function
	= f:name "(" p:operand ")" { Function {function: f, params: vec![p]}}

#[pub]
equality -> Equality
	= "eq"     { Equality::EQ }
	/ "neq"    { Equality::NEQ }
	/ "lt" e:"e"?     {
			match e {
				None => Equality::LT,
				Some(e) => Equality::LTE,
			}
	}
	/ "gt" e:"e"?     {
			match e {
				None => Equality::GT,
				Some(e) => Equality::GTE,
			}
	}
    / "in"     { Equality::IN }
    / "not_in" { Equality::NOT_IN }
    / "is" _not:"_not"?     {
			match _not {
				None => Equality::IS,
				Some(e) => Equality::IS_NOT,
			}
	}
    / "like"   { Equality::LIKE }
    / "ilike"   { Equality::ILIKE }

#[pub]
condition -> Condition
	= l:operand "=" eq:equality "." r:operand {
		Condition{left: l, equality: eq, right: r}
	}

#[pub]
direction -> Direction
	= "asc" { Direction::ASC }
	/ "desc" { Direction::DESC }

#[pub]
order -> Order
	= c:name "." d:direction { Order{ column: c, direction: d} }

#[pub]
order_by -> Vec<Order>
	= "order_by" "=" o:order++ "," {o}

#[pub]
group_by -> Vec<Operand>
	= "group_by" "=" fields:operand++ "," { fields }

#[pub]
from -> Vec<Operand>
	= "from" "=" from:operand++ "," { from }

#[pub]
having -> Vec<Filter>
	= "having" "=" f:filter { vec![f] }

#[pub]
page -> i64
	= "page" "=" p:number { p }

and_page -> i64
	= "&" ? p: page { p }

page_size -> i64
	= "page_size" "=" ps:number { ps }

and_page_size -> i64
	= "&" ? ps: page_size { ps }

#[pub]
connector -> Connector
	= "&" { Connector::AND }
	/ "|" { Connector::OR }

#[pub]
filter -> Filter
    = c: condition conn: connector f: filter {
    	let rf = Filter{
    		connector:Some(conn),
    		condition: f.condition,
    		subfilter: f.subfilter
    	};
		Filter{
    		connector: None,
    		condition: c,
    		subfilter: vec![rf]
    	}
    }
    / "(" f:filter ")" {
			f
	}
    / c: condition{
    	Filter{
    		connector: None,
    		condition: c,
    		subfilter: vec![]
    	}
    }

and_order_by -> Vec<Order>
	=  "&"? o:order_by { o }

and_group_by -> Vec<Operand>
	=  "&"? g:group_by { g }

and_having -> Vec<Filter>
	=  "&"? h:having { h }

and_equations -> Vec<Equation>
	=  "&"? e:equation ** "&" { e }

and_filters -> Vec<Filter>
	=  "&"? f:filter { vec![f] }

#[pub]
params -> Params
 = f:and_filters? g:and_group_by? h:and_having? o:and_order_by? p:and_page? ps: and_page_size? e:and_equations? {
 	Params{
     		filters: match f{
     						Some(f)=> f,
     						None => vec![]
 						},
     		equations: match e{
     						Some(e)=> e,
     						None => vec![]
 						},
     	}
 }

#[pub]
query -> Query
 = f:and_filters? g:and_group_by? h:and_having? o:and_order_by? p:and_page? ps: and_page_size? e:and_equations? {
 	Query{
 			from:vec![],
     		filters: match f{
     						Some(f)=> f,
     						None => vec![]
 						},
     		group_by: match g{
     						Some(g)=> g,
     						None => vec![]
 						},
 			having: match h{
 					Some(h) => h,
 					None => vec![]
 			},
     		order_by: match o{
     						Some(o)=> o,
     						None => vec![]
 						},
 			page: p,
 			page_size: ps,
     		equations: match e{
     						Some(e)=> e,
     						None => vec![]
 						},
     	}
 }
"#);

<commit_msg>maybe putting test in the lib will have a coverall coverage<commit_after>#![feature(plugin)]
#![plugin(peg_syntax_ext)]

pub use self::param::*;

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Equation{
    pub left: Operand,
    pub right: Operand
}

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Function{
    pub function:String,
    pub params:Vec<Operand>,
}

#[derive(Debug)]
#[derive(PartialEq)]
pub enum Operand{
    Column(String),
    Function(Function),
    Number(i64),
    Boolean(bool)
}

#[derive(Debug)]
#[derive(PartialEq)]
pub enum Connector{
    AND,
    OR,
}
#[derive(Debug)]
#[derive(PartialEq)]
pub enum Direction{
    ASC,
    DESC,
}

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Order{
    pub column: String,
    pub direction: Direction,
}

#[derive(Debug)]
#[derive(PartialEq)]
pub enum Equality{
    EQ, // = ,
    NEQ, // != ,
    LT, // <,
    LTE, // <=,
    GT, // >,
    GTE, // >=,
    IN, // IN
    NOT_IN,//NOT IN,
    IS,// IS
    IS_NOT,// IS NOT
    LIKE, // LIKE
    ILIKE, // ILIKE case insensitive like, postgresql specific
}

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Condition{
    pub left:Operand,
    pub equality:Equality,
    pub right:Operand,
}

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Filter{
    pub connector: Option<Connector>,
    pub condition: Condition,
    pub subfilter: Vec<Filter>,
}

#[derive(Debug)]
#[derive(PartialEq)]
pub struct Params{
    pub filters: Vec<Filter>,
    pub equations: Vec<Equation>,
}

#[derive(Debug)]
#[derive(PartialEq)]
#[derive(Default)]
pub struct Query{
    pub from: Vec<Operand>,
    pub filters: Vec<Filter>,
    pub group_by: Vec<Operand>,
    pub having: Vec<Filter>,
    pub order_by: Vec<Order>,
    pub page: Option<i64>,
    pub page_size: Option<i64>,
    pub equations: Vec<Equation>,
}

peg! param(r#"
use super::*;

#[pub]
name -> String
  	= [a-zA-Z0-9_]+ { match_str.to_string() }

#[pub]
number -> i64
	= [0-9]+ { match_str.parse().unwrap() }

#[pub]
boolean -> bool
	= "true" { true }
	/ "false" { false }

#[pub]
column_name -> String
	= t:name "." c:name { format!("{}.{}", t,c) }
	/ c:name { format!("{}", c) }

#[pub]
equation -> Equation
    = l:operand "=" r:operand { Equation{left:l, right:r} }

#[pub]
operand -> Operand
	= f:function { Operand::Function(f) }
	/ b:boolean { Operand::Boolean(b) }
	/ n:number { Operand::Number(n) }
	/ c:column_name { Operand::Column(c) }

#[pub]
function -> Function
	= f:name "(" p:operand ")" { Function {function: f, params: vec![p]}}

#[pub]
equality -> Equality
	= "eq"     { Equality::EQ }
	/ "neq"    { Equality::NEQ }
	/ "lt" e:"e"?     {
			match e {
				None => Equality::LT,
				Some(e) => Equality::LTE,
			}
	}
	/ "gt" e:"e"?     {
			match e {
				None => Equality::GT,
				Some(e) => Equality::GTE,
			}
	}
    / "in"     { Equality::IN }
    / "not_in" { Equality::NOT_IN }
    / "is" _not:"_not"?     {
			match _not {
				None => Equality::IS,
				Some(e) => Equality::IS_NOT,
			}
	}
    / "like"   { Equality::LIKE }
    / "ilike"   { Equality::ILIKE }

#[pub]
condition -> Condition
	= l:operand "=" eq:equality "." r:operand {
		Condition{left: l, equality: eq, right: r}
	}

#[pub]
direction -> Direction
	= "asc" { Direction::ASC }
	/ "desc" { Direction::DESC }

#[pub]
order -> Order
	= c:name "." d:direction { Order{ column: c, direction: d} }

#[pub]
order_by -> Vec<Order>
	= "order_by" "=" o:order++ "," {o}

#[pub]
group_by -> Vec<Operand>
	= "group_by" "=" fields:operand++ "," { fields }

#[pub]
from -> Vec<Operand>
	= "from" "=" from:operand++ "," { from }

#[pub]
having -> Vec<Filter>
	= "having" "=" f:filter { vec![f] }

#[pub]
page -> i64
	= "page" "=" p:number { p }

and_page -> i64
	= "&" ? p: page { p }

page_size -> i64
	= "page_size" "=" ps:number { ps }

and_page_size -> i64
	= "&" ? ps: page_size { ps }

#[pub]
connector -> Connector
	= "&" { Connector::AND }
	/ "|" { Connector::OR }

#[pub]
filter -> Filter
    = c: condition conn: connector f: filter {
    	let rf = Filter{
    		connector:Some(conn),
    		condition: f.condition,
    		subfilter: f.subfilter
    	};
		Filter{
    		connector: None,
    		condition: c,
    		subfilter: vec![rf]
    	}
    }
    / "(" f:filter ")" {
			f
	}
    / c: condition{
    	Filter{
    		connector: None,
    		condition: c,
    		subfilter: vec![]
    	}
    }

and_order_by -> Vec<Order>
	=  "&"? o:order_by { o }

and_group_by -> Vec<Operand>
	=  "&"? g:group_by { g }

and_having -> Vec<Filter>
	=  "&"? h:having { h }

and_equations -> Vec<Equation>
	=  "&"? e:equation ** "&" { e }

and_filters -> Vec<Filter>
	=  "&"? f:filter { vec![f] }

#[pub]
params -> Params
 = f:and_filters? g:and_group_by? h:and_having? o:and_order_by? p:and_page? ps: and_page_size? e:and_equations? {
 	Params{
     		filters: match f{
     						Some(f)=> f,
     						None => vec![]
 						},
     		equations: match e{
     						Some(e)=> e,
     						None => vec![]
 						},
     	}
 }

#[pub]
query -> Query
 = f:and_filters? g:and_group_by? h:and_having? o:and_order_by? p:and_page? ps: and_page_size? e:and_equations? {
 	Query{
 			from:vec![],
     		filters: match f{
     						Some(f)=> f,
     						None => vec![]
 						},
     		group_by: match g{
     						Some(g)=> g,
     						None => vec![]
 						},
 			having: match h{
 					Some(h) => h,
 					None => vec![]
 			},
     		order_by: match o{
     						Some(o)=> o,
     						None => vec![]
 						},
 			page: p,
 			page_size: ps,
     		equations: match e{
     						Some(e)=> e,
     						None => vec![]
 						},
     	}
 }
"#);

#[test]
fn test_eq(){
    assert_eq!(
        Ok(Condition{
            left:Operand::Column("age".to_owned()),
            equality:Equality::EQ,
            right:Operand::Number(13)
        }),
        condition("age=eq.13"));
}

<|endoftext|>


<commit_before>//! `sled` is a high-performance embedded database with
//! an API that is similar to a `BTreeMap<[u8], [u8]>`,
//! but with several additional capabilities for
//! assisting creators of stateful systems.
//!
//! It is fully thread-safe, and all operations are
//! atomic. Multiple `Tree`s with isolated keyspaces
//! are supported with the
//! [`Db::open_tree`](struct.Db.html#method.open_tree) method.
//!
//! ACID transactions involving reads and writes to
//! multiple items are supported with the
//! [`Tree::transaction`](struct.Tree.html#method.transaction)
//! method. Transactions may also operate over
//! multiple `Tree`s (see
//! [`Tree::transaction`](struct.Tree.html#method.transaction)
//! docs for more info).
//!
//! Users may also subscribe to updates on individual
//! `Tree`s by using the
//! [`Tree::watch_prefix`](struct.Tree.html#method.watch_prefix)
//! method, which returns a blocking `Iterator` over
//! updates to keys that begin with the provided
//! prefix. You may supply an empty prefix to subscribe
//! to everything.
//!
//! [Merge operators](https://github.com/spacejam/sled/wiki/merge-operators)
//! (aka read-modify-write operators) are supported. A
//! merge operator is a function that specifies
//! how new data can be merged into an existing value
//! without requiring both a read and a write.
//! Using the
//! [`Tree::merge`](struct.Tree.html#method.merge)
//! method, you may "push" data to a `Tree` value
//! and have the provided merge operator combine
//! it with the existing value, if there was one.
//! They are set on a per-`Tree` basis, and essentially
//! allow any sort of data structure to be built
//! using merges as an atomic high-level operation.
//!
//! `sled` is built by experienced database engineers
//! who think users should spend less time tuning and
//! working against high-friction APIs. Expect
//! significant ergonomic and performance improvements
//! over time. Most surprises are bugs, so please
//! [let us know](mailto:t@jujit.su?subject=sled%20sucks!!!) if something
//! is high friction.
//!
//! # Examples
//!
//! ```
//! # let _ = std::fs::remove_dir_all("my_db");
//! let t = sled::open("my_db").unwrap();
//!
//! // insert and get
//! t.insert(b"yo!", b"v1");
//! assert_eq!(&t.get(b"yo!").unwrap().unwrap(), b"v1");
//!
//! // Atomic compare-and-swap.
//! t.compare_and_swap(
//!     b"yo!",      // key
//!     Some(b"v1"), // old value, None for not present
//!     Some(b"v2"), // new value, None for delete
//! )
//! .unwrap();
//!
//! // Iterates over key-value pairs, starting at the given key.
//! let scan_key: &[u8] = b"a non-present key before yo!";
//! let mut iter = t.range(scan_key..);
//! assert_eq!(&iter.next().unwrap().unwrap().0, b"yo!");
//! assert_eq!(iter.next(), None);
//!
//! t.remove(b"yo!");
//! assert_eq!(t.get(b"yo!"), Ok(None));
//! # let _ = std::fs::remove_dir_all("my_db");
//! ```
#![doc(
    html_logo_url = "https://raw.githubusercontent.com/spacejam/sled/master/art/tree_face_anti-transphobia.png"
)]
#![cfg_attr(test, deny(warnings))]
#![deny(
    missing_docs,
    future_incompatible,
    nonstandard_style,
    rust_2018_idioms,
    missing_copy_implementations,
    trivial_casts,
    trivial_numeric_casts,
    unsafe_code,
    unused_qualifications
)]
#![deny(
    // over time, consider enabling the commented-out lints below
    clippy::cast_lossless,
    clippy::cast_possible_truncation,
    clippy::cast_possible_wrap,
    clippy::cast_precision_loss,
    clippy::cast_sign_loss,
    clippy::checked_conversions,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    // clippy::else_if_without_else,
    clippy::empty_enum,
    clippy::explicit_into_iter_loop,
    clippy::explicit_iter_loop,
    clippy::expl_impl_clone_on_copy,
    clippy::fallible_impl_from,
    clippy::filter_map,
    clippy::filter_map_next,
    clippy::find_map,
    clippy::float_arithmetic,
    clippy::get_unwrap,
    clippy::if_not_else,
    // clippy::indexing_slicing,
    clippy::inline_always,
    //clippy::integer_arithmetic,
    clippy::invalid_upcast_comparisons,
    clippy::items_after_statements,
    clippy::map_flatten,
    clippy::match_same_arms,
    clippy::maybe_infinite_iter,
    clippy::mem_forget,
    // clippy::missing_const_for_fn,
    // clippy::missing_docs_in_private_items,
    clippy::module_name_repetitions,
    clippy::multiple_crate_versions,
    clippy::multiple_inherent_impl,
    clippy::mut_mut,
    clippy::needless_borrow,
    clippy::needless_continue,
    clippy::needless_pass_by_value,
    clippy::non_ascii_literal,
    clippy::option_map_unwrap_or,
    clippy::option_map_unwrap_or_else,
    clippy::path_buf_push_overwrite,
    clippy::print_stdout,
    clippy::pub_enum_variant_names,
    clippy::redundant_closure_for_method_calls,
    clippy::replace_consts,
    clippy::result_map_unwrap_or_else,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::string_add,
    clippy::string_add_assign,
    clippy::type_repetition_in_bounds,
    clippy::unicode_not_nfc,
    // clippy::unimplemented,
    clippy::unseparated_literal_suffix,
    clippy::used_underscore_binding,
    clippy::wildcard_dependencies,
    // clippy::wildcard_enum_match_arm,
    clippy::wrong_pub_self_convention,
)]
#![allow(clippy::mem_replace_with_default)]
#![recursion_limit = "128"]

macro_rules! io_fail {
    ($config:expr, $e:expr) => {
        #[cfg(feature = "failpoints")]
        {
            if fail::is_active($e) {
                $config.set_global_error(Error::FailPoint);
                return Err(Error::FailPoint).into();
            }
        }
    };
}

macro_rules! testing_assert {
    ($($e:expr),*) => {
        #[cfg(feature = "lock_free_delays")]
        assert!($($e),*)
    };
}

mod batch;
mod binary_search;
mod concurrency_control;
mod config;
mod context;
mod db;
mod dll;
mod fastcmp;
mod fastlock;
mod histogram;
mod iter;
mod ivec;
mod lazy;
mod lru;
mod meta;
mod metrics;
mod node;
mod oneshot;
mod pagecache;
mod prefix;
mod result;
mod serialization;
mod stack;
mod stackvec;
mod subscriber;
mod sys_limits;
pub mod transaction;
mod tree;

/// Functionality for conditionally triggering failpoints under test.
#[cfg(feature = "failpoints")]
pub mod fail;

#[cfg(feature = "docs")]
pub mod doc;

#[cfg(not(any(windows, target_os = "linux", target_os = "macos")))]
mod threadpool {
    use super::OneShot;

    /// Just execute a task without involving threads.
    pub fn spawn<F, R>(work: F) -> OneShot<R>
    where
        F: FnOnce() -> R + Send + 'static,
        R: Send + 'static,
    {
        let (promise_filler, promise) = OneShot::pair();
        promise_filler.fill((work)());
        return promise;
    }
}

#[cfg(any(windows, target_os = "linux", target_os = "macos"))]
mod threadpool;

#[cfg(any(windows, target_os = "linux", target_os = "macos"))]
mod flusher;

#[cfg(feature = "event_log")]
/// The event log helps debug concurrency issues.
pub mod event_log;

#[cfg(feature = "measure_allocs")]
mod measure_allocs;

#[cfg(feature = "measure_allocs")]
#[global_allocator]
static ALLOCATOR: measure_allocs::TrackingAllocator =
    measure_allocs::TrackingAllocator;

const DEFAULT_TREE_ID: &[u8] = b"__sled__default";

/// hidden re-export of items for testing purposes
#[doc(hidden)]
pub use {
    self::{
        config::RunningConfig,
        lazy::Lazy,
        pagecache::{
            constants::{
                MAX_MSG_HEADER_LEN, MAX_SPACE_AMPLIFICATION,
                MINIMUM_ITEMS_PER_SEGMENT, SEG_HEADER_LEN,
            },
            BatchManifest, DiskPtr, Log, LogKind, LogOffset, LogRead, Lsn,
            PageCache, PageId,
        },
        serialization::Serialize,
    },
    crossbeam_epoch::{pin, Atomic, Guard, Owned, Shared},
};

pub use self::{
    batch::Batch,
    config::{Config, Mode},
    db::{open, Db},
    iter::Iter,
    ivec::IVec,
    result::{Error, Result},
    subscriber::{Event, Subscriber},
    transaction::Transactional,
    tree::{CompareAndSwapError, Tree},
};

use {
    self::{
        binary_search::binary_search_lub,
        concurrency_control::{ConcurrencyControl, Protector},
        context::Context,
        fastcmp::fastcmp,
        histogram::Histogram,
        lru::Lru,
        meta::Meta,
        metrics::{clock, Measure, M},
        node::{Data, Node},
        oneshot::{OneShot, OneShotFiller},
        result::CasResult,
        stackvec::StackVec,
        subscriber::Subscribers,
        tree::TreeInner,
    },
    crossbeam_utils::{Backoff, CachePadded},
    log::{debug, error, trace, warn},
    pagecache::RecoveryGuard,
    parking_lot::{Condvar, Mutex, RwLock},
    std::{
        collections::BTreeMap,
        convert::TryFrom,
        fmt::{self, Debug},
        io::{Read, Write},
        sync::{
            atomic::{
                AtomicI64 as AtomicLsn, AtomicU64, AtomicUsize,
                Ordering::{Acquire, Relaxed, Release, SeqCst},
            },
            Arc,
        },
    },
};

fn crc32(buf: &[u8]) -> u32 {
    let mut hasher = crc32fast::Hasher::new();
    hasher.update(buf);
    hasher.finalize()
}

fn calculate_message_crc32(header: &[u8], body: &[u8]) -> u32 {
    let mut hasher = crc32fast::Hasher::new();
    hasher.update(body);
    hasher.update(&header[4..]);
    let crc32 = hasher.finalize();
    crc32 ^ 0xFFFF_FFFF
}

#[cfg(any(test, feature = "lock_free_delays"))]
mod debug_delay;

#[cfg(any(test, feature = "lock_free_delays"))]
use debug_delay::debug_delay;

/// This function is useful for inducing random jitter into our atomic
/// operations, shaking out more possible interleavings quickly. It gets
/// fully eliminated by the compiler in non-test code.
#[cfg(not(any(test, feature = "lock_free_delays")))]
const fn debug_delay() {}

/// Link denotes a tree node or its modification fragment such as
/// key addition or removal.
#[derive(Clone, Debug, PartialEq)]
pub(crate) enum Link {
    /// A new value is set for a given key
    Set(IVec, IVec),
    /// The associated value is removed for a given key
    Del(IVec),
    /// A child of this Index node is marked as mergable
    ParentMergeIntention(PageId),
    /// The merging child has been completely merged into its left sibling
    ParentMergeConfirm,
    /// A Node is marked for being merged into its left sibling
    ChildMergeCap,
}

/// A fast map that is not resistant to collision attacks. Works
/// on 8 bytes at a time.
pub(crate) type FastMap8<K, V> = std::collections::HashMap<
    K,
    V,
    std::hash::BuildHasherDefault<fxhash::FxHasher64>,
>;

/// A fast set that is not resistant to collision attacks. Works
/// on 8 bytes at a time.
pub(crate) type FastSet8<V> = std::collections::HashSet<
    V,
    std::hash::BuildHasherDefault<fxhash::FxHasher64>,
>;

/// Allows arbitrary logic to be injected into mere operations of the
/// `PageCache`.
pub trait MergeOperator:
    Fn(&[u8], Option<&[u8]>, &[u8]) -> Option<Vec<u8>>
{
}
impl<F> MergeOperator for F where
    F: Fn(&[u8], Option<&[u8]>, &[u8]) -> Option<Vec<u8>>
{
}
<commit_msg>Add comment regarding MSRV to relevant #![allow()]<commit_after>//! `sled` is a high-performance embedded database with
//! an API that is similar to a `BTreeMap<[u8], [u8]>`,
//! but with several additional capabilities for
//! assisting creators of stateful systems.
//!
//! It is fully thread-safe, and all operations are
//! atomic. Multiple `Tree`s with isolated keyspaces
//! are supported with the
//! [`Db::open_tree`](struct.Db.html#method.open_tree) method.
//!
//! ACID transactions involving reads and writes to
//! multiple items are supported with the
//! [`Tree::transaction`](struct.Tree.html#method.transaction)
//! method. Transactions may also operate over
//! multiple `Tree`s (see
//! [`Tree::transaction`](struct.Tree.html#method.transaction)
//! docs for more info).
//!
//! Users may also subscribe to updates on individual
//! `Tree`s by using the
//! [`Tree::watch_prefix`](struct.Tree.html#method.watch_prefix)
//! method, which returns a blocking `Iterator` over
//! updates to keys that begin with the provided
//! prefix. You may supply an empty prefix to subscribe
//! to everything.
//!
//! [Merge operators](https://github.com/spacejam/sled/wiki/merge-operators)
//! (aka read-modify-write operators) are supported. A
//! merge operator is a function that specifies
//! how new data can be merged into an existing value
//! without requiring both a read and a write.
//! Using the
//! [`Tree::merge`](struct.Tree.html#method.merge)
//! method, you may "push" data to a `Tree` value
//! and have the provided merge operator combine
//! it with the existing value, if there was one.
//! They are set on a per-`Tree` basis, and essentially
//! allow any sort of data structure to be built
//! using merges as an atomic high-level operation.
//!
//! `sled` is built by experienced database engineers
//! who think users should spend less time tuning and
//! working against high-friction APIs. Expect
//! significant ergonomic and performance improvements
//! over time. Most surprises are bugs, so please
//! [let us know](mailto:t@jujit.su?subject=sled%20sucks!!!) if something
//! is high friction.
//!
//! # Examples
//!
//! ```
//! # let _ = std::fs::remove_dir_all("my_db");
//! let t = sled::open("my_db").unwrap();
//!
//! // insert and get
//! t.insert(b"yo!", b"v1");
//! assert_eq!(&t.get(b"yo!").unwrap().unwrap(), b"v1");
//!
//! // Atomic compare-and-swap.
//! t.compare_and_swap(
//!     b"yo!",      // key
//!     Some(b"v1"), // old value, None for not present
//!     Some(b"v2"), // new value, None for delete
//! )
//! .unwrap();
//!
//! // Iterates over key-value pairs, starting at the given key.
//! let scan_key: &[u8] = b"a non-present key before yo!";
//! let mut iter = t.range(scan_key..);
//! assert_eq!(&iter.next().unwrap().unwrap().0, b"yo!");
//! assert_eq!(iter.next(), None);
//!
//! t.remove(b"yo!");
//! assert_eq!(t.get(b"yo!"), Ok(None));
//! # let _ = std::fs::remove_dir_all("my_db");
//! ```
#![doc(
    html_logo_url = "https://raw.githubusercontent.com/spacejam/sled/master/art/tree_face_anti-transphobia.png"
)]
#![cfg_attr(test, deny(warnings))]
#![deny(
    missing_docs,
    future_incompatible,
    nonstandard_style,
    rust_2018_idioms,
    missing_copy_implementations,
    trivial_casts,
    trivial_numeric_casts,
    unsafe_code,
    unused_qualifications
)]
#![deny(
    // over time, consider enabling the commented-out lints below
    clippy::cast_lossless,
    clippy::cast_possible_truncation,
    clippy::cast_possible_wrap,
    clippy::cast_precision_loss,
    clippy::cast_sign_loss,
    clippy::checked_conversions,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    // clippy::else_if_without_else,
    clippy::empty_enum,
    clippy::explicit_into_iter_loop,
    clippy::explicit_iter_loop,
    clippy::expl_impl_clone_on_copy,
    clippy::fallible_impl_from,
    clippy::filter_map,
    clippy::filter_map_next,
    clippy::find_map,
    clippy::float_arithmetic,
    clippy::get_unwrap,
    clippy::if_not_else,
    // clippy::indexing_slicing,
    clippy::inline_always,
    //clippy::integer_arithmetic,
    clippy::invalid_upcast_comparisons,
    clippy::items_after_statements,
    clippy::map_flatten,
    clippy::match_same_arms,
    clippy::maybe_infinite_iter,
    clippy::mem_forget,
    // clippy::missing_const_for_fn,
    // clippy::missing_docs_in_private_items,
    clippy::module_name_repetitions,
    clippy::multiple_crate_versions,
    clippy::multiple_inherent_impl,
    clippy::mut_mut,
    clippy::needless_borrow,
    clippy::needless_continue,
    clippy::needless_pass_by_value,
    clippy::non_ascii_literal,
    clippy::option_map_unwrap_or,
    clippy::option_map_unwrap_or_else,
    clippy::path_buf_push_overwrite,
    clippy::print_stdout,
    clippy::pub_enum_variant_names,
    clippy::redundant_closure_for_method_calls,
    clippy::replace_consts,
    clippy::result_map_unwrap_or_else,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::string_add,
    clippy::string_add_assign,
    clippy::type_repetition_in_bounds,
    clippy::unicode_not_nfc,
    // clippy::unimplemented,
    clippy::unseparated_literal_suffix,
    clippy::used_underscore_binding,
    clippy::wildcard_dependencies,
    // clippy::wildcard_enum_match_arm,
    clippy::wrong_pub_self_convention,
)]
#![allow(clippy::mem_replace_with_default)] // Not using std::mem::take() due to MSRV of 1.37
#![recursion_limit = "128"]

macro_rules! io_fail {
    ($config:expr, $e:expr) => {
        #[cfg(feature = "failpoints")]
        {
            if fail::is_active($e) {
                $config.set_global_error(Error::FailPoint);
                return Err(Error::FailPoint).into();
            }
        }
    };
}

macro_rules! testing_assert {
    ($($e:expr),*) => {
        #[cfg(feature = "lock_free_delays")]
        assert!($($e),*)
    };
}

mod batch;
mod binary_search;
mod concurrency_control;
mod config;
mod context;
mod db;
mod dll;
mod fastcmp;
mod fastlock;
mod histogram;
mod iter;
mod ivec;
mod lazy;
mod lru;
mod meta;
mod metrics;
mod node;
mod oneshot;
mod pagecache;
mod prefix;
mod result;
mod serialization;
mod stack;
mod stackvec;
mod subscriber;
mod sys_limits;
pub mod transaction;
mod tree;

/// Functionality for conditionally triggering failpoints under test.
#[cfg(feature = "failpoints")]
pub mod fail;

#[cfg(feature = "docs")]
pub mod doc;

#[cfg(not(any(windows, target_os = "linux", target_os = "macos")))]
mod threadpool {
    use super::OneShot;

    /// Just execute a task without involving threads.
    pub fn spawn<F, R>(work: F) -> OneShot<R>
    where
        F: FnOnce() -> R + Send + 'static,
        R: Send + 'static,
    {
        let (promise_filler, promise) = OneShot::pair();
        promise_filler.fill((work)());
        return promise;
    }
}

#[cfg(any(windows, target_os = "linux", target_os = "macos"))]
mod threadpool;

#[cfg(any(windows, target_os = "linux", target_os = "macos"))]
mod flusher;

#[cfg(feature = "event_log")]
/// The event log helps debug concurrency issues.
pub mod event_log;

#[cfg(feature = "measure_allocs")]
mod measure_allocs;

#[cfg(feature = "measure_allocs")]
#[global_allocator]
static ALLOCATOR: measure_allocs::TrackingAllocator =
    measure_allocs::TrackingAllocator;

const DEFAULT_TREE_ID: &[u8] = b"__sled__default";

/// hidden re-export of items for testing purposes
#[doc(hidden)]
pub use {
    self::{
        config::RunningConfig,
        lazy::Lazy,
        pagecache::{
            constants::{
                MAX_MSG_HEADER_LEN, MAX_SPACE_AMPLIFICATION,
                MINIMUM_ITEMS_PER_SEGMENT, SEG_HEADER_LEN,
            },
            BatchManifest, DiskPtr, Log, LogKind, LogOffset, LogRead, Lsn,
            PageCache, PageId,
        },
        serialization::Serialize,
    },
    crossbeam_epoch::{pin, Atomic, Guard, Owned, Shared},
};

pub use self::{
    batch::Batch,
    config::{Config, Mode},
    db::{open, Db},
    iter::Iter,
    ivec::IVec,
    result::{Error, Result},
    subscriber::{Event, Subscriber},
    transaction::Transactional,
    tree::{CompareAndSwapError, Tree},
};

use {
    self::{
        binary_search::binary_search_lub,
        concurrency_control::{ConcurrencyControl, Protector},
        context::Context,
        fastcmp::fastcmp,
        histogram::Histogram,
        lru::Lru,
        meta::Meta,
        metrics::{clock, Measure, M},
        node::{Data, Node},
        oneshot::{OneShot, OneShotFiller},
        result::CasResult,
        stackvec::StackVec,
        subscriber::Subscribers,
        tree::TreeInner,
    },
    crossbeam_utils::{Backoff, CachePadded},
    log::{debug, error, trace, warn},
    pagecache::RecoveryGuard,
    parking_lot::{Condvar, Mutex, RwLock},
    std::{
        collections::BTreeMap,
        convert::TryFrom,
        fmt::{self, Debug},
        io::{Read, Write},
        sync::{
            atomic::{
                AtomicI64 as AtomicLsn, AtomicU64, AtomicUsize,
                Ordering::{Acquire, Relaxed, Release, SeqCst},
            },
            Arc,
        },
    },
};

fn crc32(buf: &[u8]) -> u32 {
    let mut hasher = crc32fast::Hasher::new();
    hasher.update(buf);
    hasher.finalize()
}

fn calculate_message_crc32(header: &[u8], body: &[u8]) -> u32 {
    let mut hasher = crc32fast::Hasher::new();
    hasher.update(body);
    hasher.update(&header[4..]);
    let crc32 = hasher.finalize();
    crc32 ^ 0xFFFF_FFFF
}

#[cfg(any(test, feature = "lock_free_delays"))]
mod debug_delay;

#[cfg(any(test, feature = "lock_free_delays"))]
use debug_delay::debug_delay;

/// This function is useful for inducing random jitter into our atomic
/// operations, shaking out more possible interleavings quickly. It gets
/// fully eliminated by the compiler in non-test code.
#[cfg(not(any(test, feature = "lock_free_delays")))]
const fn debug_delay() {}

/// Link denotes a tree node or its modification fragment such as
/// key addition or removal.
#[derive(Clone, Debug, PartialEq)]
pub(crate) enum Link {
    /// A new value is set for a given key
    Set(IVec, IVec),
    /// The associated value is removed for a given key
    Del(IVec),
    /// A child of this Index node is marked as mergable
    ParentMergeIntention(PageId),
    /// The merging child has been completely merged into its left sibling
    ParentMergeConfirm,
    /// A Node is marked for being merged into its left sibling
    ChildMergeCap,
}

/// A fast map that is not resistant to collision attacks. Works
/// on 8 bytes at a time.
pub(crate) type FastMap8<K, V> = std::collections::HashMap<
    K,
    V,
    std::hash::BuildHasherDefault<fxhash::FxHasher64>,
>;

/// A fast set that is not resistant to collision attacks. Works
/// on 8 bytes at a time.
pub(crate) type FastSet8<V> = std::collections::HashSet<
    V,
    std::hash::BuildHasherDefault<fxhash::FxHasher64>,
>;

/// Allows arbitrary logic to be injected into mere operations of the
/// `PageCache`.
pub trait MergeOperator:
    Fn(&[u8], Option<&[u8]>, &[u8]) -> Option<Vec<u8>>
{
}
impl<F> MergeOperator for F where
    F: Fn(&[u8], Option<&[u8]>, &[u8]) -> Option<Vec<u8>>
{
}
<|endoftext|>


<commit_before>#![feature(custom_derive)]

#![allow(dead_code, unused_imports, non_upper_case_globals)]
#![recursion_limit = "1024"]

// #[macro_use]
// extern crate diesel;
#[macro_use]
extern crate error_chain;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate serde;
#[macro_use]
extern crate log;
// extern crate dotenv;
extern crate hyper_rustls;
extern crate hyper;
// extern crate rocksdb;
extern crate serde_json;

pub mod errors;
pub mod lru;
pub mod update_client;
pub mod query_client;
pub mod updater;
pub mod database;
<commit_msg>Update lib.rs<commit_after>#![feature(custom_derive)]

#![allow(dead_code, unused_imports, non_upper_case_globals)]
#![recursion_limit = "1024"]

// #[macro_use]
// extern crate diesel;
#[macro_use]
extern crate error_chain;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate serde;
#[macro_use]
extern crate log;
// extern crate dotenv;
extern crate lru_cache;

extern crate hyper_rustls;
extern crate hyper;
// extern crate rocksdb;
extern crate serde_json;

pub mod errors;
pub mod lru;
pub mod update_client;
pub mod query_client;
pub mod updater;
pub mod database;
<|endoftext|>


<commit_before>extern crate rand;
use HaltCondition::{ Epochs, MSE };
use UpdateRule::{ Stochastic, Batch };
use std::iter::{Zip, Enumerate};
use std::slice;
use std::num::Float;

static DEFAULT_LEARNING_RATE: f64 = 0.3f64;
static DEFAULT_MOMENTUM: f64 = 0f64;
static DEFAULT_EPOCHS: usize = 1000;

type Node = Vec<f64>;
type Layer = Vec<Node>;

#[derive(Debug)]
#[derive(Clone)]
pub enum HaltCondition {
    Epochs(usize),
    MSE(f64),
}

#[derive(Debug)]
#[derive(Clone)]
pub enum UpdateRule {
    Stochastic,
    Batch
}

#[derive(Debug)]
pub struct Trainer<'a,'b> {
    examples: &'b [(Vec<f64>, Vec<f64>)],
    rate: f64,
    momentum: f64,
    log_interval: Option<usize>,
    halt_condition: HaltCondition,
    update_rule: UpdateRule,
    threads: usize,
    nn: &'a mut NN,
}

impl<'a,'b> Trainer<'a,'b>  {

    pub fn rate(&mut self, rate: f64) -> &mut Trainer<'a,'b> {
        self.rate = rate;
        self
    }

    pub fn momentum(&mut self, momentum: f64) -> &mut Trainer<'a,'b> {
        self.momentum = momentum;
        self
    }

    pub fn log_interval(&mut self, log_interval: Option<usize>) -> &mut Trainer<'a,'b> {
        self.log_interval = log_interval;
        self
    }

    pub fn halt_condition(&mut self, halt_condition: HaltCondition) -> &mut Trainer<'a,'b> {
        self.halt_condition = halt_condition;
        self
    }

    pub fn update_rule(&mut self, update_rule: UpdateRule) -> &mut Trainer<'a,'b> {
        self.update_rule = update_rule;
        self
    }

    pub fn threads(&mut self, threads: usize) -> &mut Trainer<'a,'b> {
        self.threads = threads;
        self
    }

    pub fn go(&mut self) -> f64 {
        self.nn.train_details(
            self.examples,
            self.rate,
            self.momentum,
            self.log_interval,
            self.halt_condition.clone(),
            self.update_rule.clone(),
            self.threads
        )
    }

}

#[derive(Debug)]
#[derive(Clone)]
pub struct NN {
    layers: Vec<Layer>,
    num_inputs: usize,
}

impl NN {
    pub fn run(&self, inputs: &Vec<f64>) -> Vec<f64> {
        if inputs.len() != self.num_inputs {
            panic!("input has a different length than the network's input layer");
        }
        self.do_run(inputs).pop().unwrap()
    }

    pub fn train<'b>(&'b mut self, examples: &'b [(Vec<f64>, Vec<f64>)]) -> Trainer {
        Trainer {
            examples: examples,
            rate: DEFAULT_LEARNING_RATE,
            momentum: DEFAULT_MOMENTUM,
            log_interval: None,
            halt_condition: Epochs(DEFAULT_EPOCHS),
            update_rule: Stochastic,
            threads: 1,
            nn: self
        }
    }

    fn train_details(&mut self, examples: &[(Vec<f64>, Vec<f64>)], rate: f64, momentum: f64, log_interval: Option<usize>,
                    halt_condition: HaltCondition, update_rule: UpdateRule, threads: usize) -> f64 {

        let mut epochs = 0;
        let mut error_rate = 0f64;

        // check that input and output sizes are correct
        let input_layer_size = self.num_inputs;
        let output_layer_size = self.layers[self.layers.len() - 1].len();
        for &(ref inputs, ref outputs) in examples.iter() {
            if inputs.len() != input_layer_size {
                panic!("input has a different length than the network's input layer");
            }
            if outputs.len() != output_layer_size {
                panic!("output has a different length than the network's output layer");
            }
        }

        loop {

            // log error rate if neccessary
            match log_interval {
                Some(interval) if epochs>0 && epochs % interval == 0 => {
                    println!("error rate: {}", error_rate);
                },
                _ => (),
            }

            // check if we've met the halt condition yet
            match halt_condition {
                Epochs(epochs_halt) => {
                    if epochs == epochs_halt { return error_rate }
                },
                MSE(target_error) => {
                    if target_error == error_rate { return error_rate }
                }
            }

            error_rate = 0f64;

            let mut prev_deltas = self.make_weights_tracker(0.0f64);
            let mut batch_data = self.make_weights_tracker(0.0f64);

            for &(ref inputs, ref targets) in examples.iter() {
                let results = self.do_run(&inputs);
                let weight_updates = self.calculate_weight_updates(&results, &targets);
                error_rate += calculate_error(&results, &targets);
                match update_rule {
                    Batch => update_batch_data(&mut batch_data, &weight_updates),
                    Stochastic => self.update_weights(&weight_updates, &mut prev_deltas, rate, momentum),
                }
            }

            // if we're in batch mode, update the weights now
            match update_rule {
                Batch => self.update_weights(&batch_data, &mut prev_deltas, rate, momentum),
                Stochastic => (),
            }

            epochs += 1;
        }
    }

    fn do_run(&self, inputs: &Vec<f64>) -> Vec<Vec<f64>> {
        let mut results = Vec::new();
        results.push(inputs.clone());
        for (layer_index, layer) in self.layers.iter().enumerate() {
            let mut layer_results = Vec::new();
            for node in layer.iter() {
                layer_results.push( sigmoid(modified_dotprod(&node, &results[layer_index])) )
            }
            results.push(layer_results.clone());
        }
        results
    }

    // updates all weights in the network
    fn update_weights(&mut self, network_weight_updates: &Vec<Vec<Vec<f64>>>, prev_deltas: &mut Vec<Vec<Vec<f64>>>, rate: f64, momentum: f64) {
        for layer_index in 0..self.layers.len() {
            let mut layer = &mut self.layers[layer_index];
            let layer_weight_updates = &network_weight_updates[layer_index];
            for node_index in 0..layer.len() {
                let mut node = &mut layer[node_index];
                let node_weight_updates = &layer_weight_updates[node_index];
                for weight_index in 0..node.len() {
                    let weight_update = node_weight_updates[weight_index];
                    let prev_delta = prev_deltas[layer_index][node_index][weight_index];
                    let delta = (rate * weight_update) + (momentum * prev_delta);
                    node[weight_index] += delta;
                    prev_deltas[layer_index][node_index][weight_index] = delta;
                }
            }
        }
    }

    // calculates all weight updates by backpropagation
    fn calculate_weight_updates(&self, results: &Vec<Vec<f64>>, targets: &[f64]) -> Vec<Vec<Vec<f64>>> {
        let mut network_errors:Vec<Vec<f64>> = Vec::new();
        let mut network_weight_updates = Vec::new();
        let layers = &self.layers;
        let network_results = &results[1..]; // skip the input layer

        let mut next_layer_nodes: Option<&Layer> = None;

        for (layer_index, (layer_nodes, layer_results)) in iter_zip_enum(layers, network_results).rev() {
            let prev_layer_results = &results[layer_index];
            let mut layer_errors = Vec::new();
            let mut layer_weight_updates = Vec::new();

            for (node_index, (node, &result)) in iter_zip_enum(layer_nodes, layer_results) {
                let mut node_weight_updates = Vec::new();
                let mut node_error;

                // calculate error for this node
                if layer_index == layers.len() - 1 {
                    node_error = result * (1f64 - result) * (targets[node_index] - result);
                } else {
                    let mut sum = 0f64;
                    for (next_node, &next_node_error_data) in next_layer_nodes.unwrap().iter().zip((&network_errors[layer_index]).iter()) {
                        sum += next_node[node_index+1] * next_node_error_data; // +1 because the 0th weight is the threshold
                    }
                    node_error = result * (1f64 - result) * sum;
                }

                // calculate weight updates for this node
                for weight_index in 0..node.len() {
                    let mut prev_layer_result;
                    if weight_index == 0 {
                        prev_layer_result = 1f64; // theshold
                    } else {
                        prev_layer_result = prev_layer_results[weight_index-1];
                    }
                    let weight_update = node_error * prev_layer_result;
                    node_weight_updates.push(weight_update);
                }

                layer_errors.push(node_error);
                layer_weight_updates.push(node_weight_updates);
            }

            network_errors.push(layer_errors);
            network_weight_updates.push(layer_weight_updates);
            next_layer_nodes = Some(&layer_nodes);
        }

        // updates were built by backpropagation so reverse them
        network_weight_updates.reverse();

        network_weight_updates
    }

    fn make_weights_tracker<T: Clone>(&self, place_holder: T) -> Vec<Vec<Vec<T>>> {
        let mut network_level = Vec::new();
        for layer in self.layers.iter() {
            let mut layer_level = Vec::new();
            for node in layer.iter() {
                let mut node_level = Vec::new();
                for _ in node.iter() {
                    node_level.push(place_holder.clone());
                }
                layer_level.push(node_level);
            }
            network_level.push(layer_level);
        }

        network_level
    }
}

fn modified_dotprod(node: &Node, values: &Vec<f64>) -> f64 {
    let mut it = node.iter();
    let mut total = *it.next().unwrap(); // for the threshold weight
    for (weight, value) in it.zip(values.iter()) {
        total += weight * value;
    }
    total
}

fn sigmoid(y: f64) -> f64 {
    1f64 / (1f64 + Float::exp(-y))
}

// adds new network weight updates into the update updates already collected
fn update_batch_data(batch_data: &mut Vec<Vec<Vec<f64>>> , network_weight_updates: &Vec<Vec<Vec<f64>>>) {
    for layer_index in 0..batch_data.len() {
        let mut batch_layer = &mut batch_data[layer_index];
        let layer_weight_updates = &network_weight_updates[layer_index];
        for node_index in 0..batch_layer.len() {
            let mut batch_node = &mut batch_layer[node_index];
            let node_weight_updates = &layer_weight_updates[node_index];
            for weight_index in 0..batch_node.len() {
                batch_node[weight_index] += node_weight_updates[weight_index];
            }
        }
    }
}

// takes two arrays and enumerates the iterator produced by zipping each of
// their iterators together
fn iter_zip_enum<'s, 't, S: 's, T: 't>(s: &'s [S], t: &'t [T]) ->
    Enumerate<Zip<slice::Iter<'s, S>, slice::Iter<'t, T>>>  {
    s.iter().zip(t.iter()).enumerate()
}

fn calculate_error(results: &Vec<Vec<f64>>, targets: &[f64]) -> f64 {
    let ref last_resutls = results[results.len()-1];
    let mut total:f64 = 0f64;
    for (&result, &target) in last_resutls.iter().zip(targets.iter()) {
        total += (target - result).powi(2);
    }
    0.5 * total
}

pub fn new(layers_sizes: &[usize]) -> NN {
    if layers_sizes.len() < 2 {
        panic!("must have at least two layers");
    }
    let mut layers = Vec::new();
    let mut it = layers_sizes.iter();
    // get the first layer size
    let first_layer_size = *it.next().unwrap();

    // setup the rest of the layers
    let mut prev_layer_size = first_layer_size;
    for &layer_size in it {
        let mut layer: Layer = Vec::new();
        for _ in 0..layer_size {
            let mut node: Node = Vec::new();
            for _ in 0..prev_layer_size+1 {
                let random_weight: f64 = rand::random();
                node.push(random_weight);
            }
            node.shrink_to_fit();
            layer.push(node)
        }
        layer.shrink_to_fit();
        layers.push(layer);
        prev_layer_size = layer_size;
    }
    layers.shrink_to_fit();
    NN { layers: layers, num_inputs: first_layer_size }
}<commit_msg>removed unneccsary clone in results build<commit_after>extern crate rand;
use HaltCondition::{ Epochs, MSE };
use UpdateRule::{ Stochastic, Batch };
use std::iter::{Zip, Enumerate};
use std::slice;
use std::num::Float;

static DEFAULT_LEARNING_RATE: f64 = 0.3f64;
static DEFAULT_MOMENTUM: f64 = 0f64;
static DEFAULT_EPOCHS: usize = 1000;

type Node = Vec<f64>;
type Layer = Vec<Node>;

#[derive(Debug)]
#[derive(Clone)]
pub enum HaltCondition {
    Epochs(usize),
    MSE(f64),
}

#[derive(Debug)]
#[derive(Clone)]
pub enum UpdateRule {
    Stochastic,
    Batch
}

#[derive(Debug)]
pub struct Trainer<'a,'b> {
    examples: &'b [(Vec<f64>, Vec<f64>)],
    rate: f64,
    momentum: f64,
    log_interval: Option<usize>,
    halt_condition: HaltCondition,
    update_rule: UpdateRule,
    threads: usize,
    nn: &'a mut NN,
}

impl<'a,'b> Trainer<'a,'b>  {

    pub fn rate(&mut self, rate: f64) -> &mut Trainer<'a,'b> {
        self.rate = rate;
        self
    }

    pub fn momentum(&mut self, momentum: f64) -> &mut Trainer<'a,'b> {
        self.momentum = momentum;
        self
    }

    pub fn log_interval(&mut self, log_interval: Option<usize>) -> &mut Trainer<'a,'b> {
        self.log_interval = log_interval;
        self
    }

    pub fn halt_condition(&mut self, halt_condition: HaltCondition) -> &mut Trainer<'a,'b> {
        self.halt_condition = halt_condition;
        self
    }

    pub fn update_rule(&mut self, update_rule: UpdateRule) -> &mut Trainer<'a,'b> {
        self.update_rule = update_rule;
        self
    }

    pub fn threads(&mut self, threads: usize) -> &mut Trainer<'a,'b> {
        self.threads = threads;
        self
    }

    pub fn go(&mut self) -> f64 {
        self.nn.train_details(
            self.examples,
            self.rate,
            self.momentum,
            self.log_interval,
            self.halt_condition.clone(),
            self.update_rule.clone(),
            self.threads
        )
    }

}

#[derive(Debug)]
#[derive(Clone)]
pub struct NN {
    layers: Vec<Layer>,
    num_inputs: usize,
}

impl NN {
    pub fn run(&self, inputs: &Vec<f64>) -> Vec<f64> {
        if inputs.len() != self.num_inputs {
            panic!("input has a different length than the network's input layer");
        }
        self.do_run(inputs).pop().unwrap()
    }

    pub fn train<'b>(&'b mut self, examples: &'b [(Vec<f64>, Vec<f64>)]) -> Trainer {
        Trainer {
            examples: examples,
            rate: DEFAULT_LEARNING_RATE,
            momentum: DEFAULT_MOMENTUM,
            log_interval: None,
            halt_condition: Epochs(DEFAULT_EPOCHS),
            update_rule: Stochastic,
            threads: 1,
            nn: self
        }
    }

    fn train_details(&mut self, examples: &[(Vec<f64>, Vec<f64>)], rate: f64, momentum: f64, log_interval: Option<usize>,
                    halt_condition: HaltCondition, update_rule: UpdateRule, threads: usize) -> f64 {

        let mut epochs = 0;
        let mut error_rate = 0f64;

        // check that input and output sizes are correct
        let input_layer_size = self.num_inputs;
        let output_layer_size = self.layers[self.layers.len() - 1].len();
        for &(ref inputs, ref outputs) in examples.iter() {
            if inputs.len() != input_layer_size {
                panic!("input has a different length than the network's input layer");
            }
            if outputs.len() != output_layer_size {
                panic!("output has a different length than the network's output layer");
            }
        }

        loop {

            // log error rate if neccessary
            match log_interval {
                Some(interval) if epochs>0 && epochs % interval == 0 => {
                    println!("error rate: {}", error_rate);
                },
                _ => (),
            }

            // check if we've met the halt condition yet
            match halt_condition {
                Epochs(epochs_halt) => {
                    if epochs == epochs_halt { return error_rate }
                },
                MSE(target_error) => {
                    if target_error == error_rate { return error_rate }
                }
            }

            error_rate = 0f64;

            let mut prev_deltas = self.make_weights_tracker(0.0f64);
            let mut batch_data = self.make_weights_tracker(0.0f64);

            for &(ref inputs, ref targets) in examples.iter() {
                let results = self.do_run(&inputs);
                let weight_updates = self.calculate_weight_updates(&results, &targets);
                error_rate += calculate_error(&results, &targets);
                match update_rule {
                    Batch => update_batch_data(&mut batch_data, &weight_updates),
                    Stochastic => self.update_weights(&weight_updates, &mut prev_deltas, rate, momentum),
                }
            }

            // if we're in batch mode, update the weights now
            match update_rule {
                Batch => self.update_weights(&batch_data, &mut prev_deltas, rate, momentum),
                Stochastic => (),
            }

            epochs += 1;
        }
    }

    fn do_run(&self, inputs: &Vec<f64>) -> Vec<Vec<f64>> {
        let mut results = Vec::new();
        results.push(inputs.clone());
        for (layer_index, layer) in self.layers.iter().enumerate() {
            let mut layer_results = Vec::new();
            for node in layer.iter() {
                layer_results.push( sigmoid(modified_dotprod(&node, &results[layer_index])) )
            }
            results.push(layer_results);
        }
        results
    }

    // updates all weights in the network
    fn update_weights(&mut self, network_weight_updates: &Vec<Vec<Vec<f64>>>, prev_deltas: &mut Vec<Vec<Vec<f64>>>, rate: f64, momentum: f64) {
        for layer_index in 0..self.layers.len() {
            let mut layer = &mut self.layers[layer_index];
            let layer_weight_updates = &network_weight_updates[layer_index];
            for node_index in 0..layer.len() {
                let mut node = &mut layer[node_index];
                let node_weight_updates = &layer_weight_updates[node_index];
                for weight_index in 0..node.len() {
                    let weight_update = node_weight_updates[weight_index];
                    let prev_delta = prev_deltas[layer_index][node_index][weight_index];
                    let delta = (rate * weight_update) + (momentum * prev_delta);
                    node[weight_index] += delta;
                    prev_deltas[layer_index][node_index][weight_index] = delta;
                }
            }
        }
    }

    // calculates all weight updates by backpropagation
    fn calculate_weight_updates(&self, results: &Vec<Vec<f64>>, targets: &[f64]) -> Vec<Vec<Vec<f64>>> {
        let mut network_errors:Vec<Vec<f64>> = Vec::new();
        let mut network_weight_updates = Vec::new();
        let layers = &self.layers;
        let network_results = &results[1..]; // skip the input layer

        let mut next_layer_nodes: Option<&Layer> = None;

        for (layer_index, (layer_nodes, layer_results)) in iter_zip_enum(layers, network_results).rev() {
            let prev_layer_results = &results[layer_index];
            let mut layer_errors = Vec::new();
            let mut layer_weight_updates = Vec::new();

            for (node_index, (node, &result)) in iter_zip_enum(layer_nodes, layer_results) {
                let mut node_weight_updates = Vec::new();
                let mut node_error;

                // calculate error for this node
                if layer_index == layers.len() - 1 {
                    node_error = result * (1f64 - result) * (targets[node_index] - result);
                } else {
                    let mut sum = 0f64;
                    for (next_node, &next_node_error_data) in next_layer_nodes.unwrap().iter().zip((&network_errors[layer_index]).iter()) {
                        sum += next_node[node_index+1] * next_node_error_data; // +1 because the 0th weight is the threshold
                    }
                    node_error = result * (1f64 - result) * sum;
                }

                // calculate weight updates for this node
                for weight_index in 0..node.len() {
                    let mut prev_layer_result;
                    if weight_index == 0 {
                        prev_layer_result = 1f64; // theshold
                    } else {
                        prev_layer_result = prev_layer_results[weight_index-1];
                    }
                    let weight_update = node_error * prev_layer_result;
                    node_weight_updates.push(weight_update);
                }

                layer_errors.push(node_error);
                layer_weight_updates.push(node_weight_updates);
            }

            network_errors.push(layer_errors);
            network_weight_updates.push(layer_weight_updates);
            next_layer_nodes = Some(&layer_nodes);
        }

        // updates were built by backpropagation so reverse them
        network_weight_updates.reverse();

        network_weight_updates
    }

    fn make_weights_tracker<T: Clone>(&self, place_holder: T) -> Vec<Vec<Vec<T>>> {
        let mut network_level = Vec::new();
        for layer in self.layers.iter() {
            let mut layer_level = Vec::new();
            for node in layer.iter() {
                let mut node_level = Vec::new();
                for _ in node.iter() {
                    node_level.push(place_holder.clone());
                }
                layer_level.push(node_level);
            }
            network_level.push(layer_level);
        }

        network_level
    }
}

fn modified_dotprod(node: &Node, values: &Vec<f64>) -> f64 {
    let mut it = node.iter();
    let mut total = *it.next().unwrap(); // for the threshold weight
    for (weight, value) in it.zip(values.iter()) {
        total += weight * value;
    }
    total
}

fn sigmoid(y: f64) -> f64 {
    1f64 / (1f64 + Float::exp(-y))
}

// adds new network weight updates into the update updates already collected
fn update_batch_data(batch_data: &mut Vec<Vec<Vec<f64>>> , network_weight_updates: &Vec<Vec<Vec<f64>>>) {
    for layer_index in 0..batch_data.len() {
        let mut batch_layer = &mut batch_data[layer_index];
        let layer_weight_updates = &network_weight_updates[layer_index];
        for node_index in 0..batch_layer.len() {
            let mut batch_node = &mut batch_layer[node_index];
            let node_weight_updates = &layer_weight_updates[node_index];
            for weight_index in 0..batch_node.len() {
                batch_node[weight_index] += node_weight_updates[weight_index];
            }
        }
    }
}

// takes two arrays and enumerates the iterator produced by zipping each of
// their iterators together
fn iter_zip_enum<'s, 't, S: 's, T: 't>(s: &'s [S], t: &'t [T]) ->
    Enumerate<Zip<slice::Iter<'s, S>, slice::Iter<'t, T>>>  {
    s.iter().zip(t.iter()).enumerate()
}

fn calculate_error(results: &Vec<Vec<f64>>, targets: &[f64]) -> f64 {
    let ref last_resutls = results[results.len()-1];
    let mut total:f64 = 0f64;
    for (&result, &target) in last_resutls.iter().zip(targets.iter()) {
        total += (target - result).powi(2);
    }
    0.5 * total
}

pub fn new(layers_sizes: &[usize]) -> NN {
    if layers_sizes.len() < 2 {
        panic!("must have at least two layers");
    }
    let mut layers = Vec::new();
    let mut it = layers_sizes.iter();
    // get the first layer size
    let first_layer_size = *it.next().unwrap();

    // setup the rest of the layers
    let mut prev_layer_size = first_layer_size;
    for &layer_size in it {
        let mut layer: Layer = Vec::new();
        for _ in 0..layer_size {
            let mut node: Node = Vec::new();
            for _ in 0..prev_layer_size+1 {
                let random_weight: f64 = rand::random();
                node.push(random_weight);
            }
            node.shrink_to_fit();
            layer.push(node)
        }
        layer.shrink_to_fit();
        layers.push(layer);
        prev_layer_size = layer_size;
    }
    layers.shrink_to_fit();
    NN { layers: layers, num_inputs: first_layer_size }
}<|endoftext|>