<commit_before>extern crate std;

#[derive(Debug, Clone)]
pub struct MagicRule {
    pub indent_level: u32,
    pub start_off: u32,
    pub val_len: u16,
    pub val: Vec<u8>,
    pub mask: Option<Vec<u8>>,
    pub word_len: u32,
    pub region_len: u32
}

pub mod ruleset {
    extern crate nom;
    extern crate std;
    use std::str;
    use nom::*;
    use std::collections::HashMap;

    // This is the catch-all when all else fails
    // (but it's not that bad.)
    pub fn can_check(mime: String) -> bool {
        true
    }

    // Below functions from https://github.com/badboy/iso8601/blob/master/src/helper.rs
    // but modified to be safe and provide defaults
    pub fn to_string(s: &[u8]) -> std::result::Result<&str, std::str::Utf8Error> {
        str::from_utf8(s)
    }
    pub fn to_u32(s: std::result::Result<&str, std::str::Utf8Error>, def: u32) -> u32 {

        match s {
            Ok (t) => {str::FromStr::from_str(t).unwrap_or(def)},
            Err (_) => def
        }
    }

    pub fn buf_to_u32(s: &[u8], def: u32) -> u32 {
        to_u32(to_string(s), def)
    }

    // Initial mime string
    // Format: [priority: mime]
    named!(mime<&str>,
        map_res!(
            delimited!(
                delimited!(
                    char!('['),
                    is_not!(":"),
                    char!(':')
                ),
                is_not!("]"), // the mime
                tag!("]\n")
            ),
            str::from_utf8
        )
    );

    #[test]
    fn mime_test() {
        assert_eq!(mime(&b"[90:text/plain]\n"[..]), IResult::Done(&b""[..], "text/plain"));
    }

    // Indent levels sub-parser for magic_rules
    // Default value 0
    named!(magic_rules_indent_level<u32>,
        do_parse!(
            ret: take_until!(">") >>
            (buf_to_u32(ret, 0))
        )
    );

    #[test]
    fn indent_level_test() {
        assert_eq!(magic_rules_indent_level(&b"0>fgh"[..]).to_result().unwrap(), 0);
        assert_eq!(magic_rules_indent_level(&b"42>fgh"[..]).to_result().unwrap(), 42);
        assert_eq!(magic_rules_indent_level(&b">fgh"[..]).to_result().unwrap(), 0);
        assert_eq!(magic_rules_indent_level(&b"xyz>fgh"[..]).to_result().is_err(), true);
    }

    // Start offset sub-parser for magic_rules
    named!(magic_rules_start_off<u32>,
        do_parse!(
            ret: take_until!("=") >>
            (buf_to_u32(ret, 0))
        )
    );

    #[test]
    fn start_off_test() {
        assert_eq!(magic_rules_start_off(&b"0="[..]).to_result().unwrap(), 0);
        assert_eq!(magic_rules_start_off(&b"42="[..]).to_result().unwrap(), 42);
    }

    // Singular magic ruleset
    named!(magic_rules<super::MagicRule>,

        do_parse!(
            peek!(is_a!("012345689>")) >>
            _indent_level: magic_rules_indent_level >>
            tag!(">") >>
            _start_off: magic_rules_start_off >>
            tag!("=") >>
            _val_len: u16!(nom::Endianness::Big) >> // length of value
            _val: do_parse!(
                ret: take!(_val_len) >>
                (ret.iter().map(|&x| x).collect())
            ) >> // value

            _mask: opt!(
                do_parse!(
                    char!('&') >>
                    ret: take!(_val_len) >> // mask (default 0xFF)
                    (ret.iter().map(|&x| x).collect())
                )
            ) >>

            // word size (default 1)
            _word_len: opt!(
                do_parse!(
                    tag!("~") >>
                    ret: take_until!("+") >>
                    (buf_to_u32(ret, 1))
                )
            ) >>

            // length of region in file to check (default 1)
            _region_len: opt!(
                do_parse!(
                    tag!("+") >>
                    ret: take_until!("\n") >>
                    (buf_to_u32(ret, 1))
                )
            ) >>

            take_until_and_consume!("\n") >>

            (super::MagicRule{
                indent_level: _indent_level,
                start_off: _start_off,
                val: _val,
                val_len: _val_len,
                mask: _mask,
                word_len: _word_len.unwrap_or(1),
                region_len: _region_len.unwrap_or(1)
            })
        )

    );

    // Singular magic entry
    named!(magic_entry<(String, Vec<super::MagicRule>)>,
        do_parse!(
            _mime: do_parse!(
                ret: mime >>
                (ret.to_string())
            ) >>

            _rules: many0!(magic_rules) >>

            (_mime, _rules)
        )
    );

    /// Converts a magic file given as a &[u8] array
    /// to a vector of MagicEntry structs
    named!(from_u8_to_tuple_vec<Vec<(String, Vec<super::MagicRule>)>>,
        do_parse!(
            tag!("MIME-Magic\0\n") >>
            ret: many0!(magic_entry) >>
            (ret)
        )
    );

    pub fn from_u8(b: &[u8]) -> Result<HashMap<String, Vec<super::MagicRule>>, String> {
        let tuplevec = from_u8_to_tuple_vec(b).to_result().map_err(|e| e.to_string())?;
        let mut res = HashMap::<String, Vec<super::MagicRule>>::new();

        for x in tuplevec {
            res.insert(x.0, x.1);
        }

        Ok(res)

    }

    /// Loads the given magic file and outputs a vector of MagicEntry structs
    pub fn from_filepath(filepath: &str) -> Result<HashMap<String, Vec<super::MagicRule>>, String>{
        use std::io::prelude::*;
        use std::io::BufReader;
        use std::fs::File;

        let fmagic = File::open(filepath).map_err(|e| e.to_string())?;
        let mut rmagic = BufReader::new(fmagic);
        let mut bmagic = Vec::<u8>::new();
        rmagic.read_to_end(&mut bmagic).map_err(|e| e.to_string())?;

        let magic_ruleset = from_u8(
            bmagic.as_slice()
        ).map_err(|e| e.to_string())?;

        Ok(magic_ruleset)
    }

}

/// Init whatever is needed for test mod to work
pub mod init {

    extern crate std;
    use std::io::prelude::*;
    use std::io::BufReader;
    use std::fs::File;
    use std::collections::HashMap;

    // Get list of known system filetypes
    fn read_mimelist() -> Result<Vec<String>, std::io::Error> {
        let ftypes = File::open("/usr/share/mime/types")?;
        let rtypes = BufReader::new(ftypes);
        let mut mimelist = Vec::<String>::new();

        for line in rtypes.lines() {
            let mime = line?.split_whitespace().nth(0).unwrap_or("").to_string();
            mimelist.push(mime);
        }

        let mimelist = mimelist;
        Ok(mimelist)
    }

    /// Read all subclass lines from file
    fn read_subclasses() -> Result<Vec<(String, String)>, std::io::Error> {

        let f = File::open("/usr/share/mime/subclasses")?;
        let r = BufReader::new(f);
        let mut subclasses = Vec::<(String, String)>::new();

        for x in r.lines() {
            let line = x?;

            let mut child_raw = line.split_whitespace().nth(0).unwrap_or("").to_string();
            let mut parent_raw = line.split_whitespace().nth(1).unwrap_or("").to_string();

            subclasses.push( (parent_raw, child_raw) );
        }

        Ok(subclasses)
    }

    // Get filetype aliases
    fn read_aliaslist() -> Result<HashMap<String, String>, std::io::Error> {
        let faliases = File::open("/usr/share/mime/aliases")?;
        let raliases = BufReader::new(faliases);
        let mut aliaslist = HashMap::<String, String>::new();

        for line in raliases.lines() {
            let line_raw = line?;

            let a = line_raw.split_whitespace().nth(0).unwrap_or("").to_string();
            let b = line_raw.split_whitespace().nth(1).unwrap_or("").to_string();
            aliaslist.insert(a,b);
        }

        let aliaslist = aliaslist;
        Ok(aliaslist)
    }

    /// Get list of supported MIME types
    pub fn get_supported() -> Vec<String> {
        read_mimelist().unwrap_or(Vec::<String>::new())
    }

    /// Get list of parent -> child subclass links
    pub fn get_subclasses() -> Vec<(String, String)> {

        let mut subclasses = read_subclasses().unwrap_or(Vec::<(String, String)>::new());
        let aliaslist = read_aliaslist().unwrap_or(HashMap::<String, String>::new());

        // If child or parent refers to an alias, change it to the real type
        for x in 0..subclasses.iter().count(){
            match aliaslist.get(&subclasses[x].0) {
                Some(alias) => {subclasses[x].0 = alias.clone();}
                None => {}
            }
            match aliaslist.get(&subclasses[x].1) {
                Some(alias) => {subclasses[x].1 = alias.clone();}
                None => {}
            }
        }

        subclasses
    }
}

// Functions to check if a file matches a magic entry
pub mod test {

    extern crate std;

    fn from_vec_u8_singlerule(file: &Vec<u8>, rule: super::MagicRule) -> bool {

        // Check if we're even in bounds
        let bound_min = std::cmp::min(
            rule.start_off as usize,
            rule.val.iter().count()
        );
        let bound_max =
            std::cmp::min(
            (
                rule.start_off as usize +
                rule.val_len as usize +
                rule.region_len as usize
            ),
            rule.val.iter().count()
        );

        if file.iter().count() < bound_max {
            return false;
        }

        // Define our testing slice
        let ref x: Vec<u8> = *file;
        let testarea: Vec<u8> = x[bound_min .. bound_max].to_vec();
        //println!("{:?}, {:?}, {:?}\n", file, testarea, rule.val);

        // Search down until we find a hit
        for x in testarea.windows(rule.val_len as usize) {

            // Apply mask to value
            let mut y: Vec<u8>;

            match rule.mask.clone() {
                Some(mask) => {
                    y = Vec::<u8>::new();
                    for i in 0..rule.val_len {
                        y.push(x[i as usize] & mask[i as usize]);
                    }
                },
                None => y = x.to_vec(),
            }

            if y.iter().eq(rule.val.iter()) {
                return true;
            }
        }

        false
    }

    pub fn can_check(mimetype: &str) -> bool {
        /*let list = super::init::get_supported();

        list.contains(&mimetype.to_string())*/

        true
    }

    /// Test against all rules
    pub fn from_vec_u8(
        file: Vec<u8>, mimetype: &str, magic_rules: Vec<super::MagicRule>
    ) -> bool {

        if magic_rules.iter().count() < 2 {
            for x in magic_rules {
                match from_vec_u8_singlerule(&file, x) {
                    true => return true,
                    false => continue,
                }
            }

        } else {

            for x in magic_rules.windows(2) {
                    match from_vec_u8_singlerule(&file, x[0].clone()) {
                        true => {
                            if x[1].indent_level >= x[0].indent_level {
                                continue;
                            } else {
                                return true;
                            }
                        },
                        false => continue,
                    }
            }
        }

        false
    }

    pub fn from_filepath(filepath: &str, mimetype: &str, magic_rules: Vec<super::MagicRule>) -> Result<bool, std::io::Error>{
        use std::io::prelude::*;
        use std::io::BufReader;
        use std::fs::File;

        // Get # of bytes to read
        let mut scanlen:u64 = 0;
        for x in &magic_rules {
            let tmplen:u64 =
                x.start_off as u64 +
                x.val_len as u64 +
                x.region_len as u64;

            if tmplen > scanlen {
                scanlen = tmplen;
            }
        }

        let f = File::open(filepath)?;
        let r = BufReader::new(f);
        let mut b = Vec::<u8>::new();
        r.take(scanlen).read_to_end(&mut b)?;

        Ok(from_vec_u8(b, mimetype, magic_rules))
    }

}

<commit_msg>Turns out it was kind of broken. Added indent level parsing for magic module.<commit_after>extern crate std;

#[derive(Debug, Clone)]
pub struct MagicRule {
    pub indent_level: u32,
    pub start_off: u32,
    pub val_len: u16,
    pub val: Vec<u8>,
    pub mask: Option<Vec<u8>>,
    pub word_len: u32,
    pub region_len: u32
}

pub mod ruleset {
    extern crate nom;
    extern crate std;
    use std::str;
    use nom::*;
    use std::collections::HashMap;

    // This is the catch-all when all else fails
    // (but it's not that bad.)
    pub fn can_check(mime: String) -> bool {
        true
    }

    // Below functions from https://github.com/badboy/iso8601/blob/master/src/helper.rs
    // but modified to be safe and provide defaults
    pub fn to_string(s: &[u8]) -> std::result::Result<&str, std::str::Utf8Error> {
        str::from_utf8(s)
    }
    pub fn to_u32(s: std::result::Result<&str, std::str::Utf8Error>, def: u32) -> u32 {

        match s {
            Ok (t) => {str::FromStr::from_str(t).unwrap_or(def)},
            Err (_) => def
        }
    }

    pub fn buf_to_u32(s: &[u8], def: u32) -> u32 {
        to_u32(to_string(s), def)
    }

    // Initial mime string
    // Format: [priority: mime]
    named!(mime<&str>,
        map_res!(
            delimited!(
                delimited!(
                    char!('['),
                    is_not!(":"),
                    char!(':')
                ),
                is_not!("]"), // the mime
                tag!("]\n")
            ),
            str::from_utf8
        )
    );

    #[test]
    fn mime_test() {
        assert_eq!(mime(&b"[90:text/plain]\n"[..]), IResult::Done(&b""[..], "text/plain"));
    }

    // Indent levels sub-parser for magic_rules
    // Default value 0
    named!(magic_rules_indent_level<u32>,
        do_parse!(
            ret: take_until!(">") >>
            (buf_to_u32(ret, 0))
        )
    );

    #[test]
    fn indent_level_test() {
        assert_eq!(magic_rules_indent_level(&b"0>fgh"[..]).to_result().unwrap(), 0);
        assert_eq!(magic_rules_indent_level(&b"42>fgh"[..]).to_result().unwrap(), 42);
        assert_eq!(magic_rules_indent_level(&b">fgh"[..]).to_result().unwrap(), 0);
        assert_eq!(magic_rules_indent_level(&b"xyz>fgh"[..]).to_result().is_err(), true);
    }

    // Start offset sub-parser for magic_rules
    named!(magic_rules_start_off<u32>,
        do_parse!(
            ret: take_until!("=") >>
            (buf_to_u32(ret, 0))
        )
    );

    #[test]
    fn start_off_test() {
        assert_eq!(magic_rules_start_off(&b"0="[..]).to_result().unwrap(), 0);
        assert_eq!(magic_rules_start_off(&b"42="[..]).to_result().unwrap(), 42);
    }

    // Singular magic ruleset
    named!(magic_rules<super::MagicRule>,

        do_parse!(
            peek!(is_a!("012345689>")) >>
            _indent_level: magic_rules_indent_level >>
            tag!(">") >>
            _start_off: magic_rules_start_off >>
            tag!("=") >>
            _val_len: u16!(nom::Endianness::Big) >> // length of value
            _val: do_parse!(
                ret: take!(_val_len) >>
                (ret.iter().map(|&x| x).collect())
            ) >> // value

            _mask: opt!(
                do_parse!(
                    char!('&') >>
                    ret: take!(_val_len) >> // mask (default 0xFF)
                    (ret.iter().map(|&x| x).collect())
                )
            ) >>

            // word size (default 1)
            _word_len: opt!(
                do_parse!(
                    tag!("~") >>
                    ret: take_until!("+") >>
                    (buf_to_u32(ret, 1))
                )
            ) >>

            // length of region in file to check (default 1)
            _region_len: opt!(
                do_parse!(
                    tag!("+") >>
                    ret: take_until!("\n") >>
                    (buf_to_u32(ret, 1))
                )
            ) >>

            take_until_and_consume!("\n") >>

            (super::MagicRule{
                indent_level: _indent_level,
                start_off: _start_off,
                val: _val,
                val_len: _val_len,
                mask: _mask,
                word_len: _word_len.unwrap_or(1),
                region_len: _region_len.unwrap_or(1)
            })
        )

    );

    // Singular magic entry
    named!(magic_entry<(String, Vec<super::MagicRule>)>,
        do_parse!(
            _mime: do_parse!(
                ret: mime >>
                (ret.to_string())
            ) >>

            _rules: many0!(magic_rules) >>

            (_mime, _rules)
        )
    );

    /// Converts a magic file given as a &[u8] array
    /// to a vector of MagicEntry structs
    named!(from_u8_to_tuple_vec<Vec<(String, Vec<super::MagicRule>)>>,
        do_parse!(
            tag!("MIME-Magic\0\n") >>
            ret: many0!(magic_entry) >>
            (ret)
        )
    );

    pub fn from_u8(b: &[u8]) -> Result<HashMap<String, Vec<super::MagicRule>>, String> {
        let tuplevec = from_u8_to_tuple_vec(b).to_result().map_err(|e| e.to_string())?;
        let mut res = HashMap::<String, Vec<super::MagicRule>>::new();

        for x in tuplevec {
            res.insert(x.0, x.1);
        }

        Ok(res)

    }

    /// Loads the given magic file and outputs a vector of MagicEntry structs
    pub fn from_filepath(filepath: &str) -> Result<HashMap<String, Vec<super::MagicRule>>, String>{
        use std::io::prelude::*;
        use std::io::BufReader;
        use std::fs::File;

        let fmagic = File::open(filepath).map_err(|e| e.to_string())?;
        let mut rmagic = BufReader::new(fmagic);
        let mut bmagic = Vec::<u8>::new();
        rmagic.read_to_end(&mut bmagic).map_err(|e| e.to_string())?;

        let magic_ruleset = from_u8(
            bmagic.as_slice()
        ).map_err(|e| e.to_string())?;

        Ok(magic_ruleset)
    }

}

/// Init whatever is needed for test mod to work
pub mod init {

    extern crate std;
    use std::io::prelude::*;
    use std::io::BufReader;
    use std::fs::File;
    use std::collections::HashMap;

    // Get list of known system filetypes
    fn read_mimelist() -> Result<Vec<String>, std::io::Error> {
        let ftypes = File::open("/usr/share/mime/types")?;
        let rtypes = BufReader::new(ftypes);
        let mut mimelist = Vec::<String>::new();

        for line in rtypes.lines() {
            let mime = line?.split_whitespace().nth(0).unwrap_or("").to_string();
            mimelist.push(mime);
        }

        let mimelist = mimelist;
        Ok(mimelist)
    }

    /// Read all subclass lines from file
    fn read_subclasses() -> Result<Vec<(String, String)>, std::io::Error> {

        let f = File::open("/usr/share/mime/subclasses")?;
        let r = BufReader::new(f);
        let mut subclasses = Vec::<(String, String)>::new();

        for x in r.lines() {
            let line = x?;

            let child_raw = line.split_whitespace().nth(0).unwrap_or("").to_string();
            let parent_raw = line.split_whitespace().nth(1).unwrap_or("").to_string();

            subclasses.push( (parent_raw, child_raw) );
        }

        Ok(subclasses)
    }

    // Get filetype aliases
    fn read_aliaslist() -> Result<HashMap<String, String>, std::io::Error> {
        let faliases = File::open("/usr/share/mime/aliases")?;
        let raliases = BufReader::new(faliases);
        let mut aliaslist = HashMap::<String, String>::new();

        for line in raliases.lines() {
            let line_raw = line?;

            let a = line_raw.split_whitespace().nth(0).unwrap_or("").to_string();
            let b = line_raw.split_whitespace().nth(1).unwrap_or("").to_string();
            aliaslist.insert(a,b);
        }

        let aliaslist = aliaslist;
        Ok(aliaslist)
    }

    /// Get list of supported MIME types
    pub fn get_supported() -> Vec<String> {
        read_mimelist().unwrap_or(Vec::<String>::new())
    }

    /// Get list of parent -> child subclass links
    pub fn get_subclasses() -> Vec<(String, String)> {

        let mut subclasses = read_subclasses().unwrap_or(Vec::<(String, String)>::new());
        let aliaslist = read_aliaslist().unwrap_or(HashMap::<String, String>::new());

        // If child or parent refers to an alias, change it to the real type
        for x in 0..subclasses.iter().count(){
            match aliaslist.get(&subclasses[x].0) {
                Some(alias) => {subclasses[x].0 = alias.clone();}
                None => {}
            }
            match aliaslist.get(&subclasses[x].1) {
                Some(alias) => {subclasses[x].1 = alias.clone();}
                None => {}
            }
        }

        subclasses
    }
}

// Functions to check if a file matches a magic entry
pub mod test {

    extern crate std;

    fn from_vec_u8_singlerule(file: &Vec<u8>, rule: super::MagicRule) -> bool {

        // Check if we're even in bounds
        let bound_min = std::cmp::min(
            rule.start_off as usize,
            rule.val.iter().count()
        );
        let bound_max =
            std::cmp::min(
            (
                rule.start_off as usize +
                rule.val_len as usize +
                rule.region_len as usize
            ),
            rule.val.iter().count()
        );

        if file.iter().count() < bound_max {
            return false;
        }

        // Define our testing slice
        let ref x: Vec<u8> = *file;
        let testarea: Vec<u8> = x[bound_min .. bound_max].to_vec();
        //println!("{:?}, {:?}, {:?}\n", file, testarea, rule.val);

        // Search down until we find a hit
        for x in testarea.windows(rule.val_len as usize) {

            // Apply mask to value
            let mut y: Vec<u8>;

            let ref rule_mask = rule.mask;
            match *rule_mask {
                Some(ref mask) => {
                    y = Vec::<u8>::new();
                    for i in 0..rule.val_len {
                        y.push(x[i as usize] & mask[i as usize]);
                    }
                },
                None => y = x.to_vec(),
            }

            if y.iter().eq(rule.val.iter()) {
                return true;
            }
        }

        false
    }

    pub fn can_check(mimetype: &str) -> bool {
        /*let list = super::init::get_supported();

        list.contains(&mimetype.to_string())*/

        true
    }

    /// Test against all rules
    // This got really complicated really fast...
    pub fn from_vec_u8(
        file: Vec<u8>, mimetype: &str, magic_rules: Vec<super::MagicRule>
    ) -> bool {

        // Test every given rule
        for i in 0..magic_rules.iter().count() {

            // If there aren't any rules ahead of us, just test the rule
            if magic_rules.iter().count() - i < 2 {
                let ref x = magic_rules[i];
                match from_vec_u8_singlerule(&file, x.clone()) {
                    true => return true,
                    false => continue,
                };

            // If there are rules ahead of us...
            } else {
                let x = magic_rules[i..magic_rules.iter().count()].windows(2).next();
                let y;

                // Make sure that assumption was true
                match x {
                    Some(out) => y = out,
                    None => continue,
                }

                // Test the current rule
                match from_vec_u8_singlerule(&file, y[0].clone()) {
                    true => {
                        // Check next indent level if needed
                        if y[1].indent_level >= y[0].indent_level {
                            continue;
                        // Next indent level is lower, so this must be it
                        } else {
                            return true;
                        }
                    },
                    // No match, so keep searching
                    false => continue,
                };
            }
        }

        false
    }

    pub fn from_filepath(filepath: &str, mimetype: &str, magic_rules: Vec<super::MagicRule>) -> Result<bool, std::io::Error>{
        use std::io::prelude::*;
        use std::io::BufReader;
        use std::fs::File;

        // Get # of bytes to read
        let mut scanlen:u64 = 0;
        for x in &magic_rules {
            let tmplen:u64 =
                x.start_off as u64 +
                x.val_len as u64 +
                x.region_len as u64;

            if tmplen > scanlen {
                scanlen = tmplen;
            }
        }

        let f = File::open(filepath)?;
        let r = BufReader::new(f);
        let mut b = Vec::<u8>::new();
        r.take(scanlen).read_to_end(&mut b)?;

        Ok(from_vec_u8(b, mimetype, magic_rules))
    }

}

<|endoftext|>


<commit_before>//! `sled` is a flash-sympathetic persistent lock-free B+ tree.
//!
//! # Examples
//!
//! ```
//! let t = sled::Tree::start_default("my_db").unwrap();
//!
//! t.set(b"yo!", b"v1".to_vec());
//! assert!(t.get(b"yo!").unwrap().unwrap() == &*b"v1".to_vec());
//!
//! t.cas(
//!     b"yo!",                // key
//!     Some(b"v1"),           // old value, None for not present
//!     Some(b"v2".to_vec()),  // new value, None for delete
//! ).unwrap();
//!
//! let mut iter = t.scan(b"a non-present key before yo!");
//! // assert_eq!(iter.next(), Some(Ok((b"yo!".to_vec(), b"v2".to_vec()))));
//! // assert_eq!(iter.next(), None);
//!
//! t.del(b"yo!");
//! assert_eq!(t.get(b"yo!"), Ok(None));
//! ```

#![deny(missing_docs)]
#![cfg_attr(test, deny(clippy::warnings))]
#![cfg_attr(test, deny(clippy::bad_style))]
#![cfg_attr(test, deny(clippy::future_incompatible))]
#![cfg_attr(test, deny(clippy::nonstandard_style))]
#![cfg_attr(test, deny(clippy::rust_2018_compatibility))]
#![cfg_attr(test, deny(clippy::rust_2018_idioms))]

mod binary_search;
mod data;
mod db;
mod frag;
mod iter;
mod materializer;
mod node;
mod pinned_value;
mod prefix;
mod recovery;
mod subscription;
mod tree;

const META_PID: PageId = 0;
const COUNTER_PID: PageId = 1;
const ORIGINAL_ROOT_PID: PageId = 3;

pub use self::{
    db::Db,
    iter::{Iter, Keys, Values},
    pinned_value::PinnedValue,
    subscription::{Event, Subscriber},
    tree::Tree,
};

use pagecache::{
    Materializer, Measure, MergeOperator, PageCache, PageGet, PageId,
    M,
};

pub use pagecache::{Config, ConfigBuilder, Error, Meta, Result};

use self::{
    binary_search::{
        binary_search_gt, binary_search_lt, binary_search_lub,
        leaf_search,
    },
    data::Data,
    frag::{ChildSplit, ParentSplit},
    node::Node,
    prefix::{
        prefix_cmp, prefix_cmp_encoded, prefix_decode, prefix_encode,
    },
    recovery::Recovery,
    subscription::Subscriptions,
};

use sled_sync::{debug_delay, pin, Guard};

use log::{debug, error, trace};
use serde::{Deserialize, Serialize};

pub(crate) use self::frag::Frag;
pub(crate) use self::materializer::BLinkMaterializer;

type Key = Vec<u8>;
type KeyRef<'a> = &'a [u8];
type Value = Vec<u8>;

type TreePtr<'g> = pagecache::PagePtr<'g, Frag>;
<commit_msg>Fix doctest for sled<commit_after>//! `sled` is a flash-sympathetic persistent lock-free B+ tree.
//!
//! # Examples
//!
//! ```
//! let t = sled::Db::start_default("my_db").unwrap();
//!
//! t.set(b"yo!", b"v1".to_vec());
//! assert!(t.get(b"yo!").unwrap().unwrap() == &*b"v1".to_vec());
//!
//! t.cas(
//!     b"yo!",                // key
//!     Some(b"v1"),           // old value, None for not present
//!     Some(b"v2".to_vec()),  // new value, None for delete
//! ).unwrap();
//!
//! let mut iter = t.scan(b"a non-present key before yo!");
//! // assert_eq!(iter.next(), Some(Ok((b"yo!".to_vec(), b"v2".to_vec()))));
//! // assert_eq!(iter.next(), None);
//!
//! t.del(b"yo!");
//! assert_eq!(t.get(b"yo!"), Ok(None));
//! ```

#![deny(missing_docs)]
#![cfg_attr(test, deny(clippy::warnings))]
#![cfg_attr(test, deny(clippy::bad_style))]
#![cfg_attr(test, deny(clippy::future_incompatible))]
#![cfg_attr(test, deny(clippy::nonstandard_style))]
#![cfg_attr(test, deny(clippy::rust_2018_compatibility))]
#![cfg_attr(test, deny(clippy::rust_2018_idioms))]

mod binary_search;
mod data;
mod db;
mod frag;
mod iter;
mod materializer;
mod node;
mod pinned_value;
mod prefix;
mod recovery;
mod subscription;
mod tree;

const META_PID: PageId = 0;
const COUNTER_PID: PageId = 1;
const ORIGINAL_ROOT_PID: PageId = 3;

pub use self::{
    db::Db,
    iter::{Iter, Keys, Values},
    pinned_value::PinnedValue,
    subscription::{Event, Subscriber},
    tree::Tree,
};

use pagecache::{
    Materializer, Measure, MergeOperator, PageCache, PageGet, PageId,
    M,
};

pub use pagecache::{Config, ConfigBuilder, Error, Meta, Result};

use self::{
    binary_search::{
        binary_search_gt, binary_search_lt, binary_search_lub,
        leaf_search,
    },
    data::Data,
    frag::{ChildSplit, ParentSplit},
    node::Node,
    prefix::{
        prefix_cmp, prefix_cmp_encoded, prefix_decode, prefix_encode,
    },
    recovery::Recovery,
    subscription::Subscriptions,
};

use sled_sync::{debug_delay, pin, Guard};

use log::{debug, error, trace};
use serde::{Deserialize, Serialize};

pub(crate) use self::frag::Frag;
pub(crate) use self::materializer::BLinkMaterializer;

type Key = Vec<u8>;
type KeyRef<'a> = &'a [u8];
type Value = Vec<u8>;

type TreePtr<'g> = pagecache::PagePtr<'g, Frag>;
<|endoftext|>


<commit_before>//! A basic subtractive synthesizer

extern crate oxcable;

use oxcable::adsr::{self, Adsr};
use oxcable::filters::{first_order, second_order};
use oxcable::oscillator::{self, Oscillator, Waveform};
use oxcable::tremolo::{self, Tremolo};
use oxcable::types::{AudioDevice, MidiDevice, MidiEvent, MidiMessage, Time, Sample};
use oxcable::utils::helpers::midi_note_to_freq;
use oxcable::voice_array::VoiceArray;

#[derive(Copy, Clone, Debug)]
pub enum SubtractiveSynthMessage {
    SetOsc1(Waveform),
    SetOsc2(Waveform),
    SetOsc1Transpose(f32),
    SetOsc2Transpose(f32),
    SetAttack(f32),
    SetDecay(f32),
    SetSustain(f32),
    SetRelease(f32),
    SetLFOFreq(f32),
    SetVibrato(f32),
    SetTremolo(f32),
    SetFilterFirstOrder(first_order::FilterMode),
    SetFilterSecondOrder(second_order::FilterMode),
}
pub use self::SubtractiveSynthMessage::*;

#[derive(Copy, Clone, Debug)]
enum FilterType { FirstOrder, SecondOrder }

/// A polyphonic subtractive synthesizer
pub struct SubtractiveSynth<M: MidiDevice> {
    voices: VoiceArray<SubtractiveSynthVoice>,
    controls: Option<Box<Fn(MidiEvent) -> Option<SubtractiveSynthMessage>>>,
    midi: M,
    lfo: Oscillator,
    filter: FilterType,
    first_filter: first_order::Filter,
    second_filter: second_order::Filter,
    tremolo: Tremolo,
    lfo_buf: [Sample; 1],
    filter_input_buf: [Sample; 1],
    first_filter_buf: [Sample; 1],
    second_filter_buf: [Sample; 1],
    tremolo_buf: [Sample; 1],
    gain: f32,
}

impl<M> SubtractiveSynth<M> where M: MidiDevice {
    /// Returns a new subtractive synth that can play `num_voices` notes at one
    /// time.
    pub fn new(midi: M, num_voices: usize) -> SubtractiveSynth<M> {
        let mut voices = Vec::with_capacity(num_voices);
        for _i in (0 .. num_voices) {
            voices.push(SubtractiveSynthVoice::new());
        }
        let voice_array = VoiceArray::new(voices);

        SubtractiveSynth {
            voices: voice_array,
            controls: None,
            midi: midi,
            lfo: Oscillator::new(oscillator::Sine).freq(10.0),
            filter: FilterType::FirstOrder,
            first_filter: first_order::Filter::new(
                first_order::LowPass(20000.0), 1),
            second_filter: second_order::Filter::new(
                second_order::LowPass(20000.0), 1),
            tremolo: Tremolo::new(0.0),
            lfo_buf: [0.0],
            filter_input_buf: [0.0],
            first_filter_buf: [0.0],
            second_filter_buf: [0.0],
            tremolo_buf: [0.0],
            gain: -12.0,
        }
    }

    pub fn control_map<F>(mut self, map: F) -> SubtractiveSynth<M>
            where F: 'static+Fn(MidiEvent) -> Option<SubtractiveSynthMessage> {
        self.controls = Some(Box::new(map));
        self
    }

    pub fn osc1(mut self, waveform: Waveform) -> SubtractiveSynth<M> {
        self.handle_message(SetOsc1(waveform));
        self
    }

    pub fn osc2(mut self, waveform: Waveform) -> SubtractiveSynth<M> {
        self.handle_message(SetOsc2(waveform));
        self
    }

    pub fn osc1_transpose(mut self, steps: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetOsc1Transpose(steps));
        self
    }

    pub fn osc2_transpose(mut self, steps: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetOsc2Transpose(steps));
        self
    }

    pub fn adsr(mut self, attack_time: f32, decay_time: f32, sustain_level: f32,
               release_time: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetAttack(attack_time));
        self.handle_message(SetDecay(decay_time));
        self.handle_message(SetSustain(sustain_level));
        self.handle_message(SetRelease(release_time));
        self
    }

    pub fn lfo(mut self, freq: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetLFOFreq(freq));
        self
    }

    pub fn vibrato(mut self, vibrato: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetVibrato(vibrato));
        self
    }

    pub fn tremolo(mut self, tremolo: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetTremolo(tremolo));
        self
    }

    pub fn first_order(mut self, mode: first_order::FilterMode)
            -> SubtractiveSynth<M> {
        self.handle_message(SetFilterFirstOrder(mode));
        self
    }

    pub fn second_order(mut self, mode: second_order::FilterMode)
            -> SubtractiveSynth<M> {
        self.handle_message(SetFilterSecondOrder(mode));
        self
    }

    fn handle_message(&mut self, msg: SubtractiveSynthMessage) {
        match msg {
            SubtractiveSynthMessage::SetOsc1(waveform) => {
                for voice in self.voices.iter_mut() {
                    voice.osc1.handle_message(oscillator::SetWaveform(waveform));
                }
            },
            SubtractiveSynthMessage::SetOsc2(waveform) => {
                for voice in self.voices.iter_mut() {
                    voice.osc2.handle_message(oscillator::SetWaveform(waveform));
                }
            },
            SubtractiveSynthMessage::SetOsc1Transpose(steps) => {
                for voice in self.voices.iter_mut() {
                    voice.osc1.handle_message(oscillator::SetTranspose(steps));
                }
            },
            SubtractiveSynthMessage::SetOsc2Transpose(steps) => {
                for voice in self.voices.iter_mut() {
                    voice.osc2.handle_message(oscillator::SetTranspose(steps));
                }
            },
            SubtractiveSynthMessage::SetAttack(attack) => {
                for voice in self.voices.iter_mut() {
                    voice.adsr.handle_message(adsr::SetAttack(attack));
                }
            },
            SubtractiveSynthMessage::SetDecay(decay) => {
                for voice in self.voices.iter_mut() {
                    voice.adsr.handle_message(adsr::SetDecay(decay));
                }
            },
            SubtractiveSynthMessage::SetSustain(sustain) => {
                for voice in self.voices.iter_mut() {
                    voice.adsr.handle_message(adsr::SetSustain(sustain));
                }
            },
            SubtractiveSynthMessage::SetRelease(release) => {
                for voice in self.voices.iter_mut() {
                    voice.adsr.handle_message(adsr::SetRelease(release));
                }
            },
            SubtractiveSynthMessage::SetLFOFreq(freq) => {
                self.lfo.handle_message(oscillator::SetFreq(freq));
            },
            SubtractiveSynthMessage::SetVibrato(intensity) => {
                for voice in self.voices.iter_mut() {
                    voice.osc1.handle_message(
                        oscillator::SetLFOIntensity(intensity));
                    voice.osc2.handle_message(
                        oscillator::SetLFOIntensity(intensity));
                }
            },
            SubtractiveSynthMessage::SetTremolo(intensity) => {
                self.tremolo.handle_message(tremolo::SetIntensity(intensity));
            },
            SubtractiveSynthMessage::SetFilterFirstOrder(mode) => {
                self.filter = FilterType::FirstOrder;
                self.first_filter.set_mode(mode);
            },
            SubtractiveSynthMessage::SetFilterSecondOrder(mode) => {
                self.filter = FilterType::SecondOrder;
                self.second_filter.set_mode(mode);
            },
        }
    }

    fn handle_event(&mut self, event: MidiEvent) {
        match event.payload {
            MidiMessage::NoteOn(note, _) => {
                self.voices.note_on(note).handle_event(event);
            },
            MidiMessage::NoteOff(note, _) => {
                self.voices.note_off(note).map_or((), |d| d.handle_event(event));
            },
            MidiMessage::ControlChange(_, _) => {
                let msg = match self.controls {
                    Some(ref f) => f(event),
                    None => None
                };
                msg.map(|m| self.handle_message(m));
            },
            _ => {
                for voice in self.voices.iter_mut() {
                    voice.handle_event(event);
                }
            }
        }
    }
}

impl<M> AudioDevice for SubtractiveSynth<M> where M: MidiDevice {
    fn num_inputs(&self) -> usize {
        0
    }

    fn num_outputs(&self) -> usize {
        1
    }

    fn tick(&mut self, t: Time, _: &[Sample], outputs: &mut[Sample]) {
        for event in self.midi.get_events(t) {
            self.handle_event(event);
        }

        self.lfo.tick(t, &[0.0;0], &mut self.lfo_buf);
        self.filter_input_buf[0] = 0.0;
        for voice in self.voices.iter_mut() {
            self.filter_input_buf[0] += voice.tick(t, &self.lfo_buf);
        }
        self.first_filter.tick(t, &self.filter_input_buf,
                              &mut self.first_filter_buf);
        self.second_filter.tick(t, &self.filter_input_buf,
                               &mut self.second_filter_buf);
        match self.filter {
            FilterType::FirstOrder =>
                self.tremolo.tick(t, &self.first_filter_buf, &mut self.tremolo_buf),
            FilterType::SecondOrder =>
                self.tremolo.tick(t, &self.second_filter_buf, &mut self.tremolo_buf)
        };
        outputs[0] = self.gain * self.tremolo_buf[0];
    }
}

/// The container for a single voice
struct SubtractiveSynthVoice {
    key_held: bool,
    sustain_held: bool,
    osc1: Oscillator,
    osc2: Oscillator,
    adsr: Adsr,
    osc1_buf: [Sample; 1],
    osc2_buf: [Sample; 1],
    osc_out: [Sample; 1],
    adsr_buf: [Sample; 1],
}

impl SubtractiveSynthVoice {
    fn new() -> SubtractiveSynthVoice {
        SubtractiveSynthVoice {
            key_held: false,
            sustain_held: false,
            osc1: Oscillator::new(oscillator::Sine),
            osc2: Oscillator::new(oscillator::Sine),
            adsr: Adsr::default(1),
            osc1_buf: [0.0],
            osc2_buf: [0.0],
            osc_out: [0.0],
            adsr_buf: [0.0],
        }
    }

    fn handle_event(&mut self, event: MidiEvent) {
        match event.payload {
            MidiMessage::NoteOn(note, _) => {
                self.key_held = true;
                let freq = midi_note_to_freq(note);
                self.osc1.handle_message(oscillator::SetFreq(freq));
                self.osc2.handle_message(oscillator::SetFreq(freq));
                self.adsr.handle_message(adsr::NoteDown);
            },
            MidiMessage::NoteOff(_, _) => {
                self.key_held = false;
                if !self.sustain_held {
                    self.adsr.handle_message(adsr::NoteUp);
                }
            },
            MidiMessage::SustainPedal(true) => {
                self.sustain_held = true;
            },
            MidiMessage::SustainPedal(false) => {
                self.sustain_held = false;
                if !self.key_held {
                    self.adsr.handle_message(adsr::NoteUp);
                }
            },
            MidiMessage::PitchBend(value) => {
                let bend = 2.0*value;
                self.osc1.handle_message(oscillator::SetBend(bend));
                self.osc2.handle_message(oscillator::SetBend(bend));
            },
            _ => ()
        }
    }

    fn tick(&mut self, t: Time, lfo: &[Sample]) -> Sample {
        self.osc1.tick(t, lfo, &mut self.osc1_buf);
        self.osc2.tick(t, lfo, &mut self.osc2_buf);
        self.osc_out[0] = self.osc1_buf[0] + self.osc2_buf[0];
        self.adsr.tick(t, &self.osc_out, &mut self.adsr_buf);
        self.adsr_buf[0]
    }
}
<commit_msg>Fix tremolo input to properly feed in lfo<commit_after>//! A basic subtractive synthesizer

extern crate oxcable;

use oxcable::adsr::{self, Adsr};
use oxcable::filters::{first_order, second_order};
use oxcable::oscillator::{self, Oscillator, Waveform};
use oxcable::tremolo::{self, Tremolo};
use oxcable::types::{AudioDevice, MidiDevice, MidiEvent, MidiMessage, Time, Sample};
use oxcable::utils::helpers::midi_note_to_freq;
use oxcable::voice_array::VoiceArray;

#[derive(Copy, Clone, Debug)]
pub enum SubtractiveSynthMessage {
    SetOsc1(Waveform),
    SetOsc2(Waveform),
    SetOsc1Transpose(f32),
    SetOsc2Transpose(f32),
    SetAttack(f32),
    SetDecay(f32),
    SetSustain(f32),
    SetRelease(f32),
    SetLFOFreq(f32),
    SetVibrato(f32),
    SetTremolo(f32),
    SetFilterFirstOrder(first_order::FilterMode),
    SetFilterSecondOrder(second_order::FilterMode),
}
pub use self::SubtractiveSynthMessage::*;

#[derive(Copy, Clone, Debug)]
enum FilterType { FirstOrder, SecondOrder }

/// A polyphonic subtractive synthesizer
pub struct SubtractiveSynth<M: MidiDevice> {
    voices: VoiceArray<SubtractiveSynthVoice>,
    controls: Option<Box<Fn(MidiEvent) -> Option<SubtractiveSynthMessage>>>,
    midi: M,
    lfo: Oscillator,
    filter: FilterType,
    first_filter: first_order::Filter,
    second_filter: second_order::Filter,
    tremolo: Tremolo,
    lfo_buf: [Sample; 1],
    filter_input_buf: [Sample; 1],
    first_filter_buf: [Sample; 1],
    second_filter_buf: [Sample; 1],
    tremolo_in: [Sample; 2],
    tremolo_out: [Sample; 1],
    gain: f32,
}

impl<M> SubtractiveSynth<M> where M: MidiDevice {
    /// Returns a new subtractive synth that can play `num_voices` notes at one
    /// time.
    pub fn new(midi: M, num_voices: usize) -> SubtractiveSynth<M> {
        let mut voices = Vec::with_capacity(num_voices);
        for _i in (0 .. num_voices) {
            voices.push(SubtractiveSynthVoice::new());
        }
        let voice_array = VoiceArray::new(voices);

        SubtractiveSynth {
            voices: voice_array,
            controls: None,
            midi: midi,
            lfo: Oscillator::new(oscillator::Sine).freq(10.0),
            filter: FilterType::FirstOrder,
            first_filter: first_order::Filter::new(
                first_order::LowPass(20000.0), 1),
            second_filter: second_order::Filter::new(
                second_order::LowPass(20000.0), 1),
            tremolo: Tremolo::new(0.0),
            lfo_buf: [0.0],
            filter_input_buf: [0.0],
            first_filter_buf: [0.0],
            second_filter_buf: [0.0],
            gain: -12.0,
            tremolo_in: [0.0, 0.0],
            tremolo_out: [0.0],
        }
    }

    pub fn control_map<F>(mut self, map: F) -> SubtractiveSynth<M>
            where F: 'static+Fn(MidiEvent) -> Option<SubtractiveSynthMessage> {
        self.controls = Some(Box::new(map));
        self
    }

    pub fn osc1(mut self, waveform: Waveform) -> SubtractiveSynth<M> {
        self.handle_message(SetOsc1(waveform));
        self
    }

    pub fn osc2(mut self, waveform: Waveform) -> SubtractiveSynth<M> {
        self.handle_message(SetOsc2(waveform));
        self
    }

    pub fn osc1_transpose(mut self, steps: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetOsc1Transpose(steps));
        self
    }

    pub fn osc2_transpose(mut self, steps: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetOsc2Transpose(steps));
        self
    }

    pub fn adsr(mut self, attack_time: f32, decay_time: f32, sustain_level: f32,
               release_time: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetAttack(attack_time));
        self.handle_message(SetDecay(decay_time));
        self.handle_message(SetSustain(sustain_level));
        self.handle_message(SetRelease(release_time));
        self
    }

    pub fn lfo(mut self, freq: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetLFOFreq(freq));
        self
    }

    pub fn vibrato(mut self, vibrato: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetVibrato(vibrato));
        self
    }

    pub fn tremolo(mut self, tremolo: f32) -> SubtractiveSynth<M> {
        self.handle_message(SetTremolo(tremolo));
        self
    }

    pub fn first_order(mut self, mode: first_order::FilterMode)
            -> SubtractiveSynth<M> {
        self.handle_message(SetFilterFirstOrder(mode));
        self
    }

    pub fn second_order(mut self, mode: second_order::FilterMode)
            -> SubtractiveSynth<M> {
        self.handle_message(SetFilterSecondOrder(mode));
        self
    }

    fn handle_message(&mut self, msg: SubtractiveSynthMessage) {
        match msg {
            SubtractiveSynthMessage::SetOsc1(waveform) => {
                for voice in self.voices.iter_mut() {
                    voice.osc1.handle_message(oscillator::SetWaveform(waveform));
                }
            },
            SubtractiveSynthMessage::SetOsc2(waveform) => {
                for voice in self.voices.iter_mut() {
                    voice.osc2.handle_message(oscillator::SetWaveform(waveform));
                }
            },
            SubtractiveSynthMessage::SetOsc1Transpose(steps) => {
                for voice in self.voices.iter_mut() {
                    voice.osc1.handle_message(oscillator::SetTranspose(steps));
                }
            },
            SubtractiveSynthMessage::SetOsc2Transpose(steps) => {
                for voice in self.voices.iter_mut() {
                    voice.osc2.handle_message(oscillator::SetTranspose(steps));
                }
            },
            SubtractiveSynthMessage::SetAttack(attack) => {
                for voice in self.voices.iter_mut() {
                    voice.adsr.handle_message(adsr::SetAttack(attack));
                }
            },
            SubtractiveSynthMessage::SetDecay(decay) => {
                for voice in self.voices.iter_mut() {
                    voice.adsr.handle_message(adsr::SetDecay(decay));
                }
            },
            SubtractiveSynthMessage::SetSustain(sustain) => {
                for voice in self.voices.iter_mut() {
                    voice.adsr.handle_message(adsr::SetSustain(sustain));
                }
            },
            SubtractiveSynthMessage::SetRelease(release) => {
                for voice in self.voices.iter_mut() {
                    voice.adsr.handle_message(adsr::SetRelease(release));
                }
            },
            SubtractiveSynthMessage::SetLFOFreq(freq) => {
                self.lfo.handle_message(oscillator::SetFreq(freq));
            },
            SubtractiveSynthMessage::SetVibrato(intensity) => {
                for voice in self.voices.iter_mut() {
                    voice.osc1.handle_message(
                        oscillator::SetLFOIntensity(intensity));
                    voice.osc2.handle_message(
                        oscillator::SetLFOIntensity(intensity));
                }
            },
            SubtractiveSynthMessage::SetTremolo(intensity) => {
                self.tremolo.handle_message(tremolo::SetIntensity(intensity));
            },
            SubtractiveSynthMessage::SetFilterFirstOrder(mode) => {
                self.filter = FilterType::FirstOrder;
                self.first_filter.set_mode(mode);
            },
            SubtractiveSynthMessage::SetFilterSecondOrder(mode) => {
                self.filter = FilterType::SecondOrder;
                self.second_filter.set_mode(mode);
            },
        }
    }

    fn handle_event(&mut self, event: MidiEvent) {
        match event.payload {
            MidiMessage::NoteOn(note, _) => {
                self.voices.note_on(note).handle_event(event);
            },
            MidiMessage::NoteOff(note, _) => {
                self.voices.note_off(note).map_or((), |d| d.handle_event(event));
            },
            MidiMessage::ControlChange(_, _) => {
                let msg = match self.controls {
                    Some(ref f) => f(event),
                    None => None
                };
                msg.map(|m| self.handle_message(m));
            },
            _ => {
                for voice in self.voices.iter_mut() {
                    voice.handle_event(event);
                }
            }
        }
    }
}

impl<M> AudioDevice for SubtractiveSynth<M> where M: MidiDevice {
    fn num_inputs(&self) -> usize {
        0
    }

    fn num_outputs(&self) -> usize {
        1
    }

    fn tick(&mut self, t: Time, _: &[Sample], outputs: &mut[Sample]) {
        for event in self.midi.get_events(t) {
            self.handle_event(event);
        }

        self.lfo.tick(t, &[0.0;0], &mut self.lfo_buf);

        self.filter_input_buf[0] = 0.0;
        for voice in self.voices.iter_mut() {
            self.filter_input_buf[0] += voice.tick(t, &self.lfo_buf);
        }
        self.first_filter.tick(t, &self.filter_input_buf,
                              &mut self.first_filter_buf);
        self.second_filter.tick(t, &self.filter_input_buf,
                               &mut self.second_filter_buf);

        self.tremolo_in[0] = match self.filter {
            FilterType::FirstOrder => self.first_filter_buf[0],
            FilterType::SecondOrder => self.second_filter_buf[0]
        };
        self.tremolo_in[1] = self.lfo_buf[0];
        self.tremolo.tick(t, &self.tremolo_in, &mut self.tremolo_out);

        outputs[0] = self.gain * self.tremolo_out[0];
    }
}

/// The container for a single voice
struct SubtractiveSynthVoice {
    key_held: bool,
    sustain_held: bool,
    osc1: Oscillator,
    osc2: Oscillator,
    adsr: Adsr,
    osc1_buf: [Sample; 1],
    osc2_buf: [Sample; 1],
    osc_out: [Sample; 1],
    adsr_buf: [Sample; 1],
}

impl SubtractiveSynthVoice {
    fn new() -> SubtractiveSynthVoice {
        SubtractiveSynthVoice {
            key_held: false,
            sustain_held: false,
            osc1: Oscillator::new(oscillator::Sine),
            osc2: Oscillator::new(oscillator::Sine),
            adsr: Adsr::default(1),
            osc1_buf: [0.0],
            osc2_buf: [0.0],
            osc_out: [0.0],
            adsr_buf: [0.0],
        }
    }

    fn handle_event(&mut self, event: MidiEvent) {
        match event.payload {
            MidiMessage::NoteOn(note, _) => {
                self.key_held = true;
                let freq = midi_note_to_freq(note);
                self.osc1.handle_message(oscillator::SetFreq(freq));
                self.osc2.handle_message(oscillator::SetFreq(freq));
                self.adsr.handle_message(adsr::NoteDown);
            },
            MidiMessage::NoteOff(_, _) => {
                self.key_held = false;
                if !self.sustain_held {
                    self.adsr.handle_message(adsr::NoteUp);
                }
            },
            MidiMessage::SustainPedal(true) => {
                self.sustain_held = true;
            },
            MidiMessage::SustainPedal(false) => {
                self.sustain_held = false;
                if !self.key_held {
                    self.adsr.handle_message(adsr::NoteUp);
                }
            },
            MidiMessage::PitchBend(value) => {
                let bend = 2.0*value;
                self.osc1.handle_message(oscillator::SetBend(bend));
                self.osc2.handle_message(oscillator::SetBend(bend));
            },
            _ => ()
        }
    }

    fn tick(&mut self, t: Time, lfo: &[Sample]) -> Sample {
        self.osc1.tick(t, lfo, &mut self.osc1_buf);
        self.osc2.tick(t, lfo, &mut self.osc2_buf);
        self.osc_out[0] = self.osc1_buf[0] + self.osc2_buf[0];
        self.adsr.tick(t, &self.osc_out, &mut self.adsr_buf);
        self.adsr_buf[0]
    }
}
<|endoftext|>


<commit_before>use std::cmp;
use std::iter;
use std::time::{Duration, Instant};

use rand::{self, Rng};
use rand::distributions::IndependentSample;
use rand::distributions::gamma::Gamma;

use board::{Board, LegalMove, GameState};
use player::Player;

// Jeffrey's prior
const PRIOR: f64 = 0.5;

pub struct MCTSPlayer {
    dur: Duration,
}

impl MCTSPlayer {
    pub fn new(dur: Duration) -> Self {
        MCTSPlayer { dur }
    }
}

impl Player for MCTSPlayer {
    fn choose(&self, b: &Board) -> LegalMove {
        let now = Instant::now();
        let mut rng = rand::thread_rng();
        let mut node = Node::Unvisited;
        for i in 0.. {
            node.explore(&mut rng, b.clone());
            if node.is_certain() { break; }
            if now.elapsed() >= self.dur {
                println!("Choosing move after {} play-throughs", i);
                break;
            }
        }
        node.best_move(b)
    }
}

#[derive(Clone, Debug, PartialEq)]
enum Node {
    Unvisited,
    CertainLoss(Certain),
    CertainWin(Certain),
    CertainDraw(Certain),
    Probabilistic(Probabilistic),
}

impl Node {
    pub fn is_certain(&self) -> bool {
        match *self {
            Node::Unvisited => false,
            Node::CertainLoss(..) => true,
            Node::CertainWin(..) => true,
            Node::CertainDraw(..) => true,
            Node::Probabilistic(..) => false,
        }
    }

    pub fn best_move(&self, b: &Board) -> LegalMove {
        match *self {
            Node::Unvisited => panic!("node is unvisited"),
            Node::CertainLoss(ref c) => {
                println!("Certain loss in {} move(s)", c.depth);
                c.best_move(b)
            },
            Node::CertainWin(ref c) => {
                println!("Certain win in {} move(s)", c.depth);
                c.best_move(b)
            },
            Node::CertainDraw(ref c) => c.best_move(b),
            Node::Probabilistic(ref p) => p.best_move(b),
        }
    }

    pub fn explore<R: Rng>(&mut self, rng: &mut R, b: Board) -> f64 {
        let result = match *self {
            Node::Unvisited => self.explore_unvisted(rng, b),
            Node::Probabilistic(ref mut p) => p.explore(rng, b),
            _ => Ok(self.score()),
        };
        match result {
            Ok(score) => score,
            Err(node) => {
                *self = node;
                self.score()
            },
        }
    }

    fn score(&self) -> f64 {
        match *self {
            Node::Unvisited => panic!("node is unvisited"),
            Node::CertainLoss(..) => 1.0,
            Node::CertainWin(..) => 0.0,
            Node::CertainDraw(..) => 0.5,
            Node::Probabilistic(ref p) => p.score(),
        }
    }

    fn explore_unvisted<R: Rng>(&mut self, rng: &mut R, mut b: Board)
                                -> Result<f64, Node> {
        let (n, i, m) = Node::choose_unvisited_first(rng, &b);
        match b.make_legal_move(m) {
            GameState::Won => Err(Node::CertainWin(Certain::new(1, i))),
            GameState::Drawn => Err(Node::CertainDraw(Certain::new(1, i))),
            GameState::Ongoing => {
                let score = Node::choose_unvisited_rest(rng, b);
                Err(Node::Probabilistic(Probabilistic::new(n, score)))
            }
        }
    }

    fn choose_unvisited_first<R: Rng>(rng: &mut R, b: &Board) -> (usize, usize, LegalMove) {
        let mut m = None;
        let mut i = 0;
        let mut n = 0;
        for (i1, m1) in b.legal_moves_iter().enumerate() {
            n += 1;
            if m1.is_winning() { return (n, i1, m1); }
            if rng.gen_range(0, n) == 0 {
                i = i1;
                m = Some(m1);
            }
        }
        (n, i, m.unwrap())
    }

    fn choose_unvisited_rest<R: Rng>(rng: &mut R, mut b: Board) -> f64 {
        let mut score = 1.0;
        loop {
            let m = super::choose_winning_or_random(&b, rng);
            match b.make_legal_move(m) {
                GameState::Won => return score,
                GameState::Drawn => return 0.5,
                GameState::Ongoing => score = 1.0 - score,
            }
        }
    }

    fn p_parent_win(&self) -> f64 {
        match *self {
            Node::Unvisited => 0.5,
            Node::CertainLoss(..) => 1.0,
            Node::CertainWin(..) => 0.0,
            Node::CertainDraw(..) => 0.5,
            Node::Probabilistic(ref p) => p.p_parent_win(),
        }
    }

    fn p_parent_win_sample<R: Rng>(&self, rng: &mut R) -> f64 {
        match *self {
            Node::Unvisited => beta_sample(rng, PRIOR, PRIOR),
            Node::Probabilistic(ref p) => p.p_parent_win_sample(rng),
            _ => self.p_parent_win(),
        }
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
struct Certain {
    depth: usize,
    index: usize,
}

impl Certain {
    fn new(depth: usize, index: usize) -> Self {
        Certain { depth, index }
    }

    fn parent(&self, index: usize) -> Self {
        Certain::new(self.depth + 1, index)
    }

    fn best_move(&self, b: &Board) -> LegalMove {
        b.legal_moves_iter().nth(self.index).unwrap()
    }
}

#[derive(Clone, Debug, PartialEq)]
struct Probabilistic {
    score: f64,
    nplay: f64,
    children: Box<[Node]>,
}

impl Probabilistic {
    fn new(nchildren: usize, score: f64) -> Self {
        let score = PRIOR + score;
        let nplay = PRIOR + PRIOR + 1.0;
        let children = iter::repeat(Node::Unvisited).
            take(nchildren).collect::<Vec<Node>>().into_boxed_slice();
        Probabilistic { score, nplay, children }
    }

    fn best_move(&self, b: &Board) -> LegalMove {
        self.children.iter().zip(b.legal_moves_iter()).max_by(|&(n1, _), &(n2, _)| {
            n1.p_parent_win().partial_cmp(&n2.p_parent_win()).unwrap()
        }).map(|(_, m)| m).unwrap()
    }

    fn p_parent_win(&self) -> f64 {
        self.score / self.nplay
    }

    fn p_parent_win_sample<R: Rng>(&self, rng: &mut R) -> f64 {
        beta_sample(rng, self.score, self.nplay - self.score)
    }

    fn explore<R: Rng>(&mut self, rng: &mut R, mut b: Board)
                       -> Result<f64, Node> {
        let mut closs = None;
        let mut nwin = 0;
        let mut cwin = None;
        let mut ndraw = 0;
        let mut cdraw = None;
        for (i, node) in self.children.iter().enumerate() {
            match *node {
                Node::CertainLoss(ref c1) => {
                    match closs {
                        None => closs = Some(c1.parent(i)),
                        Some(ref mut c) => *c = cmp::min(*c, c1.parent(i)),
                    }
                },
                Node::CertainWin(ref c1) => {
                    nwin += 1;
                    match cwin {
                        None => cwin = Some(c1.parent(i)),
                        Some(ref mut c) => *c = cmp::max(*c, c1.parent(i)),
                    }
                },
                Node::CertainDraw(ref c1) => {
                    ndraw += 1;
                    match cdraw {
                        None => cdraw = Some(c1.parent(i)),
                        Some(ref mut c) => *c = cmp::max(*c, c1.parent(i)),
                    }
                },
                _ => (),
            }
        }
        let n = self.children.len();
        if let Some(c) = closs { return Err(Node::CertainWin(c)); }
        if let Some(c) = cwin { if n == nwin { return Err(Node::CertainLoss(c)); } }
        if let Some(c) = cdraw { if n == nwin + ndraw { return Err(Node::CertainDraw(c)); } }
        let (_, node, m) = self.children.iter_mut().zip(b.legal_moves_iter()).
            map(|(node, m)| (node.p_parent_win_sample(rng), node, m)).
            max_by(|&(p1, _, _), &(p2, _, _)| p1.partial_cmp(&p2).unwrap()).
            unwrap();
        b.make_legal_move(m);
        let score = 1.0 - node.explore(rng, b);
        self.score += score;
        self.nplay += 1.0;
        Ok(score)
    }

    fn score(&self) -> f64 {
        self.score - PRIOR as f64
    }
}

fn beta_sample<R: Rng>(rng: &mut R, alpha: f64, beta: f64) -> f64 {
    if alpha <= 1.0 && beta <= 1.0 {
        loop {
            let u = rng.next_f64();
            let v = rng.next_f64();
            let x = u.powf(1.0 / alpha);
            let y = v.powf(1.0 / beta);

            if x + y > 1.0 { continue; }
            if x + y > 0.0 { return x / (x + y); }

            let ln_x = u.ln() / alpha;
            let ln_y = v.ln() / beta;
            let ln_m = if ln_x > ln_y { ln_x } else { ln_y };
            let ln_x = ln_x - ln_m;
            let ln_y = ln_y - ln_m;
            return (ln_x - (ln_x.exp() + ln_y.exp()).ln()).exp();
        }
    } else {
        let a = Gamma::new(alpha, 1.0).ind_sample(rng);
        let b = Gamma::new(beta, 1.0).ind_sample(rng);
        a / (a + b)
    }
}
<commit_msg>Refactor probabilistic exploration<commit_after>use std::iter;
use std::time::{Duration, Instant};

use rand::{self, Rng};
use rand::distributions::IndependentSample;
use rand::distributions::gamma::Gamma;

use board::{Board, LegalMove, GameState};
use player::Player;

// Jeffrey's prior
const PRIOR: f64 = 0.5;

pub struct MCTSPlayer {
    dur: Duration,
}

impl MCTSPlayer {
    pub fn new(dur: Duration) -> Self {
        MCTSPlayer { dur }
    }
}

impl Player for MCTSPlayer {
    fn choose(&self, b: &Board) -> LegalMove {
        let now = Instant::now();
        let mut rng = rand::thread_rng();
        let mut node = Node::Unvisited;
        for i in 0.. {
            node.explore(&mut rng, b.clone());
            if node.is_certain() { break; }
            if now.elapsed() >= self.dur {
                println!("Choosing move after {} play-throughs", i);
                break;
            }
        }
        node.best_move(b)
    }
}

#[derive(Clone, Debug, PartialEq)]
enum Node {
    Unvisited,
    Probabilistic(Probabilistic),
    CertainLoss(Certain),
    CertainWin(Certain),
    CertainDraw(Certain),
}

impl Node {
    pub fn is_certain(&self) -> bool {
        match *self {
            Node::Unvisited => false,
            Node::Probabilistic(..) => false,
            Node::CertainLoss(..) => true,
            Node::CertainWin(..) => true,
            Node::CertainDraw(..) => true,
        }
    }

    pub fn best_move(&self, b: &Board) -> LegalMove {
        match *self {
            Node::Unvisited => panic!("node is unvisited"),
            Node::Probabilistic(ref p) => p.best_move(b),
            Node::CertainLoss(ref c) => {
                println!("Certain loss in {} move(s)", c.depth);
                c.best_move(b)
            },
            Node::CertainWin(ref c) => {
                println!("Certain win in {} move(s)", c.depth);
                c.best_move(b)
            },
            Node::CertainDraw(ref c) => c.best_move(b),
        }
    }

    pub fn explore<R: Rng>(&mut self, rng: &mut R, b: Board) -> f64 {
        let result = match *self {
            Node::Unvisited => self.explore_unvisted(rng, b),
            Node::Probabilistic(ref mut p) => p.explore(rng, b),
            _ => Ok(self.score()),
        };
        match result {
            Ok(score) => score,
            Err(node) => {
                *self = node;
                self.score()
            },
        }
    }

    fn score(&self) -> f64 {
        match *self {
            Node::Unvisited => panic!("node is unvisited"),
            Node::Probabilistic(ref p) => p.score(),
            Node::CertainLoss(..) => 1.0,
            Node::CertainWin(..) => 0.0,
            Node::CertainDraw(..) => 0.5,
        }
    }

    fn explore_unvisted<R: Rng>(&mut self, rng: &mut R, mut b: Board)
                                -> Result<f64, Node> {
        let (n, i, m) = Node::choose_unvisited_first(rng, &b);
        match b.make_legal_move(m) {
            GameState::Won => Err(Node::CertainWin(Certain::new(1, i))),
            GameState::Drawn => Err(Node::CertainDraw(Certain::new(1, i))),
            GameState::Ongoing => {
                let score = Node::choose_unvisited_rest(rng, b);
                Err(Node::Probabilistic(Probabilistic::new(n, score)))
            }
        }
    }

    fn choose_unvisited_first<R: Rng>(rng: &mut R, b: &Board) -> (usize, usize, LegalMove) {
        let mut m = None;
        let mut i = 0;
        let mut n = 0;
        for (i1, m1) in b.legal_moves_iter().enumerate() {
            n += 1;
            if m1.is_winning() { return (n, i1, m1); }
            if rng.gen_range(0, n) == 0 {
                i = i1;
                m = Some(m1);
            }
        }
        (n, i, m.unwrap())
    }

    fn choose_unvisited_rest<R: Rng>(rng: &mut R, mut b: Board) -> f64 {
        let mut score = 1.0;
        loop {
            let m = super::choose_winning_or_random(&b, rng);
            match b.make_legal_move(m) {
                GameState::Won => return score,
                GameState::Drawn => return 0.5,
                GameState::Ongoing => score = 1.0 - score,
            }
        }
    }

    fn p_parent_win(&self) -> f64 {
        match *self {
            Node::Unvisited => 0.5,
            Node::Probabilistic(ref p) => p.p_parent_win(),
            Node::CertainLoss(..) => 1.0,
            Node::CertainWin(..) => 0.0,
            Node::CertainDraw(..) => 0.5,
        }
    }

    fn p_parent_win_sample<R: Rng>(&self, rng: &mut R) -> f64 {
        match *self {
            Node::Unvisited => beta_sample(rng, PRIOR, PRIOR),
            Node::Probabilistic(ref p) => p.p_parent_win_sample(rng),
            _ => self.p_parent_win(),
        }
    }

    fn rank_ordinal(&self) -> usize {
        match *self {
            Node::Unvisited => 2,
            Node::Probabilistic(..) => 2,
            Node::CertainLoss(..) => 3,
            Node::CertainWin(..) => 0,
            Node::CertainDraw(..) => 1,
        }
    }

    fn rank_discriminator(&self) -> isize {
        match *self {
            Node::Unvisited => 0,
            Node::Probabilistic(..) => 0,
            Node::CertainLoss(ref c) => -(c.depth as isize),
            Node::CertainWin(ref c) => c.depth as isize,
            Node::CertainDraw(ref c) => c.depth as isize,
        }
    }

    fn rank_key<R: Rng>(&self, rng: &mut R) -> (usize, f64, isize) {
        let o = self.rank_ordinal();
        let p = self.p_parent_win_sample(rng);
        let d = self.rank_discriminator();
        (o, p, d)
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
struct Certain {
    depth: usize,
    index: usize,
}

impl Certain {
    fn new(depth: usize, index: usize) -> Self {
        Certain { depth, index }
    }

    fn parent(&self, index: usize) -> Self {
        Certain::new(self.depth + 1, index)
    }

    fn best_move(&self, b: &Board) -> LegalMove {
        b.legal_moves_iter().nth(self.index).unwrap()
    }
}

#[derive(Clone, Debug, PartialEq)]
struct Probabilistic {
    score: f64,
    nplay: f64,
    children: Box<[Node]>,
}

impl Probabilistic {
    fn new(nchildren: usize, score: f64) -> Self {
        let score = PRIOR + score;
        let nplay = PRIOR + PRIOR + 1.0;
        let children = iter::repeat(Node::Unvisited).
            take(nchildren).collect::<Vec<Node>>().into_boxed_slice();
        Probabilistic { score, nplay, children }
    }

    fn best_move(&self, b: &Board) -> LegalMove {
        self.children.iter().zip(b.legal_moves_iter()).max_by(|&(n1, _), &(n2, _)| {
            n1.p_parent_win().partial_cmp(&n2.p_parent_win()).unwrap()
        }).map(|(_, m)| m).unwrap()
    }

    fn p_parent_win(&self) -> f64 {
        self.score / self.nplay
    }

    fn p_parent_win_sample<R: Rng>(&self, rng: &mut R) -> f64 {
        beta_sample(rng, self.score, self.nplay - self.score)
    }

    fn explore<R: Rng>(&mut self, rng: &mut R, mut b: Board) -> Result<f64, Node> {
        let (_, i, node) = self.children.iter_mut().enumerate().map(|(i, node)| {
            (node.rank_key(rng), i, node)
        }).max_by(|&(k1, _, _), &(k2, _, _)| {
            k1.partial_cmp(&k2).unwrap()
        }).unwrap();
        match *node {
            Node::CertainLoss(ref c) => Err(Node::CertainWin(c.parent(i))),
            Node::CertainWin(ref c) => Err(Node::CertainLoss(c.parent(i))),
            Node::CertainDraw(ref c) => Err(Node::CertainDraw(c.parent(i))),
            _ => {
                let m = b.legal_moves_iter().nth(i).unwrap();
                b.make_legal_move(m);
                let score = 1.0 - node.explore(rng, b);
                self.score += score;
                self.nplay += 1.0;
                Ok(score)
            }
        }
    }

    fn score(&self) -> f64 {
        self.score - PRIOR as f64
    }
}

fn beta_sample<R: Rng>(rng: &mut R, alpha: f64, beta: f64) -> f64 {
    if alpha <= 1.0 && beta <= 1.0 {
        loop {
            let u = rng.next_f64();
            let v = rng.next_f64();
            let x = u.powf(1.0 / alpha);
            let y = v.powf(1.0 / beta);

            if x + y > 1.0 { continue; }
            if x + y > 0.0 { return x / (x + y); }

            let ln_x = u.ln() / alpha;
            let ln_y = v.ln() / beta;
            let ln_m = if ln_x > ln_y { ln_x } else { ln_y };
            let ln_x = ln_x - ln_m;
            let ln_y = ln_y - ln_m;
            return (ln_x - (ln_x.exp() + ln_y.exp()).ln()).exp();
        }
    } else {
        let a = Gamma::new(alpha, 1.0).ind_sample(rng);
        let b = Gamma::new(beta, 1.0).ind_sample(rng);
        a / (a + b)
    }
}
<|endoftext|>


<commit_before><commit_msg>Bug 1748499 - Switch WR's scale_factors to operate in double precision r=nical<commit_after><|endoftext|>


<commit_before>mod sortdir;
mod entity;

use std::io;
use std::ops::Range;
use std::path::PathBuf;
use std::fs::{ Metadata, ReadDir };
use futures::{ stream, Stream, Future };
use futures::future::Executor;
use hyper::{ header, Request, Response, Head, Body, StatusCode };
use mime_guess::guess_mime_type;
use maud::Render;
use slog::Logger;
use ::utils::{ path_canonicalize, decode_path };
use ::{ error, file, Httpd };
use self::sortdir::{ SortDir, up };
use self::entity::{ Entity, EntifyResult };

pub struct Process<'a> {
    httpd: &'a Httpd,
    log: &'a Logger,
    req: &'a Request,
    is_root: bool,
    path: PathBuf
}

impl<'a> Process<'a> {
    #[inline]
    pub fn new(httpd: &'a Httpd, log: &'a Logger, req: &'a Request) -> Process<'a> {
        let path_buf = decode_path(req.path());
        let (depth, path) = path_canonicalize(&httpd.root, path_buf);
        Process { httpd, log, req, path, is_root: depth == 0 }
    }

    #[inline]
    pub fn process(&self) -> io::Result<Response> {
        let metadata = self.path.metadata()?;

        if let Ok(dir) = self.path.read_dir() {
            self.process_dir(dir)
        } else {
            self.process_file(&metadata)
        }
    }
}

impl<'a> Process<'a> {
    fn process_dir(&self, dir: ReadDir) -> io::Result<Response> {
        const HTML_HEADER: &str = "

";
        const HTML_FOOTER: &str = "
</table";

        let mut res = Response::new();

        if self.req.method() == &Head {
            res.set_body(Body::empty());
        } else {
            let log = self.log.clone();
            let (send, body) = Body::pair();
            res.set_body(body);

            debug!(self.log, "process"; "method" => "senddir");

            let done = stream::once::<_, error::Error>(Ok(chunk!(HTML_HEADER)))
                .chain(stream::once(Ok(chunk!(into up(self.is_root)))))
                .chain(stream::iter_ok(SortDir::new(dir))
                    .map(|p| p.and_then(|m| chunk!(into m.render())).map_err(Into::into))
                )
                .chain(stream::once(Ok(chunk!(HTML_FOOTER))))
                .map_err(error::Error::from)
                .forward(send)
                .map(drop)
                .map_err(move |err| error!(log, "send"; "err" => format_args!("{}", err)));

            self.httpd.remote.execute(done).unwrap();
        }

        // TODO https://github.com/hyperium/mime/issues/52
        let mime = "text/html; charset=utf-8".parse().unwrap();
        Ok(res.with_header(header::ContentType(mime)))
    }

    fn process_file(&self, metadata: &Metadata) -> io::Result<Response> {
        let entity = Entity::new(&self.path, metadata, self.log, self.httpd.chunk_length);

        match entity.check(self.req.headers()) {
            EntifyResult::Err(resp) => Ok(resp.with_headers(entity.headers(false))),
            EntifyResult::None => {
                let fd = entity.open()?;
                let length = fd.length;
                self.send(fd, None)
                    .map(|res| res
                        .with_headers(entity.headers(false))
                        .with_header(header::ContentLength(length))
                    )
            },
            EntifyResult::One(range) => {
                debug!(self.log, "process"; "range" => format_args!("{:?}", range));

                let fd = entity.open()?;
                let length = fd.length;
                self.send(fd, Some(range.clone()))
                    .map(|res| res
                         .with_status(StatusCode::PartialContent)
                         .with_headers(entity.headers(false))
                         .with_header(header::ContentLength(range.end - range.start))
                         .with_header(header::ContentRange(header::ContentRangeSpec::Bytes {
                            range: Some((range.start, range.end - 1)), instance_length: Some(length)
                        }))
                    )
            },
            EntifyResult::Vec(ranges) => {
                const BOUNDARY_LINE: &str = concat!("--", boundary!(), "\r\n");

                debug!(self.log, "process"; "ranges" => format_args!("{:?}", ranges));

                let mut res = Response::new();

                if self.req.method() == &Head {
                    return Ok(res
                        .with_status(StatusCode::PartialContent)
                        .with_headers(entity.headers(true))
                        .with_body(Body::empty())
                    );
                }

                let log = self.log.clone();
                let (send, body) = Body::pair();
                res.set_body(body);

                let fd = entity.open()?;
                let mime_type = guess_mime_type(&self.path);

                let done = stream::iter_ok::<_, error::Error>(ranges.into_iter())
                    .and_then(move |range| {
                        let length = range.end - range.start;
                        let mut headers = header::Headers::new();
                        headers.set(header::ContentType(mime_type.clone()));
                        headers.set(header::ContentRange(header::ContentRangeSpec::Bytes {
                            range: Some((range.start, range.end - 1)), instance_length: Some(length)
                        }));

                        fd.try_clone()?
                            .read(range)
                            .map(|fut| {
                                stream::once(Ok(chunk!(BOUNDARY_LINE)))
                                    .chain(stream::once(Ok(chunk!(format!("{}\r\n", headers)))))
                                    .chain(fut)
                                    .chain(stream::once(Ok(chunk!("\r\n"))))
                            })
                            .map_err(Into::into)
                    })
                    .flatten()
                    .forward(send)
                    .map(drop)
                    .map_err(move |err| error!(log, "send"; "err" => format_args!("{}", err)));

                self.httpd.remote.execute(done).unwrap();

                Ok(res
                    .with_status(StatusCode::PartialContent)
                    .with_headers(entity.headers(true))
                )
            }
        }
    }

    #[cfg(not(feature = "sendfile"))]
    fn send(&self, fd: file::File, range: Option<Range<u64>>) -> io::Result<Response> {
        let mut res = Response::new();

        if self.req.method() == &Head {
            res.set_body(Body::empty());
        } else {
            let range = range.unwrap_or_else(|| 0..fd.length);

            let log = self.log.clone();
            let (send, body) = Body::pair();
            res.set_body(body);

            debug!(self.log, "process"; "method" => "readchunk");

            let done = fd.read(range)?
                .forward(send)
                .map(drop)
                .map_err(move |err| error!(log, "send"; "err" => format_args!("{}", err)));

            self.httpd.remote.execute(done).unwrap();
        }

        Ok(res)
    }

    #[cfg(feature = "sendfile")]
    fn send(&self, fd: file::File, range: Option<Range<u64>>) -> io::Result<Response> {
        use futures::future;
        let mut res = Response::new();

        if self.req.method() == &Head {
            res.set_body(Body::empty());
        } else if let (&Some(ref socket), true) = (&self.httpd.socket, self.httpd.chunk_length >= fd.length as _) {
            let range = range.unwrap_or_else(|| 0..fd.length);
            let log = self.log.clone();
            res.set_body(Body::empty());

            debug!(self.log, "process"; "method" => "sendfile");

            let done = fd.sendfile(range, socket.clone())?
                .for_each(|_| future::ok(()))
                .map_err(move |err| error!(log, "send"; "err" => format_args!("{}", err)));

            self.httpd.remote.execute(done).unwrap();
        } else {
            let range = range.unwrap_or_else(|| 0..fd.length);

            let log = self.log.clone();
            let (send, body) = Body::pair();
            res.set_body(body);

            debug!(self.log, "process"; "method" => "readchunk");

            let done = fd.read(range)?
                .forward(send)
                .map(drop)
                .map_err(move |err| error!(log, "send"; "err" => format_args!("{}", err)));

            self.httpd.remote.execute(done).unwrap();
        }

        Ok(res)
    }
}
<commit_msg>[Fixed] html footer<commit_after>mod sortdir;
mod entity;

use std::io;
use std::ops::Range;
use std::path::PathBuf;
use std::fs::{ Metadata, ReadDir };
use futures::{ stream, Stream, Future };
use futures::future::Executor;
use hyper::{ header, Request, Response, Head, Body, StatusCode };
use mime_guess::guess_mime_type;
use maud::Render;
use slog::Logger;
use ::utils::{ path_canonicalize, decode_path };
use ::{ error, file, Httpd };
use self::sortdir::{ SortDir, up };
use self::entity::{ Entity, EntifyResult };

pub struct Process<'a> {
    httpd: &'a Httpd,
    log: &'a Logger,
    req: &'a Request,
    is_root: bool,
    path: PathBuf
}

impl<'a> Process<'a> {
    #[inline]
    pub fn new(httpd: &'a Httpd, log: &'a Logger, req: &'a Request) -> Process<'a> {
        let path_buf = decode_path(req.path());
        let (depth, path) = path_canonicalize(&httpd.root, path_buf);
        Process { httpd, log, req, path, is_root: depth == 0 }
    }

    #[inline]
    pub fn process(&self) -> io::Result<Response> {
        let metadata = self.path.metadata()?;

        if let Ok(dir) = self.path.read_dir() {
            self.process_dir(dir)
        } else {
            self.process_file(&metadata)
        }
    }
}

impl<'a> Process<'a> {
    fn process_dir(&self, dir: ReadDir) -> io::Result<Response> {
        const HTML_HEADER: &str = "

";
        const HTML_FOOTER: &str = "

";

        let mut res = Response::new();

        if self.req.method() == &Head {
            res.set_body(Body::empty());
        } else {
            let log = self.log.clone();
            let (send, body) = Body::pair();
            res.set_body(body);

            debug!(self.log, "process"; "method" => "senddir");

            let done = stream::once::<_, error::Error>(Ok(chunk!(HTML_HEADER)))
                .chain(stream::once(Ok(chunk!(into up(self.is_root)))))
                .chain(stream::iter_ok(SortDir::new(dir))
                    .map(|p| p.and_then(|m| chunk!(into m.render())).map_err(Into::into))
                )
                .chain(stream::once(Ok(chunk!(HTML_FOOTER))))
                .map_err(error::Error::from)
                .forward(send)
                .map(drop)
                .map_err(move |err| error!(log, "send"; "err" => format_args!("{}", err)));

            self.httpd.remote.execute(done).unwrap();
        }

        // TODO https://github.com/hyperium/mime/issues/52
        let mime = "text/html; charset=utf-8".parse().unwrap();
        Ok(res.with_header(header::ContentType(mime)))
    }

    fn process_file(&self, metadata: &Metadata) -> io::Result<Response> {
        let entity = Entity::new(&self.path, metadata, self.log, self.httpd.chunk_length);

        match entity.check(self.req.headers()) {
            EntifyResult::Err(resp) => Ok(resp.with_headers(entity.headers(false))),
            EntifyResult::None => {
                let fd = entity.open()?;
                let length = fd.length;
                self.send(fd, None)
                    .map(|res| res
                        .with_headers(entity.headers(false))
                        .with_header(header::ContentLength(length))
                    )
            },
            EntifyResult::One(range) => {
                debug!(self.log, "process"; "range" => format_args!("{:?}", range));

                let fd = entity.open()?;
                let length = fd.length;
                self.send(fd, Some(range.clone()))
                    .map(|res| res
                         .with_status(StatusCode::PartialContent)
                         .with_headers(entity.headers(false))
                         .with_header(header::ContentLength(range.end - range.start))
                         .with_header(header::ContentRange(header::ContentRangeSpec::Bytes {
                            range: Some((range.start, range.end - 1)), instance_length: Some(length)
                        }))
                    )
            },
            EntifyResult::Vec(ranges) => {
                const BOUNDARY_LINE: &str = concat!("--", boundary!(), "\r\n");

                debug!(self.log, "process"; "ranges" => format_args!("{:?}", ranges));

                let mut res = Response::new();

                if self.req.method() == &Head {
                    return Ok(res
                        .with_status(StatusCode::PartialContent)
                        .with_headers(entity.headers(true))
                        .with_body(Body::empty())
                    );
                }

                let log = self.log.clone();
                let (send, body) = Body::pair();
                res.set_body(body);

                let fd = entity.open()?;
                let mime_type = guess_mime_type(&self.path);

                let done = stream::iter_ok::<_, error::Error>(ranges.into_iter())
                    .and_then(move |range| {
                        let length = range.end - range.start;
                        let mut headers = header::Headers::new();
                        headers.set(header::ContentType(mime_type.clone()));
                        headers.set(header::ContentRange(header::ContentRangeSpec::Bytes {
                            range: Some((range.start, range.end - 1)), instance_length: Some(length)
                        }));

                        fd.try_clone()?
                            .read(range)
                            .map(|fut| {
                                stream::once(Ok(chunk!(BOUNDARY_LINE)))
                                    .chain(stream::once(Ok(chunk!(format!("{}\r\n", headers)))))
                                    .chain(fut)
                                    .chain(stream::once(Ok(chunk!("\r\n"))))
                            })
                            .map_err(Into::into)
                    })
                    .flatten()
                    .forward(send)
                    .map(drop)
                    .map_err(move |err| error!(log, "send"; "err" => format_args!("{}", err)));

                self.httpd.remote.execute(done).unwrap();

                Ok(res
                    .with_status(StatusCode::PartialContent)
                    .with_headers(entity.headers(true))
                )
            }
        }
    }

    #[cfg(not(feature = "sendfile"))]
    fn send(&self, fd: file::File, range: Option<Range<u64>>) -> io::Result<Response> {
        let mut res = Response::new();

        if self.req.method() == &Head {
            res.set_body(Body::empty());
        } else {
            let range = range.unwrap_or_else(|| 0..fd.length);

            let log = self.log.clone();
            let (send, body) = Body::pair();
            res.set_body(body);

            debug!(self.log, "process"; "method" => "readchunk");

            let done = fd.read(range)?
                .forward(send)
                .map(drop)
                .map_err(move |err| error!(log, "send"; "err" => format_args!("{}", err)));

            self.httpd.remote.execute(done).unwrap();
        }

        Ok(res)
    }

    #[cfg(feature = "sendfile")]
    fn send(&self, fd: file::File, range: Option<Range<u64>>) -> io::Result<Response> {
        use futures::future;
        let mut res = Response::new();

        if self.req.method() == &Head {
            res.set_body(Body::empty());
        } else if let (&Some(ref socket), true) = (&self.httpd.socket, self.httpd.chunk_length >= fd.length as _) {
            let range = range.unwrap_or_else(|| 0..fd.length);
            let log = self.log.clone();
            res.set_body(Body::empty());

            debug!(self.log, "process"; "method" => "sendfile");

            let done = fd.sendfile(range, socket.clone())?
                .for_each(|_| future::ok(()))
                .map_err(move |err| error!(log, "send"; "err" => format_args!("{}", err)));

            self.httpd.remote.execute(done).unwrap();
        } else {
            let range = range.unwrap_or_else(|| 0..fd.length);

            let log = self.log.clone();
            let (send, body) = Body::pair();
            res.set_body(body);

            debug!(self.log, "process"; "method" => "readchunk");

            let done = fd.read(range)?
                .forward(send)
                .map(drop)
                .map_err(move |err| error!(log, "send"; "err" => format_args!("{}", err)));

            self.httpd.remote.execute(done).unwrap();
        }

        Ok(res)
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    constants, error::Error, layout::Layout, secure_backend::StorageLocation::RemoteStorage,
    SingleBackend,
};
use libra_crypto::ed25519::Ed25519PublicKey;
use libra_global_constants::{ASSOCIATION_KEY, OPERATOR_KEY};
use libra_secure_storage::KVStorage;
use libra_types::transaction::{Transaction, TransactionPayload};
use std::{fs::File, io::Write, path::PathBuf};
use structopt::StructOpt;
use vm_genesis::ValidatorRegistration;

// TODO(davidiw) add operator_address, since that will eventually be the identity producing this.
/// Note, it is implicitly expected that the storage supports
/// a namespace but one has not been set.
#[derive(Debug, StructOpt)]
pub struct Genesis {
    #[structopt(flatten)]
    pub backend: SingleBackend,
    #[structopt(long)]
    pub path: Option<PathBuf>,
}

impl Genesis {
    pub fn execute(self) -> Result<Transaction, Error> {
        let layout = self.layout()?;
        let association_key = self.association(&layout)?;
        let validators = self.validators(&layout)?;

        let genesis = vm_genesis::encode_genesis_transaction_with_validator(
            association_key,
            &validators,
            Some(libra_types::on_chain_config::VMPublishingOption::Open),
        );

        if let Some(path) = self.path {
            let mut file = File::create(path).map_err(|e| {
                Error::UnexpectedError(format!("Unable to create genesis file: {}", e.to_string()))
            })?;
            let bytes = lcs::to_bytes(&genesis).map_err(|e| {
                Error::UnexpectedError(format!("Unable to serialize genesis: {}", e.to_string()))
            })?;
            file.write_all(&bytes).map_err(|e| {
                Error::UnexpectedError(format!("Unable to write genesis file: {}", e.to_string()))
            })?;
        }

        Ok(genesis)
    }

    /// Retrieves association key from the remote storage. Note, at this point in time, genesis
    /// only supports a single association key.
    pub fn association(&self, layout: &Layout) -> Result<Ed25519PublicKey, Error> {
        let association_config = self.backend.backend.clone();
        let association_config = association_config.set_namespace(layout.association[0].clone());

        let association_storage = association_config.create_storage(RemoteStorage)?;

        let association_key = association_storage
            .get(ASSOCIATION_KEY)
            .map_err(|e| Error::RemoteStorageReadError(ASSOCIATION_KEY, e.to_string()))?;
        association_key
            .value
            .ed25519_public_key()
            .map_err(|e| Error::RemoteStorageReadError(ASSOCIATION_KEY, e.to_string()))
    }

    /// Retrieves a layout from the remote storage.
    pub fn layout(&self) -> Result<Layout, Error> {
        let common_config = self.backend.backend.clone();
        let common_config = common_config.set_namespace(constants::COMMON_NS.into());

        let common_storage = common_config.create_storage(RemoteStorage)?;

        let layout = common_storage
            .get(constants::LAYOUT)
            .and_then(|v| v.value.string())
            .map_err(|e| Error::RemoteStorageReadError(constants::LAYOUT, e.to_string()))?;
        Layout::parse(&layout)
            .map_err(|e| Error::RemoteStorageReadError(constants::LAYOUT, e.to_string()))
    }

    /// Produces a set of ValidatorRegistration from the remote storage.
    pub fn validators(&self, layout: &Layout) -> Result<Vec<ValidatorRegistration>, Error> {
        let mut validators = Vec::new();
        for operator in layout.operators.iter() {
            let validator_config = self.backend.backend.clone();
            let validator_config = validator_config.set_namespace(operator.into());

            let validator_storage = validator_config.create_storage(RemoteStorage)?;

            let key = validator_storage
                .get(OPERATOR_KEY)
                .map_err(|e| Error::RemoteStorageReadError(OPERATOR_KEY, e.to_string()))?
                .value
                .ed25519_public_key()
                .map_err(|e| Error::RemoteStorageReadError(OPERATOR_KEY, e.to_string()))?;

            let txn = validator_storage
                .get(constants::VALIDATOR_CONFIG)
                .map_err(|e| {
                    Error::RemoteStorageReadError(constants::VALIDATOR_CONFIG, e.to_string())
                })?
                .value;
            let txn = txn.transaction().unwrap();
            let txn = txn.as_signed_user_txn().unwrap().payload();
            let txn = if let TransactionPayload::Script(script) = txn {
                script.clone()
            } else {
                return Err(Error::UnexpectedError("Found invalid registration".into()));
            };

            validators.push((key, txn));
        }

        Ok(validators)
    }
}
<commit_msg>Revert "[management] allow custom modules/scripts temporarily for smoke test"<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    constants, error::Error, layout::Layout, secure_backend::StorageLocation::RemoteStorage,
    SingleBackend,
};
use libra_crypto::ed25519::Ed25519PublicKey;
use libra_global_constants::{ASSOCIATION_KEY, OPERATOR_KEY};
use libra_secure_storage::KVStorage;
use libra_types::transaction::{Transaction, TransactionPayload};
use std::{fs::File, io::Write, path::PathBuf};
use structopt::StructOpt;
use vm_genesis::ValidatorRegistration;

// TODO(davidiw) add operator_address, since that will eventually be the identity producing this.
/// Note, it is implicitly expected that the storage supports
/// a namespace but one has not been set.
#[derive(Debug, StructOpt)]
pub struct Genesis {
    #[structopt(flatten)]
    pub backend: SingleBackend,
    #[structopt(long)]
    pub path: Option<PathBuf>,
}

impl Genesis {
    pub fn execute(self) -> Result<Transaction, Error> {
        let layout = self.layout()?;
        let association_key = self.association(&layout)?;
        let validators = self.validators(&layout)?;

        let genesis = vm_genesis::encode_genesis_transaction_with_validator(
            association_key,
            &validators,
            Some(libra_types::on_chain_config::VMPublishingOption::CustomScripts),
        );

        if let Some(path) = self.path {
            let mut file = File::create(path).map_err(|e| {
                Error::UnexpectedError(format!("Unable to create genesis file: {}", e.to_string()))
            })?;
            let bytes = lcs::to_bytes(&genesis).map_err(|e| {
                Error::UnexpectedError(format!("Unable to serialize genesis: {}", e.to_string()))
            })?;
            file.write_all(&bytes).map_err(|e| {
                Error::UnexpectedError(format!("Unable to write genesis file: {}", e.to_string()))
            })?;
        }

        Ok(genesis)
    }

    /// Retrieves association key from the remote storage. Note, at this point in time, genesis
    /// only supports a single association key.
    pub fn association(&self, layout: &Layout) -> Result<Ed25519PublicKey, Error> {
        let association_config = self.backend.backend.clone();
        let association_config = association_config.set_namespace(layout.association[0].clone());

        let association_storage = association_config.create_storage(RemoteStorage)?;

        let association_key = association_storage
            .get(ASSOCIATION_KEY)
            .map_err(|e| Error::RemoteStorageReadError(ASSOCIATION_KEY, e.to_string()))?;
        association_key
            .value
            .ed25519_public_key()
            .map_err(|e| Error::RemoteStorageReadError(ASSOCIATION_KEY, e.to_string()))
    }

    /// Retrieves a layout from the remote storage.
    pub fn layout(&self) -> Result<Layout, Error> {
        let common_config = self.backend.backend.clone();
        let common_config = common_config.set_namespace(constants::COMMON_NS.into());

        let common_storage = common_config.create_storage(RemoteStorage)?;

        let layout = common_storage
            .get(constants::LAYOUT)
            .and_then(|v| v.value.string())
            .map_err(|e| Error::RemoteStorageReadError(constants::LAYOUT, e.to_string()))?;
        Layout::parse(&layout)
            .map_err(|e| Error::RemoteStorageReadError(constants::LAYOUT, e.to_string()))
    }

    /// Produces a set of ValidatorRegistration from the remote storage.
    pub fn validators(&self, layout: &Layout) -> Result<Vec<ValidatorRegistration>, Error> {
        let mut validators = Vec::new();
        for operator in layout.operators.iter() {
            let validator_config = self.backend.backend.clone();
            let validator_config = validator_config.set_namespace(operator.into());

            let validator_storage = validator_config.create_storage(RemoteStorage)?;

            let key = validator_storage
                .get(OPERATOR_KEY)
                .map_err(|e| Error::RemoteStorageReadError(OPERATOR_KEY, e.to_string()))?
                .value
                .ed25519_public_key()
                .map_err(|e| Error::RemoteStorageReadError(OPERATOR_KEY, e.to_string()))?;

            let txn = validator_storage
                .get(constants::VALIDATOR_CONFIG)
                .map_err(|e| {
                    Error::RemoteStorageReadError(constants::VALIDATOR_CONFIG, e.to_string())
                })?
                .value;
            let txn = txn.transaction().unwrap();
            let txn = txn.as_signed_user_txn().unwrap().payload();
            let txn = if let TransactionPayload::Script(script) = txn {
                script.clone()
            } else {
                return Err(Error::UnexpectedError("Found invalid registration".into()));
            };

            validators.push((key, txn));
        }

        Ok(validators)
    }
}
<|endoftext|>