<commit_before>use std::error::Error;
use std::fmt::Error as FmtError;
use std::fmt::{Display, Formatter};

generate_error_types!(NoteError, NoteErrorKind,
    StoreWriteError       => "Error writing store",
    StoreReadError        => "Error reading store",
    HeaderTypeError       => "Header type error",
    NoteToEntryConversion => "Error converting Note instance to Entry instance"
);

<commit_msg>libimagnotes: Replace error module imports with macro helper<commit_after>generate_error_imports!();

generate_error_types!(NoteError, NoteErrorKind,
    StoreWriteError       => "Error writing store",
    StoreReadError        => "Error reading store",
    HeaderTypeError       => "Header type error",
    NoteToEntryConversion => "Error converting Note instance to Entry instance"
);

<|endoftext|>


<commit_before>use regex::Regex;
use std::env;
use std::fs::File;
use std::io::BufReader;

#[macro_use]
extern crate lazy_static;
extern crate regex;

//Maybe do not need regex
lazy_static! {
    static ref line_re: Regex = Regex::new(r"^\d ").unwrap();
    static ref symbols_re: Regex = Regex::new(r"").unwrap();
    static ref addres_re: Regex = Regex::new(r"").unwrap();
}

#[derive(Debug)]
enum Content {
    Symbols(Vec<String>),
    Addrs(Vec<String>),
}

#[derive(Debug)]
struct SingleLine {
    num: i32,
    content: Option<Content>,
}

fn read_linker_line(li: String) -> SingleLine {
    let mut cut_str: Vec<&str> = li.split_whitespace().collect();
    let num = cut_str[0].parse::<i32>().unwrap();

    if cut_str[1] != "A" && cut_str[1] != "E" && cut_str[1] != "R" && cut_str[1] != "I" {
        return SingleLine {
            num: num,
            content: Some(Content::Symbols(
                cut_str.split_off(1).iter().map(|s| s.to_string()).collect(),
            )),
        };
    } else {
        return SingleLine {
            num: num,
            content: Some(Content::Addrs(
                cut_str.split_off(1).iter().map(|s| s.to_string()).collect(),
            )),
        };
    }

    return SingleLine {
        num: num,
        content: None,
    };
}

#[test]
fn regex_test() {
    let mut fields: Vec<&str> = line_re
        .split("5 R 1004  I 5678  E 2000  R 8002  E 7001")
        .collect();
    //println!("{:?}", fields);

    fields = line_re
        .split("5 R 1004  I 5678  E 2000  R 8002  E 7001")
        .collect();
}

#[test]
fn read_linker_line_test() {
    println!(
        "{:?}",
        read_linker_line(String::from("5 R 1004  I 5678  E 2000  R 8002  E 7001"))
    );

    println!("{:?}", read_linker_line(String::from("2 xy z")));
}

fn main() {
    let fname = env::args().nth(1).unwrap();
    let mut f = File::open(fname).expect("cannot find file");
    let mut file = BufReader::new(&f);
}
<commit_msg>add a bit<commit_after>use regex::Regex;
use std::env;
use std::fs::File;
use std::io::BufReader;

#[macro_use]
extern crate lazy_static;
extern crate regex;

//Maybe do not need regex
lazy_static! {
    static ref line_re: Regex = Regex::new(r"^\d ").unwrap();
    static ref symbols_re: Regex = Regex::new(r"").unwrap();
    static ref addres_re: Regex = Regex::new(r"").unwrap();
}

#[derive(Debug)]
enum Content {
    DefSym(Vec<String>),
    Symbols(Vec<String>),
    Addrs(Vec<String>),
}

#[derive(Debug)]
struct SingleLine {
    num: i32,
    content: Option<Content>,
}

impl Content {
    fn num_change(&mut self) -> Vec<(String, i32)> {
        let mut result: Vec<(String, i32)> = vec![];
        match self {
            Content::DefSym(l) | Content::Addrs(l) => {
                let (_, l2) = l.split_at(1);
                for (i, (x, y)) in l.iter().zip(l2.iter()).enumerate() {
                    if i % 2 == 0 {
                        result.push((x.clone(), y.parse::<i32>().unwrap()));
                    }
                }
            }
            _ => (),
        }
        return result;
    }
}

#[test]
fn test_num_change() {
    let a = read_linker_line(String::from("5 R 1004  I 5678  E 2000  R 8002  E 7001"));
    println!("{:?}", a.content.unwrap().num_change());
}

fn read_linker_line(li: String) -> SingleLine {
    let mut cut_str: Vec<&str> = li.split_whitespace().collect();
    let num = cut_str[0].parse::<i32>().unwrap();

    if cut_str.len() == 1 {
        return SingleLine {
            num: num,
            content: None,
        };
    }

    if cut_str[1] != "A" && cut_str[1] != "E" && cut_str[1] != "R" && cut_str[1] != "I" {
        if let Ok(_) = cut_str[2].parse::<i32>() {
            return SingleLine {
                num: num,
                content: Some(Content::DefSym(
                    cut_str.split_off(1).iter().map(|s| s.to_string()).collect(),
                )),
            };
        }
        return SingleLine {
            num: num,
            content: Some(Content::Symbols(
                cut_str.split_off(1).iter().map(|s| s.to_string()).collect(),
            )),
        };
    } else {
        return SingleLine {
            num: num,
            content: Some(Content::Addrs(
                cut_str.split_off(1).iter().map(|s| s.to_string()).collect(),
            )),
        };
    }
}

#[test]
#[ignore]
fn read_linker_line_test() {
    println!(
        "{:?}",
        read_linker_line(String::from("5 R 1004  I 5678  E 2000  R 8002  E 7001"))
    );

    println!("{:?}", read_linker_line(String::from("2 xy z")));

    println!("{:?}", read_linker_line(String::from("0")));

    println!("{:?}", read_linker_line(String::from("2 xy 2")));
}

fn main() {
    let fname = env::args().nth(1).unwrap();
    let mut f = File::open(fname).expect("cannot find file");
    let mut file = BufReader::new(&f);
}
<|endoftext|>


<commit_before>use std::cmp::min;
use std::env;
use std::fs::{File};
use std::io::{BufReader, BufRead};
use std::str::FromStr;
use std::str::from_utf8;
use std::path::{Path, PathBuf};
use unshare::{Command, Stdio};

use libc::{geteuid, getegid};
use libc::{uid_t, gid_t};

use config::Range;
use config::Settings;
use self::Uidmap::*;
use process_util::{env_path_find, capture_stdout};

#[derive(Clone)]
pub enum Uidmap {
    Singleton(uid_t, gid_t),
    Ranges(Vec<(uid_t, uid_t, uid_t)>, Vec<(gid_t, gid_t, gid_t)>),
}

fn read_uid_map(username: &str) -> Result<Vec<Range>,String> {
    let file = try_msg!(File::open(&Path::new("/etc/subuid")),
        "Can't open /etc/subuid: {err}");
    let mut res = Vec::new();
    let reader = BufReader::new(file);
    for (num, line) in reader.lines().enumerate() {
        let line = try_msg!(line, "Error reading /etc/subuid: {err}");
        let parts: Vec<&str> = line[..].split(':').collect();
        if parts.len() == 0 || parts[0].trim().starts_with('#') {
            continue;
        }
        if parts.len() != 3 {
            return Err(format!("/etc/subuid:{}: Bad syntax: {:?}",
                num+1, line));
        }
        let start = FromStr::from_str(parts[1]);
        let count: Result<uid_t, _> = FromStr::from_str(parts[2].trim_right());
        if parts.len() != 3 || start.is_err() || count.is_err() {
            return Err(format!("/etc/subuid:{}: Bad syntax: {:?}",
                num+1, line));
        }
        if parts[0].eq(username) {
            let start: uid_t = start.unwrap();
            let end = start + count.unwrap() - 1;
            res.push(Range::new(start, end));
        }
    }
    return Ok(res);
}

fn read_gid_map(username: &str) -> Result<Vec<Range>,String> {
    let file = try_msg!(File::open(&Path::new("/etc/subgid")),
        "Can't open /etc/subgid: {err}");
    let mut res = Vec::new();
    let reader = BufReader::new(file);
    for (num, line) in reader.lines().enumerate() {
        let line = try_msg!(line, "Error reading /etc/subgid: {err}");
        let parts: Vec<&str> = line[..].split(':').collect();
        if parts.len() == 0 || parts[0].trim().starts_with('#') {
            continue;
        }
        if parts.len() != 3 {
            return Err(format!("/etc/subgid:{}: Bad syntax: {:?}",
                num+1, line));
        }
        let start = FromStr::from_str(parts[1]);
        let count: Result<uid_t, _> = FromStr::from_str(parts[2].trim_right());
        if parts.len() != 3 || start.is_err() || count.is_err() {
            return Err(format!("/etc/subgid:{}: Bad syntax", num+1));
        }
        if parts[0].eq(username) {
            let start: gid_t = start.unwrap();
            let end = start + count.unwrap() - 1;
            res.push(Range::new(start, end));
        }
    }
    return Ok(res);
}

pub fn match_ranges(req: &Vec<Range>, allowed: &Vec<Range>, own_id: uid_t)
    -> Result<Vec<(uid_t, uid_t, uid_t)>, ()>
{
    let mut res = vec!((0, own_id, 1));
    let mut reqiter = req.iter();
    let mut reqval = *reqiter.next().unwrap();
    let mut allowiter = allowed.iter();
    let mut allowval = *allowiter.next().unwrap();
    loop {
        if reqval.start() == 0 {
            reqval = reqval.shift(1);
        }
        if allowval.start() == 0 {
            allowval = allowval.shift(1);
        }
        let clen = min(reqval.len(), allowval.len());
        if clen > 0 {
            res.push((reqval.start(), allowval.start(), clen));
        }
        reqval = reqval.shift(clen);
        allowval = allowval.shift(clen);
        if reqval.len() == 0 {
            reqval = match reqiter.next() {
                Some(val) => *val,
                None => break,
            };
        }
        if allowval.len() == 0 {
            allowval = match allowiter.next() {
                Some(val) => *val,
                None => return Err(()),
            };
        }
    }
    return Ok(res);
}

pub fn get_max_uidmap() -> Result<Uidmap, String>
{
    let mut cmd = Command::new(env_path_find("id")
                               .unwrap_or(PathBuf::from("/usr/bin/id")));
    cmd.arg("--user").arg("--name");
    if let Ok(path) = env::var("_VAGGA_PATH") {
        cmd.env("PATH", path);
    }
    cmd.stdin(Stdio::null()).stderr(Stdio::inherit());
    let username = try!(capture_stdout(cmd)
        .map_err(|e| format!("Error running `id --user --name`: {}", e))
        .and_then(|val| from_utf8(&val).map(|x| x.trim().to_string())
                   .map_err(|e| format!("Can't decode username: {}", e))));
    let uid_map = read_uid_map(&username)
        .map_err(|e| error!("Error reading uidmap: {}", e));
    let gid_map = read_gid_map(&username)
        .map_err(|e| error!("Error reading gidmap: {}", e));

    let uid = unsafe { geteuid() };
    let gid = unsafe { getegid() };
    if let (Ok(uid_map), Ok(gid_map)) = (uid_map, gid_map) {
        if uid_map.len() == 0 && gid_map.len() == 0 && uid == 0 {
            let uid_rng = try!(read_uid_ranges("/proc/self/uid_map", true));
            let gid_rng = try!(read_uid_ranges("/proc/self/gid_map", true));
            return Ok(Ranges(
                uid_rng.into_iter()
                    .map(|r| (r.start(), r.start(), r.len())).collect(),
                gid_rng.into_iter()
                    .map(|r| (r.start(), r.start(), r.len())).collect()));
        }
        let mut uids = vec!((0, uid, 1));
        for &rng in uid_map.iter() {
            let mut rng = rng;
            if uid >= rng.start() && uid <= rng.end() {
                // TODO(tailhook) implement better heuristic
                assert!(uid == rng.start());
                rng = rng.shift(1);
                if rng.len() == 0 { continue; }
            }
            uids.push((rng.start(), rng.start(), rng.len()));
        }

        let mut gids = vec!((0, gid, 1));
        for &rng in gid_map.iter() {
            let mut rng = rng;
            if gid >= rng.start() && gid <= rng.end() {
                // TODO(tailhook) implement better heuristic
                assert!(gid == rng.start());
                rng = rng.shift(1);
                if rng.len() == 0 { continue; }
            }
            gids.push((rng.start(), rng.start(), rng.len()));
        }

        return Ok(Ranges(uids, gids));
    } else {
        warn!("Could not read /etc/subuid or /etc/subgid \
            (see http://bit.ly/err_subuid)");
        return Ok(Singleton(uid, gid));
    }
}

fn read_uid_ranges(path: &str, read_inside: bool) -> Result<Vec<Range>, String>
{
    let file = BufReader::new(try!(File::open(&Path::new(path))
        .map_err(|e| format!("Error reading uid/gid map: {}", e))));
    let mut result = vec!();
    for line in file.lines() {
        let line = try!(line
            .map_err(|_| format!("Error reading uid/gid map")));
        let mut words = line[..].split_whitespace();
        let inside: u32 = try!(words.next().and_then(|x| FromStr::from_str(x).ok())
            .ok_or(format!("uid/gid map format error")));
        let outside: u32 = try!(words.next().and_then(|x| FromStr::from_str(x).ok())
            .ok_or(format!("uid/gid map format error")));
        let count: u32 = try!(words.next().and_then(|x| FromStr::from_str(x).ok())
            .ok_or(format!("uid/gid map format error")));
        if read_inside {
            result.push(Range::new(inside, inside+count-1));
        } else {
            result.push(Range::new(outside, outside+count-1));
        }
    }
    return Ok(result);
}

pub fn map_users(settings: &Settings, uids: &Vec<Range>, gids: &Vec<Range>)
    -> Result<Uidmap, String>
{
    let default_uids = vec!(Range::new(0, 0));
    let default_gids = vec!(Range::new(0, 0));
    let uids = if uids.len() > 0 { uids } else { &default_uids };
    let gids = if gids.len() > 0 { gids } else { &default_gids };
    if settings.uid_map.is_none() {
        let ranges = try!(read_uid_ranges("/proc/self/uid_map", true));
        let uid_map = try!(match_ranges(uids, &ranges, 0)
            .map_err(|()| {
                return format!("Number of allowed subuids is too small. \
                    Required {:?}, allowed {:?}. You either need to increase \
                    allowed numbers in /etc/subuid (preferred) or decrease \
                    needed ranges in vagga.yaml by adding `uids` key \
                    to container config", uids, ranges);
            }));
        let ranges = try!(read_uid_ranges("/proc/self/gid_map", true));
        let gid_map = try!(match_ranges(gids, &ranges, 0)
            .map_err(|()| {
                return format!("Number of allowed subgids is too small. \
                    Required {:?}, allowed {:?}. You either need to increase \
                    allowed numbers in /etc/subgid (preferred) or decrease \
                    needed ranges in vagga.yaml by adding `gids` key \
                    to container config", gids, ranges);
            }));
        return Ok(Ranges(uid_map, gid_map));
    } else {
        let &(ref uids, ref gids) = settings.uid_map.as_ref().unwrap();
        return Ok(Ranges(uids.clone(), gids.clone()));
    }
}
<commit_msg>Use `id -u -n` instead of `id --user --name` as former is more compatible<commit_after>use std::cmp::min;
use std::env;
use std::fs::{File};
use std::io::{BufReader, BufRead};
use std::str::FromStr;
use std::str::from_utf8;
use std::path::{Path, PathBuf};
use unshare::{Command, Stdio};

use libc::{geteuid, getegid};
use libc::{uid_t, gid_t};

use config::Range;
use config::Settings;
use self::Uidmap::*;
use process_util::{env_path_find, capture_stdout};

#[derive(Clone)]
pub enum Uidmap {
    Singleton(uid_t, gid_t),
    Ranges(Vec<(uid_t, uid_t, uid_t)>, Vec<(gid_t, gid_t, gid_t)>),
}

fn read_uid_map(username: &str) -> Result<Vec<Range>,String> {
    let file = try_msg!(File::open(&Path::new("/etc/subuid")),
        "Can't open /etc/subuid: {err}");
    let mut res = Vec::new();
    let reader = BufReader::new(file);
    for (num, line) in reader.lines().enumerate() {
        let line = try_msg!(line, "Error reading /etc/subuid: {err}");
        let parts: Vec<&str> = line[..].split(':').collect();
        if parts.len() == 0 || parts[0].trim().starts_with('#') {
            continue;
        }
        if parts.len() != 3 {
            return Err(format!("/etc/subuid:{}: Bad syntax: {:?}",
                num+1, line));
        }
        let start = FromStr::from_str(parts[1]);
        let count: Result<uid_t, _> = FromStr::from_str(parts[2].trim_right());
        if parts.len() != 3 || start.is_err() || count.is_err() {
            return Err(format!("/etc/subuid:{}: Bad syntax: {:?}",
                num+1, line));
        }
        if parts[0].eq(username) {
            let start: uid_t = start.unwrap();
            let end = start + count.unwrap() - 1;
            res.push(Range::new(start, end));
        }
    }
    return Ok(res);
}

fn read_gid_map(username: &str) -> Result<Vec<Range>,String> {
    let file = try_msg!(File::open(&Path::new("/etc/subgid")),
        "Can't open /etc/subgid: {err}");
    let mut res = Vec::new();
    let reader = BufReader::new(file);
    for (num, line) in reader.lines().enumerate() {
        let line = try_msg!(line, "Error reading /etc/subgid: {err}");
        let parts: Vec<&str> = line[..].split(':').collect();
        if parts.len() == 0 || parts[0].trim().starts_with('#') {
            continue;
        }
        if parts.len() != 3 {
            return Err(format!("/etc/subgid:{}: Bad syntax: {:?}",
                num+1, line));
        }
        let start = FromStr::from_str(parts[1]);
        let count: Result<uid_t, _> = FromStr::from_str(parts[2].trim_right());
        if parts.len() != 3 || start.is_err() || count.is_err() {
            return Err(format!("/etc/subgid:{}: Bad syntax", num+1));
        }
        if parts[0].eq(username) {
            let start: gid_t = start.unwrap();
            let end = start + count.unwrap() - 1;
            res.push(Range::new(start, end));
        }
    }
    return Ok(res);
}

pub fn match_ranges(req: &Vec<Range>, allowed: &Vec<Range>, own_id: uid_t)
    -> Result<Vec<(uid_t, uid_t, uid_t)>, ()>
{
    let mut res = vec!((0, own_id, 1));
    let mut reqiter = req.iter();
    let mut reqval = *reqiter.next().unwrap();
    let mut allowiter = allowed.iter();
    let mut allowval = *allowiter.next().unwrap();
    loop {
        if reqval.start() == 0 {
            reqval = reqval.shift(1);
        }
        if allowval.start() == 0 {
            allowval = allowval.shift(1);
        }
        let clen = min(reqval.len(), allowval.len());
        if clen > 0 {
            res.push((reqval.start(), allowval.start(), clen));
        }
        reqval = reqval.shift(clen);
        allowval = allowval.shift(clen);
        if reqval.len() == 0 {
            reqval = match reqiter.next() {
                Some(val) => *val,
                None => break,
            };
        }
        if allowval.len() == 0 {
            allowval = match allowiter.next() {
                Some(val) => *val,
                None => return Err(()),
            };
        }
    }
    return Ok(res);
}

pub fn get_max_uidmap() -> Result<Uidmap, String>
{
    let mut cmd = Command::new(env_path_find("id")
                               .unwrap_or(PathBuf::from("/usr/bin/id")));
    cmd.arg("-u").arg("-n");
    if let Ok(path) = env::var("_VAGGA_PATH") {
        cmd.env("PATH", path);
    }
    cmd.stdin(Stdio::null()).stderr(Stdio::inherit());
    let username = try!(capture_stdout(cmd)
        .map_err(|e| format!("Error running `id -u -n`: {}", e))
        .and_then(|val| from_utf8(&val).map(|x| x.trim().to_string())
                   .map_err(|e| format!("Can't decode username: {}", e))));
    let uid_map = read_uid_map(&username)
        .map_err(|e| error!("Error reading uidmap: {}", e));
    let gid_map = read_gid_map(&username)
        .map_err(|e| error!("Error reading gidmap: {}", e));

    let uid = unsafe { geteuid() };
    let gid = unsafe { getegid() };
    if let (Ok(uid_map), Ok(gid_map)) = (uid_map, gid_map) {
        if uid_map.len() == 0 && gid_map.len() == 0 && uid == 0 {
            let uid_rng = try!(read_uid_ranges("/proc/self/uid_map", true));
            let gid_rng = try!(read_uid_ranges("/proc/self/gid_map", true));
            return Ok(Ranges(
                uid_rng.into_iter()
                    .map(|r| (r.start(), r.start(), r.len())).collect(),
                gid_rng.into_iter()
                    .map(|r| (r.start(), r.start(), r.len())).collect()));
        }
        let mut uids = vec!((0, uid, 1));
        for &rng in uid_map.iter() {
            let mut rng = rng;
            if uid >= rng.start() && uid <= rng.end() {
                // TODO(tailhook) implement better heuristic
                assert!(uid == rng.start());
                rng = rng.shift(1);
                if rng.len() == 0 { continue; }
            }
            uids.push((rng.start(), rng.start(), rng.len()));
        }

        let mut gids = vec!((0, gid, 1));
        for &rng in gid_map.iter() {
            let mut rng = rng;
            if gid >= rng.start() && gid <= rng.end() {
                // TODO(tailhook) implement better heuristic
                assert!(gid == rng.start());
                rng = rng.shift(1);
                if rng.len() == 0 { continue; }
            }
            gids.push((rng.start(), rng.start(), rng.len()));
        }

        return Ok(Ranges(uids, gids));
    } else {
        warn!("Could not read /etc/subuid or /etc/subgid \
            (see http://bit.ly/err_subuid)");
        return Ok(Singleton(uid, gid));
    }
}

fn read_uid_ranges(path: &str, read_inside: bool) -> Result<Vec<Range>, String>
{
    let file = BufReader::new(try!(File::open(&Path::new(path))
        .map_err(|e| format!("Error reading uid/gid map: {}", e))));
    let mut result = vec!();
    for line in file.lines() {
        let line = try!(line
            .map_err(|_| format!("Error reading uid/gid map")));
        let mut words = line[..].split_whitespace();
        let inside: u32 = try!(words.next().and_then(|x| FromStr::from_str(x).ok())
            .ok_or(format!("uid/gid map format error")));
        let outside: u32 = try!(words.next().and_then(|x| FromStr::from_str(x).ok())
            .ok_or(format!("uid/gid map format error")));
        let count: u32 = try!(words.next().and_then(|x| FromStr::from_str(x).ok())
            .ok_or(format!("uid/gid map format error")));
        if read_inside {
            result.push(Range::new(inside, inside+count-1));
        } else {
            result.push(Range::new(outside, outside+count-1));
        }
    }
    return Ok(result);
}

pub fn map_users(settings: &Settings, uids: &Vec<Range>, gids: &Vec<Range>)
    -> Result<Uidmap, String>
{
    let default_uids = vec!(Range::new(0, 0));
    let default_gids = vec!(Range::new(0, 0));
    let uids = if uids.len() > 0 { uids } else { &default_uids };
    let gids = if gids.len() > 0 { gids } else { &default_gids };
    if settings.uid_map.is_none() {
        let ranges = try!(read_uid_ranges("/proc/self/uid_map", true));
        let uid_map = try!(match_ranges(uids, &ranges, 0)
            .map_err(|()| {
                return format!("Number of allowed subuids is too small. \
                    Required {:?}, allowed {:?}. You either need to increase \
                    allowed numbers in /etc/subuid (preferred) or decrease \
                    needed ranges in vagga.yaml by adding `uids` key \
                    to container config", uids, ranges);
            }));
        let ranges = try!(read_uid_ranges("/proc/self/gid_map", true));
        let gid_map = try!(match_ranges(gids, &ranges, 0)
            .map_err(|()| {
                return format!("Number of allowed subgids is too small. \
                    Required {:?}, allowed {:?}. You either need to increase \
                    allowed numbers in /etc/subgid (preferred) or decrease \
                    needed ranges in vagga.yaml by adding `gids` key \
                    to container config", gids, ranges);
            }));
        return Ok(Ranges(uid_map, gid_map));
    } else {
        let &(ref uids, ref gids) = settings.uid_map.as_ref().unwrap();
        return Ok(Ranges(uids.clone(), gids.clone()));
    }
}
<|endoftext|>


<commit_before>extern crate clap;
extern crate glob;
#[macro_use] extern crate log;
extern crate serde_json;
extern crate semver;
extern crate toml;
#[macro_use] extern crate version;

extern crate task_hookrs;

extern crate libimagrt;
extern crate libimagstore;
extern crate libimagerror;
extern crate libimagtodo;

use std::process::exit;
use std::process::{Command, Stdio};
use std::io::stdin;
use std::io::BufRead;

use task_hookrs::import::{import_task, import_tasks};

use libimagrt::runtime::Runtime;
use libimagtodo::task::IntoTask;
use libimagtodo::task::Task;
use libimagerror::trace::trace_error;

mod ui;

use ui::build_ui;
fn main() {
    let name    = "imag-todo";
    let version = &version!()[..];
    let about   = "Interface with taskwarrior";
    let ui      = build_ui(Runtime::get_default_cli_builder(name, version, about));

    let rt = {
        let rt = Runtime::new(ui);
        if rt.is_ok() {
            rt.unwrap()
        } else {
            println!("Could not set up Runtime");
            println!("{:?}", rt.unwrap_err());
            exit(1);
        }
    };

    match rt.cli().subcommand_name() {
        Some("tw-hook") => tw_hook(&rt),
        Some("list") => list(&rt),
        _ => unimplemented!(),
    } // end match scmd
} // end main

fn tw_hook(rt: &Runtime) {
    let subcmd = rt.cli().subcommand_matches("tw-hook").unwrap();
    if subcmd.is_present("add") {
        let stdin     = stdin();
        let mut stdin = stdin.lock();
        let mut line  = String::new();

        if let Err(e) = stdin.read_line(&mut line) {
            trace_error(&e);
            exit(1);
        };

        if let Ok(ttask) = import_task(&line.as_str()) {
            match serde_json::ser::to_string(&ttask) {
                Ok(val) => println!("{}", val),
                Err(e) => {
                    trace_error(&e);
                    exit(1);
                }
            }

            let uuid = *ttask.uuid();
            match ttask.into_task(rt.store()) {
                Ok(val) => {
                    println!("Task {} stored in imag", uuid);
                    val
                },
                Err(e) => {
                    trace_error(&e);
                    exit(1);
                }
            };
        } else {
            error!("No usable input");
            exit(1);
        }
    } else if subcmd.is_present("delete") {
        // The used hook is "on-modify". This hook gives two json-objects
        // per usage und wants one (the second one) back.
        let mut counter   = 0;
        let stdin         = stdin();
        let stdin         = stdin.lock();

        match import_tasks(stdin) {
            Ok(ttasks) => for ttask in ttasks {
                if counter % 2 == 1 {
                    // Only every second task is needed, the first one is the
                    // task before the change, and the second one after
                    // the change. The (maybe modified) second one is
                    // expected by taskwarrior.
                    match serde_json::ser::to_string(&ttask) {
                        Ok(val) => println!("{}", val),
                        Err(e) => {
                            trace_error(&e);
                            exit(1);
                        }
                    }

                    match ttask.status() {
                        &task_hookrs::status::TaskStatus::Deleted => {
                            match Task::delete_by_uuid(rt.store(), *ttask.uuid()) {
                                Ok(_) => println!("Deleted task {}", *ttask.uuid()),
                                Err(e) => {
                                    trace_error(&e);
                                    exit(1);
                                }
                            }
                        }
                        _ => {
                        }
                    } // end match ttask.status()
                } // end if c % 2
                counter += 1;
            },
            Err(e) => {
                trace_error(&e);
                exit(1);
            },
        }
    } else {
        // Should not be possible, as one argument is required via
        // ArgGroup
        unreachable!();
    }
}

fn list(rt: &Runtime) {
    let subcmd   = rt.cli().subcommand_matches("list").unwrap();
    let mut args = Vec::new();
    let verbose  = subcmd.is_present("verbose");
    let iter     = match Task::all(rt.store()) {
        Ok(iter) => iter,
        Err(e)   => {
            trace_error(&e);
            exit(1);
        },
    };

    for task in iter {
        match task {
            Ok(val) => {
                let uuid = match val.get_header().read("todo.uuid") {
                    Ok(Some(u)) => u,
                    Ok(None)    => continue,
                    Err(e)      => {
                        trace_error(&e);
                        continue;
                    }
                };

                if verbose {
                    args.clear();
                    args.push(format!("uuid:{} information", uuid));

                    let tw_process = Command::new("task")
                        .stdin(Stdio::null())
                        .args(&args)
                        .spawn()
                        .unwrap_or_else(|e| {
                            trace_error(&e);
                            panic!("failed");
                        });
                    let output = tw_process
                        .wait_with_output()
                        .unwrap_or_else(|e| panic!("failed to unwrap output: {}", e));
                    let outstring = String::from_utf8(output.stdout)
                        .unwrap_or_else(|e| panic!("failed to execute: {}", e));

                    println!("{}", outstring);
                } else {
                    println!("{}", match uuid {
                        toml::Value::String(s) => s,
                        _ => {
                            error!("Unexpected type for todo.uuid: {}", uuid);
                            continue;
                        },
                    });
                }
            }
            Err(e) => {
                trace_error(&e);
                continue;
            }
        } // end match task
    } // end for
}

<commit_msg>Use generate_runtime_setup() instead of doing it manually<commit_after>extern crate clap;
extern crate glob;
#[macro_use] extern crate log;
extern crate serde_json;
extern crate semver;
extern crate toml;
#[macro_use] extern crate version;

extern crate task_hookrs;

extern crate libimagrt;
extern crate libimagstore;
extern crate libimagerror;
extern crate libimagtodo;

use std::process::exit;
use std::process::{Command, Stdio};
use std::io::stdin;
use std::io::BufRead;

use task_hookrs::import::{import_task, import_tasks};

use libimagrt::runtime::Runtime;
use libimagrt::setup::generate_runtime_setup;
use libimagtodo::task::IntoTask;
use libimagtodo::task::Task;
use libimagerror::trace::trace_error;

mod ui;

use ui::build_ui;
fn main() {
    let rt = generate_runtime_setup("imag-todo",
                                    &version!()[..],
                                    "Interface with taskwarrior",
                                    build_ui);

    match rt.cli().subcommand_name() {
        Some("tw-hook") => tw_hook(&rt),
        Some("list") => list(&rt),
        _ => unimplemented!(),
    } // end match scmd
} // end main

fn tw_hook(rt: &Runtime) {
    let subcmd = rt.cli().subcommand_matches("tw-hook").unwrap();
    if subcmd.is_present("add") {
        let stdin     = stdin();
        let mut stdin = stdin.lock();
        let mut line  = String::new();

        if let Err(e) = stdin.read_line(&mut line) {
            trace_error(&e);
            exit(1);
        };

        if let Ok(ttask) = import_task(&line.as_str()) {
            match serde_json::ser::to_string(&ttask) {
                Ok(val) => println!("{}", val),
                Err(e) => {
                    trace_error(&e);
                    exit(1);
                }
            }

            let uuid = *ttask.uuid();
            match ttask.into_task(rt.store()) {
                Ok(val) => {
                    println!("Task {} stored in imag", uuid);
                    val
                },
                Err(e) => {
                    trace_error(&e);
                    exit(1);
                }
            };
        } else {
            error!("No usable input");
            exit(1);
        }
    } else if subcmd.is_present("delete") {
        // The used hook is "on-modify". This hook gives two json-objects
        // per usage und wants one (the second one) back.
        let mut counter   = 0;
        let stdin         = stdin();
        let stdin         = stdin.lock();

        match import_tasks(stdin) {
            Ok(ttasks) => for ttask in ttasks {
                if counter % 2 == 1 {
                    // Only every second task is needed, the first one is the
                    // task before the change, and the second one after
                    // the change. The (maybe modified) second one is
                    // expected by taskwarrior.
                    match serde_json::ser::to_string(&ttask) {
                        Ok(val) => println!("{}", val),
                        Err(e) => {
                            trace_error(&e);
                            exit(1);
                        }
                    }

                    match ttask.status() {
                        &task_hookrs::status::TaskStatus::Deleted => {
                            match Task::delete_by_uuid(rt.store(), *ttask.uuid()) {
                                Ok(_) => println!("Deleted task {}", *ttask.uuid()),
                                Err(e) => {
                                    trace_error(&e);
                                    exit(1);
                                }
                            }
                        }
                        _ => {
                        }
                    } // end match ttask.status()
                } // end if c % 2
                counter += 1;
            },
            Err(e) => {
                trace_error(&e);
                exit(1);
            },
        }
    } else {
        // Should not be possible, as one argument is required via
        // ArgGroup
        unreachable!();
    }
}

fn list(rt: &Runtime) {
    let subcmd   = rt.cli().subcommand_matches("list").unwrap();
    let mut args = Vec::new();
    let verbose  = subcmd.is_present("verbose");
    let iter     = match Task::all(rt.store()) {
        Ok(iter) => iter,
        Err(e)   => {
            trace_error(&e);
            exit(1);
        },
    };

    for task in iter {
        match task {
            Ok(val) => {
                let uuid = match val.get_header().read("todo.uuid") {
                    Ok(Some(u)) => u,
                    Ok(None)    => continue,
                    Err(e)      => {
                        trace_error(&e);
                        continue;
                    }
                };

                if verbose {
                    args.clear();
                    args.push(format!("uuid:{} information", uuid));

                    let tw_process = Command::new("task")
                        .stdin(Stdio::null())
                        .args(&args)
                        .spawn()
                        .unwrap_or_else(|e| {
                            trace_error(&e);
                            panic!("failed");
                        });
                    let output = tw_process
                        .wait_with_output()
                        .unwrap_or_else(|e| panic!("failed to unwrap output: {}", e));
                    let outstring = String::from_utf8(output.stdout)
                        .unwrap_or_else(|e| panic!("failed to execute: {}", e));

                    println!("{}", outstring);
                } else {
                    println!("{}", match uuid {
                        toml::Value::String(s) => s,
                        _ => {
                            error!("Unexpected type for todo.uuid: {}", uuid);
                            continue;
                        },
                    });
                }
            }
            Err(e) => {
                trace_error(&e);
                continue;
            }
        } // end match task
    } // end for
}

<|endoftext|>


<commit_before>//! Song decoding / analysis module.
//!
//! Use decoding, and features-extraction functions from other modules
//! e.g. tempo features, spectral features, etc to build a Song and its
//! corresponding Analysis.
#[cfg(feature = "aubio-lib")]
extern crate aubio_lib;

extern crate crossbeam;
extern crate ffmpeg_next as ffmpeg;
extern crate ndarray;
extern crate ndarray_npy;

use super::CHANNELS;
use crate::chroma::ChromaDesc;
use crate::misc::LoudnessDesc;
use crate::temporal::BPMDesc;
use crate::timbral::{SpectralDesc, ZeroCrossingRateDesc};
use crate::SAMPLE_RATE;
use crate::{Analysis, Song};
use crossbeam::thread;
use ffmpeg::util;
use ffmpeg::util::format::sample::{Sample, Type};
use std::sync::mpsc;
use std::thread as std_thread;

pub fn push_to_sample_array(frame: &ffmpeg::frame::Audio, sample_array: &mut Vec<f32>) {
    if frame.samples() == 0 {
        return;
    }
    // Account for the padding
    let actual_size = util::format::sample::Buffer::size(
        Sample::F32(Type::Packed),
        CHANNELS,
        frame.samples(),
        false,
    );
    let f32_frame: Vec<f32> = frame.data(0)[..actual_size]
        .chunks_exact(4)
        .map(|x| {
            let mut a: [u8; 4] = [0; 4];
            a.copy_from_slice(x);
            f32::from_le_bytes(a)
        })
        .collect();
    sample_array.extend_from_slice(&f32_frame);
}

impl Song {
    pub fn new(path: &str) -> Result<Self, String> {
        // TODO error handling here
        let mut song = Song::decode(&path)?;

        song.analysis = (&song).analyze();
        Ok(song)
    }

    pub fn analyze(&self) -> Analysis {
        thread::scope(|s| {
            let child_chroma = s.spawn(|_| {
                let mut chroma_desc = ChromaDesc::new(self.sample_rate, 12);
                chroma_desc.do_(&self.sample_array);
                chroma_desc.get_values()
            });

            // These descriptors can be streamed
            let child_timbral = s.spawn(|_| {
                let mut spectral_desc = SpectralDesc::new(self.sample_rate);
                let mut zcr_desc = ZeroCrossingRateDesc::default();
                let windows = self
                    .sample_array
                    .windows(SpectralDesc::WINDOW_SIZE)
                    .step_by(SpectralDesc::HOP_SIZE);
                for window in windows {
                    spectral_desc.do_(&window);
                    zcr_desc.do_(&window);
                }
                let centroid = spectral_desc.get_centroid();
                let rolloff = spectral_desc.get_rolloff();
                let flatness = spectral_desc.get_flatness();
                let zcr = zcr_desc.get_value();
                (centroid, rolloff, flatness, zcr)
            });

            let child_tempo = s.spawn(|_| {
                let mut tempo_desc = BPMDesc::new(self.sample_rate);
                let windows = self
                    .sample_array
                    .windows(BPMDesc::WINDOW_SIZE)
                    .step_by(BPMDesc::HOP_SIZE);

                for window in windows {
                    tempo_desc.do_(&window);
                }
                tempo_desc.get_value()
            });

            let child_loudness = s.spawn(|_| {
                let mut loudness_desc = LoudnessDesc::default();
                let windows = self.sample_array.chunks(LoudnessDesc::WINDOW_SIZE);

                for window in windows {
                    loudness_desc.do_(&window);
                }
                loudness_desc.get_value()
            });

            // Non-streaming approach for that one
            let (is_major, fifth) = child_chroma.join().unwrap();
            let (centroid, rolloff, flatness, zcr) = child_timbral.join().unwrap();
            let tempo = child_tempo.join().unwrap();
            let loudness = child_loudness.join().unwrap();

            Analysis {
                tempo,
                spectral_centroid: centroid,
                zero_crossing_rate: zcr,
                spectral_rolloff: rolloff,
                spectral_flatness: flatness,
                loudness,
                is_major,
                fifth,
            }
        })
        .unwrap()
    }

    pub fn decode(path: &str) -> Result<Song, String> {
        ffmpeg::init().map_err(|e| format!("FFmpeg init error: {:?}.", e))?;

        let mut song = Song::default();
        song.path = path.to_string();
        let mut format = ffmpeg::format::input(&path)
            .map_err(|e| format!("FFmpeg error while opening format: {:?}.", e))?;
        let (mut codec, stream, expected_sample_number) = {
            let stream = format
                .streams()
                .find(|s| s.codec().medium() == ffmpeg::media::Type::Audio)
                .ok_or("No audio stream found.")?;
            let codec = stream
                .codec()
                .decoder()
                .audio()
                .map_err(|e| format!("FFmpeg error when finding codec: {:?}.", e))?;
            let expected_sample_number = (SAMPLE_RATE as f32 * stream.duration() as f32
                / stream.time_base().denominator() as f32)
                .ceil();
            (codec, stream.index(), expected_sample_number)
        };
        let mut sample_array: Vec<f32> = Vec::with_capacity(expected_sample_number as usize);

        if let Some(title) = format.metadata().get("title") {
            song.title = title.to_string();
        };
        if let Some(artist) = format.metadata().get("artist") {
            song.artist = artist.to_string();
        };
        if let Some(album) = format.metadata().get("album") {
            song.album = album.to_string();
        };
        if let Some(genre) = format.metadata().get("genre") {
            song.genre = genre.to_string();
        };
        if let Some(track_number) = format.metadata().get("track") {
            song.track_number = track_number.to_string();
        };

        // TODO handle WAV without a channel layout set (cf bruiblan.wav)
        let mut resample_context = ffmpeg::software::resampling::context::Context::get(
            codec.format(),
            codec.channel_layout(),
            codec.rate(),
            Sample::F32(Type::Packed),
            ffmpeg::util::channel_layout::ChannelLayout::MONO,
            SAMPLE_RATE,
        )
        .map_err(|e| {
            format!(
                "FFmpeg error trying to allocate resampling context: {:?}",
                e
            )
        })?;

        let (tx, rx) = mpsc::channel();
        let child = std_thread::spawn(move || -> Result<Vec<f32>, String> {
            let mut resampled = ffmpeg::frame::Audio::empty();
            for decoded in rx.iter() {
                resample_context
                    .run(&decoded, &mut resampled)
                    .map_err(|e| format!("FFmpeg error while trying to resample song: {:?}", e))?;
                push_to_sample_array(&resampled, &mut sample_array);
            }

            loop {
                match resample_context
                    .flush(&mut resampled)
                    .map_err(|e| format!("FFmpeg error while trying to resample song: {:?}", e))?
                {
                    Some(_) => {
                        push_to_sample_array(&resampled, &mut sample_array);
                    }
                    None => {
                        push_to_sample_array(&resampled, &mut sample_array);
                        break;
                    }
                };
            }
            Ok(sample_array)
        });
        let mut decoded = ffmpeg::frame::Audio::empty();
        for (s, packet) in format.packets() {
            if s.index() != stream {
                continue;
            }
            codec.send_packet(&packet).unwrap();
            while codec.receive_frame(&mut decoded).is_ok() {
                tx.send(decoded.clone()).map_err(|e| {
                    format!(
                        "Error while sending decoded frame to the resampling thread: {:?}",
                        e
                    )
                })?;
            }
        }

        // Flush the stream
        // TODO check that it's still how to do this
        let packet = ffmpeg::codec::packet::Packet::empty();
        loop {
            codec.send_packet(&packet).unwrap();
            while codec.receive_frame(&mut decoded).is_ok() {
                tx.send(decoded.clone()).map_err(|e| {
                    format!(
                        "Error while sending decoded frame to the resampling thread: {:?}",
                        e
                    )
                })?;
            }
            break;
        }

        drop(tx);
        song.sample_array = child.join().unwrap()?;
        song.sample_rate = SAMPLE_RATE;
        Ok(song)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use ripemd160::{Digest, Ripemd160};
    use std::f32::consts::PI;

    #[test]
    fn test_analyze() {
        let song = Song::decode("data/s16_mono_22_5kHz.flac").unwrap();
        let expected_analysis = Analysis {
            tempo: 0.37860596,
            spectral_centroid: -0.75483,
            zero_crossing_rate: -0.85036564,
            spectral_rolloff: -0.6326486,
            spectral_flatness: -0.77610075,
            loudness: 0.27126348,
            is_major: -1.,
            fifth: (f32::cos(5. * PI / 3.), f32::sin(5. * PI / 3.)),
        };
        assert!(expected_analysis.approx_eq(&song.analyze()));
    }

    fn _test_decode(path: &str, expected_hash: &[u8]) {
        let song = Song::decode(path).unwrap();
        let mut hasher = Ripemd160::new();
        for sample in song.sample_array.iter() {
            hasher.update(sample.to_le_bytes().to_vec());
        }

        assert_eq!(expected_hash, hasher.finalize().as_slice());
    }

    #[test]
    fn tags() {
        let song = Song::decode("data/s16_mono_22_5kHz.flac").unwrap();
        assert_eq!(song.artist, "David TMX");
        assert_eq!(song.title, "Renaissance");
        assert_eq!(song.album, "Renaissance");
        assert_eq!(song.track_number, "02");
        assert_eq!(song.genre, "Pop");
    }

    #[test]
    fn resample_multi() {
        let path = String::from("data/s32_stereo_44_1_kHz.flac");
        let expected_hash = [
            0xc5, 0xf8, 0x23, 0xce, 0x63, 0x2c, 0xf4, 0xa0, 0x72, 0x66, 0xbb, 0x49, 0xad, 0x84,
            0xb6, 0xea, 0x48, 0x48, 0x9c, 0x50,
        ];
        _test_decode(&path, &expected_hash);
    }

    #[test]
    fn resample_stereo() {
        let path = String::from("data/s16_stereo_22_5kHz.flac");
        let expected_hash = [
            0x24, 0xed, 0x45, 0x58, 0x06, 0xbf, 0xfb, 0x05, 0x57, 0x5f, 0xdc, 0x4d, 0xb4, 0x9b,
            0xa5, 0x2b, 0x05, 0x56, 0x10, 0x4f,
        ];
        _test_decode(&path, &expected_hash);
    }

    #[test]
    fn decode_mono() {
        let path = String::from("data/s16_mono_22_5kHz.flac");
        // Obtained through
        // ffmpeg -i data/s16_mono_22_5kHz.flac -ar 22050 -ac 1 -c:a pcm_f32le
        // -f hash -hash ripemd160 -
        let expected_hash = [
            0x9d, 0x95, 0xa5, 0xf2, 0xd2, 0x9c, 0x68, 0xe8, 0x8a, 0x70, 0xcd, 0xf3, 0x54, 0x2c,
            0x5b, 0x45, 0x98, 0xb4, 0xf3, 0xb4,
        ];
        _test_decode(&path, &expected_hash);
    }

    #[test]
    fn test_decode_right_capacity_vec() {
        let path = String::from("data/s16_mono_22_5kHz.flac");
        let song = Song::decode(&path).unwrap();
        assert_eq!(song.sample_array.len(), song.sample_array.capacity());

        let path = String::from("data/s32_stereo_44_1_kHz.flac");
        let song = Song::decode(&path).unwrap();
        assert_eq!(song.sample_array.len(), song.sample_array.capacity());

        // Not 100% sure that the number of samples for ogg is known
        // precisely in advance
        let path = String::from("data/capacity_fix.ogg");
        let song = Song::decode(&path).unwrap();
        assert!(song.sample_array.len() as f32 / song.sample_array.capacity() as f32 > 0.95);
        assert!(song.sample_array.len() as f32 / (song.sample_array.capacity() as f32) < 1.);
    }
}
<commit_msg>Enable multithreading when decoding<commit_after>//! Song decoding / analysis module.
//!
//! Use decoding, and features-extraction functions from other modules
//! e.g. tempo features, spectral features, etc to build a Song and its
//! corresponding Analysis.
#[cfg(feature = "aubio-lib")]
extern crate aubio_lib;

extern crate crossbeam;
extern crate ffmpeg_next as ffmpeg;
extern crate ndarray;
extern crate ndarray_npy;

use super::CHANNELS;
use crate::chroma::ChromaDesc;
use crate::misc::LoudnessDesc;
use crate::temporal::BPMDesc;
use crate::timbral::{SpectralDesc, ZeroCrossingRateDesc};
use crate::SAMPLE_RATE;
use crate::{Analysis, Song};
use crossbeam::thread;
use ffmpeg::codec::threading::{Config, Type as ThreadingType};
use ffmpeg::util;
use ffmpeg::util::format::sample::{Sample, Type};
use std::sync::mpsc;
use std::thread as std_thread;

pub fn push_to_sample_array(frame: &ffmpeg::frame::Audio, sample_array: &mut Vec<f32>) {
    if frame.samples() == 0 {
        return;
    }
    // Account for the padding
    let actual_size = util::format::sample::Buffer::size(
        Sample::F32(Type::Packed),
        CHANNELS,
        frame.samples(),
        false,
    );
    let f32_frame: Vec<f32> = frame.data(0)[..actual_size]
        .chunks_exact(4)
        .map(|x| {
            let mut a: [u8; 4] = [0; 4];
            a.copy_from_slice(x);
            f32::from_le_bytes(a)
        })
        .collect();
    sample_array.extend_from_slice(&f32_frame);
}

impl Song {
    pub fn new(path: &str) -> Result<Self, String> {
        // TODO error handling here
        let mut song = Song::decode(&path)?;

        song.analysis = (&song).analyze();
        Ok(song)
    }

    pub fn analyze(&self) -> Analysis {
        thread::scope(|s| {
            let child_chroma = s.spawn(|_| {
                let mut chroma_desc = ChromaDesc::new(self.sample_rate, 12);
                chroma_desc.do_(&self.sample_array);
                chroma_desc.get_values()
            });

            // These descriptors can be streamed
            let child_timbral = s.spawn(|_| {
                let mut spectral_desc = SpectralDesc::new(self.sample_rate);
                let mut zcr_desc = ZeroCrossingRateDesc::default();
                let windows = self
                    .sample_array
                    .windows(SpectralDesc::WINDOW_SIZE)
                    .step_by(SpectralDesc::HOP_SIZE);
                for window in windows {
                    spectral_desc.do_(&window);
                    zcr_desc.do_(&window);
                }
                let centroid = spectral_desc.get_centroid();
                let rolloff = spectral_desc.get_rolloff();
                let flatness = spectral_desc.get_flatness();
                let zcr = zcr_desc.get_value();
                (centroid, rolloff, flatness, zcr)
            });

            let child_tempo = s.spawn(|_| {
                let mut tempo_desc = BPMDesc::new(self.sample_rate);
                let windows = self
                    .sample_array
                    .windows(BPMDesc::WINDOW_SIZE)
                    .step_by(BPMDesc::HOP_SIZE);

                for window in windows {
                    tempo_desc.do_(&window);
                }
                tempo_desc.get_value()
            });

            let child_loudness = s.spawn(|_| {
                let mut loudness_desc = LoudnessDesc::default();
                let windows = self.sample_array.chunks(LoudnessDesc::WINDOW_SIZE);

                for window in windows {
                    loudness_desc.do_(&window);
                }
                loudness_desc.get_value()
            });

            // Non-streaming approach for that one
            let (is_major, fifth) = child_chroma.join().unwrap();
            let (centroid, rolloff, flatness, zcr) = child_timbral.join().unwrap();
            let tempo = child_tempo.join().unwrap();
            let loudness = child_loudness.join().unwrap();

            Analysis {
                tempo,
                spectral_centroid: centroid,
                zero_crossing_rate: zcr,
                spectral_rolloff: rolloff,
                spectral_flatness: flatness,
                loudness,
                is_major,
                fifth,
            }
        })
        .unwrap()
    }

    // TODO set av_logs to quiet
    pub fn decode(path: &str) -> Result<Song, String> {
        ffmpeg::init().map_err(|e| format!("FFmpeg init error: {:?}.", e))?;

        let mut song = Song::default();
        song.path = path.to_string();
        let mut format = ffmpeg::format::input(&path)
            .map_err(|e| format!("FFmpeg error while opening format: {:?}.", e))?;
        let (mut codec, stream, expected_sample_number) = {
            let stream = format
                .streams()
                .find(|s| s.codec().medium() == ffmpeg::media::Type::Audio)
                .ok_or("No audio stream found.")?;
            stream.codec().set_threading(Config {
                kind: ThreadingType::Frame,
                count: 0,
                safe: true,
            });
            let codec = stream
                .codec()
                .decoder()
                .audio()
                .map_err(|e| format!("FFmpeg error when finding codec: {:?}.", e))?;
            let expected_sample_number = (SAMPLE_RATE as f32 * stream.duration() as f32
                / stream.time_base().denominator() as f32)
                .ceil();
            (codec, stream.index(), expected_sample_number)
        };
        let mut sample_array: Vec<f32> = Vec::with_capacity(expected_sample_number as usize);

        if let Some(title) = format.metadata().get("title") {
            song.title = title.to_string();
        };
        if let Some(artist) = format.metadata().get("artist") {
            song.artist = artist.to_string();
        };
        if let Some(album) = format.metadata().get("album") {
            song.album = album.to_string();
        };
        if let Some(genre) = format.metadata().get("genre") {
            song.genre = genre.to_string();
        };
        if let Some(track_number) = format.metadata().get("track") {
            song.track_number = track_number.to_string();
        };

        // TODO handle WAV without a channel layout set (cf bruiblan.wav)
        let mut resample_context = ffmpeg::software::resampling::context::Context::get(
            codec.format(),
            codec.channel_layout(),
            codec.rate(),
            Sample::F32(Type::Packed),
            ffmpeg::util::channel_layout::ChannelLayout::MONO,
            SAMPLE_RATE,
        )
        .map_err(|e| {
            format!(
                "FFmpeg error trying to allocate resampling context: {:?}",
                e
            )
        })?;

        let (tx, rx) = mpsc::channel();
        let child = std_thread::spawn(move || -> Result<Vec<f32>, String> {
            let mut resampled = ffmpeg::frame::Audio::empty();
            for decoded in rx.iter() {
                resample_context
                    .run(&decoded, &mut resampled)
                    .map_err(|e| format!("FFmpeg error while trying to resample song: {:?}", e))?;
                push_to_sample_array(&resampled, &mut sample_array);
            }

            loop {
                match resample_context
                    .flush(&mut resampled)
                    .map_err(|e| format!("FFmpeg error while trying to resample song: {:?}", e))?
                {
                    Some(_) => {
                        push_to_sample_array(&resampled, &mut sample_array);
                    }
                    None => {
                        push_to_sample_array(&resampled, &mut sample_array);
                        break;
                    }
                };
            }
            Ok(sample_array)
        });
        let mut decoded = ffmpeg::frame::Audio::empty();
        for (s, packet) in format.packets() {
            if s.index() != stream {
                continue;
            }
            codec.send_packet(&packet).unwrap();
            while codec.receive_frame(&mut decoded).is_ok() {
                tx.send(decoded.clone()).map_err(|e| {
                    format!(
                        "Error while sending decoded frame to the resampling thread: {:?}",
                        e
                    )
                })?;
            }
        }

        // Flush the stream
        // TODO check that it's still how to do this
        let packet = ffmpeg::codec::packet::Packet::empty();
        loop {
            codec.send_packet(&packet).unwrap();
            while codec.receive_frame(&mut decoded).is_ok() {
                tx.send(decoded.clone()).map_err(|e| {
                    format!(
                        "Error while sending decoded frame to the resampling thread: {:?}",
                        e
                    )
                })?;
            }
            break;
        }

        drop(tx);
        song.sample_array = child.join().unwrap()?;
        song.sample_rate = SAMPLE_RATE;
        Ok(song)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use ripemd160::{Digest, Ripemd160};
    use std::f32::consts::PI;

    #[test]
    fn test_analyze() {
        let song = Song::decode("data/s16_mono_22_5kHz.flac").unwrap();
        let expected_analysis = Analysis {
            tempo: 0.37860596,
            spectral_centroid: -0.75483,
            zero_crossing_rate: -0.85036564,
            spectral_rolloff: -0.6326486,
            spectral_flatness: -0.77610075,
            loudness: 0.27126348,
            is_major: -1.,
            fifth: (f32::cos(5. * PI / 3.), f32::sin(5. * PI / 3.)),
        };
        assert!(expected_analysis.approx_eq(&song.analyze()));
    }

    fn _test_decode(path: &str, expected_hash: &[u8]) {
        let song = Song::decode(path).unwrap();
        let mut hasher = Ripemd160::new();
        for sample in song.sample_array.iter() {
            hasher.update(sample.to_le_bytes().to_vec());
        }

        assert_eq!(expected_hash, hasher.finalize().as_slice());
    }

    #[test]
    fn tags() {
        let song = Song::decode("data/s16_mono_22_5kHz.flac").unwrap();
        assert_eq!(song.artist, "David TMX");
        assert_eq!(song.title, "Renaissance");
        assert_eq!(song.album, "Renaissance");
        assert_eq!(song.track_number, "02");
        assert_eq!(song.genre, "Pop");
    }

    #[test]
    fn resample_multi() {
        let path = String::from("data/s32_stereo_44_1_kHz.flac");
        let expected_hash = [
            0xc5, 0xf8, 0x23, 0xce, 0x63, 0x2c, 0xf4, 0xa0, 0x72, 0x66, 0xbb, 0x49, 0xad, 0x84,
            0xb6, 0xea, 0x48, 0x48, 0x9c, 0x50,
        ];
        _test_decode(&path, &expected_hash);
    }

    #[test]
    fn resample_stereo() {
        let path = String::from("data/s16_stereo_22_5kHz.flac");
        let expected_hash = [
            0x24, 0xed, 0x45, 0x58, 0x06, 0xbf, 0xfb, 0x05, 0x57, 0x5f, 0xdc, 0x4d, 0xb4, 0x9b,
            0xa5, 0x2b, 0x05, 0x56, 0x10, 0x4f,
        ];
        _test_decode(&path, &expected_hash);
    }

    #[test]
    fn decode_mono() {
        let path = String::from("data/s16_mono_22_5kHz.flac");
        // Obtained through
        // ffmpeg -i data/s16_mono_22_5kHz.flac -ar 22050 -ac 1 -c:a pcm_f32le
        // -f hash -hash ripemd160 -
        let expected_hash = [
            0x9d, 0x95, 0xa5, 0xf2, 0xd2, 0x9c, 0x68, 0xe8, 0x8a, 0x70, 0xcd, 0xf3, 0x54, 0x2c,
            0x5b, 0x45, 0x98, 0xb4, 0xf3, 0xb4,
        ];
        _test_decode(&path, &expected_hash);
    }

    #[test]
    fn test_decode_right_capacity_vec() {
        let path = String::from("data/s16_mono_22_5kHz.flac");
        let song = Song::decode(&path).unwrap();
        assert_eq!(song.sample_array.len(), song.sample_array.capacity());

        let path = String::from("data/s32_stereo_44_1_kHz.flac");
        let song = Song::decode(&path).unwrap();
        assert_eq!(song.sample_array.len(), song.sample_array.capacity());

        // Not 100% sure that the number of samples for ogg is known
        // precisely in advance
        let path = String::from("data/capacity_fix.ogg");
        let song = Song::decode(&path).unwrap();
        assert!(song.sample_array.len() as f32 / song.sample_array.capacity() as f32 > 0.95);
        assert!(song.sample_array.len() as f32 / (song.sample_array.capacity() as f32) < 1.);
    }
}
<|endoftext|>


<commit_before><commit_msg>dao query execute<commit_after><|endoftext|>


<commit_before>use super::super::{Regexp, NoExpand};

#[test]
fn splitn() {
    let re = Regexp::new(r"\d+").unwrap();
    let text = "cauchy123plato456tyler789binks";
    let subs: Vec<&str> = re.splitn(text, 2).collect();
    assert_eq!(subs, vec!("cauchy", "plato456tyler789binks"));
}

#[test]
fn split() {
    let re = Regexp::new(r"\d+").unwrap();
    let text = "cauchy123plato456tyler789binks";
    let subs: Vec<&str> = re.split(text).collect();
    assert_eq!(subs, vec!("cauchy", "plato", "tyler", "binks"));
}

macro_rules! replace(
    ($name:ident, $which:ident, $re:expr, $search:expr, $replace:expr, $result:expr) => (
        #[test]
        fn $name() {
            let re = Regexp::new($re).unwrap();
            assert_eq!(re.$which($search, $replace), $result);
        }
    );
)

replace!(rep_first, replace, r"\d", "age: 26", "Z", ~"age: Z6")
replace!(rep_plus, replace, r"\d+", "age: 26", "Z", ~"age: Z")
replace!(rep_all, replace_all, r"\d", "age: 26", "Z", ~"age: ZZ")
replace!(rep_groups, replace, r"(\S+)\s+(\S+)", "w1 w2", "$2 $1", ~"w2 w1")
replace!(rep_double_dollar, replace,
         r"(\S+)\s+(\S+)", "w1 w2", "$2 $$1", ~"w2 $1")
replace!(rep_no_expand, replace,
         r"(\S+)\s+(\S+)", "w1 w2", NoExpand("$2 $1"), ~"$2 $1")
replace!(rep_named, replace_all,
         r"(?P<first>\S+)\s+(?P<last>\S+)(?P<space>\s*)",
         "w1 w2 w3 w4", "$last $first$space", ~"w2 w1 w4 w3")

macro_rules! fail_parse(
    ($name:ident, $re:expr) => (
        #[test]
        fn $name() {
            let re = $re;
            match Regexp::new(re) {
                Err(_) => {},
                Ok(_) => fail!("Regexp '{}' should cause a parse error.", re),
            }
        }
    );
)

fail_parse!(fail_parse_double_repeat, "a**")
fail_parse!(fail_parse_no_repeat_arg, "*")
fail_parse!(fail_parse_no_repeat_arg_begin, "^*")
fail_parse!(fail_parse_incomplete_escape, "\\")
fail_parse!(fail_parse_class_incomplete, "[A-")
fail_parse!(fail_parse_class_not_closed, "[A")
fail_parse!(fail_parse_class_no_begin, r"[\A]")
fail_parse!(fail_parse_class_no_end, r"[\z]")
fail_parse!(fail_parse_class_no_boundary, r"[\b]")
fail_parse!(fail_parse_open_paren, "(")
fail_parse!(fail_parse_close_paren, ")")
fail_parse!(fail_parse_invalid_range, "[a-Z]")
fail_parse!(fail_parse_empty_capture_name, "(?P<>a)")
fail_parse!(fail_parse_empty_capture_exp, "(?P<name>)")
fail_parse!(fail_parse_bad_flag, "(?a)a")
fail_parse!(fail_parse_empty_alt_before, "|a")
fail_parse!(fail_parse_empty_alt_after, "a|")
fail_parse!(fail_parse_counted_big_exact, "a{1001}")
fail_parse!(fail_parse_counted_big_min, "a{1001,}")
fail_parse!(fail_parse_counted_no_close, "a{1001")

macro_rules! mat(
    ($name:ident, $re:expr, $text:expr, $($loc:tt)+) => (
        #[test]
        fn $name() {
            let re = $re;
            let text = $text;
            let expected: Vec<Option<(uint, uint)>> = vec!($($loc)+);
            let r = match Regexp::new(re) {
                Ok(r) => r,
                Err(err) => fail!("Could not compile '{}': {}", re, err),
            };
            let got = match r.captures(text) {
                Some(c) => c.iter_pos().collect::<Vec<Option<(uint, uint)>>>(),
                None => vec!(None),
            };
            // The test set sometimes leave out capture groups, so truncate
            // actual capture groups to match test set.
            let (sexpect, mut sgot) = (expected.as_slice(), got.as_slice());
            if sgot.len() > sexpect.len() {
                sgot = sgot.slice(0, sexpect.len())
            }
            if sexpect != sgot {
                fail!("For RE '{}' against '{}', expected '{}' but got '{}'",
                      re, text, sexpect, sgot);
            }
        }
    );
)

mod matches;
<commit_msg>get spelling right<commit_after>use super::super::{Regexp, NoExpand};

#[test]
fn splitn() {
    let re = Regexp::new(r"\d+").unwrap();
    let text = "cauchy123plato456tyler789binx";
    let subs: Vec<&str> = re.splitn(text, 2).collect();
    assert_eq!(subs, vec!("cauchy", "plato456tyler789binx"));
}

#[test]
fn split() {
    let re = Regexp::new(r"\d+").unwrap();
    let text = "cauchy123plato456tyler789binx";
    let subs: Vec<&str> = re.split(text).collect();
    assert_eq!(subs, vec!("cauchy", "plato", "tyler", "binx"));
}

macro_rules! replace(
    ($name:ident, $which:ident, $re:expr, $search:expr, $replace:expr, $result:expr) => (
        #[test]
        fn $name() {
            let re = Regexp::new($re).unwrap();
            assert_eq!(re.$which($search, $replace), $result);
        }
    );
)

replace!(rep_first, replace, r"\d", "age: 26", "Z", ~"age: Z6")
replace!(rep_plus, replace, r"\d+", "age: 26", "Z", ~"age: Z")
replace!(rep_all, replace_all, r"\d", "age: 26", "Z", ~"age: ZZ")
replace!(rep_groups, replace, r"(\S+)\s+(\S+)", "w1 w2", "$2 $1", ~"w2 w1")
replace!(rep_double_dollar, replace,
         r"(\S+)\s+(\S+)", "w1 w2", "$2 $$1", ~"w2 $1")
replace!(rep_no_expand, replace,
         r"(\S+)\s+(\S+)", "w1 w2", NoExpand("$2 $1"), ~"$2 $1")
replace!(rep_named, replace_all,
         r"(?P<first>\S+)\s+(?P<last>\S+)(?P<space>\s*)",
         "w1 w2 w3 w4", "$last $first$space", ~"w2 w1 w4 w3")

macro_rules! fail_parse(
    ($name:ident, $re:expr) => (
        #[test]
        fn $name() {
            let re = $re;
            match Regexp::new(re) {
                Err(_) => {},
                Ok(_) => fail!("Regexp '{}' should cause a parse error.", re),
            }
        }
    );
)

fail_parse!(fail_parse_double_repeat, "a**")
fail_parse!(fail_parse_no_repeat_arg, "*")
fail_parse!(fail_parse_no_repeat_arg_begin, "^*")
fail_parse!(fail_parse_incomplete_escape, "\\")
fail_parse!(fail_parse_class_incomplete, "[A-")
fail_parse!(fail_parse_class_not_closed, "[A")
fail_parse!(fail_parse_class_no_begin, r"[\A]")
fail_parse!(fail_parse_class_no_end, r"[\z]")
fail_parse!(fail_parse_class_no_boundary, r"[\b]")
fail_parse!(fail_parse_open_paren, "(")
fail_parse!(fail_parse_close_paren, ")")
fail_parse!(fail_parse_invalid_range, "[a-Z]")
fail_parse!(fail_parse_empty_capture_name, "(?P<>a)")
fail_parse!(fail_parse_empty_capture_exp, "(?P<name>)")
fail_parse!(fail_parse_bad_flag, "(?a)a")
fail_parse!(fail_parse_empty_alt_before, "|a")
fail_parse!(fail_parse_empty_alt_after, "a|")
fail_parse!(fail_parse_counted_big_exact, "a{1001}")
fail_parse!(fail_parse_counted_big_min, "a{1001,}")
fail_parse!(fail_parse_counted_no_close, "a{1001")

macro_rules! mat(
    ($name:ident, $re:expr, $text:expr, $($loc:tt)+) => (
        #[test]
        fn $name() {
            let re = $re;
            let text = $text;
            let expected: Vec<Option<(uint, uint)>> = vec!($($loc)+);
            let r = match Regexp::new(re) {
                Ok(r) => r,
                Err(err) => fail!("Could not compile '{}': {}", re, err),
            };
            let got = match r.captures(text) {
                Some(c) => c.iter_pos().collect::<Vec<Option<(uint, uint)>>>(),
                None => vec!(None),
            };
            // The test set sometimes leave out capture groups, so truncate
            // actual capture groups to match test set.
            let (sexpect, mut sgot) = (expected.as_slice(), got.as_slice());
            if sgot.len() > sexpect.len() {
                sgot = sgot.slice(0, sexpect.len())
            }
            if sexpect != sgot {
                fail!("For RE '{}' against '{}', expected '{}' but got '{}'",
                      re, text, sexpect, sgot);
            }
        }
    );
)

mod matches;
<|endoftext|>


<commit_before>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

pub type TokenResult = result::Result<Token, String>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert!(tokenizer.next().unwrap().unwrap() == Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert!(tokenizer.next().unwrap().unwrap() == Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert!(tokenizer.next().unwrap().unwrap() == ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert!(tokenizer.next().unwrap().unwrap() == ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert!(tokenizer.next().unwrap().unwrap() == EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$1111^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(format!("  {}", symbol).as_ref());

        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
    }

    #[test]
    fn ignores_leading_whitespace() {
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }
}

<commit_msg>Finished implementing tests<commit_after>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

pub type TokenResult = result::Result<Token, String>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert!(tokenizer.next().unwrap().unwrap() == Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert!(tokenizer.next().unwrap().unwrap() == Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert!(tokenizer.next().unwrap().unwrap() == ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert!(tokenizer.next().unwrap().unwrap() == ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert!(tokenizer.next().unwrap().unwrap() == EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$1111^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().is_err());
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh#*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }
}

<|endoftext|>


<commit_before>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or https://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use vulkano::device::{Device, DeviceExtensions};
use vulkano::format::Format;
use vulkano::image::Dimensions;
use vulkano::image::ImmutableImage;
use vulkano::image::MipmapsCount;
use vulkano::instance;
use vulkano::instance::debug::{DebugCallback, MessageSeverity, MessageType};
use vulkano::instance::{Instance, InstanceExtensions, PhysicalDevice};

fn main() {
    // Vulkano Debugging Example Code
    //
    // This example code will demonstrate using the debug functions of the Vulkano API.
    //
    // There is documentation here about how to setup debugging:
    // https://vulkan.lunarg.com/doc/view/1.0.13.0/windows/layers.html
    //
    // .. but if you just want a template of code that has everything ready to go then follow
    // this example. First, enable debugging using this extension: VK_EXT_debug_utils
    let extensions = InstanceExtensions {
        ext_debug_utils: true,
        ..InstanceExtensions::none()
    };

    // You also need to specify (unless you've used the methods linked above) which debugging layers
    // your code should use. Each layer is a bunch of checks or messages that provide information of
    // some sort.
    //
    // The main layer you might want is: VK_LAYER_LUNARG_standard_validation
    // This includes a number of the other layers for you and is quite detailed.
    //
    // Additional layers can be installed (gpu vendor provided, something you found on GitHub, etc)
    // and you should verify that list for safety - Vulkano will return an error if you specify
    // any layers that are not installed on this system. That code to do could look like this:
    println!("List of Vulkan debugging layers available to use:");
    let mut layers = instance::layers_list().unwrap();
    while let Some(l) = layers.next() {
        println!("\t{}", l.name());
    }

    // NOTE: To simplify the example code we won't verify these layer(s) are actually in the layers list:
    let layer = "VK_LAYER_LUNARG_standard_validation";
    let layers = vec![layer];

    // Important: pass the extension(s) and layer(s) when creating the vulkano instance
    let instance =
        Instance::new(None, &extensions, layers).expect("failed to create Vulkan instance");

    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // After creating the instance we must register the debugging callback.                                      //
    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////

    // Note: If you let this debug_callback binding fall out of scope then the callback will stop providing events
    // Note: There is a helper method too: DebugCallback::errors_and_warnings(&instance, |msg| {...

    let severity = MessageSeverity {
        error: true,
        warning: true,
        information: true,
        verbose: true,
    };

    let ty = MessageType::all();

    let _debug_callback = DebugCallback::new(&instance, severity, ty, |msg| {
        let severity = if msg.severity.error {
            "error"
        } else if msg.severity.warning {
            "warning"
        } else if msg.severity.information {
            "information"
        } else if msg.severity.verbose {
            "verbose"
        } else {
            panic!("no-impl");
        };

        let ty = if msg.ty.general {
            "general"
        } else if msg.ty.validation {
            "validation"
        } else if msg.ty.performance {
            "performance"
        } else {
            panic!("no-impl");
        };

        println!(
            "{} {} {}: {}",
            msg.layer_prefix, ty, severity, msg.description
        );
    })
    .ok();

    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // Create Vulkan objects in the same way as the other examples                                               //
    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////

    let physical = PhysicalDevice::enumerate(&instance)
        .next()
        .expect("no device available");
    let queue_family = physical
        .queue_families()
        .next()
        .expect("couldn't find a queue family");
    let (_, mut queues) = Device::new(
        physical,
        physical.supported_features(),
        &DeviceExtensions::none(),
        vec![(queue_family, 0.5)],
    )
    .expect("failed to create device");
    let queue = queues.next().unwrap();

    // Create an image in order to generate some additional logging:
    let pixel_format = Format::R8G8B8A8Uint;
    let dimensions = Dimensions::Dim2d {
        width: 4096,
        height: 4096,
    };
    const DATA: [[u8; 4]; 4096 * 4096] = [[0; 4]; 4096 * 4096];
    let _ = ImmutableImage::from_iter(
        DATA.iter().cloned(),
        dimensions,
        MipmapsCount::One,
        pixel_format,
        queue.clone(),
    )
    .unwrap();

    // (At this point you should see a bunch of messages printed to the terminal window - have fun debugging!)
}
<commit_msg>examples: fix debug example layers on macos using "VK_LAYER_KHRONOS_validation" (#1488)<commit_after>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or https://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use vulkano::device::{Device, DeviceExtensions};
use vulkano::format::Format;
use vulkano::image::Dimensions;
use vulkano::image::ImmutableImage;
use vulkano::image::MipmapsCount;
use vulkano::instance;
use vulkano::instance::debug::{DebugCallback, MessageSeverity, MessageType};
use vulkano::instance::{Instance, InstanceExtensions, PhysicalDevice};

fn main() {
    // Vulkano Debugging Example Code
    //
    // This example code will demonstrate using the debug functions of the Vulkano API.
    //
    // There is documentation here about how to setup debugging:
    // https://vulkan.lunarg.com/doc/view/1.0.13.0/windows/layers.html
    //
    // .. but if you just want a template of code that has everything ready to go then follow
    // this example. First, enable debugging using this extension: VK_EXT_debug_utils
    let extensions = InstanceExtensions {
        ext_debug_utils: true,
        ..InstanceExtensions::none()
    };

    // You also need to specify (unless you've used the methods linked above) which debugging layers
    // your code should use. Each layer is a bunch of checks or messages that provide information of
    // some sort.
    //
    // The main layer you might want is: VK_LAYER_LUNARG_standard_validation
    // This includes a number of the other layers for you and is quite detailed.
    //
    // Additional layers can be installed (gpu vendor provided, something you found on GitHub, etc)
    // and you should verify that list for safety - Vulkano will return an error if you specify
    // any layers that are not installed on this system. That code to do could look like this:
    println!("List of Vulkan debugging layers available to use:");
    let mut layers = instance::layers_list().unwrap();
    while let Some(l) = layers.next() {
        println!("\t{}", l.name());
    }

    // NOTE: To simplify the example code we won't verify these layer(s) are actually in the layers list:
    #[cfg(not(target_os = "macos"))]
    let layers = vec!["VK_LAYER_LUNARG_standard_validation"];

    #[cfg(target_os = "macos")]
    let layers = vec!["VK_LAYER_KHRONOS_validation"];

    // Important: pass the extension(s) and layer(s) when creating the vulkano instance
    let instance =
        Instance::new(None, &extensions, layers).expect("failed to create Vulkan instance");

    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // After creating the instance we must register the debugging callback.                                      //
    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////

    // Note: If you let this debug_callback binding fall out of scope then the callback will stop providing events
    // Note: There is a helper method too: DebugCallback::errors_and_warnings(&instance, |msg| {...

    let severity = MessageSeverity {
        error: true,
        warning: true,
        information: true,
        verbose: true,
    };

    let ty = MessageType::all();

    let _debug_callback = DebugCallback::new(&instance, severity, ty, |msg| {
        let severity = if msg.severity.error {
            "error"
        } else if msg.severity.warning {
            "warning"
        } else if msg.severity.information {
            "information"
        } else if msg.severity.verbose {
            "verbose"
        } else {
            panic!("no-impl");
        };

        let ty = if msg.ty.general {
            "general"
        } else if msg.ty.validation {
            "validation"
        } else if msg.ty.performance {
            "performance"
        } else {
            panic!("no-impl");
        };

        println!(
            "{} {} {}: {}",
            msg.layer_prefix, ty, severity, msg.description
        );
    })
    .ok();

    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // Create Vulkan objects in the same way as the other examples                                               //
    ///////////////////////////////////////////////////////////////////////////////////////////////////////////////

    let physical = PhysicalDevice::enumerate(&instance)
        .next()
        .expect("no device available");
    let queue_family = physical
        .queue_families()
        .next()
        .expect("couldn't find a queue family");
    let (_, mut queues) = Device::new(
        physical,
        physical.supported_features(),
        &DeviceExtensions::none(),
        vec![(queue_family, 0.5)],
    )
    .expect("failed to create device");
    let queue = queues.next().unwrap();

    // Create an image in order to generate some additional logging:
    let pixel_format = Format::R8G8B8A8Uint;
    let dimensions = Dimensions::Dim2d {
        width: 4096,
        height: 4096,
    };
    const DATA: [[u8; 4]; 4096 * 4096] = [[0; 4]; 4096 * 4096];
    let _ = ImmutableImage::from_iter(
        DATA.iter().cloned(),
        dimensions,
        MipmapsCount::One,
        pixel_format,
        queue.clone(),
    )
    .unwrap();

    // (At this point you should see a bunch of messages printed to the terminal window - have fun debugging!)
}
<|endoftext|>


<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::common::types::{CliCommand, CliTypedResult, TransactionOptions};
use aptos_rest_client::aptos_api_types::HashValue;
use aptos_rest_client::{
    aptos_api_types::{WriteResource, WriteSetChange},
    Transaction,
};
use aptos_types::account_address::AccountAddress;
use async_trait::async_trait;
use cached_packages::aptos_stdlib;
use clap::Parser;
use serde::Serialize;
use std::collections::BTreeMap;

/// Command to transfer coins between accounts
///
#[derive(Debug, Parser)]
pub struct TransferCoins {
    /// Address of account you want to send coins to
    #[clap(long, parse(try_from_str = crate::common::types::load_account_arg))]
    pub(crate) account: AccountAddress,

    /// Amount of coins to transfer
    #[clap(long)]
    pub(crate) amount: u64,

    #[clap(flatten)]
    pub(crate) txn_options: TransactionOptions,
}

#[async_trait]
impl CliCommand<TransferSummary> for TransferCoins {
    fn command_name(&self) -> &'static str {
        "TransferCoins"
    }

    async fn execute(self) -> CliTypedResult<TransferSummary> {
        self.txn_options
            .submit_transaction(aptos_stdlib::aptos_coin_transfer(self.account, self.amount))
            .await
            .map(TransferSummary::from)
    }
}

const SUPPORTED_COINS: [&str; 1] = ["0x1::coin::CoinStore<0x1::aptos_coin::AptosCoin>"];

/// A shortened transaction output
#[derive(Clone, Debug, Serialize)]
pub struct TransferSummary {
    pub gas_unit_price: u64,
    pub gas_used: u64,
    pub balance_changes: BTreeMap<AccountAddress, serde_json::Value>,
    pub sender: AccountAddress,
    pub success: bool,
    pub version: u64,
    pub vm_status: String,
    pub transaction_hash: HashValue,
}

impl From<Transaction> for TransferSummary {
    fn from(transaction: Transaction) -> Self {
        if let Transaction::UserTransaction(txn) = transaction {
            let vm_status = txn.info.vm_status;
            let success = txn.info.success;
            let sender = *txn.request.sender.inner();
            let gas_unit_price = txn.request.gas_unit_price.0;
            let gas_used = txn.info.gas_used.0;
            let transaction_hash = txn.info.hash;
            let version = txn.info.version.0;
            let balance_changes = txn
                .info
                .changes
                .into_iter()
                .filter_map(|change| match change {
                    WriteSetChange::WriteResource(WriteResource { address, data, .. }) => {
                        if SUPPORTED_COINS.contains(&data.typ.to_string().as_str()) {
                            Some((
                                *address.inner(),
                                serde_json::to_value(data.data).unwrap_or_default(),
                            ))
                        } else {
                            None
                        }
                    }
                    _ => None,
                })
                .collect();

            TransferSummary {
                gas_unit_price,
                gas_used,
                balance_changes,
                sender,
                success,
                version,
                vm_status,
                transaction_hash,
            }
        } else {
            panic!("Can't call From<Transaction> for a non UserTransaction")
        }
    }
}
<commit_msg>[aptos-cli] Move to using account transfer over coin transfer<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::common::types::{CliCommand, CliTypedResult, TransactionOptions};
use aptos_rest_client::aptos_api_types::HashValue;
use aptos_rest_client::{
    aptos_api_types::{WriteResource, WriteSetChange},
    Transaction,
};
use aptos_types::account_address::AccountAddress;
use async_trait::async_trait;
use cached_packages::aptos_stdlib;
use clap::Parser;
use serde::Serialize;
use std::collections::BTreeMap;

/// Command to transfer coins between accounts
///
#[derive(Debug, Parser)]
pub struct TransferCoins {
    /// Address of account you want to send coins to
    #[clap(long, parse(try_from_str = crate::common::types::load_account_arg))]
    pub(crate) account: AccountAddress,

    /// Amount of coins to transfer
    #[clap(long)]
    pub(crate) amount: u64,

    #[clap(flatten)]
    pub(crate) txn_options: TransactionOptions,
}

#[async_trait]
impl CliCommand<TransferSummary> for TransferCoins {
    fn command_name(&self) -> &'static str {
        "TransferCoins"
    }

    async fn execute(self) -> CliTypedResult<TransferSummary> {
        self.txn_options
            .submit_transaction(aptos_stdlib::aptos_account_transfer(
                self.account,
                self.amount,
            ))
            .await
            .map(TransferSummary::from)
    }
}

const SUPPORTED_COINS: [&str; 1] = ["0x1::coin::CoinStore<0x1::aptos_coin::AptosCoin>"];

/// A shortened transaction output
#[derive(Clone, Debug, Serialize)]
pub struct TransferSummary {
    pub gas_unit_price: u64,
    pub gas_used: u64,
    pub balance_changes: BTreeMap<AccountAddress, serde_json::Value>,
    pub sender: AccountAddress,
    pub success: bool,
    pub version: u64,
    pub vm_status: String,
    pub transaction_hash: HashValue,
}

impl From<Transaction> for TransferSummary {
    fn from(transaction: Transaction) -> Self {
        if let Transaction::UserTransaction(txn) = transaction {
            let vm_status = txn.info.vm_status;
            let success = txn.info.success;
            let sender = *txn.request.sender.inner();
            let gas_unit_price = txn.request.gas_unit_price.0;
            let gas_used = txn.info.gas_used.0;
            let transaction_hash = txn.info.hash;
            let version = txn.info.version.0;
            let balance_changes = txn
                .info
                .changes
                .into_iter()
                .filter_map(|change| match change {
                    WriteSetChange::WriteResource(WriteResource { address, data, .. }) => {
                        if SUPPORTED_COINS.contains(&data.typ.to_string().as_str()) {
                            Some((
                                *address.inner(),
                                serde_json::to_value(data.data).unwrap_or_default(),
                            ))
                        } else {
                            None
                        }
                    }
                    _ => None,
                })
                .collect();

            TransferSummary {
                gas_unit_price,
                gas_used,
                balance_changes,
                sender,
                success,
                version,
                vm_status,
                transaction_hash,
            }
        } else {
            panic!("Can't call From<Transaction> for a non UserTransaction")
        }
    }
}
<|endoftext|>