<commit_before>extern crate regex;
use mysql::conn::pool::{MyPool,MyPooledConn};
use mysql::conn::Stmt;

use std::process::{Command, Stdio, Child};
use std::error::Error;
use std::io::prelude::*;
use std::io::BufReader;
use std::path::Path;
use lib::config::ConfigGen;
use std::convert::Into;

use lib::DownloadError;

use CONFIG;
use lib;

macro_rules! regex(
    ($s:expr) => (regex::Regex::new($s).unwrap());
);

#[derive(Clone)]
pub struct DownloadDB {
    pub url: String,
    pub quality: i16,
    pub playlist: bool,
    pub compress: bool,
    pub qid: i64,
    pub from: i32,
    pub to: i32,
    pub folder: String, // download folder, changes for playlists
    pub pool: MyPool,
}

impl DownloadDB {
    pub fn update_video(&mut self,url: String, qid: i64) -> &mut DownloadDB{
        self.qid = qid;
        self.url = url;
        self
    }
    pub fn update_folder(&mut self, folder: String){
        self.folder = folder;
    }
}

pub struct Downloader<'a> {
    pub ddb: &'a DownloadDB,
    defaults: &'a ConfigGen,
    // pool: MyPool,
}

impl<'a> Downloader<'a>{
    pub fn new(ddb: &'a DownloadDB, defaults: &'a ConfigGen) -> Downloader<'a>{
        Downloader {ddb: ddb, defaults: defaults}
    }

    ///Regex matching: [download]  13.4% of 275.27MiB at 525.36KiB/s ETA 07:52
    ///
    ///Downloads a video, updates the DB
    ///TODO: get the sql statements out of the class
    ///TODO: wrap errors
    ///Doesn't care about DMCAs, will emit errors on them
    ///download_audio: ignore quality & download audio_raw codec for split containers
    pub fn download_file(&self, file_path: &str, download_audio: bool) -> Result<bool,DownloadError> {
        println!("{:?}", self.ddb.url);

        let curr_quality = if download_audio {
            &CONFIG.codecs.audio_raw
        }else{
            &self.ddb.quality
        };

        println!("quality: {}",curr_quality);
        let mut child = try!(self.run_download_process(file_path,curr_quality));
        let stdout = BufReader::new(child.stdout.take().unwrap());

        let mut stderr_buffer = BufReader::new(child.stderr.take().unwrap());

        let mut conn = self.ddb.pool.get_conn().unwrap();
        let mut statement = self.prepare_progress_updater(&mut conn);
        let re = regex!(r"\d+\.\d%");

        for line in stdout.lines(){
            match line{
                Err(why) => panic!("couldn't read cmd stdout: {}", Error::description(&why)),
                Ok(text) => {
                        println!("Out: {}",text);
                        if !self.ddb.playlist {
                            match re.find(&text) {
                                Some(s) => { //println!("Match at {}", s.0);
                                            println!("{}", &text[s.0..s.1]); // ONLY with ASCII chars makeable!
                                            try!(self.update_progress(&mut statement, &text[s.0..s.1].to_string()));
                                        },
                                None => {},
                            }
                        }
                },
            }
        }

        try!(child.wait()); // waits for finish & then exists zombi process, fixes #10

        let mut stderr: String = String::new();
        try!(stderr_buffer.read_to_string(&mut stderr));

        if stderr.is_empty() {
            Ok(true)
        } else if stderr.contains("requested format not available") {
            Err(DownloadError::QualityNotAvailable)
        /*} else if stderr.contains("") {
            Err(DownloadError::NotAvailable) */
        } else {
            Err(DownloadError::InternalError(stderr))
        }

    }

    ///Trys to get the original name of a file, while checking for availability
    ///As an ExtractError can appear randomly, bug 11, we're retrying again 2 times if it should occour
    pub fn get_file_name(&self) -> Result<String,DownloadError> {
        for attempts in 0..2 {
            let mut child = try!(self.run_filename_process());
            let mut stdout_buffer = BufReader::new(child.stdout.take().unwrap());
            let mut stderr_buffer = BufReader::new(child.stderr.take().unwrap());

            let mut stdout: String = String::new();
            try!(stdout_buffer.read_to_string(&mut stdout));
            let mut stderr: String = String::new();
            try!(stderr_buffer.read_to_string(&mut stderr));

            try!(child.wait());
            if stderr.is_empty() {
                println!("get_file_name: {:?}", stdout);
                return Ok(stdout.trim().to_string());
            }else{
                if stderr.contains("not available in your country") || stderr.contains("contains content from") || stderr.contains("This video is available in") {
                    return Err(DownloadError::DMCAError);
                } else if stderr.contains("ExtractorError") { // #11
                    println!("ExtractorError on attempt {}", attempts +1);
                } else {
                    return Err(DownloadError::DownloadError(stderr));
                }
            }
        }
        Err(DownloadError::ExtractorError)
    }

    ///Get playlist file ids
    pub fn get_playlist_ids(&self) -> Result<Vec<String>,DownloadError> {
        let mut child = try!(self.run_playlist_extract());
        println!("retrieving playlist ids");
        let stdout = BufReader::new(child.stdout.take().unwrap());
        let mut stderr_buffer = BufReader::new(child.stderr.take().unwrap());

        let re = regex!(r#""url": "([a-zA-Z0-9_-]+)""#);

        let mut id_list: Vec<String> = Vec::new();
        for line in stdout.lines(){
            match line{
                Err(why) => panic!("couldn't read cmd stdout: {}", Error::description(&why)),
                Ok(text) => {
                        println!("Out: {}",text);
                        match re.captures(&text) {
                            Some(cap) => { //println!("Match at {}", s.0);
                                        println!("{}", cap.at(1).unwrap()); // ONLY with ASCII chars makeable!
                                        id_list.push(cap.at(1).unwrap().to_string());
                                    },
                            None => {},
                        }
                    },
            }
        }

        let mut stderr: String = String::new();
        try!(stderr_buffer.read_to_string(&mut stderr));

        try!(child.wait());

        if !stderr.is_empty() {
            println!("stderr: {:?}", stderr);
            return Err(DownloadError::InternalError(stderr));
        }

        Ok(id_list)
    }

    ///Retrive playlist name, will kill the process due to yt-dl starting detail retrieval
    pub fn get_playlist_name(&self) -> Result<String,DownloadError> {
        let mut child = try!(self.run_playlist_get_name());
        let stdout = BufReader::new(child.stdout.take().unwrap());

        let re = regex!(r"\[download\] Downloading playlist: (.*)");

        let name: String;
        for line in stdout.lines(){

            match line{
                Err(why) => panic!("couldn't read cmd stdout: {}", Error::description(&why)),
                Ok(text) => {
                        println!("Out: {}",text);
                        match re.captures(&text) {
                            Some(cap) => {
                                        println!("{}", cap.at(1).unwrap()); // ONLY with ASCII chars makeable!
                                        name = cap.at(1).unwrap().to_string();
                                        try!(child.kill());
                                        println!("killed it");
                                        return Ok(name);
                                    },
                            None => {},
                        }
                    },
            }
        }

        try!(child.wait()); // waits for finish & then exists zombi process fixes #10

        Err(DownloadError::DownloadError("no playlist name".to_string()))
    }

    fn run_download_process(&self, file_path: &str, quality: &i16) -> Result<Child,DownloadError> {
        match Command::new("youtube-dl")
                                    .arg("--newline")
                                    .arg("-r")
                                    .arg(format!("{}M",self.defaults.download_mbps))
                                    .arg("-f")
                                    .arg(quality.to_string())
                                    .arg("-o")
                                    .arg(file_path)
                                    .arg(&self.ddb.url)
                                    .stdin(Stdio::null())
                                    .stdout(Stdio::piped())
                                    .spawn() {
            Err(why) => Err(DownloadError::InternalError(Error::description(&why).into())),
            Ok(process) => Ok(process),
        }
    }

    fn run_filename_process(&self) -> Result<Child,DownloadError> {
        match Command::new("youtube-dl")
                                    .arg("--get-filename")
                                    .arg("-o")
                                    .arg("%(title)s")
                                    .arg(&self.ddb.url)
                                    .stdin(Stdio::null())
                                    .stdout(Stdio::piped())
                                    .stderr(Stdio::piped())
                                    .spawn() {
            Err(why) => Err(DownloadError::InternalError(Error::description(&why).into())),
            Ok(process) => Ok(process),
        }
    }

    fn run_playlist_extract(&self) -> Result<Child,DownloadError> {
        let mut cmd = Command::new("youtube-dl");
        cmd.arg("-s").arg("--dump-json").arg("--flat-playlist");
        if self.ddb.from > 0 {
            cmd.arg("--playlist-start");
            cmd.arg(self.ddb.from.to_string());
        }
        if self.ddb.to > 0 {
            cmd.arg("--playlist-end");
            cmd.arg(self.ddb.to.to_string());
        }
        match cmd.arg(&self.ddb.url)
                    .stdin(Stdio::null())
                    .stdout(Stdio::piped())
                    .stderr(Stdio::piped())
                    .spawn() {
            Err(why) => Err(DownloadError::InternalError(Error::description(&why).into())),
            Ok(process) => Ok(process),
        }
    }

    fn run_playlist_get_name(&self) -> Result<Child,DownloadError> {
        match Command::new("youtube-dl")
                                    .arg("-s")
                                    .arg("--no-warnings")
                                    .arg(&self.ddb.url)
                                    .stdin(Stdio::null())
                                    .stdout(Stdio::piped())
                                    .spawn() {
            Err(why) => Err(DownloadError::InternalError(Error::description(&why).into())),
            Ok(process) => Ok(process),
        }
    }

    // MyPooledConn does only live when MyOpts is alive -> lifetime needs to be declared
    fn prepare_progress_updater(&'a self,conn: &'a mut MyPooledConn) -> Stmt<'a> { // no livetime needed: struct livetime used
        conn.prepare("UPDATE querydetails SET progress = ? WHERE qid = ?").unwrap()
    }

    ///updater called from the stdout progress
    fn update_progress(&self,stmt: &mut Stmt, progress: &String) -> Result<(),DownloadError>{
        try!(stmt.execute(&[progress,&self.ddb.qid]).map(|_| Ok(())))
        //-> only return errors, ignore the return value of stmt.execute
    }

    ///This function does a 3rd party binding in case it's needed
    ///due to the country restrictions
    ///Because hyper doesn't support timeout settings atm, we're calling an external
    ///lib
    ///The returned value contains the original video name, the lib downloads & saves
    ///the file at the given folder under the given name
    pub fn lib_request_video(&self, current_steps: i32,max_steps: i32) -> Result<String,DownloadError> {
        let mut child = try!(self.lib_request_video_cmd());
        println!("Requesting video via lib..");
        let stdout = BufReader::new(child.stdout.take().unwrap());
        let mut stderr_buffer = BufReader::new(child.stderr.take().unwrap());

        let mut conn = self.ddb.pool.get_conn().unwrap();
        let mut statement = self.prepare_progress_updater(&mut conn);
        let re = regex!(r"step (\d)");

        let mut last_line = String::new();
        for line in stdout.lines(){
            match line{
                Err(why) => panic!("couldn't read cmd stdout: {}", Error::description(&why)),
                Ok(text) => {
                        println!("Out: {}",text);
                        match re.captures(&text) {
                            Some(cap) => { //println!("Match at {}", s.0);
                                        println!("{}", cap.at(1).unwrap()); // ONLY with ASCII chars makeable!
                                        if !self.ddb.playlist {
                                            lib::update_steps(&self.ddb.pool ,&self.ddb.qid, current_steps + &cap.at(1).unwrap().parse::<i32>().unwrap(), max_steps, false);
                                        }
                                    },
                            None => {last_line = text.clone()},
                        }
                    },
            }
        }

        let mut stderr: String = String::new();
        try!(stderr_buffer.read_to_string(&mut stderr));

        try!(child.wait());

        if !stderr.is_empty() {
            println!("stderr: {:?}", stderr);
            return Err(DownloadError::InternalError(stderr));
        }
        //this ONLY works because `filename ` is ascii..
        let mut out = last_line[last_line.find("filename ").unwrap()+9..].trim().to_string();
        out = lib::url_encode(&out);//stdout.trim();

        Ok(out)
    }

    ///Generate the lib-cmd `request [..]?v=asdf -folder /downloads -a -name testfile`
    fn lib_request_video_cmd(&self) -> Result<Child,DownloadError> {
        let java_path = Path::new(&self.defaults.jar_cmd);
        println!("{:?}", format!("{}/java -jar {}/offliberty.jar",self.defaults.jar_cmd,self.defaults.jar_folder));
        match Command::new("./java")
                                        .current_dir(java_path)
                                        .arg("-jar")
                                        .arg(format!("{}/offliberty.jar",&self.defaults.jar_folder))
                                        .arg("request")
                                        .arg(&self.ddb.url)
                                        .arg("-folder")
                                        .arg(&self.ddb.folder)
                                        .arg(self.gen_request_str())
                                        .arg("-name")
                                        .arg(self.ddb.qid.to_string()) //eq. format! https://botbot.me/mozilla/rust/msg/37524131/
                                        .stdin(Stdio::null())
                                        .stdout(Stdio::piped())
                                        .stderr(Stdio::piped())
                                        .spawn() {
                Err(why) => {println!("{:?}",why); Err(DownloadError::InternalError(Error::description(&why).into()))},
                Ok(process) => Ok(process),
            }
    }

    ///Generate -a or -v, based on if an audio or video quality is requested
    fn gen_request_str(&self) -> &'a str{
        if self.is_audio() && !self.is_aac() {
            "-a"
        } else {
            "-v"
        }
    }

    ///Check if the quality is 141, standing for audio or not
    pub fn is_audio(&self) -> bool {
        if self.ddb.quality == CONFIG.codecs.audio_raw {
            true
        } else if self.ddb.quality == CONFIG.codecs.audio_source_hq {
            true
        } else if self.ddb.quality == CONFIG.codecs.audio_mp3 {
            true
        } else {
            false
        }
    }

    ///Check if is AAC file
    fn is_aac(&self) -> bool {
        CONFIG.extensions.aac.contains(&self.ddb.quality)
    }
}<commit_msg>Added private video detection as of #23<commit_after>extern crate regex;
use mysql::conn::pool::{MyPool,MyPooledConn};
use mysql::conn::Stmt;

use std::process::{Command, Stdio, Child};
use std::error::Error;
use std::io::prelude::*;
use std::io::BufReader;
use std::path::Path;
use lib::config::ConfigGen;
use std::convert::Into;

use lib::DownloadError;

use CONFIG;
use lib;

macro_rules! regex(
    ($s:expr) => (regex::Regex::new($s).unwrap());
);

#[derive(Clone)]
pub struct DownloadDB {
    pub url: String,
    pub quality: i16,
    pub playlist: bool,
    pub compress: bool,
    pub qid: i64,
    pub from: i32,
    pub to: i32,
    pub folder: String, // download folder, changes for playlists
    pub pool: MyPool,
}

impl DownloadDB {
    pub fn update_video(&mut self,url: String, qid: i64) -> &mut DownloadDB{
        self.qid = qid;
        self.url = url;
        self
    }
    pub fn update_folder(&mut self, folder: String){
        self.folder = folder;
    }
}

pub struct Downloader<'a> {
    pub ddb: &'a DownloadDB,
    defaults: &'a ConfigGen,
    // pool: MyPool,
}

impl<'a> Downloader<'a>{
    pub fn new(ddb: &'a DownloadDB, defaults: &'a ConfigGen) -> Downloader<'a>{
        Downloader {ddb: ddb, defaults: defaults}
    }

    ///Regex matching: [download]  13.4% of 275.27MiB at 525.36KiB/s ETA 07:52
    ///
    ///Downloads a video, updates the DB
    ///TODO: get the sql statements out of the class
    ///TODO: wrap errors
    ///Doesn't care about DMCAs, will emit errors on them
    ///download_audio: ignore quality & download audio_raw codec for split containers
    pub fn download_file(&self, file_path: &str, download_audio: bool) -> Result<bool,DownloadError> {
        println!("{:?}", self.ddb.url);

        let curr_quality = if download_audio {
            &CONFIG.codecs.audio_raw
        }else{
            &self.ddb.quality
        };

        println!("quality: {}",curr_quality);
        let mut child = try!(self.run_download_process(file_path,curr_quality));
        let stdout = BufReader::new(child.stdout.take().unwrap());

        let mut stderr_buffer = BufReader::new(child.stderr.take().unwrap());

        let mut conn = self.ddb.pool.get_conn().unwrap();
        let mut statement = self.prepare_progress_updater(&mut conn);
        let re = regex!(r"\d+\.\d%");

        for line in stdout.lines(){
            match line{
                Err(why) => panic!("couldn't read cmd stdout: {}", Error::description(&why)),
                Ok(text) => {
                        println!("Out: {}",text);
                        if !self.ddb.playlist {
                            match re.find(&text) {
                                Some(s) => { //println!("Match at {}", s.0);
                                            println!("{}", &text[s.0..s.1]); // ONLY with ASCII chars makeable!
                                            try!(self.update_progress(&mut statement, &text[s.0..s.1].to_string()));
                                        },
                                None => {},
                            }
                        }
                },
            }
        }

        try!(child.wait()); // waits for finish & then exists zombi process, fixes #10

        let mut stderr: String = String::new();
        try!(stderr_buffer.read_to_string(&mut stderr));

        if stderr.is_empty() {
            Ok(true)
        } else if stderr.contains("requested format not available") {
            Err(DownloadError::QualityNotAvailable)
        /*} else if stderr.contains("") {
            Err(DownloadError::NotAvailable) */
        } else {
            Err(DownloadError::InternalError(stderr))
        }

    }

    ///Trys to get the original name of a file, while checking for availability
    ///As an ExtractError can appear randomly, bug 11, we're retrying again 2 times if it should occour
    pub fn get_file_name(&self) -> Result<String,DownloadError> {
        for attempts in 0..2 {
            let mut child = try!(self.run_filename_process());
            let mut stdout_buffer = BufReader::new(child.stdout.take().unwrap());
            let mut stderr_buffer = BufReader::new(child.stderr.take().unwrap());

            let mut stdout: String = String::new();
            try!(stdout_buffer.read_to_string(&mut stdout));
            let mut stderr: String = String::new();
            try!(stderr_buffer.read_to_string(&mut stderr));

            try!(child.wait());
            if stderr.is_empty() {
                println!("get_file_name: {:?}", stdout);
                return Ok(stdout.trim().to_string());
            }else{
                if stderr.contains("not available in your country") || stderr.contains("contains content from") || stderr.contains("This video is available in") {
                    return Err(DownloadError::DMCAError);
                } else if stderr.contains("Please sign in to view this video") {
                    return Err(DownloadError::NotAvailable);
                } else if stderr.contains("ExtractorError") { // #11
                    println!("ExtractorError on attempt {}", attempts +1);
                } else {
                    return Err(DownloadError::DownloadError(stderr));
                }
            }
        }
        Err(DownloadError::ExtractorError)
    }

    ///Get playlist file ids
    pub fn get_playlist_ids(&self) -> Result<Vec<String>,DownloadError> {
        let mut child = try!(self.run_playlist_extract());
        println!("retrieving playlist ids");
        let stdout = BufReader::new(child.stdout.take().unwrap());
        let mut stderr_buffer = BufReader::new(child.stderr.take().unwrap());

        let re = regex!(r#""url": "([a-zA-Z0-9_-]+)""#);

        let mut id_list: Vec<String> = Vec::new();
        for line in stdout.lines(){
            match line{
                Err(why) => panic!("couldn't read cmd stdout: {}", Error::description(&why)),
                Ok(text) => {
                        println!("Out: {}",text);
                        match re.captures(&text) {
                            Some(cap) => { //println!("Match at {}", s.0);
                                        println!("{}", cap.at(1).unwrap()); // ONLY with ASCII chars makeable!
                                        id_list.push(cap.at(1).unwrap().to_string());
                                    },
                            None => {},
                        }
                    },
            }
        }

        let mut stderr: String = String::new();
        try!(stderr_buffer.read_to_string(&mut stderr));

        try!(child.wait());

        if !stderr.is_empty() {
            println!("stderr: {:?}", stderr);
            return Err(DownloadError::InternalError(stderr));
        }

        Ok(id_list)
    }

    ///Retrive playlist name, will kill the process due to yt-dl starting detail retrieval
    pub fn get_playlist_name(&self) -> Result<String,DownloadError> {
        let mut child = try!(self.run_playlist_get_name());
        let stdout = BufReader::new(child.stdout.take().unwrap());

        let re = regex!(r"\[download\] Downloading playlist: (.*)");

        let name: String;
        for line in stdout.lines(){

            match line{
                Err(why) => panic!("couldn't read cmd stdout: {}", Error::description(&why)),
                Ok(text) => {
                        println!("Out: {}",text);
                        match re.captures(&text) {
                            Some(cap) => {
                                        println!("{}", cap.at(1).unwrap()); // ONLY with ASCII chars makeable!
                                        name = cap.at(1).unwrap().to_string();
                                        try!(child.kill());
                                        println!("killed it");
                                        return Ok(name);
                                    },
                            None => {},
                        }
                    },
            }
        }

        try!(child.wait()); // waits for finish & then exists zombi process fixes #10

        Err(DownloadError::DownloadError("no playlist name".to_string()))
    }

    fn run_download_process(&self, file_path: &str, quality: &i16) -> Result<Child,DownloadError> {
        match Command::new("youtube-dl")
                                    .arg("--newline")
                                    .arg("-r")
                                    .arg(format!("{}M",self.defaults.download_mbps))
                                    .arg("-f")
                                    .arg(quality.to_string())
                                    .arg("-o")
                                    .arg(file_path)
                                    .arg(&self.ddb.url)
                                    .stdin(Stdio::null())
                                    .stdout(Stdio::piped())
                                    .spawn() {
            Err(why) => Err(DownloadError::InternalError(Error::description(&why).into())),
            Ok(process) => Ok(process),
        }
    }

    fn run_filename_process(&self) -> Result<Child,DownloadError> {
        match Command::new("youtube-dl")
                                    .arg("--get-filename")
                                    .arg("-o")
                                    .arg("%(title)s")
                                    .arg(&self.ddb.url)
                                    .stdin(Stdio::null())
                                    .stdout(Stdio::piped())
                                    .stderr(Stdio::piped())
                                    .spawn() {
            Err(why) => Err(DownloadError::InternalError(Error::description(&why).into())),
            Ok(process) => Ok(process),
        }
    }

    fn run_playlist_extract(&self) -> Result<Child,DownloadError> {
        let mut cmd = Command::new("youtube-dl");
        cmd.arg("-s").arg("--dump-json").arg("--flat-playlist");
        if self.ddb.from > 0 {
            cmd.arg("--playlist-start");
            cmd.arg(self.ddb.from.to_string());
        }
        if self.ddb.to > 0 {
            cmd.arg("--playlist-end");
            cmd.arg(self.ddb.to.to_string());
        }
        match cmd.arg(&self.ddb.url)
                    .stdin(Stdio::null())
                    .stdout(Stdio::piped())
                    .stderr(Stdio::piped())
                    .spawn() {
            Err(why) => Err(DownloadError::InternalError(Error::description(&why).into())),
            Ok(process) => Ok(process),
        }
    }

    fn run_playlist_get_name(&self) -> Result<Child,DownloadError> {
        match Command::new("youtube-dl")
                                    .arg("-s")
                                    .arg("--no-warnings")
                                    .arg(&self.ddb.url)
                                    .stdin(Stdio::null())
                                    .stdout(Stdio::piped())
                                    .spawn() {
            Err(why) => Err(DownloadError::InternalError(Error::description(&why).into())),
            Ok(process) => Ok(process),
        }
    }

    // MyPooledConn does only live when MyOpts is alive -> lifetime needs to be declared
    fn prepare_progress_updater(&'a self,conn: &'a mut MyPooledConn) -> Stmt<'a> { // no livetime needed: struct livetime used
        conn.prepare("UPDATE querydetails SET progress = ? WHERE qid = ?").unwrap()
    }

    ///updater called from the stdout progress
    fn update_progress(&self,stmt: &mut Stmt, progress: &String) -> Result<(),DownloadError>{
        try!(stmt.execute(&[progress,&self.ddb.qid]).map(|_| Ok(())))
        //-> only return errors, ignore the return value of stmt.execute
    }

    ///This function does a 3rd party binding in case it's needed
    ///due to the country restrictions
    ///Because hyper doesn't support timeout settings atm, we're calling an external
    ///lib
    ///The returned value contains the original video name, the lib downloads & saves
    ///the file at the given folder under the given name
    pub fn lib_request_video(&self, current_steps: i32,max_steps: i32) -> Result<String,DownloadError> {
        let mut child = try!(self.lib_request_video_cmd());
        println!("Requesting video via lib..");
        let stdout = BufReader::new(child.stdout.take().unwrap());
        let mut stderr_buffer = BufReader::new(child.stderr.take().unwrap());

        let mut conn = self.ddb.pool.get_conn().unwrap();
        let mut statement = self.prepare_progress_updater(&mut conn);
        let re = regex!(r"step (\d)");

        let mut last_line = String::new();
        for line in stdout.lines(){
            match line{
                Err(why) => panic!("couldn't read cmd stdout: {}", Error::description(&why)),
                Ok(text) => {
                        println!("Out: {}",text);
                        match re.captures(&text) {
                            Some(cap) => { //println!("Match at {}", s.0);
                                        println!("{}", cap.at(1).unwrap()); // ONLY with ASCII chars makeable!
                                        if !self.ddb.playlist {
                                            lib::update_steps(&self.ddb.pool ,&self.ddb.qid, current_steps + &cap.at(1).unwrap().parse::<i32>().unwrap(), max_steps, false);
                                        }
                                    },
                            None => {last_line = text.clone()},
                        }
                    },
            }
        }

        let mut stderr: String = String::new();
        try!(stderr_buffer.read_to_string(&mut stderr));

        try!(child.wait());

        if !stderr.is_empty() {
            println!("stderr: {:?}", stderr);
            return Err(DownloadError::InternalError(stderr));
        }
        //this ONLY works because `filename ` is ascii..
        let mut out = last_line[last_line.find("filename ").unwrap()+9..].trim().to_string();
        out = lib::url_encode(&out);//stdout.trim();

        Ok(out)
    }

    ///Generate the lib-cmd `request [..]?v=asdf -folder /downloads -a -name testfile`
    fn lib_request_video_cmd(&self) -> Result<Child,DownloadError> {
        let java_path = Path::new(&self.defaults.jar_cmd);
        println!("{:?}", format!("{}/java -jar {}/offliberty.jar",self.defaults.jar_cmd,self.defaults.jar_folder));
        match Command::new("./java")
                                        .current_dir(java_path)
                                        .arg("-jar")
                                        .arg(format!("{}/offliberty.jar",&self.defaults.jar_folder))
                                        .arg("request")
                                        .arg(&self.ddb.url)
                                        .arg("-folder")
                                        .arg(&self.ddb.folder)
                                        .arg(self.gen_request_str())
                                        .arg("-name")
                                        .arg(self.ddb.qid.to_string()) //eq. format! https://botbot.me/mozilla/rust/msg/37524131/
                                        .stdin(Stdio::null())
                                        .stdout(Stdio::piped())
                                        .stderr(Stdio::piped())
                                        .spawn() {
                Err(why) => {println!("{:?}",why); Err(DownloadError::InternalError(Error::description(&why).into()))},
                Ok(process) => Ok(process),
            }
    }

    ///Generate -a or -v, based on if an audio or video quality is requested
    fn gen_request_str(&self) -> &'a str{
        if self.is_audio() && !self.is_aac() {
            "-a"
        } else {
            "-v"
        }
    }

    ///Check if the quality is 141, standing for audio or not
    pub fn is_audio(&self) -> bool {
        if self.ddb.quality == CONFIG.codecs.audio_raw {
            true
        } else if self.ddb.quality == CONFIG.codecs.audio_source_hq {
            true
        } else if self.ddb.quality == CONFIG.codecs.audio_mp3 {
            true
        } else {
            false
        }
    }

    ///Check if is AAC file
    fn is_aac(&self) -> bool {
        CONFIG.extensions.aac.contains(&self.ddb.quality)
    }
}<|endoftext|>


<commit_before>//! Complete system.

use std::collections::BinaryHeap;

use Result;
use platform::Platform;
use schedule::{Decision, Schedule};
use traffic::Traffic;
use workload::Workload;

mod event;
mod history;
mod job;

pub use self::event::{Event, EventKind};
pub use self::history::History;
pub use self::job::Job;

/// A complete system.
pub struct System<T, W, P, S> {
    traffic: T,
    workload: W,
    platform: P,
    schedule: S,
    history: History,
    queue: BinaryHeap<Event>,
}

impl<T, W, P, S> System<T, W, P, S>
    where T: Traffic, W: Workload, P: Platform, S: Schedule, S::Data: for<'l> From<&'l P::Data>
{
    /// Create a system.
    pub fn new(traffic: T, workload: W, platform: P, schedule: S) -> Result<System<T, W, P, S>> {
        Ok(System {
            traffic: traffic,
            workload: workload,
            platform: platform,
            schedule: schedule,
            history: History::default(),
            queue: BinaryHeap::new(),
        })
    }

    /// Advance to the next event and return the data accumulated since the
    /// previous call.
    pub fn next(&mut self) -> Result<Option<(Event, P::Data)>> {
        match (try!(self.traffic.peek()), self.queue.peek().map(|event| &event.time)) {
            (Some(&traffic), Some(&queue)) => if traffic < queue {
                self.next_from_traffic()
            } else {
                self.next_from_queue()
            },
            (Some(_), None) => self.next_from_traffic(),
            (None, Some(_)) => self.next_from_queue(),
            _ => Ok(None),
        }
    }

    /// Return the platform.
    #[inline(always)]
    pub fn platform(&self) -> &P {
        &self.platform
    }

    /// Return the history.
    #[inline(always)]
    pub fn history(&self) -> &History {
        &self.history
    }

    fn next_from_traffic(&mut self) -> Result<Option<(Event, P::Data)>> {
        let time = some!(try!(self.traffic.next()));
        let pattern = try!(self.workload.next(time));
        let job = Job::new(self.history.arrived, time, pattern);

        let event = Event::arrived(time, job.clone());
        self.history.count(&event);

        let data = try!(self.platform.next(time));
        try!(self.schedule.push(time, (&data).into()));

        if let Decision::Accept { start, finish, mapping } = try!(self.schedule.next(&job)) {
            self.queue.push(Event::started(start, job.clone()));
            self.queue.push(Event::finished(finish, job.clone()));
            try!(self.platform.push(&job, start, &mapping));
        }

        Ok(Some((event, data)))
    }

    fn next_from_queue(&mut self) -> Result<Option<(Event, P::Data)>> {
        let event = some!(self.queue.pop());
        self.history.count(&event);

        let data = try!(self.platform.next(event.time));
        try!(self.schedule.push(event.time, (&data).into()));

        Ok(Some((event, data)))
    }
}
<commit_msg>Adjust a couple of descriptions<commit_after>//! System modeling.

use std::collections::BinaryHeap;

use Result;
use platform::Platform;
use schedule::{Decision, Schedule};
use traffic::Traffic;
use workload::Workload;

mod event;
mod history;
mod job;

pub use self::event::{Event, EventKind};
pub use self::history::History;
pub use self::job::Job;

/// A system.
pub struct System<T, W, P, S> {
    traffic: T,
    workload: W,
    platform: P,
    schedule: S,
    history: History,
    queue: BinaryHeap<Event>,
}

impl<T, W, P, S> System<T, W, P, S>
    where T: Traffic, W: Workload, P: Platform, S: Schedule, S::Data: for<'l> From<&'l P::Data>
{
    /// Create a system.
    pub fn new(traffic: T, workload: W, platform: P, schedule: S) -> Result<System<T, W, P, S>> {
        Ok(System {
            traffic: traffic,
            workload: workload,
            platform: platform,
            schedule: schedule,
            history: History::default(),
            queue: BinaryHeap::new(),
        })
    }

    /// Advance to the next event and return the data accumulated since the
    /// previous call.
    pub fn next(&mut self) -> Result<Option<(Event, P::Data)>> {
        match (try!(self.traffic.peek()), self.queue.peek().map(|event| &event.time)) {
            (Some(&traffic), Some(&queue)) => if traffic < queue {
                self.next_from_traffic()
            } else {
                self.next_from_queue()
            },
            (Some(_), None) => self.next_from_traffic(),
            (None, Some(_)) => self.next_from_queue(),
            _ => Ok(None),
        }
    }

    /// Return the platform.
    #[inline(always)]
    pub fn platform(&self) -> &P {
        &self.platform
    }

    /// Return the history.
    #[inline(always)]
    pub fn history(&self) -> &History {
        &self.history
    }

    fn next_from_traffic(&mut self) -> Result<Option<(Event, P::Data)>> {
        let time = some!(try!(self.traffic.next()));
        let pattern = try!(self.workload.next(time));
        let job = Job::new(self.history.arrived, time, pattern);

        let event = Event::arrived(time, job.clone());
        self.history.count(&event);

        let data = try!(self.platform.next(time));
        try!(self.schedule.push(time, (&data).into()));

        if let Decision::Accept { start, finish, mapping } = try!(self.schedule.next(&job)) {
            self.queue.push(Event::started(start, job.clone()));
            self.queue.push(Event::finished(finish, job.clone()));
            try!(self.platform.push(&job, start, &mapping));
        }

        Ok(Some((event, data)))
    }

    fn next_from_queue(&mut self) -> Result<Option<(Event, P::Data)>> {
        let event = some!(self.queue.pop());
        self.history.count(&event);

        let data = try!(self.platform.next(event.time));
        try!(self.schedule.push(event.time, (&data).into()));

        Ok(Some((event, data)))
    }
}
<|endoftext|>


<commit_before>// Copyright (c) <2015> <lummax>
// Licensed under MIT (http://opensource.org/licenses/MIT)

use std::collections::{RingBuf, HashSet};
use std::mem;
use std::ptr;

use block_allocator::BlockAllocator;
use block_info::BlockInfo;
use constants::{BLOCK_SIZE, LINE_SIZE};
use gc_object::{GCObject, GCRTTI};

pub struct LineAllocator {
    block_allocator: BlockAllocator,
    object_map: HashSet<*mut GCObject>,
    unavailable_blocks: RingBuf<*mut BlockInfo>,
    recyclable_blocks: RingBuf<*mut BlockInfo>,
    current_block: Option<(*mut BlockInfo, u16, u16)>,
    overflow_block: Option<(*mut BlockInfo, u16, u16)>,
    current_live_mark: bool,
}

impl LineAllocator {
    pub fn new(block_allocator: BlockAllocator) -> LineAllocator {
        return LineAllocator {
            block_allocator: block_allocator,
            object_map: HashSet::new(),
            unavailable_blocks: RingBuf::new(),
            recyclable_blocks: RingBuf::new(),
            current_block: None,
            overflow_block: None,
            current_live_mark: false,
        };
    }

    pub fn set_gc_object(&mut self, object: *mut GCObject) {
        self.object_map.insert(object);
    }

    pub fn unset_gc_object(&mut self, object: *mut GCObject) {
        self.object_map.remove(&object);
    }

    pub fn clear_object_map(&mut self) {
        self.object_map.clear();
    }

    pub fn is_gc_object(&self, object: *mut GCObject) -> bool {
        return self.object_map.contains(&object);
    }

    pub fn allocate(&mut self, rtti: *const GCRTTI) -> Option<*mut GCObject> {
        let size = unsafe{ (*rtti).object_size() };
        debug!("Request to allocate an object of size {}", size);
        let block_tuple = if size < LINE_SIZE {
            self.current_block.and_then(|tp| self.scan_for_hole(size, tp))
                              .or_else(|| self.scan_recyclables(size))
                              .or_else(|| self.get_new_block())
        } else {
            self.overflow_block.and_then(|tp| self.scan_for_hole(size, tp))
                               .or_else(|| self.get_new_block())
        };
        return match block_tuple {
            None => None,
            Some((block, low, high)) => {
                let object = unsafe { (*block).offset(low as uint) };
                self.set_gc_object(object);
                unsafe {
                    ptr::write(object, GCObject::new(rtti, self.current_live_mark));
                }
                if size < LINE_SIZE {
                    self.current_block = Some((block, low + size as u16, high));
                    debug!("Allocated object {} of size {} in {}",
                           object, size, block);
                } else {
                    self.overflow_block = Some((block, low + size as u16, high));
                    debug!("Allocated object {} of size {} in {} (overflow)",
                           object, size, block);
                }
                Some(object)
            }
        };
    }

    fn scan_for_hole(&mut self, size: uint, tuple: (*mut BlockInfo, u16, u16))
        -> Option<(*mut BlockInfo, u16, u16)> {
            let (block, low, high) = tuple;
            return match (high - low) as uint >= size {
                true => {
                    debug!("Found hole in block {:p}", block);
                    Some(tuple)
                },
                false => match unsafe{ (*block).scan_block(high) } {
                    None => {
                        debug!("Push block {:p} into unavailable_blocks", block);
                        self.unavailable_blocks.push_back(block);
                        None
                    },
                    Some((low, high)) =>
                        self.scan_for_hole(size, (block, low, high)),
                }
            };
        }

    fn scan_recyclables(&mut self, size: uint) -> Option<(*mut BlockInfo, u16, u16)> {
        return match self.recyclable_blocks.pop_front() {
            None => None,
            Some(block) => match unsafe{ (*block).scan_block((LINE_SIZE - 1) as u16) } {
                None => {
                    debug!("Push block {:p} into unavailable_blocks", block);
                    self.unavailable_blocks.push_back(block);
                    self.scan_recyclables(size)
                },
                Some((low, high)) => self.scan_for_hole(size, (block, low, high))
                                         .or_else(|| self.scan_recyclables(size)),
            }
        };
    }

    fn get_new_block(&mut self) -> Option<(*mut BlockInfo, u16, u16)> {
        debug!("Request new block from block_allocator");
        return self.block_allocator.get_block()
                   .map(|block| (block, LINE_SIZE as u16, (BLOCK_SIZE - 1) as u16));
    }

    pub fn return_empty_blocks(&mut self) {
        let mut recyclable_blocks = RingBuf::new();
        let mut unavailable_blocks = RingBuf::new();
        for block in self.current_block.take().map(|(b, _, _)| b).into_iter()
                         .chain(self.recyclable_blocks.drain())
                         .chain(self.unavailable_blocks.drain()) {
            if unsafe{ (*block).is_empty() } {
                debug!("Return block {:p} to global block allocator", block);
                self.block_allocator.return_block(block);
            } else {
                let (holes, marked_lines) = unsafe{ (*block).count_holes_and_marked_lines() };
                debug!("Found {} holes and {} marked lines in block {}",
                       holes, marked_lines, block);
                match holes {
                    0 => {
                        debug!("Push block {:p} into unavailable_blocks", block);
                        unavailable_blocks.push_back(block);
                    },
                    _ => {
                        debug!("Push block {:p} into recyclable_blocks", block);
                        recyclable_blocks.push_back(block);
                    }
                }
            }
        }
        self.recyclable_blocks.extend(recyclable_blocks.into_iter());
        self.unavailable_blocks.extend(unavailable_blocks.into_iter());
    }

    unsafe fn get_block_ptr(&mut self, object: *mut GCObject) -> *mut BlockInfo {
        let block_offset = object as uint % BLOCK_SIZE;
        return mem::transmute((object as *mut u8).offset(-(block_offset as int)));
    }

    pub fn decrement_lines(&mut self, object: *mut GCObject) {
        unsafe{ (*self.get_block_ptr(object)).decrement_lines(object); }
    }

    pub fn increment_lines(&mut self, object: *mut GCObject) {
        unsafe{ (*self.get_block_ptr(object)).increment_lines(object); }
    }

    pub fn current_live_mark(&self) -> bool {
        return self.current_live_mark;
    }

    pub fn invert_live_mark(&mut self) {
        self.current_live_mark = !self.current_live_mark;
    }

    pub fn clear_line_counts(&mut self) {
        // This will only be called after the RCCollector did his work and
        // self.return_empty_blocks() was invoked. So every managed block is
        // in self.recyclable_blocks.
        for block in self.recyclable_blocks.iter_mut() {
            unsafe{ (**block).clear_line_counts(); }
        }
    }
}
<commit_msg>Generate mark-histogram in LineAllocator<commit_after>// Copyright (c) <2015> <lummax>
// Licensed under MIT (http://opensource.org/licenses/MIT)

use std::collections::{RingBuf, HashSet, VecMap};
use std::mem;
use std::ptr;

use block_allocator::BlockAllocator;
use block_info::BlockInfo;
use constants::{BLOCK_SIZE, LINE_SIZE, NUM_LINES_PER_BLOCK};
use gc_object::{GCObject, GCRTTI};

pub struct LineAllocator {
    block_allocator: BlockAllocator,
    object_map: HashSet<*mut GCObject>,
    mark_histogram: VecMap<u8>,
    unavailable_blocks: RingBuf<*mut BlockInfo>,
    recyclable_blocks: RingBuf<*mut BlockInfo>,
    current_block: Option<(*mut BlockInfo, u16, u16)>,
    overflow_block: Option<(*mut BlockInfo, u16, u16)>,
    current_live_mark: bool,
}

impl LineAllocator {
    pub fn new(block_allocator: BlockAllocator) -> LineAllocator {
        return LineAllocator {
            block_allocator: block_allocator,
            object_map: HashSet::new(),
            mark_histogram: VecMap::with_capacity(NUM_LINES_PER_BLOCK),
            unavailable_blocks: RingBuf::new(),
            recyclable_blocks: RingBuf::new(),
            current_block: None,
            overflow_block: None,
            current_live_mark: false,
        };
    }

    pub fn set_gc_object(&mut self, object: *mut GCObject) {
        self.object_map.insert(object);
    }

    pub fn unset_gc_object(&mut self, object: *mut GCObject) {
        self.object_map.remove(&object);
    }

    pub fn clear_object_map(&mut self) {
        self.object_map.clear();
    }

    pub fn is_gc_object(&self, object: *mut GCObject) -> bool {
        return self.object_map.contains(&object);
    }

    pub fn allocate(&mut self, rtti: *const GCRTTI) -> Option<*mut GCObject> {
        let size = unsafe{ (*rtti).object_size() };
        debug!("Request to allocate an object of size {}", size);
        let block_tuple = if size < LINE_SIZE {
            self.current_block.and_then(|tp| self.scan_for_hole(size, tp))
                              .or_else(|| self.scan_recyclables(size))
                              .or_else(|| self.get_new_block())
        } else {
            self.overflow_block.and_then(|tp| self.scan_for_hole(size, tp))
                               .or_else(|| self.get_new_block())
        };
        return match block_tuple {
            None => None,
            Some((block, low, high)) => {
                let object = unsafe { (*block).offset(low as uint) };
                self.set_gc_object(object);
                unsafe {
                    ptr::write(object, GCObject::new(rtti, self.current_live_mark));
                }
                if size < LINE_SIZE {
                    self.current_block = Some((block, low + size as u16, high));
                    debug!("Allocated object {} of size {} in {}",
                           object, size, block);
                } else {
                    self.overflow_block = Some((block, low + size as u16, high));
                    debug!("Allocated object {} of size {} in {} (overflow)",
                           object, size, block);
                }
                Some(object)
            }
        };
    }

    fn scan_for_hole(&mut self, size: uint, tuple: (*mut BlockInfo, u16, u16))
        -> Option<(*mut BlockInfo, u16, u16)> {
            let (block, low, high) = tuple;
            return match (high - low) as uint >= size {
                true => {
                    debug!("Found hole in block {:p}", block);
                    Some(tuple)
                },
                false => match unsafe{ (*block).scan_block(high) } {
                    None => {
                        debug!("Push block {:p} into unavailable_blocks", block);
                        self.unavailable_blocks.push_back(block);
                        None
                    },
                    Some((low, high)) =>
                        self.scan_for_hole(size, (block, low, high)),
                }
            };
        }

    fn scan_recyclables(&mut self, size: uint) -> Option<(*mut BlockInfo, u16, u16)> {
        return match self.recyclable_blocks.pop_front() {
            None => None,
            Some(block) => match unsafe{ (*block).scan_block((LINE_SIZE - 1) as u16) } {
                None => {
                    debug!("Push block {:p} into unavailable_blocks", block);
                    self.unavailable_blocks.push_back(block);
                    self.scan_recyclables(size)
                },
                Some((low, high)) => self.scan_for_hole(size, (block, low, high))
                                         .or_else(|| self.scan_recyclables(size)),
            }
        };
    }

    fn get_new_block(&mut self) -> Option<(*mut BlockInfo, u16, u16)> {
        debug!("Request new block from block_allocator");
        return self.block_allocator.get_block()
                   .map(|block| (block, LINE_SIZE as u16, (BLOCK_SIZE - 1) as u16));
    }

    pub fn return_empty_blocks(&mut self) {
        self.mark_histogram.clear();
        let mut recyclable_blocks = RingBuf::new();
        let mut unavailable_blocks = RingBuf::new();
        for block in self.current_block.take().map(|(b, _, _)| b).into_iter()
                         .chain(self.recyclable_blocks.drain())
                         .chain(self.unavailable_blocks.drain()) {
            if unsafe{ (*block).is_empty() } {
                debug!("Return block {:p} to global block allocator", block);
                self.block_allocator.return_block(block);
            } else {
                let (holes, marked_lines) = unsafe{ (*block).count_holes_and_marked_lines() };
                self.mark_histogram.update(holes as uint, marked_lines, |o, n| o + n);
                debug!("Found {} holes and {} marked lines in block {}",
                       holes, marked_lines, block);
                match holes {
                    0 => {
                        debug!("Push block {:p} into unavailable_blocks", block);
                        unavailable_blocks.push_back(block);
                    },
                    _ => {
                        debug!("Push block {:p} into recyclable_blocks", block);
                        recyclable_blocks.push_back(block);
                    }
                }
            }
        }
        self.recyclable_blocks.extend(recyclable_blocks.into_iter());
        self.unavailable_blocks.extend(unavailable_blocks.into_iter());
    }

    unsafe fn get_block_ptr(&mut self, object: *mut GCObject) -> *mut BlockInfo {
        let block_offset = object as uint % BLOCK_SIZE;
        return mem::transmute((object as *mut u8).offset(-(block_offset as int)));
    }

    pub fn decrement_lines(&mut self, object: *mut GCObject) {
        unsafe{ (*self.get_block_ptr(object)).decrement_lines(object); }
    }

    pub fn increment_lines(&mut self, object: *mut GCObject) {
        unsafe{ (*self.get_block_ptr(object)).increment_lines(object); }
    }

    pub fn current_live_mark(&self) -> bool {
        return self.current_live_mark;
    }

    pub fn invert_live_mark(&mut self) {
        self.current_live_mark = !self.current_live_mark;
    }

    pub fn clear_line_counts(&mut self) {
        // This will only be called after the RCCollector did his work and
        // self.return_empty_blocks() was invoked. So every managed block is
        // in self.recyclable_blocks.
        for block in self.recyclable_blocks.iter_mut() {
            unsafe{ (**block).clear_line_counts(); }
        }
    }
}
<|endoftext|>


<commit_before>// Unstable features coming in next Epoch
#![feature(match_default_bindings, nll, universal_impl_trait, conservative_impl_trait, dyn_trait)]
// Unstable features needed for light_arena
#![feature(placement_in_syntax, integer_atomics, vec_resize_default)]
// Lints
#![deny(unused_qualifications, unused_must_use)]
#![allow(non_snake_case)]

#[macro_use]
extern crate bitflags;
#[macro_use]
extern crate combine;
extern crate crossbeam;
#[macro_use]
extern crate failure;
extern crate image as img;
extern crate indicatif;
extern crate itertools as it;
#[macro_use]
extern crate lazy_static;
extern crate light_arena;
#[macro_use]
extern crate log;
extern crate num;
#[cfg(feature="exr")]
extern crate openexr;
extern crate ply_rs;
extern crate parking_lot;
extern crate state;

#[cfg(test)]
#[macro_use]
extern crate approx;
#[cfg(test)]
#[macro_use]
extern crate quickcheck;

use std::f32;
use std::ops::{Add, Mul, Sub};

use num::{Num, One, Signed};

// stats needs to be declared first so its macros can be used by the following modules
#[macro_use]
mod stats;
mod api;
mod blockedarray;
pub mod bounds;
mod bsdf;
pub mod bvh;
pub mod camera;
mod cie;
pub mod display;
pub mod efloat;
mod fileutil;
pub mod film;
pub mod filter;
mod geometry;
pub mod integrator;
mod interaction;
pub mod imageio;
pub mod light;
pub mod lightdistrib;
pub mod material;
pub mod mipmap;
mod noise;
mod paramset;
pub mod pbrt;
pub mod primitive;
pub mod ray;
pub mod renderer;
pub mod rng;
pub mod sampling;
pub mod sampler;
pub mod scene;
pub mod shapes;
pub mod spectrum;
pub mod texture;
pub mod transform;

pub fn init_stats() {
    // This one needs to be called first
    stats::init_stats();
    api::init_stats();
    bvh::init_stats();
    film::init_stats();
    integrator::init_stats();
    lightdistrib::init_stats();
    mipmap::init_stats();
    renderer::init_stats();
    scene::init_stats();
    shapes::init_stats();
}

use geometry::{Normal3, Point2, Point3, Vector2, Vector3};
use spectrum::Spectrum;

pub type Vector2f = Vector2<f32>;
pub type Vector3f = Vector3<f32>;
pub type Point2f = Point2<f32>;
pub type Point2i = Point2<i32>;
pub type Point3f = Point3<f32>;
pub type Point3i = Point3<i32>;
pub type Normal3f = Normal3<f32>;

pub use transform::Transform;

pub const MACHINE_EPSILON: f32 = f32::EPSILON * 0.5;
pub fn gamma(n: u32) -> f32 {
    (n as f32 * MACHINE_EPSILON) / (1.0 - n as f32 * MACHINE_EPSILON)
}

/// Smallest representable float strictly less than 1
pub const ONE_MINUS_EPSILON: f32 = 0.99999994f32;

/// Linear interpolation between 2 values.
///
/// This version should be generic enough to linearly interpolate between 2 Spectrums using an f32
/// parameter.
pub fn lerp<S, T>(t: S, a: T, b: T) -> T
    where S: One,
          S: Sub<S, Output = S>,
          S: Copy,
          T: Add<T, Output = T>,
          T: Mul<S, Output = T>
{
    let one: S = num::one();
    a * (one - t) + b * t
}

/// Return the dimension index (0, 1 or 2) that contains the largest component.
pub fn max_dimension<T>(v: &Vector3<T>) -> usize
    where T: Num + PartialOrd
{
    if v.x > v.y {
        if v.x > v.z { 0 } else { 2 }
    } else if v.y > v.z {
        1
    } else {
        2
    }
}

pub fn max_component(v: &Vector3f) -> f32 {
    f32::max(v.x, f32::max(v.y, v.z))
}

/// Permute the components of this vector based on the given indices for x, y and z.
pub fn permute_v<T>(v: &Vector3<T>, x: usize, y: usize, z: usize) -> Vector3<T>
    where T: Num + Copy
{
    Vector3::new(v[x], v[y], v[z])
}

/// Permute the components of this point based on the given indices for x, y and z.
pub fn permute_p<T>(v: &Point3<T>, x: usize, y: usize, z: usize) -> Point3<T>
    where T: Num + Signed + Copy
{
    Point3::new(v[x], v[y], v[z])
}

/// Create an orthogonal coordinate system from a single vector.
pub fn coordinate_system(v1: &Vector3f) -> (Vector3f, Vector3f) {
    let v2 = if v1.x.abs() > v1.y.abs() {
        Vector3::new(-v1.z, 0.0, v1.x) / (v1.x * v1.x + v1.z * v1.z).sqrt()
    } else {
        Vector3::new(0.0, v1.z, -v1.y) / (v1.y * v1.y + v1.z * v1.z).sqrt()
    };

    let v3 = v1.cross(&v2);

    (v2, v3)
}

// TODO does this exist in std?
pub fn find_interval<P>(size: usize, pred: P) -> usize
    where P: Fn(usize) -> bool
{
    let mut first = 0;
    let mut len = size;
    while len > 0 {
        let half = len >> 1;
        let middle = first + half;
        // Bisect range based on value of _pred_ at _middle_
        if pred(middle) {
            first = middle + 1;
            len -= half + 1;
        } else {
            len = half;
        }
    }
    clamp(first - 1, 0, size - 2)
}

/// Version of min() that works on `PartialOrd`, so it works for both u32 and f32.
pub fn min<T: PartialOrd + Copy>(a: T, b: T) -> T {
    if a.lt(&b) { a } else { b }
}

/// Version of max() that works on `PartialOrd`, so it works for both u32 and f32.
pub fn max<T: PartialOrd + Copy>(a: T, b: T) -> T {
    if a.gt(&b) { a } else { b }
}

#[inline]
pub fn is_power_of_2(v: i32) -> bool {
    (v != 0) && (v & (v - 1)) == 0
}

#[inline]
pub fn round_up_pow_2(v: i32) -> i32 {
    let mut v = v;
    v -= 1;
    v |= v >> 1;
    v |= v >> 2;
    v |= v >> 4;
    v |= v >> 8;
    v |= v >> 16;
    v + 1
}

#[inline]
pub fn next_float_up(v: f32) -> f32 {
    let mut v = v;
    if v.is_infinite() && v > 0.0 {
        return v;
    }

    if v == -0.0 {
        v = 0.0;
    }
    let mut ui = v.to_bits();
    if v >= 0.0 {
        ui += 1;
    } else {
        ui -= 1;
    }
    f32::from_bits(ui)
}

#[inline]
pub fn next_float_down(v: f32) -> f32 {
    let mut v = v;
    if v.is_infinite() && v < 0.0 {
        return v;
    }

    if v == 0.0 {
        v = -0.0;
    }
    let mut ui = v.to_bits();
    if v > 0.0 {
        ui -= 1;
    } else {
        ui += 1;
    }
    f32::from_bits(ui)
}

pub fn clamp<T>(val: T, low: T, high: T) -> T
    where T: PartialOrd + Copy
{
    if val < low {
        low
    } else if val > high {
        high
    } else {
        val
    }
}

pub trait Clampable {
    fn clamp(self, min: f32, max: f32) -> Self;
}

impl Clampable for f32 {
    fn clamp(self, min: f32, max: f32) -> f32 {
        clamp(self, min, max)
    }
}

impl Clampable for Spectrum {
    fn clamp(self, min: f32, max: f32) -> Spectrum {
        Spectrum::rgb(self.r.clamp(min, max),
                      self.g.clamp(min, max),
                      self.b.clamp(min, max))
    }
}

#[test]
fn test_gamma() {
    let g5 = gamma(5);
    let p = Point3f::new(-0.4, 0.9, 0.2);
    let v = g5 * p.abs();
    println!("gamma(5) = {}, p={:?}, v={:?}", gamma(5), p, v);
}

#[test]
fn test_is_power_of_2() {
    assert!(is_power_of_2(4));
    assert!(is_power_of_2(8));
    assert!(is_power_of_2(1024));
    assert!(!is_power_of_2(3));
    assert!(!is_power_of_2(7));
}

#[test]
fn test_round_up_pow_2() {
    assert_eq!(round_up_pow_2(1023), 1024);
    assert_eq!(round_up_pow_2(1024), 1024);
}
<commit_msg>Add tests for find_interval()<commit_after>// Unstable features coming in next Epoch
#![feature(match_default_bindings, nll, universal_impl_trait, conservative_impl_trait, dyn_trait)]
// Unstable features needed for light_arena
#![feature(placement_in_syntax, integer_atomics, vec_resize_default)]
// Lints
#![deny(unused_qualifications, unused_must_use)]
#![allow(non_snake_case)]

#[macro_use]
extern crate bitflags;
#[macro_use]
extern crate combine;
extern crate crossbeam;
#[macro_use]
extern crate failure;
extern crate image as img;
extern crate indicatif;
extern crate itertools as it;
#[macro_use]
extern crate lazy_static;
extern crate light_arena;
#[macro_use]
extern crate log;
extern crate num;
#[cfg(feature="exr")]
extern crate openexr;
extern crate ply_rs;
extern crate parking_lot;
extern crate state;

#[cfg(test)]
#[macro_use]
extern crate approx;
#[cfg(test)]
#[macro_use]
extern crate quickcheck;

use std::f32;
use std::ops::{Add, Mul, Sub};

use num::{Num, One, Signed};

// stats needs to be declared first so its macros can be used by the following modules
#[macro_use]
mod stats;
mod api;
mod blockedarray;
pub mod bounds;
mod bsdf;
pub mod bvh;
pub mod camera;
mod cie;
pub mod display;
pub mod efloat;
mod fileutil;
pub mod film;
pub mod filter;
mod geometry;
pub mod integrator;
mod interaction;
pub mod imageio;
pub mod light;
pub mod lightdistrib;
pub mod material;
pub mod mipmap;
mod noise;
mod paramset;
pub mod pbrt;
pub mod primitive;
pub mod ray;
pub mod renderer;
pub mod rng;
pub mod sampling;
pub mod sampler;
pub mod scene;
pub mod shapes;
pub mod spectrum;
pub mod texture;
pub mod transform;

pub fn init_stats() {
    // This one needs to be called first
    stats::init_stats();
    api::init_stats();
    bvh::init_stats();
    film::init_stats();
    integrator::init_stats();
    lightdistrib::init_stats();
    mipmap::init_stats();
    renderer::init_stats();
    scene::init_stats();
    shapes::init_stats();
}

use geometry::{Normal3, Point2, Point3, Vector2, Vector3};
use spectrum::Spectrum;

pub type Vector2f = Vector2<f32>;
pub type Vector3f = Vector3<f32>;
pub type Point2f = Point2<f32>;
pub type Point2i = Point2<i32>;
pub type Point3f = Point3<f32>;
pub type Point3i = Point3<i32>;
pub type Normal3f = Normal3<f32>;

pub use transform::Transform;

pub const MACHINE_EPSILON: f32 = f32::EPSILON * 0.5;
pub fn gamma(n: u32) -> f32 {
    (n as f32 * MACHINE_EPSILON) / (1.0 - n as f32 * MACHINE_EPSILON)
}

/// Smallest representable float strictly less than 1
pub const ONE_MINUS_EPSILON: f32 = 0.99999994f32;

/// Linear interpolation between 2 values.
///
/// This version should be generic enough to linearly interpolate between 2 Spectrums using an f32
/// parameter.
pub fn lerp<S, T>(t: S, a: T, b: T) -> T
    where S: One,
          S: Sub<S, Output = S>,
          S: Copy,
          T: Add<T, Output = T>,
          T: Mul<S, Output = T>
{
    let one: S = num::one();
    a * (one - t) + b * t
}

/// Return the dimension index (0, 1 or 2) that contains the largest component.
pub fn max_dimension<T>(v: &Vector3<T>) -> usize
    where T: Num + PartialOrd
{
    if v.x > v.y {
        if v.x > v.z { 0 } else { 2 }
    } else if v.y > v.z {
        1
    } else {
        2
    }
}

pub fn max_component(v: &Vector3f) -> f32 {
    f32::max(v.x, f32::max(v.y, v.z))
}

/// Permute the components of this vector based on the given indices for x, y and z.
pub fn permute_v<T>(v: &Vector3<T>, x: usize, y: usize, z: usize) -> Vector3<T>
    where T: Num + Copy
{
    Vector3::new(v[x], v[y], v[z])
}

/// Permute the components of this point based on the given indices for x, y and z.
pub fn permute_p<T>(v: &Point3<T>, x: usize, y: usize, z: usize) -> Point3<T>
    where T: Num + Signed + Copy
{
    Point3::new(v[x], v[y], v[z])
}

/// Create an orthogonal coordinate system from a single vector.
pub fn coordinate_system(v1: &Vector3f) -> (Vector3f, Vector3f) {
    let v2 = if v1.x.abs() > v1.y.abs() {
        Vector3::new(-v1.z, 0.0, v1.x) / (v1.x * v1.x + v1.z * v1.z).sqrt()
    } else {
        Vector3::new(0.0, v1.z, -v1.y) / (v1.y * v1.y + v1.z * v1.z).sqrt()
    };

    let v3 = v1.cross(&v2);

    (v2, v3)
}

// TODO does this exist in std?
pub fn find_interval<P>(size: usize, pred: P) -> usize
    where P: Fn(usize) -> bool
{
    let mut first = 0;
    let mut len = size;
    while len > 0 {
        let half = len >> 1;
        let middle = first + half;
        // Bisect range based on value of _pred_ at _middle_
        if pred(middle as usize) {
            first = middle + 1;
            len -= half + 1;
        } else {
            len = half;
        }
    }
    clamp(first as isize - 1, 0, size as isize - 2) as usize
}

/// Version of min() that works on `PartialOrd`, so it works for both u32 and f32.
pub fn min<T: PartialOrd + Copy>(a: T, b: T) -> T {
    if a.lt(&b) { a } else { b }
}

/// Version of max() that works on `PartialOrd`, so it works for both u32 and f32.
pub fn max<T: PartialOrd + Copy>(a: T, b: T) -> T {
    if a.gt(&b) { a } else { b }
}

#[inline]
pub fn is_power_of_2(v: i32) -> bool {
    (v != 0) && (v & (v - 1)) == 0
}

#[inline]
pub fn round_up_pow_2(v: i32) -> i32 {
    let mut v = v;
    v -= 1;
    v |= v >> 1;
    v |= v >> 2;
    v |= v >> 4;
    v |= v >> 8;
    v |= v >> 16;
    v + 1
}

#[inline]
pub fn next_float_up(v: f32) -> f32 {
    let mut v = v;
    if v.is_infinite() && v > 0.0 {
        return v;
    }

    if v == -0.0 {
        v = 0.0;
    }
    let mut ui = v.to_bits();
    if v >= 0.0 {
        ui += 1;
    } else {
        ui -= 1;
    }
    f32::from_bits(ui)
}

#[inline]
pub fn next_float_down(v: f32) -> f32 {
    let mut v = v;
    if v.is_infinite() && v < 0.0 {
        return v;
    }

    if v == 0.0 {
        v = -0.0;
    }
    let mut ui = v.to_bits();
    if v > 0.0 {
        ui -= 1;
    } else {
        ui += 1;
    }
    f32::from_bits(ui)
}

pub fn clamp<T>(val: T, low: T, high: T) -> T
    where T: PartialOrd + Copy
{
    if val < low {
        low
    } else if val > high {
        high
    } else {
        val
    }
}

pub trait Clampable {
    fn clamp(self, min: f32, max: f32) -> Self;
}

impl Clampable for f32 {
    fn clamp(self, min: f32, max: f32) -> f32 {
        clamp(self, min, max)
    }
}

impl Clampable for Spectrum {
    fn clamp(self, min: f32, max: f32) -> Spectrum {
        Spectrum::rgb(self.r.clamp(min, max),
                      self.g.clamp(min, max),
                      self.b.clamp(min, max))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_find_interval() {
        let a = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9];

        // check clamping for out of range
        assert_eq!(0, find_interval(a.len(), |index| a[index] as isize <= -1));
        assert_eq!(a.len() - 2, find_interval(a.len(), |index| a[index] <= 100));

        for i in 0..a.len() - 1 {
            assert_eq!(i, find_interval(a.len(), |index| a[index] <= i));
            assert_eq!(i, find_interval(a.len(), |index| a[index] as f32 <= i as f32 + 0.5));
            if i > 0 {
                assert_eq!(i - 1, find_interval(a.len(), |index| a[index] as f32 <= i as f32 - 0.5));
            }
        }
    }

    #[test]
    fn test_gamma() {
        let g5 = gamma(5);
        let p = Point3f::new(-0.4, 0.9, 0.2);
        let v = g5 * p.abs();
        println!("gamma(5) = {}, p={:?}, v={:?}", gamma(5), p, v);
    }

    #[test]
    fn test_is_power_of_2() {
        assert!(is_power_of_2(4));
        assert!(is_power_of_2(8));
        assert!(is_power_of_2(1024));
        assert!(!is_power_of_2(3));
        assert!(!is_power_of_2(7));
    }

    #[test]
    fn test_round_up_pow_2() {
        assert_eq!(round_up_pow_2(1023), 1024);
        assert_eq!(round_up_pow_2(1024), 1024);
    }
}

<|endoftext|>


<commit_before>// High-level bindings to Cairo.

use libc::*;
use cairo::{CAIRO_STATUS_SUCCESS, cairo_format_t, cairo_status_t, cairo_surface_t, cairo_t};
use cairo::bindgen::{cairo_create, cairo_fill, cairo_image_surface_create};
use cairo::bindgen::{cairo_image_surface_get_data, cairo_image_surface_get_format};
use cairo::bindgen::{cairo_image_surface_get_height, cairo_image_surface_get_stride};
use cairo::bindgen::{cairo_image_surface_get_width, cairo_rectangle, cairo_set_line_width};
use cairo::bindgen::{cairo_set_source_rgb, cairo_stroke, cairo_surface_destroy};
use cairo::bindgen::{cairo_surface_reference};
use cast::{reinterpret_cast, transmute};
use io::{BytesWriter, Writer};
use ptr::addr_of;
use result::{Err, Ok, Result};
use vec::raw::{buf_as_slice, from_buf_raw};

// FIXME: We should have a hierarchy of surfaces, but this needs to wait on case classes.
pub struct ImageSurface {
    cairo_surface: *cairo_surface_t,

    drop {
        unsafe {
            cairo_surface_destroy(self.cairo_surface);
        }
    }
}

impl ImageSurface {
    fn width()  -> c_int {
        unsafe {
            cairo_image_surface_get_width(self.cairo_surface)
        }
    }
    fn height() -> c_int {
        unsafe {
            cairo_image_surface_get_height(self.cairo_surface)
        }
    }
    fn stride() -> c_int {
        unsafe {
            cairo_image_surface_get_stride(self.cairo_surface)
        }
    }
    fn format() -> c_int    {
        unsafe {
            cairo_image_surface_get_format(self.cairo_surface)
        }
    }

    pure fn data(&self) -> &self/[u8] {
        unsafe {
            let buffer = cairo_image_surface_get_data(self.cairo_surface);
            let len = (self.stride() * self.height()) as uint;
            return vec::raw::buf_as_slice(buffer, len, |x| transmute(x));
        }
    }
}

// Should be private.
fn image_surface_from_cairo_surface(cairo_surface: *cairo_surface_t) -> ImageSurface {
    assert !cairo_surface.is_null();
    ImageSurface { cairo_surface: cairo_surface }
}

pub fn ImageSurface(format: cairo_format_t, width: c_int, height: c_int) -> ImageSurface {
    unsafe {
        let cairo_surface = cairo_image_surface_create(format, width, height);
        if cairo_surface.is_null() {
            fail!(~"couldn't create Cairo image surface");
        }
        return image_surface_from_cairo_surface(move cairo_surface);
    }
}

impl ImageSurface {
    /*fn write_to_png_stream(buffer: &BytesWriter) -> Result<(),cairo_status_t> unsafe {
        extern fn write_fn(closure: *c_void, data: *c_uchar, len: c_uint)
                        -> cairo_status_t unsafe {
            let writer: *BytesWriter = reinterpret_cast(&closure);
            do buf_as_slice(data, len as uint) |bytes| {
                (*writer).write(bytes);
            }
            return CAIRO_STATUS_SUCCESS;
        }

        let buffer_ptr = reinterpret_cast(&buffer);
        let status = cairo_surface_write_to_png_stream(self.cairo_surface, write_fn, buffer_ptr);
        if status != CAIRO_STATUS_SUCCESS {
            return Err(status);
        }

        return Ok(());
    }*/

    pure fn clone() -> ImageSurface {
        unsafe {
            cairo_surface_reference(self.cairo_surface);
            return image_surface_from_cairo_surface(self.cairo_surface);
        }
    }
}

struct Context {
    cairo_context: *cairo_t,
}

impl Context {
    fn set_line_width(width: c_double) {
        unsafe {
            cairo_set_line_width(self.cairo_context, width);
        }
    }

    fn set_source_rgb(r: c_double, g: c_double, b: c_double) {
        unsafe {
            cairo_set_source_rgb(self.cairo_context, r, g, b);
        }
    }

    fn rectangle(x: c_double, y: c_double, width: c_double, height: c_double) {
        unsafe {
            cairo_rectangle(self.cairo_context, x, y, width, height);
        }
    }

    fn stroke() {
        unsafe {
            cairo_stroke(self.cairo_context);
        }
    }

    fn fill() {
        unsafe {
            cairo_fill(self.cairo_context);
        }
    }
}

fn make_context(surface: &ImageSurface) -> Context {
    unsafe {
        Context {
            cairo_context: cairo_create(surface.cairo_surface)
        }
    }
}
<commit_msg>Update for language changes<commit_after>// High-level bindings to Cairo.

use libc::*;
use cairo::{CAIRO_STATUS_SUCCESS, cairo_format_t, cairo_status_t, cairo_surface_t, cairo_t};
use cairo::bindgen::{cairo_create, cairo_fill, cairo_image_surface_create};
use cairo::bindgen::{cairo_image_surface_get_data, cairo_image_surface_get_format};
use cairo::bindgen::{cairo_image_surface_get_height, cairo_image_surface_get_stride};
use cairo::bindgen::{cairo_image_surface_get_width, cairo_rectangle, cairo_set_line_width};
use cairo::bindgen::{cairo_set_source_rgb, cairo_stroke, cairo_surface_destroy};
use cairo::bindgen::{cairo_surface_reference};
use cast::{reinterpret_cast, transmute};
use io::{BytesWriter, Writer};
use ptr::addr_of;
use result::{Err, Ok, Result};
use vec::raw::{buf_as_slice, from_buf_raw};

// FIXME: We should have a hierarchy of surfaces, but this needs to wait on case classes.
pub struct ImageSurface {
    cairo_surface: *cairo_surface_t,

    drop {
        unsafe {
            cairo_surface_destroy(self.cairo_surface);
        }
    }
}

impl ImageSurface {
    fn width()  -> c_int {
        unsafe {
            cairo_image_surface_get_width(self.cairo_surface)
        }
    }
    fn height() -> c_int {
        unsafe {
            cairo_image_surface_get_height(self.cairo_surface)
        }
    }
    fn stride() -> c_int {
        unsafe {
            cairo_image_surface_get_stride(self.cairo_surface)
        }
    }
    fn format() -> c_int    {
        unsafe {
            cairo_image_surface_get_format(self.cairo_surface)
        }
    }

    pure fn data(&self) -> &self/[u8] {
        unsafe {
            let buffer = cairo_image_surface_get_data(self.cairo_surface);
            let len = (self.stride() * self.height()) as uint;
            return vec::raw::buf_as_slice(buffer, len, |x| transmute(x));
        }
    }
}

// Should be private.
fn image_surface_from_cairo_surface(cairo_surface: *cairo_surface_t) -> ImageSurface {
    assert !cairo_surface.is_null();
    ImageSurface { cairo_surface: cairo_surface }
}

pub fn ImageSurface(format: cairo_format_t, width: c_int, height: c_int) -> ImageSurface {
    unsafe {
        let cairo_surface = cairo_image_surface_create(format, width, height);
        if cairo_surface.is_null() {
            fail!(~"couldn't create Cairo image surface");
        }
        return image_surface_from_cairo_surface(cairo_surface);
    }
}

impl ImageSurface {
    /*fn write_to_png_stream(buffer: &BytesWriter) -> Result<(),cairo_status_t> unsafe {
        extern fn write_fn(closure: *c_void, data: *c_uchar, len: c_uint)
                        -> cairo_status_t unsafe {
            let writer: *BytesWriter = reinterpret_cast(&closure);
            do buf_as_slice(data, len as uint) |bytes| {
                (*writer).write(bytes);
            }
            return CAIRO_STATUS_SUCCESS;
        }

        let buffer_ptr = reinterpret_cast(&buffer);
        let status = cairo_surface_write_to_png_stream(self.cairo_surface, write_fn, buffer_ptr);
        if status != CAIRO_STATUS_SUCCESS {
            return Err(status);
        }

        return Ok(());
    }*/

    pure fn clone() -> ImageSurface {
        unsafe {
            cairo_surface_reference(self.cairo_surface);
            return image_surface_from_cairo_surface(self.cairo_surface);
        }
    }
}

struct Context {
    cairo_context: *cairo_t,
}

impl Context {
    fn set_line_width(width: c_double) {
        unsafe {
            cairo_set_line_width(self.cairo_context, width);
        }
    }

    fn set_source_rgb(r: c_double, g: c_double, b: c_double) {
        unsafe {
            cairo_set_source_rgb(self.cairo_context, r, g, b);
        }
    }

    fn rectangle(x: c_double, y: c_double, width: c_double, height: c_double) {
        unsafe {
            cairo_rectangle(self.cairo_context, x, y, width, height);
        }
    }

    fn stroke() {
        unsafe {
            cairo_stroke(self.cairo_context);
        }
    }

    fn fill() {
        unsafe {
            cairo_fill(self.cairo_context);
        }
    }
}

fn make_context(surface: &ImageSurface) -> Context {
    unsafe {
        Context {
            cairo_context: cairo_create(surface.cairo_surface)
        }
    }
}
<|endoftext|>


<commit_before>/// Generate a secret key for signing and verifying JSON Web Tokens and HMACs.
///
/// Returns a secret comprised of 64 URL and command line compatible characters
/// (e.g. so that it can easily be entered on the CLI e.g. for a `--key` option ).
///
/// Uses 64 bytes because this is the maximum size possible for JWT signing keys.
/// Using a large key for JWT signing reduces the probability of brute force attacks.
/// See <https://auth0.com/blog/brute-forcing-hs256-is-possible-the-importance-of-using-strong-keys-to-sign-jwts/>.
pub fn generate() -> String {
    use rand::Rng;
    const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ\
                            abcdefghijklmnopqrstuvwxyz\
                            0123456789";
    let mut rng = rand::thread_rng();
    (0..64)
        .map(|_| {
            let idx = rng.gen_range(0..CHARSET.len());
            CHARSET[idx] as char
        })
        .collect()
}
<commit_msg>fix(Key utilities): Generate key with `sik` identifying prefix<commit_after>/// Generate a secret key for signing and verifying JSON Web Tokens and HMACs.
///
/// Returns a secret comprised of 64 URL and command line compatible characters
/// (e.g. so that it can easily be entered on the CLI e.g. for a `--key` option ).
///
/// Is prefixed with "sik" (Stencila Instance Key) for identifiability and
/// consistency with keys generated by Stencila Cloud (e.g. "sdk" for Stencila
/// deployment key).
///
/// Uses 64 bytes because this is the maximum size possible for JWT signing keys.
/// Using a large key for JWT signing reduces the probability of brute force attacks.
/// See <https://auth0.com/blog/brute-forcing-hs256-is-possible-the-importance-of-using-strong-keys-to-sign-jwts/>.
pub fn generate() -> String {
    use rand::Rng;
    const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ\
                            abcdefghijklmnopqrstuvwxyz\
                            0123456789";
    let mut rng = rand::thread_rng();
    let rand = (0..60)
        .map(|_| {
            let idx = rng.gen_range(0..CHARSET.len());
            CHARSET[idx] as char
        })
        .collect::<String>();
    ["sik-", &rand].concat()
}
<|endoftext|>


<commit_before>#![feature(test)]

extern crate test;
extern crate xi_rope;

use test::Bencher;
use xi_rope::compare;

static EDITOR_STR: &str = include_str!("../../core-lib/src/editor.rs");
static VIEW_STR: &str = include_str!("../../core-lib/src/view.rs");

static INTERVAL_STR: &str = include_str!("../src/interval.rs");
static BREAKS_STR: &str = include_str!("../src/breaks.rs");

#[bench]
fn ne_idx_sw(b: &mut Bencher) {
    let one: String = [EDITOR_STR, VIEW_STR, INTERVAL_STR, BREAKS_STR].concat();
    let mut two = one.clone();
    unsafe {
        let b = two.as_bytes_mut();
        let idx = b.len() - 200;
        b[idx] = 0x02;
    }

    b.iter(|| {
        compare::ne_idx_fallback(one.as_bytes(), one.as_bytes());
        compare::ne_idx_fallback(one.as_bytes(), two.as_bytes());
    })
}

#[bench]
fn ne_idx_hw(b: &mut Bencher) {
    let one: String = [EDITOR_STR, VIEW_STR, INTERVAL_STR, BREAKS_STR].concat();
    let mut two = one.clone();
    unsafe {
        let b = two.as_bytes_mut();
        let idx = b.len() - 200;
        b[idx] = 0x02;
    }

    b.iter(|| {
        compare::ne_idx_simd(one.as_bytes(), one.as_bytes());
        compare::ne_idx_simd(one.as_bytes(), two.as_bytes());
    })
}

#[bench]
fn ne_idx_sw_rev(b: &mut Bencher) {
    let one: String = [EDITOR_STR, VIEW_STR, INTERVAL_STR, BREAKS_STR].concat();
    let mut two = one.clone();
    unsafe {
        let b = two.as_bytes_mut();
        let idx = 200;
        b[idx] = 0x02;
    }

    b.iter(|| {
        compare::ne_idx_rev_fallback(one.as_bytes(), one.as_bytes());
        compare::ne_idx_rev_fallback(one.as_bytes(), two.as_bytes());
    })
}

#[bench]
fn ne_idx_hw_rev(b: &mut Bencher) {
    let one: String = [EDITOR_STR, VIEW_STR, INTERVAL_STR, BREAKS_STR].concat();
    let mut two = one.clone();
    unsafe {
        let b = two.as_bytes_mut();
        let idx = 200;
        b[idx] = 0x02;
    }

    b.iter(|| {
        compare::ne_idx_rev_simd(one.as_bytes(), one.as_bytes());
        compare::ne_idx_rev_simd(one.as_bytes(), two.as_bytes());
    })
}
<commit_msg>Add license to diff test file<commit_after>// Copyright 2018 The xi-editor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![feature(test)]

extern crate test;
extern crate xi_rope;

use test::Bencher;
use xi_rope::compare;

static EDITOR_STR: &str = include_str!("../../core-lib/src/editor.rs");
static VIEW_STR: &str = include_str!("../../core-lib/src/view.rs");

static INTERVAL_STR: &str = include_str!("../src/interval.rs");
static BREAKS_STR: &str = include_str!("../src/breaks.rs");

#[bench]
fn ne_idx_sw(b: &mut Bencher) {
    let one: String = [EDITOR_STR, VIEW_STR, INTERVAL_STR, BREAKS_STR].concat();
    let mut two = one.clone();
    unsafe {
        let b = two.as_bytes_mut();
        let idx = b.len() - 200;
        b[idx] = 0x02;
    }

    b.iter(|| {
        compare::ne_idx_fallback(one.as_bytes(), one.as_bytes());
        compare::ne_idx_fallback(one.as_bytes(), two.as_bytes());
    })
}

#[bench]
fn ne_idx_hw(b: &mut Bencher) {
    let one: String = [EDITOR_STR, VIEW_STR, INTERVAL_STR, BREAKS_STR].concat();
    let mut two = one.clone();
    unsafe {
        let b = two.as_bytes_mut();
        let idx = b.len() - 200;
        b[idx] = 0x02;
    }

    b.iter(|| {
        compare::ne_idx_simd(one.as_bytes(), one.as_bytes());
        compare::ne_idx_simd(one.as_bytes(), two.as_bytes());
    })
}

#[bench]
fn ne_idx_sw_rev(b: &mut Bencher) {
    let one: String = [EDITOR_STR, VIEW_STR, INTERVAL_STR, BREAKS_STR].concat();
    let mut two = one.clone();
    unsafe {
        let b = two.as_bytes_mut();
        let idx = 200;
        b[idx] = 0x02;
    }

    b.iter(|| {
        compare::ne_idx_rev_fallback(one.as_bytes(), one.as_bytes());
        compare::ne_idx_rev_fallback(one.as_bytes(), two.as_bytes());
    })
}

#[bench]
fn ne_idx_hw_rev(b: &mut Bencher) {
    let one: String = [EDITOR_STR, VIEW_STR, INTERVAL_STR, BREAKS_STR].concat();
    let mut two = one.clone();
    unsafe {
        let b = two.as_bytes_mut();
        let idx = 200;
        b[idx] = 0x02;
    }

    b.iter(|| {
        compare::ne_idx_rev_simd(one.as_bytes(), one.as_bytes());
        compare::ne_idx_rev_simd(one.as_bytes(), two.as_bytes());
    })
}
<|endoftext|>


<commit_before>use std;
use std::fmt::{Debug, Display, Formatter};
use std::ops::*;

const P: i64 = 1_000_000_007;

/// Calculates `x^n`. O(log n) time.
/// By Fermat's little theorem, `x^(-1) = pow(x, P - 2)`.
pub fn pow(x: i64, n: i64) -> i64 {
    let (mut x, mut y, mut n) = (x % P, 1_i64, n);
    while n > 0 {
        if n % 2 != 0 {
            y = (y * x) % P;
            n -= 1;
        }

        x = (x * x) % P;
        n /= 2;
    }
    y
}

/// Calculates `a^(-1)` for each `a` in `1..n`.
pub fn inv_dp(n: usize) -> Vec<i64> {
    let mut dp = vec![0; n];
    if n >= 2 {
        dp[1] = 1;
        for i in 2..n {
            let mut z = P - dp[(P % i as i64) as usize];
            z %= P;
            z *= P / i as i64;
            z %= P;
            dp[i] = z;
        }
    }
    dp
}

/// Represents an element of finite field.
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default)]
struct Finite<T>(T);

impl<T: Debug> Debug for Finite<T> {
    fn fmt(&self, formatter: &mut Formatter) -> std::fmt::Result {
        formatter.write_fmt(format_args!("{:?}", self.0))
    }
}

impl<T: Display> Display for Finite<T> {
    fn fmt(&self, formatter: &mut Formatter) -> std::fmt::Result {
        formatter.write_fmt(format_args!("{}", self.0))
    }
}

trait Normalize {
    fn normalize(&mut self);
}

impl<T> From<T> for Finite<T>
where
    Finite<T>: Normalize,
{
    fn from(value: T) -> Self {
        let mut it = Finite(value);
        it.normalize();
        it
    }
}

// Derive binary operation trait from the `FooAssign` impl.
// Define `x + y` as `{ let mut x = x.clone(); x += y; x }`.
macro_rules! impl_binary_op_for_finite {
    ($op_trait:ident, $op:ident, $assign_trait:ident, $assign:ident) => {
        impl<T: $op_trait<T, Output = T>> $op_trait<T> for Finite<T>
        where
            Finite<T>: Clone + $assign_trait<T>,
        {
            type Output = Self;

            fn $op(self, other: T) -> Self {
                let mut it = self.clone();
                it.$assign(other);
                it
            }
        }

        impl<T: $op_trait<T, Output = T>> $op_trait<Finite<T>> for Finite<T>
        where
            Finite<T>: Clone + $assign_trait<Finite<T>>,
        {
            type Output = Self;

            fn $op(self, other: Self) -> Self {
                let mut it = self.clone();
                it.$assign(other);
                it
            }
        }
    };
}

// Derive assign operation trait by unwrapping the right hand side.
macro_rules! impl_binary_op_assign_with_finite_for_finite {
    ($op_trait:ident, $op:ident) => {
        impl<T> $op_trait<Finite<T>> for Finite<T>
        where
            Finite<T>: $op_trait<T>,
        {
            fn $op(&mut self, other: Self) {
                self.$op(other.0);
            }
        }
    };
}

// Derive assign operation trait from impl for inner type.
macro_rules! impl_binary_op_assign_with_inner_for_finite {
    ($op_trait:ident, $op:ident) => {
        impl<T: $op_trait<T>> $op_trait<T> for Finite<T>
        where
            Finite<T>: Normalize,
        {
            fn $op(&mut self, other: T) {
                let mut other = Finite::from(other);
                other.normalize();
                (self.0).$op(other.0);
                self.normalize();
            }
        }
    };
}

impl_binary_op_for_finite! {Add, add, AddAssign, add_assign}
impl_binary_op_for_finite! {Sub, sub, SubAssign, sub_assign}
impl_binary_op_for_finite! {Mul, mul, MulAssign, mul_assign}
impl_binary_op_for_finite! {Div, div, DivAssign, div_assign}
impl_binary_op_assign_with_inner_for_finite! {AddAssign, add_assign}
impl_binary_op_assign_with_inner_for_finite! {SubAssign, sub_assign}
impl_binary_op_assign_with_inner_for_finite! {MulAssign, mul_assign}
impl_binary_op_assign_with_finite_for_finite! {AddAssign, add_assign}
impl_binary_op_assign_with_finite_for_finite! {SubAssign, sub_assign}
impl_binary_op_assign_with_finite_for_finite! {MulAssign, mul_assign}
impl_binary_op_assign_with_finite_for_finite! {DivAssign, div_assign}

impl Finite<i64> {
    fn pow(self, e: i64) -> Self {
        pow(self.0, e).into()
    }
}

impl Normalize for Finite<i64> {
    fn normalize(&mut self) {
        self.0 %= P;
        self.0 += P;
        self.0 %= P;
    }
}

impl DivAssign<i64> for Finite<i64> {
    fn div_assign(&mut self, other: i64) {
        *self *= Self::from(other).pow(P - 2);
    }
}

#[cfg(test)]
mod tests {
    use super::{inv_dp, pow, Finite, P};
    use std;

    #[test]
    fn test_pow_edges() {
        assert_eq!(pow(0, 0), 1);
        assert_eq!(pow(2, 0), 1);
        assert_eq!(pow(3, 1), 3);
        assert_eq!(pow(5, 6), (5 * 5 * 5) * (5 * 5 * 5));
        assert_eq!(pow(std::i64::MAX % P, std::i64::MAX), 856225998);
    }

    #[test]
    fn test_pow_small() {
        for x in 0..100_i64 {
            for n in 0..100_i64 {
                let actual = pow(x, n);
                let expected = (0..n).fold(1_i64, |acc, _| (acc * x) % P);
                assert_eq!(actual, expected, "{}^{}", x, n);
            }
        }
    }

    #[test]
    fn test_inv_dp() {
        let n = 10000;
        let dp = inv_dp(n);
        for i in 1..n {
            let mut z = dp[i] * i as i64;
            z %= P;
            z += P;
            z %= P;
            assert_eq!(z, 1);
        }
    }

    #[test]
    fn test_finite() {
        let x = Finite::from(P + 2);

        // `from` should normalize the value.
        assert_eq!(x.0, 2);

        // Operations.
        assert_eq!(x + 7, (2 + 7).into());
        assert_eq!(x - 7, (P + (2 - 7)).into());
        assert_eq!(x * 3, (2 * 3).into());
        assert_eq!((x / 11) * 11, 2.into());

        assert_eq!(x + Finite::from(7), (2 + 7).into());
        assert_eq!(x - Finite::from(7), (P + (2 - 7)).into());
        assert_eq!(x * Finite::from(3), (2 * 3).into());
        assert_eq!((x / Finite::from(11)) * 11, 2.into());

        let mut x = x;
        x += 7;
        assert_eq!(x, 9.into());
        x -= 5;
        assert_eq!(x, 4.into());
        x *= 6;
        assert_eq!(x, 24.into());
        x /= 3;
        assert_eq!(x, 8.into());
    }
}
<commit_msg>docs: Add hint of proof to `inv_dp`<commit_after>use std;
use std::fmt::{Debug, Display, Formatter};
use std::ops::*;

const P: i64 = 1_000_000_007;

/// Calculates `x^n`. O(log n) time.
/// By Fermat's little theorem, `x^(-1) = pow(x, P - 2)`.
pub fn pow(x: i64, n: i64) -> i64 {
    let (mut x, mut y, mut n) = (x % P, 1_i64, n);
    while n > 0 {
        if n % 2 != 0 {
            y = (y * x) % P;
            n -= 1;
        }

        x = (x * x) % P;
        n /= 2;
    }
    y
}

/// Calculates `1/a` for each `a` in `1..n`.
/// Use `P = floor(P / k) * k + P % k` for proof.
pub fn inv_dp(n: usize) -> Vec<i64> {
    let mut dp = vec![0; n];
    if n >= 2 {
        dp[1] = 1;
        for i in 2..n {
            let mut z = P - dp[(P % i as i64) as usize];
            z %= P;
            z *= P / i as i64;
            z %= P;
            dp[i] = z;
        }
    }
    dp
}

/// Represents an element of finite field.
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Default)]
struct Finite<T>(T);

impl<T: Debug> Debug for Finite<T> {
    fn fmt(&self, formatter: &mut Formatter) -> std::fmt::Result {
        formatter.write_fmt(format_args!("{:?}", self.0))
    }
}

impl<T: Display> Display for Finite<T> {
    fn fmt(&self, formatter: &mut Formatter) -> std::fmt::Result {
        formatter.write_fmt(format_args!("{}", self.0))
    }
}

trait Normalize {
    fn normalize(&mut self);
}

impl<T> From<T> for Finite<T>
where
    Finite<T>: Normalize,
{
    fn from(value: T) -> Self {
        let mut it = Finite(value);
        it.normalize();
        it
    }
}

// Derive binary operation trait from the `FooAssign` impl.
// Define `x + y` as `{ let mut x = x.clone(); x += y; x }`.
macro_rules! impl_binary_op_for_finite {
    ($op_trait:ident, $op:ident, $assign_trait:ident, $assign:ident) => {
        impl<T: $op_trait<T, Output = T>> $op_trait<T> for Finite<T>
        where
            Finite<T>: Clone + $assign_trait<T>,
        {
            type Output = Self;

            fn $op(self, other: T) -> Self {
                let mut it = self.clone();
                it.$assign(other);
                it
            }
        }

        impl<T: $op_trait<T, Output = T>> $op_trait<Finite<T>> for Finite<T>
        where
            Finite<T>: Clone + $assign_trait<Finite<T>>,
        {
            type Output = Self;

            fn $op(self, other: Self) -> Self {
                let mut it = self.clone();
                it.$assign(other);
                it
            }
        }
    };
}

// Derive assign operation trait by unwrapping the right hand side.
macro_rules! impl_binary_op_assign_with_finite_for_finite {
    ($op_trait:ident, $op:ident) => {
        impl<T> $op_trait<Finite<T>> for Finite<T>
        where
            Finite<T>: $op_trait<T>,
        {
            fn $op(&mut self, other: Self) {
                self.$op(other.0);
            }
        }
    };
}

// Derive assign operation trait from impl for inner type.
macro_rules! impl_binary_op_assign_with_inner_for_finite {
    ($op_trait:ident, $op:ident) => {
        impl<T: $op_trait<T>> $op_trait<T> for Finite<T>
        where
            Finite<T>: Normalize,
        {
            fn $op(&mut self, other: T) {
                let mut other = Finite::from(other);
                other.normalize();
                (self.0).$op(other.0);
                self.normalize();
            }
        }
    };
}

impl_binary_op_for_finite! {Add, add, AddAssign, add_assign}
impl_binary_op_for_finite! {Sub, sub, SubAssign, sub_assign}
impl_binary_op_for_finite! {Mul, mul, MulAssign, mul_assign}
impl_binary_op_for_finite! {Div, div, DivAssign, div_assign}
impl_binary_op_assign_with_inner_for_finite! {AddAssign, add_assign}
impl_binary_op_assign_with_inner_for_finite! {SubAssign, sub_assign}
impl_binary_op_assign_with_inner_for_finite! {MulAssign, mul_assign}
impl_binary_op_assign_with_finite_for_finite! {AddAssign, add_assign}
impl_binary_op_assign_with_finite_for_finite! {SubAssign, sub_assign}
impl_binary_op_assign_with_finite_for_finite! {MulAssign, mul_assign}
impl_binary_op_assign_with_finite_for_finite! {DivAssign, div_assign}

impl Finite<i64> {
    fn pow(self, e: i64) -> Self {
        pow(self.0, e).into()
    }
}

impl Normalize for Finite<i64> {
    fn normalize(&mut self) {
        self.0 %= P;
        self.0 += P;
        self.0 %= P;
    }
}

impl DivAssign<i64> for Finite<i64> {
    fn div_assign(&mut self, other: i64) {
        *self *= Self::from(other).pow(P - 2);
    }
}

#[cfg(test)]
mod tests {
    use super::{inv_dp, pow, Finite, P};
    use std;

    #[test]
    fn test_pow_edges() {
        assert_eq!(pow(0, 0), 1);
        assert_eq!(pow(2, 0), 1);
        assert_eq!(pow(3, 1), 3);
        assert_eq!(pow(5, 6), (5 * 5 * 5) * (5 * 5 * 5));
        assert_eq!(pow(std::i64::MAX % P, std::i64::MAX), 856225998);
    }

    #[test]
    fn test_pow_small() {
        for x in 0..100_i64 {
            for n in 0..100_i64 {
                let actual = pow(x, n);
                let expected = (0..n).fold(1_i64, |acc, _| (acc * x) % P);
                assert_eq!(actual, expected, "{}^{}", x, n);
            }
        }
    }

    #[test]
    fn test_inv_dp() {
        let n = 10000;
        let dp = inv_dp(n);
        for i in 1..n {
            let mut z = dp[i] * i as i64;
            z %= P;
            z += P;
            z %= P;
            assert_eq!(z, 1);
        }
    }

    #[test]
    fn test_finite() {
        let x = Finite::from(P + 2);

        // `from` should normalize the value.
        assert_eq!(x.0, 2);

        // Operations.
        assert_eq!(x + 7, (2 + 7).into());
        assert_eq!(x - 7, (P + (2 - 7)).into());
        assert_eq!(x * 3, (2 * 3).into());
        assert_eq!((x / 11) * 11, 2.into());

        assert_eq!(x + Finite::from(7), (2 + 7).into());
        assert_eq!(x - Finite::from(7), (P + (2 - 7)).into());
        assert_eq!(x * Finite::from(3), (2 * 3).into());
        assert_eq!((x / Finite::from(11)) * 11, 2.into());

        let mut x = x;
        x += 7;
        assert_eq!(x, 9.into());
        x -= 5;
        assert_eq!(x, 4.into());
        x *= 6;
        assert_eq!(x, 24.into());
        x /= 3;
        assert_eq!(x, 8.into());
    }
}
<|endoftext|>