<commit_before>//! The `ndarray-linalg` crate provides linear algebra functionalities for `ArrayBase`, the n-dimensional array data structure provided by [`ndarray`](https://github.com/rust-ndarray/ndarray).
//!
//! `ndarray-linalg` leverages [LAPACK](http://www.netlib.org/lapack/)'s routines using the bindings provided by [blas-lapack-rs/lapack](https://github.com/blas-lapack-rs/lapack).
//!
//! Linear algebra methods
//! -----------------------
//! - Decomposition methods:
//!     - [QR decomposition](qr/index.html)
//!     - [Cholesky/LU decomposition](cholesky/index.html)
//!     - [Eigenvalue decomposition for Hermite matrices](eigh/index.html)
//!     - [**S**ingular **V**alue **D**ecomposition](svd/index.html)
//! - Solution of linear systems:
//!    - [General matrices](solve/index.html)
//!    - [Triangular matrices](triangular/index.html)
//!    - [Hermitian/real symmetric matrices](solveh/index.html)
//! - [Inverse matrix computation](solve/trait.Inverse.html)
//!
//! Naming Convention
//! -----------------------
//! Each routine is usually exposed as a trait, implemented by the relevant types.
//!
//! For each routine there might be multiple "variants": different traits corresponding to the different ownership possibilities of the array you intend to work on.
//!
//! For example, if you are interested in the QR decomposition of a square matrix, you can use:
//! - [QRSquare](qr/trait.QRSquare.html), if you hold an immutable reference (i.e. `&self`) to the matrix you want to decompose;
//! - [QRSquareInplace](qr/trait.QRSquareInplace.html), if you hold a mutable reference (i.e. `&mut self`) to the matrix you want to decompose;
//! - [QRSquareInto](qr/trait.QRSquareInto.html), if you can pass the matrix you want to decompose by value (e.g. `self`).
//!
//! Depending on the algorithm, each variant might require more or less copy operations of the underlying data.
//!
//! Details are provided in the description of each routine.
//!
//!  Utilities
//!  -----------
//!  - [Assertions for array](index.html#macros)
//!  - [Random matrix generators](generate/index.html)
//!  - [Scalar trait](types/trait.Scalar.html)

pub mod assert;
pub mod cholesky;
pub mod convert;
pub mod diagonal;
pub mod eigh;
pub mod error;
pub mod generate;
pub mod inner;
pub mod lapack;
pub mod layout;
pub mod mgs;
pub mod norm;
pub mod operator;
pub mod opnorm;
pub mod qr;
pub mod solve;
pub mod solveh;
pub mod svd;
pub mod trace;
pub mod triangular;
pub mod types;

pub use assert::*;
pub use cholesky::*;
pub use convert::*;
pub use diagonal::*;
pub use eigh::*;
pub use generate::*;
pub use inner::*;
pub use layout::*;
pub use mgs::*;
pub use norm::*;
pub use operator::*;
pub use opnorm::*;
pub use qr::*;
pub use solve::*;
pub use solveh::*;
pub use svd::*;
pub use trace::*;
pub use triangular::*;
pub use types::*;
<commit_msg>Do not re-export mgs:: namespace<commit_after>//! The `ndarray-linalg` crate provides linear algebra functionalities for `ArrayBase`, the n-dimensional array data structure provided by [`ndarray`](https://github.com/rust-ndarray/ndarray).
//!
//! `ndarray-linalg` leverages [LAPACK](http://www.netlib.org/lapack/)'s routines using the bindings provided by [blas-lapack-rs/lapack](https://github.com/blas-lapack-rs/lapack).
//!
//! Linear algebra methods
//! -----------------------
//! - Decomposition methods:
//!     - [QR decomposition](qr/index.html)
//!     - [Cholesky/LU decomposition](cholesky/index.html)
//!     - [Eigenvalue decomposition for Hermite matrices](eigh/index.html)
//!     - [**S**ingular **V**alue **D**ecomposition](svd/index.html)
//! - Solution of linear systems:
//!    - [General matrices](solve/index.html)
//!    - [Triangular matrices](triangular/index.html)
//!    - [Hermitian/real symmetric matrices](solveh/index.html)
//! - [Inverse matrix computation](solve/trait.Inverse.html)
//!
//! Naming Convention
//! -----------------------
//! Each routine is usually exposed as a trait, implemented by the relevant types.
//!
//! For each routine there might be multiple "variants": different traits corresponding to the different ownership possibilities of the array you intend to work on.
//!
//! For example, if you are interested in the QR decomposition of a square matrix, you can use:
//! - [QRSquare](qr/trait.QRSquare.html), if you hold an immutable reference (i.e. `&self`) to the matrix you want to decompose;
//! - [QRSquareInplace](qr/trait.QRSquareInplace.html), if you hold a mutable reference (i.e. `&mut self`) to the matrix you want to decompose;
//! - [QRSquareInto](qr/trait.QRSquareInto.html), if you can pass the matrix you want to decompose by value (e.g. `self`).
//!
//! Depending on the algorithm, each variant might require more or less copy operations of the underlying data.
//!
//! Details are provided in the description of each routine.
//!
//!  Utilities
//!  -----------
//!  - [Assertions for array](index.html#macros)
//!  - [Random matrix generators](generate/index.html)
//!  - [Scalar trait](types/trait.Scalar.html)

pub mod assert;
pub mod cholesky;
pub mod convert;
pub mod diagonal;
pub mod eigh;
pub mod error;
pub mod generate;
pub mod inner;
pub mod lapack;
pub mod layout;
pub mod mgs;
pub mod norm;
pub mod operator;
pub mod opnorm;
pub mod qr;
pub mod solve;
pub mod solveh;
pub mod svd;
pub mod trace;
pub mod triangular;
pub mod types;

pub use assert::*;
pub use cholesky::*;
pub use convert::*;
pub use diagonal::*;
pub use eigh::*;
pub use generate::*;
pub use inner::*;
pub use layout::*;
pub use norm::*;
pub use operator::*;
pub use opnorm::*;
pub use qr::*;
pub use solve::*;
pub use solveh::*;
pub use svd::*;
pub use trace::*;
pub use triangular::*;
pub use types::*;
<|endoftext|>


<commit_before>//! Simple module to store files in database.
//!
//! cratesfyi is generating more than 5 million files, they are small and mostly html files.
//! They are using so many inodes and it is better to store them in database instead of
//! filesystem. This module is adding files into database and retrieving them.

use std::path::{PathBuf, Path};
use postgres::Connection;
use rustc_serialize::json::{Json, ToJson};
use std::fs;
use std::io::Read;
use error::Result;
use failure::err_msg;
use rusoto_s3::{S3, PutObjectRequest, GetObjectRequest, S3Client};
use rusoto_core::region::Region;
use rusoto_credential::EnvironmentProvider;

fn get_file_list_from_dir<P: AsRef<Path>>(path: P,
                                          files: &mut Vec<PathBuf>)
                                          -> Result<()> {
    let path = path.as_ref();

    for file in try!(path.read_dir()) {
        let file = try!(file);

        if try!(file.file_type()).is_file() {
            files.push(file.path());
        } else if try!(file.file_type()).is_dir() {
            try!(get_file_list_from_dir(file.path(), files));
        }
    }

    Ok(())
}

pub fn get_file_list<P: AsRef<Path>>(path: P) -> Result<Vec<PathBuf>> {
    let path = path.as_ref();
    let mut files = Vec::new();

    if !path.exists() {
        return Err(err_msg("File not found"));
    } else if path.is_file() {
        files.push(PathBuf::from(path.file_name().unwrap()));
    } else if path.is_dir() {
        try!(get_file_list_from_dir(path, &mut files));
        for file_path in &mut files {
            // We want the paths in this list to not be {path}/bar.txt but just bar.txt
            *file_path = PathBuf::from(file_path.strip_prefix(path).unwrap());
        }
    }

    Ok(files)
}

pub struct Blob {
    pub path: String,
    pub mime: String,
    pub date_updated: time::Timespec,
    pub content: Vec<u8>,
}

pub fn get_path(conn: &Connection, path: &str) -> Option<Blob> {
    if let Some(client) = s3_client() {
        let res = client.get_object(GetObjectRequest {
            bucket: "rust-docs-rs".into(),
            key: path.into(),
            ..Default::default()
        }).sync();

        let res = match res {
            Ok(r) => r,
            Err(err) => {
                debug!("error fetching {}: {:?}", path, err);
                return None;
            }
        };

        let mut b = res.body.unwrap().into_blocking_read();
        let mut content = Vec::new();
        b.read_to_end(&mut content).unwrap();

        let last_modified = res.last_modified.unwrap();
        let last_modified = time::strptime(&last_modified, "%a, %d %b %Y %H:%M:%S %Z")
            .unwrap_or_else(|e| panic!("failed to parse {:?} as timespec: {:?}", last_modified, e))
            .to_timespec();

        Some(Blob {
            path: path.into(),
            mime: res.content_type.unwrap(),
            date_updated: last_modified,
            content,
        })
    } else {
        let rows = conn.query("SELECT path, mime, date_updated, content
                            FROM files
                            WHERE path = $1", &[&path]).unwrap();

        if rows.len() == 0 {
            None
        } else {
            let row = rows.get(0);

            Some(Blob {
                path: row.get(0),
                mime: row.get(1),
                date_updated: row.get(2),
                content: row.get(3),
            })
        }
    }
}

fn s3_client() -> Option<S3Client> {
    // If AWS keys aren't configured, then presume we should use the DB exclusively
    // for file storage.
    if std::env::var_os("AWS_ACCESS_KEY_ID").is_none() {
        return None;
    }
    Some(S3Client::new_with(
        rusoto_core::request::HttpClient::new().unwrap(),
        EnvironmentProvider::default(),
        std::env::var("S3_ENDPOINT").ok().map(|e| Region::Custom {
            name: "us-west-1".to_owned(),
            endpoint: e,
        }).unwrap_or(Region::UsWest1),
    ))
}

/// Adds files into database and returns list of files with their mime type in Json
pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
                                              prefix: &str,
                                              path: P)
                                              -> Result<Json> {
    use magic::{Cookie, flags};
    let cookie = try!(Cookie::open(flags::MIME_TYPE));
    try!(cookie.load::<&str>(&[]));

    let trans = try!(conn.transaction());
    let mut client = s3_client();
    let mut file_list_with_mimes: Vec<(String, PathBuf)> = Vec::new();

    for file_path in try!(get_file_list(&path)) {
        let (path, content, mime) = {
            let path = Path::new(path.as_ref()).join(&file_path);
            // Some files have insufficient permissions (like .lock file created by cargo in
            // documentation directory). We are skipping this files.
            let mut file = match fs::File::open(path) {
                Ok(f) => f,
                Err(_) => continue,
            };
            let mut content: Vec<u8> = Vec::new();
            try!(file.read_to_end(&mut content));
            let bucket_path = Path::new(prefix).join(&file_path)
                .into_os_string().into_string().unwrap();

            let mime = {
                let mime = try!(cookie.buffer(&content));
                // css's are causing some problem in browsers
                // magic will return text/plain for css file types
                // convert them to text/css
                // do the same for javascript files
                if mime == "text/plain" {
                    let e = file_path.extension().unwrap_or_default();
                    if e == "css" {
                        "text/css".to_owned()
                    } else if e == "js" {
                        "application/javascript".to_owned()
                    } else {
                        mime.to_owned()
                    }
                } else {
                    mime.to_owned()
                }
            };

            let content: Option<Vec<u8>> = if let Some(client) = &mut client {
                let mut attempts = 0;
                loop {
                    let s3_res = client.put_object(PutObjectRequest {
                        bucket: "rust-docs-rs".into(),
                        key: bucket_path.clone(),
                        body: Some(content.clone().into()),
                        content_type: Some(mime.clone()),
                        ..Default::default()
                    }).sync();
                    attempts += 1;
                    match s3_res {
                        // we've successfully uploaded the content, so steal it;
                        // we don't want to put it in the DB
                        Ok(_) => break None,
                        // Since s3 was configured, we want to panic on failure to upload.
                        Err(e) => {
                            log::error!("failed to upload to {}: {:?}", bucket_path, e);
                            // Get a new client, in case the old one's connection is stale.
                            // AWS will kill our connection if it's alive for too long; this avoids
                            // that preventing us from building the crate entirely.
                            *client = s3_client().unwrap();
                            if attempts > 3 {
                                panic!("failed to upload 3 times, exiting");
                            } else {
                                continue;
                            }
                        },
                    }
                }
            } else {
                Some(content.clone().into())
            };

            file_list_with_mimes.push((mime.clone(), file_path.clone()));

            (
                bucket_path,
                content,
                mime,
            )
        };

        // If AWS credentials are configured, don't insert/update the database
        if client.is_none() {
            // check if file already exists in database
            let rows = try!(conn.query("SELECT COUNT(*) FROM files WHERE path = $1", &[&path]));

            let content = content.expect("content never None if client is None");

            if rows.get(0).get::<usize, i64>(0) == 0 {
                try!(trans.query("INSERT INTO files (path, mime, content) VALUES ($1, $2, $3)",
                                &[&path, &mime, &content]));
            } else {
                try!(trans.query("UPDATE files SET mime = $2, content = $3, date_updated = NOW() \
                                WHERE path = $1",
                                &[&path, &mime, &content]));
            }
        }
    }

    try!(trans.commit());

    file_list_to_json(file_list_with_mimes)
}

fn file_list_to_json(file_list: Vec<(String, PathBuf)>) -> Result<Json> {

    let mut file_list_json: Vec<Json> = Vec::new();

    for file in file_list {
        let mut v: Vec<String> = Vec::new();
        v.push(file.0.clone());
        v.push(file.1.into_os_string().into_string().unwrap());
        file_list_json.push(v.to_json());
    }

    Ok(file_list_json.to_json())
}

pub fn move_to_s3(conn: &Connection, n: usize) -> Result<usize> {
    let trans = try!(conn.transaction());
    let client = s3_client().expect("configured s3");

    let rows = try!(trans.query(
            &format!("SELECT path, mime, content FROM files WHERE content != E'in-s3' LIMIT {}", n),
            &[]));
    let count = rows.len();

    let mut rt = ::tokio::runtime::current_thread::Runtime::new().unwrap();
    let mut futures = Vec::new();
    for row in &rows {
        let path: String = row.get(0);
        let mime: String = row.get(1);
        let content: Vec<u8> = row.get(2);
        let path_1 = path.clone();
        futures.push(client.put_object(PutObjectRequest {
            bucket: "rust-docs-rs".into(),
            key: path.clone(),
            body: Some(content.into()),
            content_type: Some(mime),
            ..Default::default()
        }).map(move |_| {
            path_1
        }).map_err(move |e| {
            panic!("failed to upload to {}: {:?}", path, e)
        }));
    }

    use ::futures::future::Future;
    match rt.block_on(::futures::future::join_all(futures)) {
        Ok(paths) => {
            let statement = trans.prepare("DELETE FROM files WHERE path = $1").unwrap();
            for path in paths {
                statement.execute(&[&path]).unwrap();
            }
        }
        Err(e) => {
            panic!("results err: {:?}", e);
        }
    }

    try!(trans.commit());

    Ok(count)
}

#[cfg(test)]
mod test {
    extern crate env_logger;
    use std::env;
    use super::get_file_list;

    #[test]
    fn test_get_file_list() {
        let _ = env_logger::try_init();

        let files = get_file_list(env::current_dir().unwrap());
        assert!(files.is_ok());
        assert!(files.unwrap().len() > 0);

        let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap();
        assert_eq!(files[0], std::path::Path::new("Cargo.toml"));
    }
}
<commit_msg>Silence debug logging for failed S3 queries<commit_after>//! Simple module to store files in database.
//!
//! cratesfyi is generating more than 5 million files, they are small and mostly html files.
//! They are using so many inodes and it is better to store them in database instead of
//! filesystem. This module is adding files into database and retrieving them.

use std::path::{PathBuf, Path};
use postgres::Connection;
use rustc_serialize::json::{Json, ToJson};
use std::fs;
use std::io::Read;
use error::Result;
use failure::err_msg;
use rusoto_s3::{S3, PutObjectRequest, GetObjectRequest, S3Client};
use rusoto_core::region::Region;
use rusoto_credential::EnvironmentProvider;

fn get_file_list_from_dir<P: AsRef<Path>>(path: P,
                                          files: &mut Vec<PathBuf>)
                                          -> Result<()> {
    let path = path.as_ref();

    for file in try!(path.read_dir()) {
        let file = try!(file);

        if try!(file.file_type()).is_file() {
            files.push(file.path());
        } else if try!(file.file_type()).is_dir() {
            try!(get_file_list_from_dir(file.path(), files));
        }
    }

    Ok(())
}

pub fn get_file_list<P: AsRef<Path>>(path: P) -> Result<Vec<PathBuf>> {
    let path = path.as_ref();
    let mut files = Vec::new();

    if !path.exists() {
        return Err(err_msg("File not found"));
    } else if path.is_file() {
        files.push(PathBuf::from(path.file_name().unwrap()));
    } else if path.is_dir() {
        try!(get_file_list_from_dir(path, &mut files));
        for file_path in &mut files {
            // We want the paths in this list to not be {path}/bar.txt but just bar.txt
            *file_path = PathBuf::from(file_path.strip_prefix(path).unwrap());
        }
    }

    Ok(files)
}

pub struct Blob {
    pub path: String,
    pub mime: String,
    pub date_updated: time::Timespec,
    pub content: Vec<u8>,
}

pub fn get_path(conn: &Connection, path: &str) -> Option<Blob> {
    if let Some(client) = s3_client() {
        let res = client.get_object(GetObjectRequest {
            bucket: "rust-docs-rs".into(),
            key: path.into(),
            ..Default::default()
        }).sync();

        let res = match res {
            Ok(r) => r,
            Err(err) => {
                return None;
            }
        };

        let mut b = res.body.unwrap().into_blocking_read();
        let mut content = Vec::new();
        b.read_to_end(&mut content).unwrap();

        let last_modified = res.last_modified.unwrap();
        let last_modified = time::strptime(&last_modified, "%a, %d %b %Y %H:%M:%S %Z")
            .unwrap_or_else(|e| panic!("failed to parse {:?} as timespec: {:?}", last_modified, e))
            .to_timespec();

        Some(Blob {
            path: path.into(),
            mime: res.content_type.unwrap(),
            date_updated: last_modified,
            content,
        })
    } else {
        let rows = conn.query("SELECT path, mime, date_updated, content
                            FROM files
                            WHERE path = $1", &[&path]).unwrap();

        if rows.len() == 0 {
            None
        } else {
            let row = rows.get(0);

            Some(Blob {
                path: row.get(0),
                mime: row.get(1),
                date_updated: row.get(2),
                content: row.get(3),
            })
        }
    }
}

fn s3_client() -> Option<S3Client> {
    // If AWS keys aren't configured, then presume we should use the DB exclusively
    // for file storage.
    if std::env::var_os("AWS_ACCESS_KEY_ID").is_none() {
        return None;
    }
    Some(S3Client::new_with(
        rusoto_core::request::HttpClient::new().unwrap(),
        EnvironmentProvider::default(),
        std::env::var("S3_ENDPOINT").ok().map(|e| Region::Custom {
            name: "us-west-1".to_owned(),
            endpoint: e,
        }).unwrap_or(Region::UsWest1),
    ))
}

/// Adds files into database and returns list of files with their mime type in Json
pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
                                              prefix: &str,
                                              path: P)
                                              -> Result<Json> {
    use magic::{Cookie, flags};
    let cookie = try!(Cookie::open(flags::MIME_TYPE));
    try!(cookie.load::<&str>(&[]));

    let trans = try!(conn.transaction());
    let mut client = s3_client();
    let mut file_list_with_mimes: Vec<(String, PathBuf)> = Vec::new();

    for file_path in try!(get_file_list(&path)) {
        let (path, content, mime) = {
            let path = Path::new(path.as_ref()).join(&file_path);
            // Some files have insufficient permissions (like .lock file created by cargo in
            // documentation directory). We are skipping this files.
            let mut file = match fs::File::open(path) {
                Ok(f) => f,
                Err(_) => continue,
            };
            let mut content: Vec<u8> = Vec::new();
            try!(file.read_to_end(&mut content));
            let bucket_path = Path::new(prefix).join(&file_path)
                .into_os_string().into_string().unwrap();

            let mime = {
                let mime = try!(cookie.buffer(&content));
                // css's are causing some problem in browsers
                // magic will return text/plain for css file types
                // convert them to text/css
                // do the same for javascript files
                if mime == "text/plain" {
                    let e = file_path.extension().unwrap_or_default();
                    if e == "css" {
                        "text/css".to_owned()
                    } else if e == "js" {
                        "application/javascript".to_owned()
                    } else {
                        mime.to_owned()
                    }
                } else {
                    mime.to_owned()
                }
            };

            let content: Option<Vec<u8>> = if let Some(client) = &mut client {
                let mut attempts = 0;
                loop {
                    let s3_res = client.put_object(PutObjectRequest {
                        bucket: "rust-docs-rs".into(),
                        key: bucket_path.clone(),
                        body: Some(content.clone().into()),
                        content_type: Some(mime.clone()),
                        ..Default::default()
                    }).sync();
                    attempts += 1;
                    match s3_res {
                        // we've successfully uploaded the content, so steal it;
                        // we don't want to put it in the DB
                        Ok(_) => break None,
                        // Since s3 was configured, we want to panic on failure to upload.
                        Err(e) => {
                            log::error!("failed to upload to {}: {:?}", bucket_path, e);
                            // Get a new client, in case the old one's connection is stale.
                            // AWS will kill our connection if it's alive for too long; this avoids
                            // that preventing us from building the crate entirely.
                            *client = s3_client().unwrap();
                            if attempts > 3 {
                                panic!("failed to upload 3 times, exiting");
                            } else {
                                continue;
                            }
                        },
                    }
                }
            } else {
                Some(content.clone().into())
            };

            file_list_with_mimes.push((mime.clone(), file_path.clone()));

            (
                bucket_path,
                content,
                mime,
            )
        };

        // If AWS credentials are configured, don't insert/update the database
        if client.is_none() {
            // check if file already exists in database
            let rows = try!(conn.query("SELECT COUNT(*) FROM files WHERE path = $1", &[&path]));

            let content = content.expect("content never None if client is None");

            if rows.get(0).get::<usize, i64>(0) == 0 {
                try!(trans.query("INSERT INTO files (path, mime, content) VALUES ($1, $2, $3)",
                                &[&path, &mime, &content]));
            } else {
                try!(trans.query("UPDATE files SET mime = $2, content = $3, date_updated = NOW() \
                                WHERE path = $1",
                                &[&path, &mime, &content]));
            }
        }
    }

    try!(trans.commit());

    file_list_to_json(file_list_with_mimes)
}

fn file_list_to_json(file_list: Vec<(String, PathBuf)>) -> Result<Json> {

    let mut file_list_json: Vec<Json> = Vec::new();

    for file in file_list {
        let mut v: Vec<String> = Vec::new();
        v.push(file.0.clone());
        v.push(file.1.into_os_string().into_string().unwrap());
        file_list_json.push(v.to_json());
    }

    Ok(file_list_json.to_json())
}

pub fn move_to_s3(conn: &Connection, n: usize) -> Result<usize> {
    let trans = try!(conn.transaction());
    let client = s3_client().expect("configured s3");

    let rows = try!(trans.query(
            &format!("SELECT path, mime, content FROM files WHERE content != E'in-s3' LIMIT {}", n),
            &[]));
    let count = rows.len();

    let mut rt = ::tokio::runtime::current_thread::Runtime::new().unwrap();
    let mut futures = Vec::new();
    for row in &rows {
        let path: String = row.get(0);
        let mime: String = row.get(1);
        let content: Vec<u8> = row.get(2);
        let path_1 = path.clone();
        futures.push(client.put_object(PutObjectRequest {
            bucket: "rust-docs-rs".into(),
            key: path.clone(),
            body: Some(content.into()),
            content_type: Some(mime),
            ..Default::default()
        }).map(move |_| {
            path_1
        }).map_err(move |e| {
            panic!("failed to upload to {}: {:?}", path, e)
        }));
    }

    use ::futures::future::Future;
    match rt.block_on(::futures::future::join_all(futures)) {
        Ok(paths) => {
            let statement = trans.prepare("DELETE FROM files WHERE path = $1").unwrap();
            for path in paths {
                statement.execute(&[&path]).unwrap();
            }
        }
        Err(e) => {
            panic!("results err: {:?}", e);
        }
    }

    try!(trans.commit());

    Ok(count)
}

#[cfg(test)]
mod test {
    extern crate env_logger;
    use std::env;
    use super::get_file_list;

    #[test]
    fn test_get_file_list() {
        let _ = env_logger::try_init();

        let files = get_file_list(env::current_dir().unwrap());
        assert!(files.is_ok());
        assert!(files.unwrap().len() > 0);

        let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap();
        assert_eq!(files[0], std::path::Path::new("Cargo.toml"));
    }
}
<|endoftext|>


<commit_before>#![feature(proc_macro, custom_derive, custom_attribute)]
#![feature(box_syntax)]
#[macro_use] extern crate serde_derive;
extern crate hyper;
pub mod dictionaries;
pub mod thesaureses;
pub use dictionaries::{Definition, Dictionary};
pub use thesaureses::Thesaurus;
<commit_msg>Add Error type<commit_after>#![feature(proc_macro, custom_derive, custom_attribute)]
#![feature(box_syntax)]
#[macro_use] extern crate serde_derive;
extern crate serde_json;
extern crate hyper;
pub mod dictionaries;
pub mod thesaureses;
pub use dictionaries::{Definition, Dictionary};
pub use thesaureses::Thesaurus;
mod error {
    use std::error;
    use std::fmt;
    #[derive(Debug)]
    pub enum Error {
        Hyper(::hyper::Error),
        Serde(::serde_json::Error),
        Error(Box<error::Error>)
    }
    impl fmt::Display for Error {
        fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
            fmt.write_str(self.description())?;
            Ok(())
        }
    }
    impl error::Error for Error {
        fn description(&self) -> &str {
            match *self {
                Error::Hyper(ref hyper) => hyper.description(),
                Error::Serde(ref serde) => serde.description(),
                Error::Error(ref err) => err.description()
            }
        }
        fn cause(&self) -> Option<&::std::error::Error> { None }
    }
}

pub use error::Error;

<|endoftext|>


<commit_before>//! Implementation of the Micro Transport Protocol.[^spec]
//!
//! [^spec]: http://www.bittorrent.org/beps/bep_0029.html

//   __________  ____  ____
//  /_  __/ __ \/ __ \/ __ \
//   / / / / / / / / / / / /
//  / / / /_/ / /_/ / /_/ /
// /_/  \____/_____/\____/
//
// - Lossy UDP socket for testing purposes: send and receive ops are wrappers
// that stochastically drop or reorder packets.
// - Sending FIN on drop
// - Handle packet loss
// - Path MTU discovery (RFC4821)

#![experimental]
#![feature(macro_rules)]
#![feature(phase)]
#![feature(if_let)]
#![feature(while_let)]
#![feature(globs)]
#![deny(missing_docs)]

extern crate time;
#[phase(plugin, link)] extern crate log;

// Public API
pub use socket::UtpSocket;
pub use stream::UtpStream;

mod util;
mod bit_iterator;
mod packet;
mod socket;
mod stream;
<commit_msg>Remove compiler directives merged into the latest Rust.<commit_after>//! Implementation of the Micro Transport Protocol.[^spec]
//!
//! [^spec]: http://www.bittorrent.org/beps/bep_0029.html

//   __________  ____  ____
//  /_  __/ __ \/ __ \/ __ \
//   / / / / / / / / / / / /
//  / / / /_/ / /_/ / /_/ /
// /_/  \____/_____/\____/
//
// - Lossy UDP socket for testing purposes: send and receive ops are wrappers
// that stochastically drop or reorder packets.
// - Sending FIN on drop
// - Handle packet loss
// - Path MTU discovery (RFC4821)

#![experimental]
#![feature(macro_rules)]
#![feature(phase)]
#![feature(globs)]
#![deny(missing_docs)]

extern crate time;
#[phase(plugin, link)] extern crate log;

// Public API
pub use socket::UtpSocket;
pub use stream::UtpStream;

mod util;
mod bit_iterator;
mod packet;
mod socket;
mod stream;
<|endoftext|>


<commit_before>extern crate libc;

pub mod groonga;

use libc::funcs::c95::string;
use std::ffi::CString;
use std::fs;
use std::mem;
use std::ptr;
use std::result::Result;
use std::result::Result::{Ok, Err};

#[derive(Debug)]
pub struct Error {
    code: i32
}

impl Error {
    pub fn new(code: i32) -> Error {
        Error {code: code}
    }
}

pub struct Groonga {
    closed: bool
}

impl Groonga {
    pub fn new() -> Result<Groonga, Error> {
        unsafe {
            let rc =  groonga::grn_init();
            if rc != groonga::GRN_SUCCESS {
                return Err(Error::new(rc))
            }
            Ok(Groonga{ closed: false })
        }
    }

    pub fn close(&mut self) -> Result<(), Error> {
        if self.closed {
            return Ok(())
        }
        unsafe {
            let rc = groonga::grn_fin();
            if rc != groonga::GRN_SUCCESS {
                return Err(Error::new(rc))
            }
            self.closed = true;
            Ok(())
        }
    }
}

impl Drop for Groonga {
    fn drop(&mut self) {
        self.close().unwrap()
    }
}

pub struct Context {
    ctx: *mut groonga::grn_ctx,
}

impl Drop for Context {
    fn drop(&mut self) {
        unsafe {
            let rc = groonga::grn_ctx_fin(self.ctx);
            if rc != groonga::GRN_SUCCESS {
                panic!("grn_ctx_fin(ctx) failed with rc={}", rc);
            }
        }
    }
}

impl Context {
    pub fn new() -> Result<Context, Error> {
        unsafe {
            let ctx = groonga::grn_ctx_open(0);
            if ctx.is_null() {
                return Err(Error::new(groonga::GRN_NO_MEMORY_AVAILABLE))
            }
            Ok(Context { ctx: ctx })
        }
    }

    pub fn db_create(&mut self, path: &str) -> Result<(), Error> {
        let c_path = CString::new(path).unwrap();
        unsafe {
            let db = groonga::grn_db_create(
                self.ctx, c_path.as_ptr(), mem::zeroed());
            if db.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(())
        }
    }

    pub fn db_open(&mut self, path: &str) -> Result<(), Error> {
        let c_path = CString::new(path).unwrap();
        unsafe {
            let db = groonga::grn_db_open(self.ctx, c_path.as_ptr());
            if db.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(())
        }
    }

    pub fn db_open_or_create(&mut self, path: &str) -> Result<(), Error> {
        if file_exists(path) {
            self.db_open(path)
        } else {
            self.db_create(path)
        }
    }

    pub fn ctx_at(&mut self, id: u32) -> Result<Object, Error> {
        unsafe {
            let obj = groonga::grn_ctx_at(self.ctx, id);
            if obj.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(Object { context: self, obj: obj })
        }
    }

    pub fn ctx_get(&mut self, name: &str) -> Result<Object, Error> {
        let c_name = CString::new(name).unwrap().as_ptr();
        unsafe {
            let obj = groonga::grn_ctx_get(
                self.ctx, c_name, string::strlen(c_name) as i32);
            if obj.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(Object { context: self, obj: obj })
        }
    }

    pub fn table_create(&mut self, name: &str, path: &str, flags: u32,
        key_type: &Object, value_type: Option<&Object>) -> Result<Object, Error> {
        let c_name = CString::new(name).unwrap().as_ptr();
        let c_path = if path != "" {
            CString::new(path).unwrap().as_ptr()
        } else {
            ptr::null()
        };
        unsafe {
            let c_value_type =
                match value_type {
                    Some(value_type_obj) => value_type_obj.obj,
                    None => mem::zeroed(),
                };
            let table = groonga::grn_table_create(
                self.ctx, c_name, string::strlen(c_name) as u32, c_path,
                flags as u16, key_type.obj, c_value_type);
            if table.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(Object { context: self, obj: table })
        }
    }

    pub fn table_open_or_create(&mut self, name: &str, path: &str, flags: u32,
        key_type: &Object, value_type: Option<&Object>) -> Result<Object, Error> {
        if let Ok(table) = self.ctx_get(name) {
            Ok(table)
        } else {
            self.table_create(name, path, flags,
                key_type, value_type)
        }
    }

    pub fn obj_column(&mut self, table: &Object, name: &str)
        -> Result<Object, Error> {
        let c_name = CString::new(name).unwrap().as_ptr();
        unsafe {
            let obj = groonga::grn_obj_column(
                self.ctx, table.obj, c_name, string::strlen(c_name) as u32);
            if obj.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(Object { context: self, obj: obj })
        }
    }

    pub fn column_create(&mut self, table: &Object, name: &str, path: &str,
        flags: u32, _type: &Object) -> Result<Object, Error> {
        let c_name = CString::new(name).unwrap().as_ptr();
        let c_path = if path != "" {
            CString::new(path).unwrap().as_ptr()
        } else {
            ptr::null()
        };
        unsafe {
            let column = groonga::grn_column_create(
                self.ctx, table.obj, c_name, string::strlen(c_name) as u32,
                c_path, flags as u16, _type.obj);
            if column.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(Object { context: self, obj: column })
        }
    }

    pub fn column_open_or_create(&mut self, table: &Object, name: &str,
        path: &str, flags: u32, _type: &Object) -> Result<Object, Error> {
        if let Ok(column) = self.obj_column(&table, name) {
            Ok(column)
        } else {
            self.column_create(&table, name, path, flags, _type)
        }
    }
}

pub struct Object {
    context: *mut Context,
    obj: *mut groonga::grn_obj,
}

impl Object {
    pub fn close(&mut self) {
        unsafe {
            if self.obj.is_null() {
                return
            }
            groonga::grn_obj_unlink((*self.context).ctx, self.obj)
        }
    }
}

fn file_exists(path: &str) -> bool {
    match fs::File::open(path) {
        Ok(_) => true,
        Err(_) => false,
    }
}
<commit_msg>Clear obj field after close<commit_after>extern crate libc;

pub mod groonga;

use libc::funcs::c95::string;
use std::ffi::CString;
use std::fs;
use std::mem;
use std::ptr;
use std::result::Result;
use std::result::Result::{Ok, Err};

#[derive(Debug)]
pub struct Error {
    code: i32
}

impl Error {
    pub fn new(code: i32) -> Error {
        Error {code: code}
    }
}

pub struct Groonga {
    closed: bool
}

impl Groonga {
    pub fn new() -> Result<Groonga, Error> {
        unsafe {
            let rc =  groonga::grn_init();
            if rc != groonga::GRN_SUCCESS {
                return Err(Error::new(rc))
            }
            Ok(Groonga{ closed: false })
        }
    }

    pub fn close(&mut self) -> Result<(), Error> {
        if self.closed {
            return Ok(())
        }
        unsafe {
            let rc = groonga::grn_fin();
            if rc != groonga::GRN_SUCCESS {
                return Err(Error::new(rc))
            }
            self.closed = true;
            Ok(())
        }
    }
}

impl Drop for Groonga {
    fn drop(&mut self) {
        self.close().unwrap()
    }
}

pub struct Context {
    ctx: *mut groonga::grn_ctx,
}

impl Drop for Context {
    fn drop(&mut self) {
        unsafe {
            let rc = groonga::grn_ctx_fin(self.ctx);
            if rc != groonga::GRN_SUCCESS {
                panic!("grn_ctx_fin(ctx) failed with rc={}", rc);
            }
        }
    }
}

impl Context {
    pub fn new() -> Result<Context, Error> {
        unsafe {
            let ctx = groonga::grn_ctx_open(0);
            if ctx.is_null() {
                return Err(Error::new(groonga::GRN_NO_MEMORY_AVAILABLE))
            }
            Ok(Context { ctx: ctx })
        }
    }

    pub fn db_create(&mut self, path: &str) -> Result<(), Error> {
        let c_path = CString::new(path).unwrap();
        unsafe {
            let db = groonga::grn_db_create(
                self.ctx, c_path.as_ptr(), mem::zeroed());
            if db.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(())
        }
    }

    pub fn db_open(&mut self, path: &str) -> Result<(), Error> {
        let c_path = CString::new(path).unwrap();
        unsafe {
            let db = groonga::grn_db_open(self.ctx, c_path.as_ptr());
            if db.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(())
        }
    }

    pub fn db_open_or_create(&mut self, path: &str) -> Result<(), Error> {
        if file_exists(path) {
            self.db_open(path)
        } else {
            self.db_create(path)
        }
    }

    pub fn ctx_at(&mut self, id: u32) -> Result<Object, Error> {
        unsafe {
            let obj = groonga::grn_ctx_at(self.ctx, id);
            if obj.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(Object { context: self, obj: obj })
        }
    }

    pub fn ctx_get(&mut self, name: &str) -> Result<Object, Error> {
        let c_name = CString::new(name).unwrap().as_ptr();
        unsafe {
            let obj = groonga::grn_ctx_get(
                self.ctx, c_name, string::strlen(c_name) as i32);
            if obj.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(Object { context: self, obj: obj })
        }
    }

    pub fn table_create(&mut self, name: &str, path: &str, flags: u32,
        key_type: &Object, value_type: Option<&Object>) -> Result<Object, Error> {
        let c_name = CString::new(name).unwrap().as_ptr();
        let c_path = if path != "" {
            CString::new(path).unwrap().as_ptr()
        } else {
            ptr::null()
        };
        unsafe {
            let c_value_type =
                match value_type {
                    Some(value_type_obj) => value_type_obj.obj,
                    None => mem::zeroed(),
                };
            let table = groonga::grn_table_create(
                self.ctx, c_name, string::strlen(c_name) as u32, c_path,
                flags as u16, key_type.obj, c_value_type);
            if table.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(Object { context: self, obj: table })
        }
    }

    pub fn table_open_or_create(&mut self, name: &str, path: &str, flags: u32,
        key_type: &Object, value_type: Option<&Object>) -> Result<Object, Error> {
        if let Ok(table) = self.ctx_get(name) {
            Ok(table)
        } else {
            self.table_create(name, path, flags,
                key_type, value_type)
        }
    }

    pub fn obj_column(&mut self, table: &Object, name: &str)
        -> Result<Object, Error> {
        let c_name = CString::new(name).unwrap().as_ptr();
        unsafe {
            let obj = groonga::grn_obj_column(
                self.ctx, table.obj, c_name, string::strlen(c_name) as u32);
            if obj.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(Object { context: self, obj: obj })
        }
    }

    pub fn column_create(&mut self, table: &Object, name: &str, path: &str,
        flags: u32, _type: &Object) -> Result<Object, Error> {
        let c_name = CString::new(name).unwrap().as_ptr();
        let c_path = if path != "" {
            CString::new(path).unwrap().as_ptr()
        } else {
            ptr::null()
        };
        unsafe {
            let column = groonga::grn_column_create(
                self.ctx, table.obj, c_name, string::strlen(c_name) as u32,
                c_path, flags as u16, _type.obj);
            if column.is_null() {
                return Err(Error::new((*self.ctx).rc))
            }
            Ok(Object { context: self, obj: column })
        }
    }

    pub fn column_open_or_create(&mut self, table: &Object, name: &str,
        path: &str, flags: u32, _type: &Object) -> Result<Object, Error> {
        if let Ok(column) = self.obj_column(&table, name) {
            Ok(column)
        } else {
            self.column_create(&table, name, path, flags, _type)
        }
    }
}

pub struct Object {
    context: *mut Context,
    obj: *mut groonga::grn_obj,
}

impl Object {
    pub fn close(&mut self) {
        unsafe {
            if self.obj.is_null() {
                return
            }
            groonga::grn_obj_unlink((*self.context).ctx, self.obj);
            self.obj = mem::zeroed()
        }
    }
}

fn file_exists(path: &str) -> bool {
    match fs::File::open(path) {
        Ok(_) => true,
        Err(_) => false,
    }
}
<|endoftext|>


<commit_before>extern crate serde;
extern crate serde_json;

#[macro_use]
extern crate serde_derive;

#[macro_use]
extern crate nom;

use nom::{digit, line_ending};
use nom::IResult::Done;
use nom::multispace;

use std::str::FromStr;
use std::collections::HashMap;

#[derive(Debug, Serialize)]
#[repr(C)]
pub struct Osu<'a> {
    version: u32,
    general: General<'a>,
}

#[derive(Debug, PartialEq, Default, Serialize)]
struct General<'a> {
    audio_filename: &'a str,
    audio_lead_in:  u32,
    preview_time:   u32,
    countdown:      bool,
    sample_set:     &'a str,
    stack_leniency: f32,
    mode:           u32,
    letterboxin_breaks: bool,
    widescreen_storyboard: bool,
}

impl<'a> General<'a> {
    pub fn new(pairs: Vec<(&'a str, &'a str)>) -> General<'a> {
        let map: HashMap<&str, &str> = pairs.into_iter().collect();
        let unwrap_get = |x| map.get(x).unwrap();
        let unwrap_from_str = |x| -> u32 {FromStr::from_str(unwrap_get(x)).unwrap()};
        let unwrap_from_str_f = |x| -> f32 {FromStr::from_str(unwrap_get(x)).unwrap()};
        let unwrap_from_str_b = |x| -> bool {unwrap_from_str(x) == 1};
        General{
            audio_filename: unwrap_get("AudioFilename"),
            audio_lead_in: unwrap_from_str("AudioLeadIn"),
            preview_time: unwrap_from_str("PreviewTime"),
            countdown: unwrap_from_str("Countdown") == 1,
            sample_set: unwrap_get("SampleSet"),
            stack_leniency: unwrap_from_str_f("StackLeniency"),
            mode: unwrap_from_str("Mode"),
            letterboxin_breaks: unwrap_from_str_b("LetterboxInBreaks"),
            widescreen_storyboard: unwrap_from_str_b("WidescreenStoryboard"),
            ..Default::default()
        }
    }
}

#[derive(Debug, PartialEq, Default)]
struct Editor {
    asdf: u32,
}

named!(key_value_pair<&str, (&str, &str)>,
    do_parse!(
           key: take_until_and_consume_s!(":")
        >> opt!(multispace)
        >> val: take_until_and_consume_s!("\r\n")
        >> (key, val)
    )
);

named!(section_general<&str, General>,
        do_parse!(
                                tag_s!("[General]")
           >>                   opt!(multispace)
           >> pairs_with_end:   many_till!(key_value_pair, line_ending)
           >> (General::new(pairs_with_end.0))
          )
        );

named!(section_editor<&str, Editor>,
        do_parse!(
                                tag_s!("[Editor]")
           >>                   opt!(multispace)
           >> pairs_with_end:   many_till!(key_value_pair, line_ending)
           >> (Editor {..Default::default()})
          )
        );

named!(parse_osu<&str, Osu>,
    do_parse!(
                    tag_s!("osu file format v")
    >>  version:    map_res!(digit, FromStr::from_str)
    >>              line_ending
    >>              opt!(multispace)
    >>  sections:   permutation!(call!(section_general), call!(section_editor))
    >>  (Osu {
        version: version,
        general: sections.0,
        })
    )
);

#[allow(no_mangle_generic_items)]
#[no_mangle]
pub extern "C" fn parse_osu_file<'a>() -> *const Osu<'a> {
    let input = include_str!("../test.osu");
    let res = parse_osu(input);
    match res {
        Done(_, x) => &x,
        _ => 0 as *const Osu,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn it_works() {
        let input = include_str!("../test.osu");
        let res = parse_osu(input);
        if let Done(_, osu) = res {
            println!("{:?}", osu);
            println!("{}", serde_json::to_string(&osu).unwrap());
        }
        else {
            println!("{:?}", res);
            assert!(false);
        }
    }
}
<commit_msg>small fix<commit_after>extern crate serde;
extern crate serde_json;

#[macro_use]
extern crate serde_derive;

#[macro_use]
extern crate nom;

use nom::{digit, line_ending};
use nom::IResult::Done;
use nom::multispace;

use std::str::FromStr;
use std::collections::HashMap;

#[derive(Debug, Serialize)]
#[repr(C)]
pub struct Osu<'a> {
    version: u32,
    general: General<'a>,
}

#[derive(Debug, PartialEq, Default, Serialize)]
struct General<'a> {
    audio_filename: &'a str,
    audio_lead_in:  u32,
    preview_time:   u32,
    countdown:      bool,
    sample_set:     &'a str,
    stack_leniency: f32,
    mode:           u32,
    letterboxin_breaks: bool,
    widescreen_storyboard: bool,
}

impl<'a> General<'a> {
    pub fn from_tuples(pairs: Vec<(&'a str, &'a str)>) -> General<'a> {
        let map: HashMap<&str, &str> = pairs.into_iter().collect();
        let unwrap_get = |x| map.get(x).unwrap();
        let unwrap_from_str = |x| -> u32 {FromStr::from_str(unwrap_get(x)).unwrap()};
        let unwrap_from_str_f = |x| -> f32 {FromStr::from_str(unwrap_get(x)).unwrap()};
        let unwrap_from_str_b = |x| -> bool {unwrap_from_str(x) == 1};
        General{
            audio_filename: unwrap_get("AudioFilename"),
            audio_lead_in: unwrap_from_str("AudioLeadIn"),
            preview_time: unwrap_from_str("PreviewTime"),
            countdown: unwrap_from_str_b("Countdown"),
            sample_set: unwrap_get("SampleSet"),
            stack_leniency: unwrap_from_str_f("StackLeniency"),
            mode: unwrap_from_str("Mode"),
            letterboxin_breaks: unwrap_from_str_b("LetterboxInBreaks"),
            widescreen_storyboard: unwrap_from_str_b("WidescreenStoryboard"),
            ..Default::default()
        }
    }
}

#[derive(Debug, PartialEq, Default, Serialize)]
struct Editor<'a> {
    bookmarks: &'a [u32],
    distance_spacing: f32,
    beat_divisor: u32,
    grid_size: u32,
    timeline_zoom: f32,
}

named!(key_value_pair<&str, (&str, &str)>,
    do_parse!(
           key: take_until_and_consume_s!(":")
        >> opt!(multispace)
        >> val: take_until_and_consume_s!("\r\n")
        >> (key, val)
    )
);

named!(section_general<&str, General>,
        do_parse!(
                                tag_s!("[General]")
           >>                   opt!(multispace)
           >> pairs_with_end:   many_till!(key_value_pair, line_ending)
           >> (General::from_tuples(pairs_with_end.0))
          )
        );

named!(section_editor<&str, Editor>,
        do_parse!(
                                tag_s!("[Editor]")
           >>                   opt!(multispace)
           >> pairs_with_end:   many_till!(key_value_pair, line_ending)
           >> (Editor {..Default::default()})
          )
        );

named!(parse_osu<&str, Osu>,
    do_parse!(
                    tag_s!("osu file format v")
    >>  version:    map_res!(digit, FromStr::from_str)
    >>              line_ending
    >>              opt!(multispace)
    >>  sections:   permutation!(call!(section_general), call!(section_editor))
    >>  (Osu {
        version: version,
        general: sections.0,
        })
    )
);

#[allow(no_mangle_generic_items)]
#[no_mangle]
pub extern "C" fn parse_osu_file<'a>() -> *const Osu<'a> {
    let input = include_str!("../test.osu");
    let res = parse_osu(input);
    match res {
        Done(_, x) => &x,
        _ => 0 as *const Osu,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn it_works() {
        let input = include_str!("../test.osu");
        let res = parse_osu(input);
        if let Done(_, osu) = res {
            println!("{:?}", osu);
            println!("{}", serde_json::to_string(&osu).unwrap());
        }
        else {
            println!("{:?}", res);
            assert!(false);
        }
    }
}
<|endoftext|>


<commit_before>//! A dead-simple test helper for rust.
//!
//! Example
//! --------
//!
//! ```rust
//! // tests/test.rs
//!
//! #[test]
//! fn do_some_test() {
//!     // Get a content of 'tests/fixtures/hello.obj'
//!     let content: String = fixture("hello.obj");
//!
//!     /* Do whatever you want with it */
//! }
//! ```

#![unstable]
#![doc(html_root_url = "https://simnalamburt.github.io/bear")]

#![deny(warnings)]
#![deny(missing_docs)]

use std::io::{File, BufferedReader};

/// Open a file from the 'tests/fixtures' directory, and return its content as a `String`
///
/// Example
/// --------
///
/// ```rust
/// // tests/test.rs
///
/// #[test]
/// fn do_some_test() {
///     // Get a content of 'tests/fixtures/hello.obj'
///     let content: String = fixture("hello.obj");
///
///     /* Do whatever you want with it */
/// }
/// ```
pub fn fixture(filename: &str) -> String {
    let path = Path::new("tests").join("fixtures").join(filename);
    let file = File::open(&path).unwrap();
    let mut reader = BufferedReader::new(file);
    let buf = reader.read_to_end().unwrap();

    String::from_utf8(buf).unwrap()
}
<commit_msg>doc: Add bear logo to docs<commit_after>//! A dead-simple test helper for rust.
//!
//! Example
//! --------
//!
//! ```rust
//! // tests/test.rs
//!
//! #[test]
//! fn do_some_test() {
//!     // Get a content of 'tests/fixtures/hello.obj'
//!     let content: String = fixture("hello.obj");
//!
//!     /* Do whatever you want with it */
//! }
//! ```

#![unstable]
#![doc(html_logo_url    = "https://simnalamburt.github.io/bear/cute.png",
       html_favicon_url = "https://simnalamburt.github.io/bear/cute.png",
       html_root_url    = "https://simnalamburt.github.io/bear")]

#![deny(warnings)]
#![deny(missing_docs)]

use std::io::{File, BufferedReader};

/// Open a file from the 'tests/fixtures' directory, and return its content as a `String`
///
/// Example
/// --------
///
/// ```rust
/// // tests/test.rs
///
/// #[test]
/// fn do_some_test() {
///     // Get a content of 'tests/fixtures/hello.obj'
///     let content: String = fixture("hello.obj");
///
///     /* Do whatever you want with it */
/// }
/// ```
pub fn fixture(filename: &str) -> String {
    let path = Path::new("tests").join("fixtures").join(filename);
    let file = File::open(&path).unwrap();
    let mut reader = BufferedReader::new(file);
    let buf = reader.read_to_end().unwrap();

    String::from_utf8(buf).unwrap()
}
<|endoftext|>


<commit_before>#![cfg_attr(test, feature(plugin))]
#![cfg_attr(test, plugin(quickcheck_macros))]

extern crate num;

#[cfg(test)] extern crate quickcheck;

use num::{Float, NumCast, Zero, cast};
use std::f64::consts::PI;
use std::fmt::{Display, Formatter, Error};
use std::ops::{Add, Div, Mul, Neg, Sub};

/// An angle.
///
/// Might be a value in degrees or in radians.
#[derive(Copy, Clone, Debug)]
pub enum Angle<T=f64> {
    Radian(T),
    Degree(T)
}

impl<T: Copy + NumCast> Angle<T> {
    /// Yield the value encoded in radians.
    pub fn in_radians(&self) -> T {
        match *self {
            Radian(v) => v,
            Degree(v) => cast(cast::<T, f64>(v).unwrap() / 180.0 * PI).unwrap()
        }
    }

    /// Yield the value encoded in degrees.
    pub fn in_degrees(&self) -> T {
        match *self {
            Radian(v) => cast(cast::<T, f64>(v).unwrap() / PI * 180.0).unwrap(),
            Degree(v) => v
        }
    }
}

impl<T: Float + NumCast> Angle<T> {
    /// Create a new angle by normalizing the value into the range of
    /// [0, 2π) rad.
    pub fn normalized(&self) -> Angle<T> {
        let (v, upper) = match *self {
            Radian(v) => (v, cast(2.0 * PI).unwrap()),
            Degree(v) => (v, cast(360.0).unwrap())
        };

        let normalized = if v < upper && v >= Zero::zero() {
            v
        } else {
            let v = v % upper;

            if v >= Zero::zero() {
                v
            } else {
                v + upper
            }
        };

        match *self {
            Radian(_) => Radian(normalized),
            Degree(_) => Degree(normalized)
        }
    }

    /// Compute the sine of the angle.
    pub fn sin(&self) -> T {
        self.in_radians().sin()
    }

    /// Compute the cosine of the angle.
    pub fn cos(&self) -> T {
        self.in_radians().cos()
    }

    /// Compute the tangent of the angle.
    pub fn tan(&self) -> T {
        self.in_radians().tan()
    }

    /// Simultaneously compute the sine and cosine of the number, `x`.
    /// Return `(sin(x), cos(x))`.
    pub fn sin_cos(&self) -> (T, T) {
        self.in_radians().sin_cos()
    }

    /*
    /// Computes the approximate mean of a set of angles by averaging the
    /// Cartesian coordinates of the angles on the unit circle.
    pub fn mean(angles: &[Angle<T>]) -> Angle<T> {
        let mut x: T = Zero::zero();
        let mut y: T = Zero::zero();

        for angle in angles.iter() {
            let radian = angle.as_radian();
            x = x + radian.cos();
            y = y + radian.sin();
        }

        let n = cast(angles.len()).unwrap();
        let a = (y/n).atan2(x/n);

        Radian(a).normalize()
    }

    /// Computes the minimal unsigned distance between two normalized angles.
    pub fn distance(&self, other: Angle<T>) -> Angle<T> {
        // FIXME: Once associated types are implemented fix this nightmares of casts #17841
        let pi: T = cast(f64::consts::PI).unwrap();
        let pi2: T = pi * cast(2.0).unwrap();
        let d = (self.as_radian() - other.as_radian()).abs();
        Radian(pi - ((d % (pi2)) - pi).abs())
    }*/
}

impl<T: Float> Angle<T> {
    /// Compute the arcsine of a number. Return value is in the range of
    /// [-π/2, π/2] rad or `None` if the number is outside the range [-1, 1].
    pub fn asin(value: T) -> Option<Angle<T>> {
        let value = value.asin();
        if value.is_nan() {
            None
        } else {
            Some(Radian(value))
        }
    }

    /// Compute the arccosine of a number. Return value is in the range of
    /// [0, π] rad or `None` if the number is outside the range [-1, 1].
    pub fn acos(value: T) -> Option<Angle<T>> {
        let value = value.acos();
        if value.is_nan() {
            None
        } else {
            Some(Radian(value))
        }
    }

    /// Compute the arctangent of a number. Return value is in the range of
    /// [-π/2, π/2] rad.
    pub fn atan(value: T) -> Angle<T> {
        Radian(value.atan())
    }

    // Computes the four quadrant arctangent of `y` and `x`.
    pub fn atan2(y: T, x: T) -> Angle<T> {
        Radian(y.atan2(x))
    }
}

/*
impl<T: Zero> Zero for Angle<T> {
    fn zero() -> Angle<T> {
        Radian(T::zero())
    }

    fn is_zero(&self) -> bool {
        match *self {
            Radian(v) => v == T::zero(),
            Degree(v) => v == T::zero()
        }
    }
}*/

impl<T: PartialEq + Copy + NumCast> PartialEq for Angle<T> {
    fn eq(&self, other: &Angle<T>) -> bool {
        if let (Radian(a), Radian(b)) = (*self, *other) {
            a == b
        } else if let (Degree(a), Degree(b)) = (*self, *other) {
            a == b
        } else {
            self.in_radians() == other.in_radians()
        }
    }
}

macro_rules! math_additive(
    ($bound:ident, $func:ident) => (
        impl<T: $bound + Float + NumCast> $bound for Angle<T> {
            type Output = Angle<T>;
            fn $func(self, rhs: Angle<T>) -> Self::Output {
                if let (Degree(a), Degree(b)) = (self, rhs) {
                    Degree(a.$func(b))
                } else {
                    Radian(self.in_radians().$func(rhs.in_radians()))
                }
            }
        }
    );
);

math_additive!(Add, add);
math_additive!(Sub, sub);

macro_rules! math_multiplicative(
    ($bound:ident, $func:ident, $($t:ident),*) => (
        impl<T: $bound> $bound<T> for Angle<T> {
            type Output = Angle<T::Output>;
            fn $func(self, rhs: T) -> Self::Output {
                match self {
                    Radian(v) => Radian(v.$func(rhs)),
                    Degree(v) => Degree(v.$func(rhs))
                }
            }
        }

        $(
            impl $bound<Angle<$t>> for $t {
                type Output = Angle<$t>;
                fn $func(self, rhs: Angle<$t>) -> Self::Output {
                    match rhs {
                        Radian(v) => Radian(self.$func(v)),
                        Degree(v) => Degree(self.$func(v))
                    }
                }
            }
        )*
    );
);

math_multiplicative!(Mul, mul, u8, u16, u32, u64, i8, i16, i32, i64, f32, f64);
math_multiplicative!(Div, div, u8, u16, u32, u64, i8, i16, i32, i64, f32, f64);

impl<T: Neg> Neg for Angle<T> {
    type Output = Angle<T::Output>;
    fn neg(self) -> Self::Output {
        match self {
            Radian(v) => Radian(-v),
            Degree(v) => Degree(-v)
        }
    }
}

impl<T: Display> Display for Angle<T> {
    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
        match *self {
            Radian(ref v) => write!(f, "{}rad", v),
            Degree(ref v) => write!(f, "{}°", v)
        }
    }
}

unsafe impl<T: Send> Send for Angle<T> {  }

/*
impl<T: Encodable + Float + NumCast> Encodable for Angle<T> {
    fn encode<S: Encoder>(&self,  s: &mut S) -> Result<(), S::Error> {
        self.as_radian().encode(s)
    }
}

impl<T: Decodable + Float + NumCast> Decodable for Angle<T> {
    fn decode<D: Decoder>(d: &mut D) -> Result<Self, D::Error> {
        let t = try!(T::decode(d));
        Ok(Angle::Radian(t))
    }
}
*/

// re-exports
pub use Angle::{Radian, Degree};

#[cfg(test)]
mod tests {
    use num::{Float, cast};
    use quickcheck::{Arbitrary, Gen};
    use std::f64::consts::PI;
    use super::{Angle, Radian, Degree};

    #[quickcheck]
    fn test_angle_conversions(angle: Angle<f64>) -> bool {
        are_close(angle.in_radians(), Degree(angle.in_degrees()).in_radians())
    }

    #[quickcheck]
    fn test_angle_math_multiplicative(a: Angle<f64>, x: f64) -> bool {
        match a {
            Radian(v) => (a * x).in_radians() == v * x &&
                         (a / x).in_radians() == v / x,
            Degree(v) => (a * x).in_degrees() == v * x &&
                         (a / x).in_degrees() == v / x
        }
    }

    #[quickcheck]
    fn test_angle_math_additive(a: Angle, b: Angle) -> bool {
        if let (Radian(x), Radian(y)) = (a, b) {
            (a + b).in_radians() == x + y &&
            (a - b).in_radians() == x - y
        } else if let (Degree(x), Degree(y)) = (a, b) {
            (a + b).in_degrees() == x + y &&
            (a - b).in_degrees() == x - y
        } else {
            (a + b).in_radians() == a.in_radians() + b.in_radians()
        }
    }

    #[quickcheck]
    fn test_angle_normalization(angle: Angle) -> bool {
        let v = angle.normalized();
        let rad = v.in_radians();
        let deg = v.in_degrees();

        0.0 <= rad && rad < 2.0 * PI &&
        0.0 <= deg && deg < 360.0 &&
        are_close(rad.cos(), angle.cos())
    }

    fn are_close<T: Float>(a: T, b: T) -> bool {
        (a - b).abs() < cast(1.0e-10).unwrap()
    }

    impl<T: Arbitrary> Arbitrary for Angle<T> {
        fn arbitrary<G: Gen>(g: &mut G) -> Self {
            let v = Arbitrary::arbitrary(g);
            if bool::arbitrary(g) {
                Radian(v)
            } else {
                Degree(v)
            }
        }
    }
}
<commit_msg>Forgot to add isize/usize<commit_after>#![cfg_attr(test, feature(plugin))]
#![cfg_attr(test, plugin(quickcheck_macros))]

extern crate num;

#[cfg(test)] extern crate quickcheck;

use num::{Float, NumCast, Zero, cast};
use std::f64::consts::PI;
use std::fmt::{Display, Formatter, Error};
use std::ops::{Add, Div, Mul, Neg, Sub};

/// An angle.
///
/// Might be a value in degrees or in radians.
#[derive(Copy, Clone, Debug)]
pub enum Angle<T=f64> {
    Radian(T),
    Degree(T)
}

impl<T: Copy + NumCast> Angle<T> {
    /// Yield the value encoded in radians.
    pub fn in_radians(&self) -> T {
        match *self {
            Radian(v) => v,
            Degree(v) => cast(cast::<T, f64>(v).unwrap() / 180.0 * PI).unwrap()
        }
    }

    /// Yield the value encoded in degrees.
    pub fn in_degrees(&self) -> T {
        match *self {
            Radian(v) => cast(cast::<T, f64>(v).unwrap() / PI * 180.0).unwrap(),
            Degree(v) => v
        }
    }
}

impl<T: Float + NumCast> Angle<T> {
    /// Create a new angle by normalizing the value into the range of
    /// [0, 2π) rad.
    pub fn normalized(&self) -> Angle<T> {
        let (v, upper) = match *self {
            Radian(v) => (v, cast(2.0 * PI).unwrap()),
            Degree(v) => (v, cast(360.0).unwrap())
        };

        let normalized = if v < upper && v >= Zero::zero() {
            v
        } else {
            let v = v % upper;

            if v >= Zero::zero() {
                v
            } else {
                v + upper
            }
        };

        match *self {
            Radian(_) => Radian(normalized),
            Degree(_) => Degree(normalized)
        }
    }

    /// Compute the sine of the angle.
    pub fn sin(&self) -> T {
        self.in_radians().sin()
    }

    /// Compute the cosine of the angle.
    pub fn cos(&self) -> T {
        self.in_radians().cos()
    }

    /// Compute the tangent of the angle.
    pub fn tan(&self) -> T {
        self.in_radians().tan()
    }

    /// Simultaneously compute the sine and cosine of the number, `x`.
    /// Return `(sin(x), cos(x))`.
    pub fn sin_cos(&self) -> (T, T) {
        self.in_radians().sin_cos()
    }

    /*
    /// Computes the approximate mean of a set of angles by averaging the
    /// Cartesian coordinates of the angles on the unit circle.
    pub fn mean(angles: &[Angle<T>]) -> Angle<T> {
        let mut x: T = Zero::zero();
        let mut y: T = Zero::zero();

        for angle in angles.iter() {
            let radian = angle.as_radian();
            x = x + radian.cos();
            y = y + radian.sin();
        }

        let n = cast(angles.len()).unwrap();
        let a = (y/n).atan2(x/n);

        Radian(a).normalize()
    }

    /// Computes the minimal unsigned distance between two normalized angles.
    pub fn distance(&self, other: Angle<T>) -> Angle<T> {
        // FIXME: Once associated types are implemented fix this nightmares of casts #17841
        let pi: T = cast(f64::consts::PI).unwrap();
        let pi2: T = pi * cast(2.0).unwrap();
        let d = (self.as_radian() - other.as_radian()).abs();
        Radian(pi - ((d % (pi2)) - pi).abs())
    }*/
}

impl<T: Float> Angle<T> {
    /// Compute the arcsine of a number. Return value is in the range of
    /// [-π/2, π/2] rad or `None` if the number is outside the range [-1, 1].
    pub fn asin(value: T) -> Option<Angle<T>> {
        let value = value.asin();
        if value.is_nan() {
            None
        } else {
            Some(Radian(value))
        }
    }

    /// Compute the arccosine of a number. Return value is in the range of
    /// [0, π] rad or `None` if the number is outside the range [-1, 1].
    pub fn acos(value: T) -> Option<Angle<T>> {
        let value = value.acos();
        if value.is_nan() {
            None
        } else {
            Some(Radian(value))
        }
    }

    /// Compute the arctangent of a number. Return value is in the range of
    /// [-π/2, π/2] rad.
    pub fn atan(value: T) -> Angle<T> {
        Radian(value.atan())
    }

    // Computes the four quadrant arctangent of `y` and `x`.
    pub fn atan2(y: T, x: T) -> Angle<T> {
        Radian(y.atan2(x))
    }
}

/*
impl<T: Zero> Zero for Angle<T> {
    fn zero() -> Angle<T> {
        Radian(T::zero())
    }

    fn is_zero(&self) -> bool {
        match *self {
            Radian(v) => v == T::zero(),
            Degree(v) => v == T::zero()
        }
    }
}*/

impl<T: PartialEq + Copy + NumCast> PartialEq for Angle<T> {
    fn eq(&self, other: &Angle<T>) -> bool {
        if let (Radian(a), Radian(b)) = (*self, *other) {
            a == b
        } else if let (Degree(a), Degree(b)) = (*self, *other) {
            a == b
        } else {
            self.in_radians() == other.in_radians()
        }
    }
}

macro_rules! math_additive(
    ($bound:ident, $func:ident) => (
        impl<T: $bound + Float + NumCast> $bound for Angle<T> {
            type Output = Angle<T>;
            fn $func(self, rhs: Angle<T>) -> Self::Output {
                if let (Degree(a), Degree(b)) = (self, rhs) {
                    Degree(a.$func(b))
                } else {
                    Radian(self.in_radians().$func(rhs.in_radians()))
                }
            }
        }
    );
);

math_additive!(Add, add);
math_additive!(Sub, sub);

macro_rules! math_multiplicative(
    ($bound:ident, $func:ident, $($t:ident),*) => (
        impl<T: $bound> $bound<T> for Angle<T> {
            type Output = Angle<T::Output>;
            fn $func(self, rhs: T) -> Self::Output {
                match self {
                    Radian(v) => Radian(v.$func(rhs)),
                    Degree(v) => Degree(v.$func(rhs))
                }
            }
        }

        $(
            impl $bound<Angle<$t>> for $t {
                type Output = Angle<$t>;
                fn $func(self, rhs: Angle<$t>) -> Self::Output {
                    match rhs {
                        Radian(v) => Radian(self.$func(v)),
                        Degree(v) => Degree(self.$func(v))
                    }
                }
            }
        )*
    );
);

math_multiplicative!(Mul, mul, u8, u16, u32, u64, i8, i16, i32, i64, usize, isize, f32, f64);
math_multiplicative!(Div, div, u8, u16, u32, u64, i8, i16, i32, i64, usize, isize, f32, f64);

impl<T: Neg> Neg for Angle<T> {
    type Output = Angle<T::Output>;
    fn neg(self) -> Self::Output {
        match self {
            Radian(v) => Radian(-v),
            Degree(v) => Degree(-v)
        }
    }
}

impl<T: Display> Display for Angle<T> {
    fn fmt(&self, f: &mut Formatter) -> Result<(), Error> {
        match *self {
            Radian(ref v) => write!(f, "{}rad", v),
            Degree(ref v) => write!(f, "{}°", v)
        }
    }
}

unsafe impl<T: Send> Send for Angle<T> {  }

/*
impl<T: Encodable + Float + NumCast> Encodable for Angle<T> {
    fn encode<S: Encoder>(&self,  s: &mut S) -> Result<(), S::Error> {
        self.as_radian().encode(s)
    }
}

impl<T: Decodable + Float + NumCast> Decodable for Angle<T> {
    fn decode<D: Decoder>(d: &mut D) -> Result<Self, D::Error> {
        let t = try!(T::decode(d));
        Ok(Angle::Radian(t))
    }
}
*/

// re-exports
pub use Angle::{Radian, Degree};

#[cfg(test)]
mod tests {
    use num::{Float, cast};
    use quickcheck::{Arbitrary, Gen};
    use std::f64::consts::PI;
    use super::{Angle, Radian, Degree};

    #[quickcheck]
    fn test_angle_conversions(angle: Angle<f64>) -> bool {
        are_close(angle.in_radians(), Degree(angle.in_degrees()).in_radians())
    }

    #[quickcheck]
    fn test_angle_math_multiplicative(a: Angle<f64>, x: f64) -> bool {
        match a {
            Radian(v) => (a * x).in_radians() == v * x &&
                         (a / x).in_radians() == v / x,
            Degree(v) => (a * x).in_degrees() == v * x &&
                         (a / x).in_degrees() == v / x
        }
    }

    #[quickcheck]
    fn test_angle_math_additive(a: Angle, b: Angle) -> bool {
        if let (Radian(x), Radian(y)) = (a, b) {
            (a + b).in_radians() == x + y &&
            (a - b).in_radians() == x - y
        } else if let (Degree(x), Degree(y)) = (a, b) {
            (a + b).in_degrees() == x + y &&
            (a - b).in_degrees() == x - y
        } else {
            (a + b).in_radians() == a.in_radians() + b.in_radians()
        }
    }

    #[quickcheck]
    fn test_angle_normalization(angle: Angle) -> bool {
        let v = angle.normalized();
        let rad = v.in_radians();
        let deg = v.in_degrees();

        0.0 <= rad && rad < 2.0 * PI &&
        0.0 <= deg && deg < 360.0 &&
        are_close(rad.cos(), angle.cos())
    }

    fn are_close<T: Float>(a: T, b: T) -> bool {
        (a - b).abs() < cast(1.0e-10).unwrap()
    }

    impl<T: Arbitrary> Arbitrary for Angle<T> {
        fn arbitrary<G: Gen>(g: &mut G) -> Self {
            let v = Arbitrary::arbitrary(g);
            if bool::arbitrary(g) {
                Radian(v)
            } else {
                Degree(v)
            }
        }
    }
}
<|endoftext|>


<commit_before>use std::{collections::VecDeque, io::SeekFrom};

use log::{debug, error, trace};
use nom::{self, Err, IResult, Needed, Offset};

use av_data::{
    audiosample::ChannelMap, packet::Packet, params::*, rational::Rational64, timeinfo::TimeInfo,
};
use av_format::{
    buffer::Buffered,
    common::GlobalInfo,
    demuxer::{Demuxer, Descr, Descriptor, Event},
    error::*,
    stream::Stream,
};

use crate::{
    ebml::{self, custom_error, ebml_header, EBMLHeader},
    elements::{
        segment, segment_element, simple_block, Audio, Cluster, Info, SeekHead, SegmentElement,
        TrackEntry, TrackType, Tracks, Video,
    },
};

#[derive(Debug, Clone, Default)]
pub struct MkvDemuxer {
    pub header: Option<EBMLHeader>,
    pub seek_head: Option<SeekHead>,
    pub info: Option<Info>,
    pub tracks: Option<Tracks>,
    pub queue: VecDeque<Event>,
    pub blockstream: Vec<u8>,
    pub params: Option<DemuxerParams>,
}

#[derive(Debug, Clone, Default)]
pub struct DemuxerParams {
    pub track_numbers: Option<Vec<u64>>,
}

impl MkvDemuxer {
    pub fn new() -> MkvDemuxer {
        MkvDemuxer {
            header: None,
            seek_head: None,
            info: None,
            tracks: None,
            queue: VecDeque::new(),
            blockstream: Vec::new(),
            params: None,
        }
    }

    pub fn with_params(params: DemuxerParams) -> MkvDemuxer {
        MkvDemuxer {
            params: Some(params),
            ..Default::default()
        }
    }

    pub fn parse_until_tracks<'a>(
        &mut self,
        original_input: &'a [u8],
    ) -> IResult<&'a [u8], (), ebml::Error<'a>> {
        let (i1, header) = ebml_header(original_input)?;

        self.header = Some(header);

        let (mut input, _) = segment(i1)?;

        self.seek_head = None;
        self.info = None;
        self.tracks = None;

        loop {
            if self.seek_head.is_some() && self.info.is_some() && self.tracks.is_some() {
                return Ok((input, ()));
            }

            let (i3, element) = segment_element(input)?;

            match element {
                SegmentElement::SeekHead(s) => {
                    trace!("got seek head: {:#?}", s);
                    self.seek_head = if self.seek_head.is_none() {
                        Some(s)
                    } else {
                        return Err(Err::Error(custom_error(input, 1)));
                    };
                }
                SegmentElement::Info(i) => {
                    trace!("got info: {:#?}", i);
                    self.info = if self.info.is_none() {
                        Some(i)
                    } else {
                        return Err(Err::Error(custom_error(input, 1)));
                    };
                }
                SegmentElement::Tracks(t) => {
                    trace!("got tracks: {:#?}", t);
                    self.tracks = if self.tracks.is_none() {
                        // Only keep tracks we're interested in
                        self.params.as_ref().and_then(|params| {
                            params.track_numbers.as_ref().map(|track_numbers| {
                                let mut t = t;
                                t.tracks = t
                                    .tracks
                                    .into_iter()
                                    .filter(|tr| track_numbers.contains(&tr.track_number))
                                    .collect();
                                t
                            })
                        })
                    } else {
                        return Err(Err::Error(custom_error(input, 1)));
                    };
                }
                el => {
                    debug!("got element: {:#?}", el);
                }
            }

            input = i3;
        }
    }
}

impl Demuxer for MkvDemuxer {
    fn read_headers(&mut self, buf: &Box<dyn Buffered>, info: &mut GlobalInfo) -> Result<SeekFrom> {
        match self.parse_until_tracks(buf.data()) {
            Ok((i, _)) => {
                info.duration = self
                    .info
                    .as_ref()
                    .and_then(|info| info.duration)
                    .map(|d| d as u64);
                if let Some(ref mut t) = self.tracks {
                    for tr in t.tracks.iter_mut() {
                        info.add_stream(track_to_stream(self.info.as_ref().unwrap(), tr));
                        tr.stream_index = info.streams.last().unwrap().index;
                    }
                }
                Ok(SeekFrom::Current(buf.data().offset(i) as i64))
            }
            Err(Err::Incomplete(needed)) => {
                let sz = match needed {
                    Needed::Size(size) => buf.data().len() + usize::from(size),
                    _ => 1024,
                };
                Err(Error::MoreDataNeeded(sz))
            }
            e => {
                error!("{:?}", e);
                Err(Error::InvalidData)
            }
        }
    }

    fn read_event(&mut self, buf: &Box<dyn Buffered>) -> Result<(SeekFrom, Event)> {
        if let Some(event) = self.queue.pop_front() {
            Ok((SeekFrom::Current(0), event))
        } else {
            match segment_element(buf.data()) {
                Ok((i, element)) => {
                    let seek = SeekFrom::Current(buf.data().offset(i) as i64);
                    if let SegmentElement::Cluster(c) = element {
                        debug!("got cluster element at timecode: {}", c.timecode);
                        let mut packets = c.generate_packets(self.tracks.as_ref().unwrap());
                        self.queue.extend(packets.drain(..));
                        if let Some(event) = self.queue.pop_front() {
                            return Ok((seek, event));
                        }
                    }
                    Ok((seek, Event::MoreDataNeeded(0)))
                }
                Err(Err::Incomplete(Needed::Size(size))) => {
                    Err(Error::MoreDataNeeded(usize::from(size)))
                }
                e => {
                    error!("{:?}", e);
                    Err(Error::InvalidData)
                }
            }
        }
    }
}

fn track_entry_codec_id(t: &TrackEntry) -> Option<String> {
    // TODO: Support V_QUICKTIME and V_MS/VFW/FOURCC
    match t.codec_id.as_ref() {
        "A_OPUS" => Some("opus".to_owned()),
        "A_VORBIS" => Some("vorbis".to_owned()),
        "V_AV1" => Some("av1".to_owned()),
        "V_VP8" => Some("vp8".to_owned()),
        "V_VP9" => Some("vp9".to_owned()),
        _ => None,
    }
}

fn track_entry_video_kind(video: &Video) -> Option<MediaKind> {
    let v = VideoInfo {
        width: video.pixel_width as usize,
        height: video.pixel_height as usize,
        // TODO parse Colour and/or CodecPrivate to extract the format
        format: None,
    };
    Some(MediaKind::Video(v))
}

fn track_entry_audio_kind(audio: &Audio) -> Option<MediaKind> {
    let rate = audio
        .output_sampling_frequency
        .unwrap_or(audio.sampling_frequency);
    // TODO: complete it
    let map = if audio.channel_positions.is_none() {
        Some(ChannelMap::default_map(audio.channels as usize))
    } else {
        unimplemented!("Convert matroska map to rust-av map")
    };
    let a = AudioInfo {
        rate: rate as usize,
        map,
        format: None,
    };
    Some(MediaKind::Audio(a))
}

fn track_entry_media_kind(t: &TrackEntry) -> Option<MediaKind> {
    match t.track_type.into() {
        TrackType::Video => t.video.as_ref().and_then(track_entry_video_kind),
        TrackType::Audio => t.audio.as_ref().and_then(track_entry_audio_kind),
        TrackType::Other => None,
    }
}

pub fn track_to_stream(info: &Info, t: &TrackEntry) -> Stream {
    let num = t
        .track_timecode_scale
        .map_or(info.timecode_scale as i64, |ts| {
            if ts != 0. {
                (ts * info.timecode_scale as f64) as i64
            } else {
                info.timecode_scale as i64
            }
        });

    Stream {
        id: t.track_uid as isize,
        index: t.stream_index,
        start: None,
        duration: t.default_duration,
        timebase: Rational64::new(num, 1000 * 1000 * 1000),
        // TODO: Extend CodecParams and fill it with the remaining information
        params: CodecParams {
            extradata: t.codec_private.clone(),
            bit_rate: 0,
            delay: t.codec_delay.unwrap_or(0) as usize,
            convergence_window: t.seek_pre_roll.unwrap_or(0) as usize,
            codec_id: track_entry_codec_id(t),
            kind: track_entry_media_kind(t),
        },
        user_private: None,
    }
}

impl<'a> Cluster<'a> {
    pub fn generate_packets(&self, tracks: &Tracks) -> Vec<Event> {
        let mut v = Vec::new();

        for block_data in self.simple_block.iter() {
            if let Ok((i, block)) = simple_block(block_data) {
                debug!("parsing simple block: {:?}", block);
                if let Some(index) = tracks.lookup(block.track_number) {
                    let packet = Packet {
                        data: i.into(),
                        t: TimeInfo {
                            pts: Some(i64::from(block.timecode)),
                            dts: None,
                            duration: None,
                            timebase: None,
                            user_private: None,
                        },
                        pos: None,
                        stream_index: index as isize,
                        is_key: block.keyframe,
                        is_corrupted: false,
                    };

                    v.push(Event::NewPacket(packet));
                }
            } else {
                error!("error parsing simple block");
            }
        }

        v
    }
}

struct Des {
    d: Descr,
}

impl Descriptor for Des {
    fn create(&self) -> Box<dyn Demuxer> {
        Box::new(MkvDemuxer::new())
    }
    fn describe(&self) -> &Descr {
        &self.d
    }
    fn probe(&self, data: &[u8]) -> u8 {
        ebml_header(&data[..100]).map_or(0, |_| 100)
    }
}

pub const MKV_DESC: &dyn Descriptor = &Des {
    d: Descr {
        name: "matroska",
        demuxer: "mkv",
        description: "Nom-based Matroska demuxer",
        extensions: &["mkv", "webm", "mka"],
        mime: &["video/x-matroska", "audio/x-matroska"],
    },
};

#[cfg(test)]
#[allow(non_upper_case_globals)]
mod tests {
    use std::io::Cursor;

    use log::info;
    use nom::Offset;

    use av_format::{buffer::*, demuxer::Context};

    use super::*;

    const webm: &[u8] = include_bytes!("../assets/bbb-vp9-opus.webm");

    #[test]
    fn parse_headers() {
        let mut demuxer = MkvDemuxer::new();

        let res = demuxer.parse_until_tracks(webm);
        info!("got parsing res: {:?}", res);
        match res {
            Ok((i, _)) => {
                info!("offset: {} bytes", webm.offset(i));
            }
            e => {
                info!("could not parse: {:?}", e);
            }
        }

        info!("demuxer: {:#?}", demuxer);
    }

    #[test]
    fn parse_headers_incremental_buffer() {
        let mut demuxer = MkvDemuxer::new();

        for n in 100..2000 {
            let res = demuxer.parse_until_tracks(&webm[0..n]);
            match res {
                Ok(_) => info!("Size {} ok", n),
                Err(Err::Incomplete(needed)) => info!("Incomplete {} needs {:?}", n, needed),
                Err(e) => {
                    panic!("Error at size {}: {:?}", n, e);
                }
            }
        }
    }

    #[test]
    fn context() {
        let mut context = Context::new(
            Box::new(MkvDemuxer::new()),
            Box::new(AccReader::new(Cursor::new(webm))),
        );
        info!("read headers: {:?}", context.read_headers().unwrap());
        info!("streams: {:?}", context.info.streams);
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
    }
}
<commit_msg>demuxer: Fix tracks parsing<commit_after>use std::{collections::VecDeque, io::SeekFrom};

use log::{debug, error, trace};
use nom::{self, Err, IResult, Needed, Offset};

use av_data::{
    audiosample::ChannelMap, packet::Packet, params::*, rational::Rational64, timeinfo::TimeInfo,
};
use av_format::{
    buffer::Buffered,
    common::GlobalInfo,
    demuxer::{Demuxer, Descr, Descriptor, Event},
    error::*,
    stream::Stream,
};

use crate::{
    ebml::{self, custom_error, ebml_header, EBMLHeader},
    elements::{
        segment, segment_element, simple_block, Audio, Cluster, Info, SeekHead, SegmentElement,
        TrackEntry, TrackType, Tracks, Video,
    },
};

#[derive(Debug, Clone, Default)]
pub struct MkvDemuxer {
    pub header: Option<EBMLHeader>,
    pub seek_head: Option<SeekHead>,
    pub info: Option<Info>,
    pub tracks: Option<Tracks>,
    pub queue: VecDeque<Event>,
    pub blockstream: Vec<u8>,
    pub params: Option<DemuxerParams>,
}

#[derive(Debug, Clone, Default)]
pub struct DemuxerParams {
    pub track_numbers: Option<Vec<u64>>,
}

impl MkvDemuxer {
    pub fn new() -> MkvDemuxer {
        MkvDemuxer {
            header: None,
            seek_head: None,
            info: None,
            tracks: None,
            queue: VecDeque::new(),
            blockstream: Vec::new(),
            params: None,
        }
    }

    pub fn with_params(params: DemuxerParams) -> MkvDemuxer {
        MkvDemuxer {
            params: Some(params),
            ..Default::default()
        }
    }

    pub fn parse_until_tracks<'a>(
        &mut self,
        original_input: &'a [u8],
    ) -> IResult<&'a [u8], (), ebml::Error<'a>> {
        let (i1, header) = ebml_header(original_input)?;

        self.header = Some(header);

        let (mut input, _) = segment(i1)?;

        self.seek_head = None;
        self.info = None;
        self.tracks = None;

        loop {
            if self.seek_head.is_some() && self.info.is_some() && self.tracks.is_some() {
                return Ok((input, ()));
            }

            let (i3, element) = segment_element(input)?;

            match element {
                SegmentElement::SeekHead(s) => {
                    trace!("got seek head: {:#?}", s);
                    self.seek_head = if self.seek_head.is_none() {
                        Some(s)
                    } else {
                        return Err(Err::Error(custom_error(input, 1)));
                    };
                }
                SegmentElement::Info(i) => {
                    trace!("got info: {:#?}", i);
                    self.info = if self.info.is_none() {
                        Some(i)
                    } else {
                        return Err(Err::Error(custom_error(input, 1)));
                    };
                }
                SegmentElement::Tracks(t) => {
                    trace!("got tracks: {:#?}", t);
                    self.tracks = if self.tracks.is_none() {
                        let mut t = t;

                        // Only keep tracks we're interested in
                        if let Some(params) = &self.params {
                            if let Some(track_numbers) = &params.track_numbers {
                                t.tracks = t
                                    .tracks
                                    .into_iter()
                                    .filter(|tr| track_numbers.contains(&tr.track_number))
                                    .collect();
                            }
                        };

                        Some(t)
                    } else {
                        return Err(Err::Error(custom_error(input, 1)));
                    }
                }
                el => {
                    debug!("got element: {:#?}", el);
                }
            }

            input = i3;
        }
    }
}

impl Demuxer for MkvDemuxer {
    fn read_headers(&mut self, buf: &Box<dyn Buffered>, info: &mut GlobalInfo) -> Result<SeekFrom> {
        match self.parse_until_tracks(buf.data()) {
            Ok((i, _)) => {
                info.duration = self
                    .info
                    .as_ref()
                    .and_then(|info| info.duration)
                    .map(|d| d as u64);
                if let Some(ref mut t) = self.tracks {
                    for tr in t.tracks.iter_mut() {
                        info.add_stream(track_to_stream(self.info.as_ref().unwrap(), tr));
                        tr.stream_index = info.streams.last().unwrap().index;
                    }
                }
                Ok(SeekFrom::Current(buf.data().offset(i) as i64))
            }
            Err(Err::Incomplete(needed)) => {
                let sz = match needed {
                    Needed::Size(size) => buf.data().len() + usize::from(size),
                    _ => 1024,
                };
                Err(Error::MoreDataNeeded(sz))
            }
            e => {
                error!("{:?}", e);
                Err(Error::InvalidData)
            }
        }
    }

    fn read_event(&mut self, buf: &Box<dyn Buffered>) -> Result<(SeekFrom, Event)> {
        if let Some(event) = self.queue.pop_front() {
            Ok((SeekFrom::Current(0), event))
        } else {
            match segment_element(buf.data()) {
                Ok((i, element)) => {
                    let seek = SeekFrom::Current(buf.data().offset(i) as i64);
                    if let SegmentElement::Cluster(c) = element {
                        debug!("got cluster element at timecode: {}", c.timecode);
                        let mut packets = c.generate_packets(self.tracks.as_ref().unwrap());
                        self.queue.extend(packets.drain(..));
                        if let Some(event) = self.queue.pop_front() {
                            return Ok((seek, event));
                        }
                    }
                    Ok((seek, Event::MoreDataNeeded(0)))
                }
                Err(Err::Incomplete(Needed::Size(size))) => {
                    Err(Error::MoreDataNeeded(usize::from(size)))
                }
                e => {
                    error!("{:?}", e);
                    Err(Error::InvalidData)
                }
            }
        }
    }
}

fn track_entry_codec_id(t: &TrackEntry) -> Option<String> {
    // TODO: Support V_QUICKTIME and V_MS/VFW/FOURCC
    match t.codec_id.as_ref() {
        "A_OPUS" => Some("opus".to_owned()),
        "A_VORBIS" => Some("vorbis".to_owned()),
        "V_AV1" => Some("av1".to_owned()),
        "V_VP8" => Some("vp8".to_owned()),
        "V_VP9" => Some("vp9".to_owned()),
        _ => None,
    }
}

fn track_entry_video_kind(video: &Video) -> Option<MediaKind> {
    let v = VideoInfo {
        width: video.pixel_width as usize,
        height: video.pixel_height as usize,
        // TODO parse Colour and/or CodecPrivate to extract the format
        format: None,
    };
    Some(MediaKind::Video(v))
}

fn track_entry_audio_kind(audio: &Audio) -> Option<MediaKind> {
    let rate = audio
        .output_sampling_frequency
        .unwrap_or(audio.sampling_frequency);
    // TODO: complete it
    let map = if audio.channel_positions.is_none() {
        Some(ChannelMap::default_map(audio.channels as usize))
    } else {
        unimplemented!("Convert matroska map to rust-av map")
    };
    let a = AudioInfo {
        rate: rate as usize,
        map,
        format: None,
    };
    Some(MediaKind::Audio(a))
}

fn track_entry_media_kind(t: &TrackEntry) -> Option<MediaKind> {
    match t.track_type.into() {
        TrackType::Video => t.video.as_ref().and_then(track_entry_video_kind),
        TrackType::Audio => t.audio.as_ref().and_then(track_entry_audio_kind),
        TrackType::Other => None,
    }
}

pub fn track_to_stream(info: &Info, t: &TrackEntry) -> Stream {
    let num = t
        .track_timecode_scale
        .map_or(info.timecode_scale as i64, |ts| {
            if ts != 0. {
                (ts * info.timecode_scale as f64) as i64
            } else {
                info.timecode_scale as i64
            }
        });

    Stream {
        id: t.track_uid as isize,
        index: t.stream_index,
        start: None,
        duration: t.default_duration,
        timebase: Rational64::new(num, 1000 * 1000 * 1000),
        // TODO: Extend CodecParams and fill it with the remaining information
        params: CodecParams {
            extradata: t.codec_private.clone(),
            bit_rate: 0,
            delay: t.codec_delay.unwrap_or(0) as usize,
            convergence_window: t.seek_pre_roll.unwrap_or(0) as usize,
            codec_id: track_entry_codec_id(t),
            kind: track_entry_media_kind(t),
        },
        user_private: None,
    }
}

impl<'a> Cluster<'a> {
    pub fn generate_packets(&self, tracks: &Tracks) -> Vec<Event> {
        let mut v = Vec::new();

        for block_data in self.simple_block.iter() {
            if let Ok((i, block)) = simple_block(block_data) {
                debug!("parsing simple block: {:?}", block);
                if let Some(index) = tracks.lookup(block.track_number) {
                    let packet = Packet {
                        data: i.into(),
                        t: TimeInfo {
                            pts: Some(i64::from(block.timecode)),
                            dts: None,
                            duration: None,
                            timebase: None,
                            user_private: None,
                        },
                        pos: None,
                        stream_index: index as isize,
                        is_key: block.keyframe,
                        is_corrupted: false,
                    };

                    v.push(Event::NewPacket(packet));
                }
            } else {
                error!("error parsing simple block");
            }
        }

        v
    }
}

struct Des {
    d: Descr,
}

impl Descriptor for Des {
    fn create(&self) -> Box<dyn Demuxer> {
        Box::new(MkvDemuxer::new())
    }
    fn describe(&self) -> &Descr {
        &self.d
    }
    fn probe(&self, data: &[u8]) -> u8 {
        ebml_header(&data[..100]).map_or(0, |_| 100)
    }
}

pub const MKV_DESC: &dyn Descriptor = &Des {
    d: Descr {
        name: "matroska",
        demuxer: "mkv",
        description: "Nom-based Matroska demuxer",
        extensions: &["mkv", "webm", "mka"],
        mime: &["video/x-matroska", "audio/x-matroska"],
    },
};

#[cfg(test)]
#[allow(non_upper_case_globals)]
mod tests {
    use std::io::Cursor;

    use log::info;
    use nom::Offset;

    use av_format::{buffer::*, demuxer::Context};

    use super::*;

    const webm: &[u8] = include_bytes!("../assets/bbb-vp9-opus.webm");

    #[test]
    fn parse_headers() {
        let mut demuxer = MkvDemuxer::new();

        let res = demuxer.parse_until_tracks(webm);
        info!("got parsing res: {:?}", res);
        match res {
            Ok((i, _)) => {
                info!("offset: {} bytes", webm.offset(i));
            }
            e => {
                info!("could not parse: {:?}", e);
            }
        }

        info!("demuxer: {:#?}", demuxer);
    }

    #[test]
    fn parse_headers_incremental_buffer() {
        let mut demuxer = MkvDemuxer::new();

        for n in 100..2000 {
            let res = demuxer.parse_until_tracks(&webm[0..n]);
            match res {
                Ok(_) => info!("Size {} ok", n),
                Err(Err::Incomplete(needed)) => info!("Incomplete {} needs {:?}", n, needed),
                Err(e) => {
                    panic!("Error at size {}: {:?}", n, e);
                }
            }
        }
    }

    #[test]
    fn context() {
        let mut context = Context::new(
            Box::new(MkvDemuxer::new()),
            Box::new(AccReader::new(Cursor::new(webm))),
        );
        info!("read headers: {:?}", context.read_headers().unwrap());
        info!("streams: {:?}", context.info.streams);
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
        info!("event: {:?}", context.read_event().unwrap());
    }
}
<|endoftext|>