<commit_before>//! Simple module to store files in database.
//!
//! cratesfyi is generating more than 5 million files, they are small and mostly html files.
//! They are using so many inodes and it is better to store them in database instead of
//! filesystem. This module is adding files into database and retrieving them.

use std::path::{PathBuf, Path};
use postgres::Connection;
use rustc_serialize::json::{Json, ToJson};
use std::cmp;
use std::fs;
use std::io::Read;
use error::Result;
use failure::err_msg;
use rusoto_s3::{S3, PutObjectRequest, GetObjectRequest, S3Client};
use rusoto_core::region::Region;
use rusoto_credential::DefaultCredentialsProvider;
use std::ffi::OsStr;
#[cfg(not(windows))]
use magic::{Cookie, flags};

const MAX_CONCURRENT_UPLOADS: usize = 1000;

pub(super) static S3_BUCKET_NAME: &str = "rust-docs-rs";

fn get_file_list_from_dir<P: AsRef<Path>>(path: P,
                                          files: &mut Vec<PathBuf>)
                                          -> Result<()> {
    let path = path.as_ref();

    for file in path.read_dir()? {
        let file = file?;

        if file.file_type()?.is_file() {
            files.push(file.path());
        } else if file.file_type()?.is_dir() {
            get_file_list_from_dir(file.path(), files)?;
        }
    }

    Ok(())
}

fn get_file_list<P: AsRef<Path>>(path: P) -> Result<Vec<PathBuf>> {
    let path = path.as_ref();
    let mut files = Vec::new();

    if !path.exists() {
        return Err(err_msg("File not found"));
    } else if path.is_file() {
        files.push(PathBuf::from(path.file_name().unwrap()));
    } else if path.is_dir() {
        get_file_list_from_dir(path, &mut files)?;
        for file_path in &mut files {
            // We want the paths in this list to not be {path}/bar.txt but just bar.txt
            *file_path = PathBuf::from(file_path.strip_prefix(path).unwrap());
        }
    }

    Ok(files)
}

pub struct Blob {
    pub path: String,
    pub mime: String,
    pub date_updated: time::Timespec,
    pub content: Vec<u8>,
}

pub fn get_path(conn: &Connection, path: &str) -> Option<Blob> {
    if let Some(client) = s3_client() {
        let res = client.get_object(GetObjectRequest {
            bucket: S3_BUCKET_NAME.into(),
            key: path.into(),
            ..Default::default()
        }).sync();

        let res = match res {
            Ok(r) => r,
            Err(_) => {
                return None;
            }
        };

        let mut b = res.body.unwrap().into_blocking_read();
        let mut content = Vec::new();
        b.read_to_end(&mut content).unwrap();

        let last_modified = res.last_modified.unwrap();
        let last_modified = time::strptime(&last_modified, "%a, %d %b %Y %H:%M:%S %Z")
            .unwrap_or_else(|e| panic!("failed to parse {:?} as timespec: {:?}", last_modified, e))
            .to_timespec();

        Some(Blob {
            path: path.into(),
            mime: res.content_type.unwrap(),
            date_updated: last_modified,
            content,
        })
    } else {
        let rows = conn.query("SELECT path, mime, date_updated, content
                            FROM files
                            WHERE path = $1", &[&path]).unwrap();

        if rows.len() == 0 {
            None
        } else {
            let row = rows.get(0);

            Some(Blob {
                path: row.get(0),
                mime: row.get(1),
                date_updated: row.get(2),
                content: row.get(3),
            })
        }
    }
}

pub(super) fn s3_client() -> Option<S3Client> {
    // If AWS keys aren't configured, then presume we should use the DB exclusively
    // for file storage.
    if std::env::var_os("AWS_ACCESS_KEY_ID").is_none() && std::env::var_os("FORCE_S3").is_none() {
        return None;
    }
    let creds = match DefaultCredentialsProvider::new() {
        Ok(creds) => creds,
        Err(err) => {
            warn!("failed to retrieve AWS credentials: {}", err);
            return None;
        }
    };
    Some(S3Client::new_with(
        rusoto_core::request::HttpClient::new().unwrap(),
        creds,
        std::env::var("S3_ENDPOINT").ok().map(|e| Region::Custom {
            name: std::env::var("S3_REGION")
                .unwrap_or_else(|_| "us-west-1".to_owned()),
            endpoint: e,
        }).unwrap_or(Region::UsWest1),
    ))
}

/// Store all files in a directory and return [[mimetype, filename]] as Json
///
/// If there is an S3 Client configured, store files into an S3 bucket;
/// otherwise, stores files into the 'files' table of the local database.
///
/// The mimetype is detected using `magic`.
///
/// Note that this function is used for uploading both sources
/// and files generated by rustdoc.
pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
                                              prefix: &str,
                                              path: P)
                                              -> Result<Json> {
    let trans = conn.transaction()?;
    #[cfg(not(windows))]
    let mime_data = load_mime_data()?;
    use std::collections::HashMap;
    let mut file_paths_and_mimes: HashMap<PathBuf, String> = HashMap::new();
    use futures::future::Future;

    let mut rt = ::tokio::runtime::Runtime::new().unwrap();

    let mut to_upload = get_file_list(&path)?;
    let mut batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
    let mut currently_uploading: Vec<_> = to_upload.drain(..batch_size).collect();
    let mut attempts = 0;

    while !to_upload.is_empty() || !currently_uploading.is_empty() {

        let mut futures = Vec::new();
        let client = s3_client();

        for file_path in &currently_uploading {
            let path = Path::new(path.as_ref()).join(&file_path);
            // Some files have insufficient permissions (like .lock file created by cargo in
            // documentation directory). We are skipping this files.
            let mut file = match fs::File::open(path) {
                Ok(f) => f,
                Err(_) => continue,
            };
            let mut content: Vec<u8> = Vec::new();
            file.read_to_end(&mut content)?;

            let bucket_path = Path::new(prefix).join(&file_path);

            #[cfg(windows)] // On windows, we need to normalize \\ to / so the route logic works
            let bucket_path = path_slash::PathBufExt::to_slash(&bucket_path).unwrap();
            #[cfg(not(windows))]
            let bucket_path = bucket_path.into_os_string().into_string().unwrap();

            #[cfg(windows)]
            let mime = detect_mime(&content, &file_path)?;
            #[cfg(not(windows))]
            let mime = detect_mime(&content, &file_path, &mime_data)?;

            if let Some(client) = &client {
                futures.push(client.put_object(PutObjectRequest {
                    bucket: S3_BUCKET_NAME.into(),
                    key: bucket_path.clone(),
                    body: Some(content.clone().into()),
                    content_type: Some(mime.clone()),
                    ..Default::default()
                }).inspect(|_| {
                    crate::web::metrics::UPLOADED_FILES_TOTAL.inc_by(1);
                }));
            } else {
                // If AWS credentials are configured, don't insert/update the database
                // check if file already exists in database
                let rows = conn.query("SELECT COUNT(*) FROM files WHERE path = $1", &[&bucket_path])?;

                if rows.get(0).get::<usize, i64>(0) == 0 {
                    trans.query("INSERT INTO files (path, mime, content) VALUES ($1, $2, $3)",
                                    &[&bucket_path, &mime, &content])?;
                } else {
                    trans.query("UPDATE files SET mime = $2, content = $3, date_updated = NOW() \
                                    WHERE path = $1",
                                    &[&bucket_path, &mime, &content])?;
                }
            }

            file_paths_and_mimes.insert(file_path.clone(), mime.clone());
        }

        if !futures.is_empty() {
            attempts += 1;

            match rt.block_on(::futures::future::join_all(futures)) {
                Ok(_) => {
                    // this batch was successful, start another batch if there are still more files
                    batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
                    currently_uploading = to_upload.drain(..batch_size).collect();
                    attempts = 0;
                },
                Err(err) => {
                    error!("failed to upload to s3: {:?}", err);
                    // if any futures error, leave `currently_uploading` in tact so that we can retry the batch
                    if attempts > 2 {
                        panic!("failed to upload 3 times, exiting");
                    }
                }
            }
        } else {
            batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
            currently_uploading = to_upload.drain(..batch_size).collect();
        }
    }

    trans.commit()?;

    let file_list_with_mimes: Vec<(String, PathBuf)> = file_paths_and_mimes
        .into_iter()
        .map(|(file_path, mime)| (mime, file_path))
        .collect();
    file_list_to_json(file_list_with_mimes)
}

#[cfg(not(windows))]
fn load_mime_data() -> Result<Cookie> {
    let cookie = Cookie::open(flags::MIME_TYPE)?;
    cookie.load::<&str>(&[])?;
    Ok(cookie)
}

#[cfg(not(windows))]
fn detect_mime(content: &Vec<u8>, file_path: &Path, cookie: &Cookie) -> Result<String> {
    let mime = cookie.buffer(&content)?;
    correct_mime(&mime, &file_path)
}

#[cfg(windows)]
fn detect_mime(_content: &Vec<u8>, file_path: &Path) -> Result<String> {
    let mime = mime_guess::from_path(file_path).first_raw().map(|m| m).unwrap_or("text/plain");
    correct_mime(&mime, &file_path)
}

fn correct_mime(mime: &str, file_path: &Path) -> Result<String> {
    Ok(match mime {
        "text/plain" | "text/troff" | "text/x-markdown" | "text/x-rust" | "text/x-toml" => {
            match file_path.extension().and_then(OsStr::to_str) {
                Some("md") => "text/markdown",
                Some("rs") => "text/rust",
                Some("markdown") => "text/markdown",
                Some("css") => "text/css",
                Some("toml") => "text/toml",
                Some("js") => "application/javascript",
                Some("json") => "application/json",
                _ => mime
            }
        }
        _ => mime
    }.to_owned())
}

fn file_list_to_json(file_list: Vec<(String, PathBuf)>) -> Result<Json> {

    let mut file_list_json: Vec<Json> = Vec::new();

    for file in file_list {
        let mut v: Vec<String> = Vec::new();
        v.push(file.0.clone());
        v.push(file.1.into_os_string().into_string().unwrap());
        file_list_json.push(v.to_json());
    }

    Ok(file_list_json.to_json())
}

pub fn move_to_s3(conn: &Connection, n: usize) -> Result<usize> {
    let trans = conn.transaction()?;
    let client = s3_client().expect("configured s3");

    let rows = trans.query(
            &format!("SELECT path, mime, content FROM files WHERE content != E'in-s3' LIMIT {}", n),
            &[])?;
    let count = rows.len();

    let mut rt = ::tokio::runtime::Runtime::new().unwrap();
    let mut futures = Vec::new();
    for row in &rows {
        let path: String = row.get(0);
        let mime: String = row.get(1);
        let content: Vec<u8> = row.get(2);
        let path_1 = path.clone();
        futures.push(client.put_object(PutObjectRequest {
            bucket: S3_BUCKET_NAME.into(),
            key: path.clone(),
            body: Some(content.into()),
            content_type: Some(mime),
            ..Default::default()
        }).map(move |_| {
            path_1
        }).map_err(move |e| {
            panic!("failed to upload to {}: {:?}", path, e)
        }));
    }

    use ::futures::future::Future;
    match rt.block_on(::futures::future::join_all(futures)) {
        Ok(paths) => {
            let statement = trans.prepare("DELETE FROM files WHERE path = $1").unwrap();
            for path in paths {
                statement.execute(&[&path]).unwrap();
            }
        }
        Err(e) => {
            panic!("results err: {:?}", e);
        }
    }

    trans.commit()?;

    Ok(count)
}

#[cfg(test)]
mod test {
    extern crate env_logger;
    use std::env;
    use super::*;

    #[test]
    fn test_get_file_list() {
        let _ = env_logger::try_init();

        let files = get_file_list(env::current_dir().unwrap());
        assert!(files.is_ok());
        assert!(files.unwrap().len() > 0);

        let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap();
        assert_eq!(files[0], std::path::Path::new("Cargo.toml"));
    }
    #[test]
    fn test_mime_types() {
        check_mime("/ignored", ".gitignore", "text/plain");
        check_mime("[package]", "hello.toml","text/toml");
        check_mime(".ok { color:red; }", "hello.css","text/css");
        check_mime("var x = 1", "hello.js","application/javascript");
        check_mime("<html>", "hello.html","text/html");
        check_mime("## HELLO", "hello.hello.md","text/markdown");
        check_mime("## WORLD", "hello.markdown","text/markdown");
        check_mime("{}", "hello.json","application/json");
        check_mime("hello world", "hello.txt","text/plain");
        check_mime("//! Simple module to ...", "file.rs", "text/rust");
    }

    fn check_mime(content: &str, path: &str, expected_mime: &str) {
        #[cfg(not(windows))]
        let mime_data = load_mime_data().unwrap();
        #[cfg(windows)]
        let detected_mime = detect_mime(&content.as_bytes().to_vec(), Path::new(&path));
        #[cfg(not(windows))]
        let detected_mime = detect_mime(&content.as_bytes().to_vec(), Path::new(&path), &mime_data);
        let detected_mime = detected_mime.expect("no mime was given");
        assert_eq!(detected_mime, expected_mime);
    }
}

<commit_msg>Fix svg mimes not being saved correctly<commit_after>//! Simple module to store files in database.
//!
//! cratesfyi is generating more than 5 million files, they are small and mostly html files.
//! They are using so many inodes and it is better to store them in database instead of
//! filesystem. This module is adding files into database and retrieving them.

use std::path::{PathBuf, Path};
use postgres::Connection;
use rustc_serialize::json::{Json, ToJson};
use std::cmp;
use std::fs;
use std::io::Read;
use error::Result;
use failure::err_msg;
use rusoto_s3::{S3, PutObjectRequest, GetObjectRequest, S3Client};
use rusoto_core::region::Region;
use rusoto_credential::DefaultCredentialsProvider;
use std::ffi::OsStr;
#[cfg(not(windows))]
use magic::{Cookie, flags};

const MAX_CONCURRENT_UPLOADS: usize = 1000;

pub(super) static S3_BUCKET_NAME: &str = "rust-docs-rs";

fn get_file_list_from_dir<P: AsRef<Path>>(path: P,
                                          files: &mut Vec<PathBuf>)
                                          -> Result<()> {
    let path = path.as_ref();

    for file in path.read_dir()? {
        let file = file?;

        if file.file_type()?.is_file() {
            files.push(file.path());
        } else if file.file_type()?.is_dir() {
            get_file_list_from_dir(file.path(), files)?;
        }
    }

    Ok(())
}

fn get_file_list<P: AsRef<Path>>(path: P) -> Result<Vec<PathBuf>> {
    let path = path.as_ref();
    let mut files = Vec::new();

    if !path.exists() {
        return Err(err_msg("File not found"));
    } else if path.is_file() {
        files.push(PathBuf::from(path.file_name().unwrap()));
    } else if path.is_dir() {
        get_file_list_from_dir(path, &mut files)?;
        for file_path in &mut files {
            // We want the paths in this list to not be {path}/bar.txt but just bar.txt
            *file_path = PathBuf::from(file_path.strip_prefix(path).unwrap());
        }
    }

    Ok(files)
}

pub struct Blob {
    pub path: String,
    pub mime: String,
    pub date_updated: time::Timespec,
    pub content: Vec<u8>,
}

pub fn get_path(conn: &Connection, path: &str) -> Option<Blob> {
    if let Some(client) = s3_client() {
        let res = client.get_object(GetObjectRequest {
            bucket: S3_BUCKET_NAME.into(),
            key: path.into(),
            ..Default::default()
        }).sync();

        let res = match res {
            Ok(r) => r,
            Err(_) => {
                return None;
            }
        };

        let mut b = res.body.unwrap().into_blocking_read();
        let mut content = Vec::new();
        b.read_to_end(&mut content).unwrap();

        let last_modified = res.last_modified.unwrap();
        let last_modified = time::strptime(&last_modified, "%a, %d %b %Y %H:%M:%S %Z")
            .unwrap_or_else(|e| panic!("failed to parse {:?} as timespec: {:?}", last_modified, e))
            .to_timespec();

        Some(Blob {
            path: path.into(),
            mime: res.content_type.unwrap(),
            date_updated: last_modified,
            content,
        })
    } else {
        let rows = conn.query("SELECT path, mime, date_updated, content
                            FROM files
                            WHERE path = $1", &[&path]).unwrap();

        if rows.len() == 0 {
            None
        } else {
            let row = rows.get(0);

            Some(Blob {
                path: row.get(0),
                mime: row.get(1),
                date_updated: row.get(2),
                content: row.get(3),
            })
        }
    }
}

pub(super) fn s3_client() -> Option<S3Client> {
    // If AWS keys aren't configured, then presume we should use the DB exclusively
    // for file storage.
    if std::env::var_os("AWS_ACCESS_KEY_ID").is_none() && std::env::var_os("FORCE_S3").is_none() {
        return None;
    }
    let creds = match DefaultCredentialsProvider::new() {
        Ok(creds) => creds,
        Err(err) => {
            warn!("failed to retrieve AWS credentials: {}", err);
            return None;
        }
    };
    Some(S3Client::new_with(
        rusoto_core::request::HttpClient::new().unwrap(),
        creds,
        std::env::var("S3_ENDPOINT").ok().map(|e| Region::Custom {
            name: std::env::var("S3_REGION")
                .unwrap_or_else(|_| "us-west-1".to_owned()),
            endpoint: e,
        }).unwrap_or(Region::UsWest1),
    ))
}

/// Store all files in a directory and return [[mimetype, filename]] as Json
///
/// If there is an S3 Client configured, store files into an S3 bucket;
/// otherwise, stores files into the 'files' table of the local database.
///
/// The mimetype is detected using `magic`.
///
/// Note that this function is used for uploading both sources
/// and files generated by rustdoc.
pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
                                              prefix: &str,
                                              path: P)
                                              -> Result<Json> {
    let trans = conn.transaction()?;
    #[cfg(not(windows))]
    let mime_data = load_mime_data()?;
    use std::collections::HashMap;
    let mut file_paths_and_mimes: HashMap<PathBuf, String> = HashMap::new();
    use futures::future::Future;

    let mut rt = ::tokio::runtime::Runtime::new().unwrap();

    let mut to_upload = get_file_list(&path)?;
    let mut batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
    let mut currently_uploading: Vec<_> = to_upload.drain(..batch_size).collect();
    let mut attempts = 0;

    while !to_upload.is_empty() || !currently_uploading.is_empty() {

        let mut futures = Vec::new();
        let client = s3_client();

        for file_path in &currently_uploading {
            let path = Path::new(path.as_ref()).join(&file_path);
            // Some files have insufficient permissions (like .lock file created by cargo in
            // documentation directory). We are skipping this files.
            let mut file = match fs::File::open(path) {
                Ok(f) => f,
                Err(_) => continue,
            };
            let mut content: Vec<u8> = Vec::new();
            file.read_to_end(&mut content)?;

            let bucket_path = Path::new(prefix).join(&file_path);

            #[cfg(windows)] // On windows, we need to normalize \\ to / so the route logic works
            let bucket_path = path_slash::PathBufExt::to_slash(&bucket_path).unwrap();
            #[cfg(not(windows))]
            let bucket_path = bucket_path.into_os_string().into_string().unwrap();

            #[cfg(windows)]
            let mime = detect_mime(&content, &file_path)?;
            #[cfg(not(windows))]
            let mime = detect_mime(&content, &file_path, &mime_data)?;

            if let Some(client) = &client {
                futures.push(client.put_object(PutObjectRequest {
                    bucket: S3_BUCKET_NAME.into(),
                    key: bucket_path.clone(),
                    body: Some(content.clone().into()),
                    content_type: Some(mime.clone()),
                    ..Default::default()
                }).inspect(|_| {
                    crate::web::metrics::UPLOADED_FILES_TOTAL.inc_by(1);
                }));
            } else {
                // If AWS credentials are configured, don't insert/update the database
                // check if file already exists in database
                let rows = conn.query("SELECT COUNT(*) FROM files WHERE path = $1", &[&bucket_path])?;

                if rows.get(0).get::<usize, i64>(0) == 0 {
                    trans.query("INSERT INTO files (path, mime, content) VALUES ($1, $2, $3)",
                                    &[&bucket_path, &mime, &content])?;
                } else {
                    trans.query("UPDATE files SET mime = $2, content = $3, date_updated = NOW() \
                                    WHERE path = $1",
                                    &[&bucket_path, &mime, &content])?;
                }
            }

            file_paths_and_mimes.insert(file_path.clone(), mime.clone());
        }

        if !futures.is_empty() {
            attempts += 1;

            match rt.block_on(::futures::future::join_all(futures)) {
                Ok(_) => {
                    // this batch was successful, start another batch if there are still more files
                    batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
                    currently_uploading = to_upload.drain(..batch_size).collect();
                    attempts = 0;
                },
                Err(err) => {
                    error!("failed to upload to s3: {:?}", err);
                    // if any futures error, leave `currently_uploading` in tact so that we can retry the batch
                    if attempts > 2 {
                        panic!("failed to upload 3 times, exiting");
                    }
                }
            }
        } else {
            batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
            currently_uploading = to_upload.drain(..batch_size).collect();
        }
    }

    trans.commit()?;

    let file_list_with_mimes: Vec<(String, PathBuf)> = file_paths_and_mimes
        .into_iter()
        .map(|(file_path, mime)| (mime, file_path))
        .collect();
    file_list_to_json(file_list_with_mimes)
}

#[cfg(not(windows))]
fn load_mime_data() -> Result<Cookie> {
    let cookie = Cookie::open(flags::MIME_TYPE)?;
    cookie.load::<&str>(&[])?;
    Ok(cookie)
}

#[cfg(not(windows))]
fn detect_mime(content: &Vec<u8>, file_path: &Path, cookie: &Cookie) -> Result<String> {
    let mime = cookie.buffer(&content)?;
    correct_mime(&mime, &file_path)
}

#[cfg(windows)]
fn detect_mime(_content: &Vec<u8>, file_path: &Path) -> Result<String> {
    let mime = mime_guess::from_path(file_path).first_raw().map(|m| m).unwrap_or("text/plain");
    correct_mime(&mime, &file_path)
}

fn correct_mime(mime: &str, file_path: &Path) -> Result<String> {
    Ok(match mime {
        "text/plain" | "text/troff" | "text/x-markdown" | "text/x-rust" | "text/x-toml" => {
            match file_path.extension().and_then(OsStr::to_str) {
                Some("md") => "text/markdown",
                Some("rs") => "text/rust",
                Some("markdown") => "text/markdown",
                Some("css") => "text/css",
                Some("toml") => "text/toml",
                Some("js") => "application/javascript",
                Some("json") => "application/json",
                _ => mime
            }
        },
        "image/svg" => "image/svg+xml",
        _ => mime
    }.to_owned())
}

fn file_list_to_json(file_list: Vec<(String, PathBuf)>) -> Result<Json> {

    let mut file_list_json: Vec<Json> = Vec::new();

    for file in file_list {
        let mut v: Vec<String> = Vec::new();
        v.push(file.0.clone());
        v.push(file.1.into_os_string().into_string().unwrap());
        file_list_json.push(v.to_json());
    }

    Ok(file_list_json.to_json())
}

pub fn move_to_s3(conn: &Connection, n: usize) -> Result<usize> {
    let trans = conn.transaction()?;
    let client = s3_client().expect("configured s3");

    let rows = trans.query(
            &format!("SELECT path, mime, content FROM files WHERE content != E'in-s3' LIMIT {}", n),
            &[])?;
    let count = rows.len();

    let mut rt = ::tokio::runtime::Runtime::new().unwrap();
    let mut futures = Vec::new();
    for row in &rows {
        let path: String = row.get(0);
        let mime: String = row.get(1);
        let content: Vec<u8> = row.get(2);
        let path_1 = path.clone();
        futures.push(client.put_object(PutObjectRequest {
            bucket: S3_BUCKET_NAME.into(),
            key: path.clone(),
            body: Some(content.into()),
            content_type: Some(mime),
            ..Default::default()
        }).map(move |_| {
            path_1
        }).map_err(move |e| {
            panic!("failed to upload to {}: {:?}", path, e)
        }));
    }

    use ::futures::future::Future;
    match rt.block_on(::futures::future::join_all(futures)) {
        Ok(paths) => {
            let statement = trans.prepare("DELETE FROM files WHERE path = $1").unwrap();
            for path in paths {
                statement.execute(&[&path]).unwrap();
            }
        }
        Err(e) => {
            panic!("results err: {:?}", e);
        }
    }

    trans.commit()?;

    Ok(count)
}

#[cfg(test)]
mod test {
    extern crate env_logger;
    use std::env;
    use super::*;

    #[test]
    fn test_get_file_list() {
        let _ = env_logger::try_init();

        let files = get_file_list(env::current_dir().unwrap());
        assert!(files.is_ok());
        assert!(files.unwrap().len() > 0);

        let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap();
        assert_eq!(files[0], std::path::Path::new("Cargo.toml"));
    }
    #[test]
    fn test_mime_types() {
        check_mime("/ignored", ".gitignore", "text/plain");
        check_mime("[package]", "hello.toml","text/toml");
        check_mime(".ok { color:red; }", "hello.css","text/css");
        check_mime("var x = 1", "hello.js","application/javascript");
        check_mime("<html>", "hello.html","text/html");
        check_mime("## HELLO", "hello.hello.md","text/markdown");
        check_mime("## WORLD", "hello.markdown","text/markdown");
        check_mime("{}", "hello.json","application/json");
        check_mime("hello world", "hello.txt","text/plain");
        check_mime("//! Simple module to ...", "file.rs", "text/rust");
    }

    fn check_mime(content: &str, path: &str, expected_mime: &str) {
        #[cfg(not(windows))]
        let mime_data = load_mime_data().unwrap();
        #[cfg(windows)]
        let detected_mime = detect_mime(&content.as_bytes().to_vec(), Path::new(&path));
        #[cfg(not(windows))]
        let detected_mime = detect_mime(&content.as_bytes().to_vec(), Path::new(&path), &mime_data);
        let detected_mime = detected_mime.expect("no mime was given");
        assert_eq!(detected_mime, expected_mime);
    }
}

<|endoftext|>


<commit_before>use std::path::Path;
use std::fs::File;
use std::env;

use std::io::Write;

const MIN_DIMS: usize = 2;
const MIN_ORDER: usize = 2;

const MAX_DIMS: usize = 4;
const MAX_ORDER: usize = 6;

fn main() {
    let out = env::var("OUT_DIR").unwrap();
    let mut file = File::create(&Path::new(&out).join("macro_invocs.rs")).unwrap();

    let dim_tags = ['x', 'y', 'z', 'w'];

    // Create points and vectors
    for dim in MIN_DIMS..(MAX_DIMS + 1) {
        writeln!(file, "n_pointvector!{{{0}; Point{0}d, Vector{0}d {{", dim).unwrap();

        for (i, dt) in dim_tags[0..dim].iter().enumerate() {
            write!(file, "    {}", dt).unwrap();

            if i == dim - 1 {
                writeln!(file, "")
            } else {
                writeln!(file, ",")
            }.unwrap();
        }

        writeln!(file, "}}}}").unwrap();
    }

    // Create one-dimensional bezier polynomials
    for order in MIN_ORDER..(MAX_ORDER + 1) {
        writeln!(file, "n_bezier!{{BezPoly{}o {{", order).unwrap();
        for o in 0..(order + 1) {
            write!(file, "    {}: {}", get_param_name(o, order), combination(order, o)).unwrap();

            // Insert commas necessary to seperate parameter names.
            if o != order {
                writeln!(file, ",")
            } else {
                writeln!(file, "")
            }.unwrap();
        }

        // Order of the derivative of the polynomial
        let dorder = order - 1;

        writeln!(file, "}} derived {{").unwrap();
        for o in 0..order {
            write!(file, "    {} - {}: {}", get_param_name(o+1, order), get_param_name(o, order), combination(dorder, o)).unwrap();

            if o != dorder {
                writeln!(file, ",")
            } else {
                writeln!(file, "")
            }.unwrap();
        }
        if order == MAX_ORDER {
            writeln!(file, "}} elevated NBezPoly<F, [F; {}]> {{", order + 2).unwrap();
        } else {
            writeln!(file, "}} elevated BezPoly{}o<F> {{", order + 1).unwrap();
        }

        writeln!(file, "    start;").unwrap();
        for o in 0..order {
            write!(file, "    {} + {}", get_param_name(o, order), get_param_name(o+1, order)).unwrap();
            if o == order - 1 {
                writeln!(file, ";")
            } else {
                writeln!(file, ",")
            }.unwrap();
        }
        writeln!(file, "    end;\n}}}}").unwrap();
    }

    // Create composite curves
    for dim in MIN_DIMS..(MAX_DIMS + 1) {
        for order in MIN_ORDER..(MAX_ORDER + 1) {
            writeln!(file, "bez_composite!{{Bez{0}o{1}d<BezPoly{0}o> {{", order, dim).unwrap();

            for o in 0..(order + 1) {
                let param_name = get_param_name(o, order);
                write!(file, "    {}:", param_name).unwrap();

                for dt in &dim_tags[0..dim] {
                    write!(file, " {}_{}", dt, param_name).unwrap();

                    if *dt != dim_tags[dim-1] {
                        write!(file, ",").unwrap();
                    }
                }
                writeln!(file, ";").unwrap();
            }

            writeln!(file, "}} -> <Point{0}d; Vector{0}d> {{", dim).unwrap();

            for dt in &dim_tags[0..dim] {
                write!(file, "    {} =", dt).unwrap();

                for o in 0..(order + 1) {
                    write!(file, " {}", get_param_name(o, order)).unwrap();
                    if o != order {
                        write!(file, ",").unwrap();
                    }
                }
                writeln!(file, ";").unwrap();
            }

            if order == MAX_ORDER {
                writeln!(file, "}} elevated NBez<Point{0}d<F>, Vector{0}d<F>, F, [Point{0}d<F>; {1}]> {{", dim, order + 2).unwrap();
            } else {
                writeln!(file, "}} elevated Bez{}o{}d<F> {{", order + 1, dim).unwrap();
            }
            for o in 0..(order + 2) {
                write!(file, "    {} =>", o).unwrap();
                for dt in &dim_tags[0..dim] {
                    write!(file, " {}", dt).unwrap();
                    if *dt != dim_tags[dim-1] {
                        write!(file, ",").unwrap();
                    }
                }
                writeln!(file, ";").unwrap();
            }
            writeln!(file, "}} chained BezChain{}o{}d}}", order, dim).unwrap();
        }
    }
}

fn get_param_name(param_number: usize, poly_order: usize) -> String {
    if 0 == param_number {
        "start".to_owned()
    } else if param_number == poly_order {
        "end".to_owned()
    } else if 2 == poly_order {
        "ctrl".to_owned()
    } else {
        format!("ctrl{}", param_number - 1)
    }
}

fn combination(n: usize, k: usize) -> usize {
    factorial(n) / (factorial(k) * factorial(n - k))
}

fn factorial(n: usize) -> usize {
    match n {
        0 => 1,
        _ => n * factorial(n-1)
    }
}<commit_msg>Lowered minimum bezier curve order<commit_after>use std::path::Path;
use std::fs::File;
use std::env;

use std::io::Write;

const MIN_DIMS: usize = 2;
const MIN_ORDER: usize = 1;

const MAX_DIMS: usize = 4;
const MAX_ORDER: usize = 6;

fn main() {
    let out = env::var("OUT_DIR").unwrap();
    let mut file = File::create(&Path::new(&out).join("macro_invocs.rs")).unwrap();

    let dim_tags = ['x', 'y', 'z', 'w'];

    // Create points and vectors
    for dim in MIN_DIMS..(MAX_DIMS + 1) {
        writeln!(file, "n_pointvector!{{{0}; Point{0}d, Vector{0}d {{", dim).unwrap();

        for (i, dt) in dim_tags[0..dim].iter().enumerate() {
            write!(file, "    {}", dt).unwrap();

            if i == dim - 1 {
                writeln!(file, "")
            } else {
                writeln!(file, ",")
            }.unwrap();
        }

        writeln!(file, "}}}}").unwrap();
    }

    // Create one-dimensional bezier polynomials
    for order in MIN_ORDER..(MAX_ORDER + 1) {
        writeln!(file, "n_bezier!{{BezPoly{}o {{", order).unwrap();
        for o in 0..(order + 1) {
            write!(file, "    {}: {}", get_param_name(o, order), combination(order, o)).unwrap();

            // Insert commas necessary to seperate parameter names.
            if o != order {
                writeln!(file, ",")
            } else {
                writeln!(file, "")
            }.unwrap();
        }

        // Order of the derivative of the polynomial
        let dorder = order - 1;

        writeln!(file, "}} derived {{").unwrap();
        for o in 0..order {
            write!(file, "    {} - {}: {}", get_param_name(o+1, order), get_param_name(o, order), combination(dorder, o)).unwrap();

            if o != dorder {
                writeln!(file, ",")
            } else {
                writeln!(file, "")
            }.unwrap();
        }
        if order == MAX_ORDER {
            writeln!(file, "}} elevated NBezPoly<F, [F; {}]> {{", order + 2).unwrap();
        } else {
            writeln!(file, "}} elevated BezPoly{}o<F> {{", order + 1).unwrap();
        }

        writeln!(file, "    start;").unwrap();
        for o in 0..order {
            write!(file, "    {} + {}", get_param_name(o, order), get_param_name(o+1, order)).unwrap();
            if o == order - 1 {
                writeln!(file, ";")
            } else {
                writeln!(file, ",")
            }.unwrap();
        }
        writeln!(file, "    end;\n}}}}").unwrap();
    }

    // Create composite curves
    for dim in MIN_DIMS..(MAX_DIMS + 1) {
        for order in MIN_ORDER..(MAX_ORDER + 1) {
            writeln!(file, "bez_composite!{{Bez{0}o{1}d<BezPoly{0}o> {{", order, dim).unwrap();

            for o in 0..(order + 1) {
                let param_name = get_param_name(o, order);
                write!(file, "    {}:", param_name).unwrap();

                for dt in &dim_tags[0..dim] {
                    write!(file, " {}_{}", dt, param_name).unwrap();

                    if *dt != dim_tags[dim-1] {
                        write!(file, ",").unwrap();
                    }
                }
                writeln!(file, ";").unwrap();
            }

            writeln!(file, "}} -> <Point{0}d; Vector{0}d> {{", dim).unwrap();

            for dt in &dim_tags[0..dim] {
                write!(file, "    {} =", dt).unwrap();

                for o in 0..(order + 1) {
                    write!(file, " {}", get_param_name(o, order)).unwrap();
                    if o != order {
                        write!(file, ",").unwrap();
                    }
                }
                writeln!(file, ";").unwrap();
            }

            if order == MAX_ORDER {
                writeln!(file, "}} elevated NBez<Point{0}d<F>, Vector{0}d<F>, F, [Point{0}d<F>; {1}]> {{", dim, order + 2).unwrap();
            } else {
                writeln!(file, "}} elevated Bez{}o{}d<F> {{", order + 1, dim).unwrap();
            }
            for o in 0..(order + 2) {
                write!(file, "    {} =>", o).unwrap();
                for dt in &dim_tags[0..dim] {
                    write!(file, " {}", dt).unwrap();
                    if *dt != dim_tags[dim-1] {
                        write!(file, ",").unwrap();
                    }
                }
                writeln!(file, ";").unwrap();
            }
            writeln!(file, "}} chained BezChain{}o{}d}}", order, dim).unwrap();
        }
    }
}

fn get_param_name(param_number: usize, poly_order: usize) -> String {
    if 0 == param_number {
        "start".to_owned()
    } else if param_number == poly_order {
        "end".to_owned()
    } else if 2 == poly_order {
        "ctrl".to_owned()
    } else {
        format!("ctrl{}", param_number - 1)
    }
}

fn combination(n: usize, k: usize) -> usize {
    factorial(n) / (factorial(k) * factorial(n - k))
}

fn factorial(n: usize) -> usize {
    match n {
        0 => 1,
        _ => n * factorial(n-1)
    }
}<|endoftext|>


<commit_before>extern crate gcc;
#[macro_use] extern crate lazy_static;
extern crate regex;
extern crate semver;

use regex::Regex;
use semver::Version;
use std::env;
use std::ffi::OsStr;
use std::io::{self, ErrorKind};
use std::path::PathBuf;
use std::process::Command;

lazy_static!{
    /// LLVM version used by this version of the crate.
    static ref CRATE_VERSION: Version = {
        let crate_version = Version::parse(env!("CARGO_PKG_VERSION"))
            .expect("Crate version is somehow not valid semver");
        Version {
            major: crate_version.major / 10,
            minor: crate_version.major % 10,
            .. crate_version
        }
    };

    /// Filesystem path to an llvm-config binary for the correct version.
    static ref LLVM_CONFIG_PATH: PathBuf = {
        // Try llvm-config via PATH first.
        if let Some(name) = locate_system_llvm_config() {
            return name.into();
        } else {
            println!("Didn't find usable system-wide LLVM.");
        }

        // Did the user give us a binary path to use? If yes, try
        // to use that and fail if it doesn't work.
        let binary_prefix_var = format!("LLVM_SYS_{}_PREFIX",
                                        env!("CARGO_PKG_VERSION_MAJOR"));
        if let Some(path) = env::var_os(&binary_prefix_var) {
            let mut pb: PathBuf = path.into();
            pb.push("bin");
            pb.push("llvm-config");

            let ver = llvm_version(&pb)
                .expect(&format!("Failed to execute {:?}", &pb));
            if is_compatible_llvm(&ver) {
                return pb;
            } else {
                panic!("LLVM binaries specified by {} are the wrong version.
                        (Found {}, need {}.)", binary_prefix_var, ver, *CRATE_VERSION);
            }
        }

        println!("No suitable version of LLVM was found system-wide or pointed
                  to by {}.

                  Consider using `llvmenv` to compile an appropriate copy of LLVM, and
                  refer to the llvm-sys documentation for more information.

                  llvm-sys: https://crates.io/crates/llvm-sys
                  llvmenv: https://crates.io/crates/llvmenv", binary_prefix_var);
        panic!("Could not find a compatible version of LLVM");
    };
}

/// Try to find a system-wide version of llvm-config that is compatible with
/// this crate.
///
/// Returns None on failure.
fn locate_system_llvm_config() -> Option<&'static str> {
    match llvm_version("llvm-config") {
        Ok(ref version) if is_compatible_llvm(version) => {
            // Compatible version found. Nice.
            return Some("llvm-config");
        },
        Ok(version) => {
            // Version mismatch. Will try further searches, but warn that
            // we're not using the system one.
            println!("Found LLVM version {} on PATH, but need {}.",
                     version, *CRATE_VERSION);
        },
        Err(ref e) if e.kind() == ErrorKind::NotFound => {
            // Looks like we failed to execute any llvm-config. Keep
            // searching.
        },
        // Some other error, probably a weird failure. Give up.
        Err(e) => panic!("Failed to search PATH for llvm-config: {}", e),
    }

    None
}

/// Check whether the given version of LLVM is blacklisted,
/// returning `Some(reason)` if it is.
fn is_blacklisted_llvm(llvm_version: &Version) -> Option<&'static str> {
    static BLACKLIST: &'static [(u64, u64, u64, &'static str)] = &[];

    let blacklist_var = format!("LLVM_SYS_{}_IGNORE_BLACKLIST",
                                env!("CARGO_PKG_VERSION_MAJOR"));
    if let Some(x) = env::var_os(&blacklist_var) {
        if &x == "YES" {
            println!("cargo:warning=Ignoring blacklist entry for LLVM {}", llvm_version);
            return None;
        } else {
            println!("cargo:warning={} is set but not exactly \"YES\"; blacklist is still honored.",
                     &blacklist_var);
        }
    }

    for &(major, minor, patch, reason) in BLACKLIST.iter() {
        let bad_version = Version {
            major: major, minor: minor, patch: patch,
            pre: vec![], build: vec![],
        };

        if &bad_version == llvm_version {
            return Some(reason);
        }
    }
    None
}

/// Check whether the given LLVM version is compatible with this version of
/// the crate.
fn is_compatible_llvm(llvm_version: &Version) -> bool {
    if let Some(reason) = is_blacklisted_llvm(llvm_version) {
        println!("Found LLVM {}, which is blacklisted: {}", llvm_version, reason);
        return false;
    }

    let strict = env::var_os(format!("LLVM_SYS_{}_STRICT_VERSIONING",
                                     env!("CARGO_PKG_VERSION_MAJOR"))).is_some()
        || cfg!(feature="strict-versioning");
    if strict {
        llvm_version.major == CRATE_VERSION.major &&
            llvm_version.minor == CRATE_VERSION.minor
    } else {
        llvm_version.major >= CRATE_VERSION.major ||
            (llvm_version.major == CRATE_VERSION.major &&
             llvm_version.minor >= CRATE_VERSION.minor)
    }
}

/// Get the output from running `llvm-config` with the given argument.
///
/// Lazily searches for or compiles LLVM as configured by the environment
/// variables.
fn llvm_config(arg: &str) -> String {
    llvm_config_ex(&*LLVM_CONFIG_PATH, arg)
        .expect("Surprising failure from llvm-config")
}

/// Invoke the specified binary as llvm-config.
///
/// Explicit version of the `llvm_config` function that bubbles errors
/// up.
fn llvm_config_ex<S: AsRef<OsStr>>(binary: S, arg: &str)
        -> io::Result<String> {
    Command::new(binary)
        .arg(arg)
        .arg("--link-static")   // Don't use dylib for >= 3.9
        .output()
        .map(|output| String::from_utf8(output.stdout)
            .expect("Output from llvm-config was not valid UTF-8"))
}

/// Get the LLVM version using llvm-config.
fn llvm_version<S: AsRef<OsStr>>(binary: S) -> io::Result<Version> {
    let version_str = try!(llvm_config_ex(binary.as_ref(), "--version"));

    // LLVM isn't really semver and uses version suffixes to build
    // version strings like '3.8.0svn', so limit what we try to parse
    // to only the numeric bits.
    let re = Regex::new(r"^(?P<major>\d+)\.(?P<minor>\d+)(?:\.(?P<patch>\d+))??")
        .unwrap();
    let c = re.captures(&version_str).expect(
        "Could not determine LLVM version from llvm-config.",
    );

    // some systems don't have a patch number but Version wants it so we just append .0 if it isn't
    // there
    let s = match c.name("patch") {
        None => format!("{}.0", &c[0]),
        Some(_) => c[0].to_string()
    };
    Ok(Version::parse(&s).unwrap())
}

/// Get the names of the dylibs required by LLVM, including the C++ standard
/// library.
fn get_system_libraries() -> Vec<String> {
    llvm_config("--system-libs")
        .split(&[' ', '\n'] as &[char])
        .filter(|s| !s.is_empty())
        .map(|flag| {
            if cfg!(target_env = "msvc") {
                // Same as --libnames, foo.lib
                assert!(flag.ends_with(".lib"));
                &flag[..flag.len() - 4]
            } else {
                // Linker flags style, -lfoo
                assert!(flag.starts_with("-l"));
                &flag[2..]
            }
        })
        .chain(get_system_libcpp())
        .map(str::to_owned)
        .collect::<Vec<String>>()
}

/// Get the library that must be linked for C++, if any.
fn get_system_libcpp() -> Option<&'static str> {
    if cfg!(target_env = "msvc") {
        // MSVC doesn't need an explicit one.
        None
    } else if cfg!(target_os = "macos") {
        // On OS X 10.9 and later, LLVM's libc++ is the default. On earlier
        // releases GCC's libstdc++ is default. Unfortunately we can't
        // reasonably detect which one we need (on older ones libc++ is
        // available and can be selected with -stdlib=lib++), so assume the
        // latest, at the cost of breaking the build on older OS releases
        // when LLVM was built against libstdc++.
        Some("c++")
    } else {
        // Otherwise assume GCC's libstdc++.
        // This assumption is probably wrong on some platforms, but would need
        // testing on them.
        Some("stdc++")
    }
}

/// Get the names of libraries to link against.
fn get_link_libraries() -> Vec<String> {
    // Using --libnames in conjunction with --libdir is particularly important
    // for MSVC when LLVM is in a path with spaces, but it is generally less of
    // a hack than parsing linker flags output from --libs and --ldflags.
    llvm_config("--libnames")
        .split(&[' ', '\n'] as &[char])
        .filter(|s| !s.is_empty())
        .map(|name| {
            // --libnames gives library filenames. Extract only the name that
            // we need to pass to the linker.
            if cfg!(target_env = "msvc") {
                // LLVMfoo.lib
                assert!(name.ends_with(".lib"));
                &name[..name.len() - 4]
            } else {
                // libLLVMfoo.a
                assert!(name.starts_with("lib") && name.ends_with(".a"));
                &name[3..name.len() - 2]
            }
        })
        .map(str::to_owned)
        .collect::<Vec<String>>()
}

fn get_llvm_cflags() -> String {
    let output = llvm_config("--cflags");

    // llvm-config includes cflags from its own compilation with --cflags that
    // may not be relevant to us. In particularly annoying cases, these might
    // include flags that aren't understood by the default compiler we're
    // using. Unless requested otherwise, clean CFLAGS of options that are
    // known to be possibly-harmful.
    let no_clean = env::var_os(format!("LLVM_SYS_{}_NO_CLEAN_CFLAGS",
                                       env!("CARGO_PKG_VERSION_MAJOR"))).is_some();
    if no_clean || cfg!(target_env = "msvc") {
        // MSVC doesn't accept -W... options, so don't try to strip them and
        // possibly strip something that should be retained. Also do nothing if
        // the user requests it.
        return output;
    }

    llvm_config("--cflags").split(&[' ', '\n'][..])
        .filter(|word| !word.starts_with("-W"))
        .collect::<Vec<_>>()
        .join(" ")
}

fn main() {
    if cfg!(feature = "no-llvm-linking") {
        return;
    }

    let libdir = llvm_config("--libdir");

    // Export information to other crates
    println!("cargo:config_path={}", LLVM_CONFIG_PATH.display()); // will be DEP_LLVM_CONFIG_PATH
    println!("cargo:libdir={}", libdir); // DEP_LLVM_LIBDIR

    // Link LLVM libraries
    println!("cargo:rustc-link-search=native={}", libdir);
    for name in get_link_libraries() {
        println!("cargo:rustc-link-lib=static={}", name);
    }

    // Link system libraries
    for name in get_system_libraries() {
        println!("cargo:rustc-link-lib=dylib={}", name);
    }

    // Link libffi if the user requested this workaround.
    // See https://bitbucket.org/tari/llvm-sys.rs/issues/12/
    let force_ffi = env::var_os(format!("LLVM_SYS_{}_FFI_WORKAROUND",
                                        env!("CARGO_PKG_VERSION_MAJOR")))
        .is_some();
    if force_ffi {
        println!("cargo:rustc-link-lib=dylib={}", "ffi");
    }

    // Build the extra wrapper functions.
    std::env::set_var("CFLAGS", get_llvm_cflags());
    gcc::Build::new()
        .file("wrappers/target.c")
        .compile("targetwrappers");
}
<commit_msg>Correct empty lines in multiline println<commit_after>extern crate gcc;
#[macro_use] extern crate lazy_static;
extern crate regex;
extern crate semver;

use regex::Regex;
use semver::Version;
use std::env;
use std::ffi::OsStr;
use std::io::{self, ErrorKind};
use std::path::PathBuf;
use std::process::Command;

lazy_static!{
    /// LLVM version used by this version of the crate.
    static ref CRATE_VERSION: Version = {
        let crate_version = Version::parse(env!("CARGO_PKG_VERSION"))
            .expect("Crate version is somehow not valid semver");
        Version {
            major: crate_version.major / 10,
            minor: crate_version.major % 10,
            .. crate_version
        }
    };

    /// Filesystem path to an llvm-config binary for the correct version.
    static ref LLVM_CONFIG_PATH: PathBuf = {
        // Try llvm-config via PATH first.
        if let Some(name) = locate_system_llvm_config() {
            return name.into();
        } else {
            println!("Didn't find usable system-wide LLVM.");
        }

        // Did the user give us a binary path to use? If yes, try
        // to use that and fail if it doesn't work.
        let binary_prefix_var = format!("LLVM_SYS_{}_PREFIX",
                                        env!("CARGO_PKG_VERSION_MAJOR"));
        if let Some(path) = env::var_os(&binary_prefix_var) {
            let mut pb: PathBuf = path.into();
            pb.push("bin");
            pb.push("llvm-config");

            let ver = llvm_version(&pb)
                .expect(&format!("Failed to execute {:?}", &pb));
            if is_compatible_llvm(&ver) {
                return pb;
            } else {
                panic!("LLVM binaries specified by {} are the wrong version.
                        (Found {}, need {}.)", binary_prefix_var, ver, *CRATE_VERSION);
            }
        }

        println!("No suitable version of LLVM was found system-wide or pointed
                  to by {}.

                  Consider using `llvmenv` to compile an appropriate copy of LLVM, and
                  refer to the llvm-sys documentation for more information.

                  llvm-sys: https://crates.io/crates/llvm-sys
                  llvmenv: https://crates.io/crates/llvmenv", binary_prefix_var);
        panic!("Could not find a compatible version of LLVM");
    };
}

/// Try to find a system-wide version of llvm-config that is compatible with
/// this crate.
///
/// Returns None on failure.
fn locate_system_llvm_config() -> Option<&'static str> {
    match llvm_version("llvm-config") {
        Ok(ref version) if is_compatible_llvm(version) => {
            // Compatible version found. Nice.
            return Some("llvm-config");
        },
        Ok(version) => {
            // Version mismatch. Will try further searches, but warn that
            // we're not using the system one.
            println!("Found LLVM version {} on PATH, but need {}.",
                     version, *CRATE_VERSION);
        },
        Err(ref e) if e.kind() == ErrorKind::NotFound => {
            // Looks like we failed to execute any llvm-config. Keep
            // searching.
        },
        // Some other error, probably a weird failure. Give up.
        Err(e) => panic!("Failed to search PATH for llvm-config: {}", e),
    }

    None
}

/// Check whether the given version of LLVM is blacklisted,
/// returning `Some(reason)` if it is.
fn is_blacklisted_llvm(llvm_version: &Version) -> Option<&'static str> {
    static BLACKLIST: &'static [(u64, u64, u64, &'static str)] = &[];

    let blacklist_var = format!("LLVM_SYS_{}_IGNORE_BLACKLIST",
                                env!("CARGO_PKG_VERSION_MAJOR"));
    if let Some(x) = env::var_os(&blacklist_var) {
        if &x == "YES" {
            println!("cargo:warning=Ignoring blacklist entry for LLVM {}", llvm_version);
            return None;
        } else {
            println!("cargo:warning={} is set but not exactly \"YES\"; blacklist is still honored.",
                     &blacklist_var);
        }
    }

    for &(major, minor, patch, reason) in BLACKLIST.iter() {
        let bad_version = Version {
            major: major, minor: minor, patch: patch,
            pre: vec![], build: vec![],
        };

        if &bad_version == llvm_version {
            return Some(reason);
        }
    }
    None
}

/// Check whether the given LLVM version is compatible with this version of
/// the crate.
fn is_compatible_llvm(llvm_version: &Version) -> bool {
    if let Some(reason) = is_blacklisted_llvm(llvm_version) {
        println!("Found LLVM {}, which is blacklisted: {}", llvm_version, reason);
        return false;
    }

    let strict = env::var_os(format!("LLVM_SYS_{}_STRICT_VERSIONING",
                                     env!("CARGO_PKG_VERSION_MAJOR"))).is_some()
        || cfg!(feature="strict-versioning");
    if strict {
        llvm_version.major == CRATE_VERSION.major &&
            llvm_version.minor == CRATE_VERSION.minor
    } else {
        llvm_version.major >= CRATE_VERSION.major ||
            (llvm_version.major == CRATE_VERSION.major &&
             llvm_version.minor >= CRATE_VERSION.minor)
    }
}

/// Get the output from running `llvm-config` with the given argument.
///
/// Lazily searches for or compiles LLVM as configured by the environment
/// variables.
fn llvm_config(arg: &str) -> String {
    llvm_config_ex(&*LLVM_CONFIG_PATH, arg)
        .expect("Surprising failure from llvm-config")
}

/// Invoke the specified binary as llvm-config.
///
/// Explicit version of the `llvm_config` function that bubbles errors
/// up.
fn llvm_config_ex<S: AsRef<OsStr>>(binary: S, arg: &str)
        -> io::Result<String> {
    Command::new(binary)
        .arg(arg)
        .arg("--link-static")   // Don't use dylib for >= 3.9
        .output()
        .map(|output| String::from_utf8(output.stdout)
            .expect("Output from llvm-config was not valid UTF-8"))
}

/// Get the LLVM version using llvm-config.
fn llvm_version<S: AsRef<OsStr>>(binary: S) -> io::Result<Version> {
    let version_str = try!(llvm_config_ex(binary.as_ref(), "--version"));

    // LLVM isn't really semver and uses version suffixes to build
    // version strings like '3.8.0svn', so limit what we try to parse
    // to only the numeric bits.
    let re = Regex::new(r"^(?P<major>\d+)\.(?P<minor>\d+)(?:\.(?P<patch>\d+))??")
        .unwrap();
    let c = re.captures(&version_str).expect(
        "Could not determine LLVM version from llvm-config.",
    );

    // some systems don't have a patch number but Version wants it so we just append .0 if it isn't
    // there
    let s = match c.name("patch") {
        None => format!("{}.0", &c[0]),
        Some(_) => c[0].to_string()
    };
    Ok(Version::parse(&s).unwrap())
}

/// Get the names of the dylibs required by LLVM, including the C++ standard
/// library.
fn get_system_libraries() -> Vec<String> {
    llvm_config("--system-libs")
        .split(&[' ', '\n'] as &[char])
        .filter(|s| !s.is_empty())
        .map(|flag| {
            if cfg!(target_env = "msvc") {
                // Same as --libnames, foo.lib
                assert!(flag.ends_with(".lib"));
                &flag[..flag.len() - 4]
            } else {
                // Linker flags style, -lfoo
                assert!(flag.starts_with("-l"));
                &flag[2..]
            }
        })
        .chain(get_system_libcpp())
        .map(str::to_owned)
        .collect::<Vec<String>>()
}

/// Get the library that must be linked for C++, if any.
fn get_system_libcpp() -> Option<&'static str> {
    if cfg!(target_env = "msvc") {
        // MSVC doesn't need an explicit one.
        None
    } else if cfg!(target_os = "macos") {
        // On OS X 10.9 and later, LLVM's libc++ is the default. On earlier
        // releases GCC's libstdc++ is default. Unfortunately we can't
        // reasonably detect which one we need (on older ones libc++ is
        // available and can be selected with -stdlib=lib++), so assume the
        // latest, at the cost of breaking the build on older OS releases
        // when LLVM was built against libstdc++.
        Some("c++")
    } else {
        // Otherwise assume GCC's libstdc++.
        // This assumption is probably wrong on some platforms, but would need
        // testing on them.
        Some("stdc++")
    }
}

/// Get the names of libraries to link against.
fn get_link_libraries() -> Vec<String> {
    // Using --libnames in conjunction with --libdir is particularly important
    // for MSVC when LLVM is in a path with spaces, but it is generally less of
    // a hack than parsing linker flags output from --libs and --ldflags.
    llvm_config("--libnames")
        .split(&[' ', '\n'] as &[char])
        .filter(|s| !s.is_empty())
        .map(|name| {
            // --libnames gives library filenames. Extract only the name that
            // we need to pass to the linker.
            if cfg!(target_env = "msvc") {
                // LLVMfoo.lib
                assert!(name.ends_with(".lib"));
                &name[..name.len() - 4]
            } else {
                // libLLVMfoo.a
                assert!(name.starts_with("lib") && name.ends_with(".a"));
                &name[3..name.len() - 2]
            }
        })
        .map(str::to_owned)
        .collect::<Vec<String>>()
}

fn get_llvm_cflags() -> String {
    let output = llvm_config("--cflags");

    // llvm-config includes cflags from its own compilation with --cflags that
    // may not be relevant to us. In particularly annoying cases, these might
    // include flags that aren't understood by the default compiler we're
    // using. Unless requested otherwise, clean CFLAGS of options that are
    // known to be possibly-harmful.
    let no_clean = env::var_os(format!("LLVM_SYS_{}_NO_CLEAN_CFLAGS",
                                       env!("CARGO_PKG_VERSION_MAJOR"))).is_some();
    if no_clean || cfg!(target_env = "msvc") {
        // MSVC doesn't accept -W... options, so don't try to strip them and
        // possibly strip something that should be retained. Also do nothing if
        // the user requests it.
        return output;
    }

    llvm_config("--cflags").split(&[' ', '\n'][..])
        .filter(|word| !word.starts_with("-W"))
        .collect::<Vec<_>>()
        .join(" ")
}

fn main() {
    if cfg!(feature = "no-llvm-linking") {
        return;
    }

    let libdir = llvm_config("--libdir");

    // Export information to other crates
    println!("cargo:config_path={}", LLVM_CONFIG_PATH.display()); // will be DEP_LLVM_CONFIG_PATH
    println!("cargo:libdir={}", libdir); // DEP_LLVM_LIBDIR

    // Link LLVM libraries
    println!("cargo:rustc-link-search=native={}", libdir);
    for name in get_link_libraries() {
        println!("cargo:rustc-link-lib=static={}", name);
    }

    // Link system libraries
    for name in get_system_libraries() {
        println!("cargo:rustc-link-lib=dylib={}", name);
    }

    // Link libffi if the user requested this workaround.
    // See https://bitbucket.org/tari/llvm-sys.rs/issues/12/
    let force_ffi = env::var_os(format!("LLVM_SYS_{}_FFI_WORKAROUND",
                                        env!("CARGO_PKG_VERSION_MAJOR")))
        .is_some();
    if force_ffi {
        println!("cargo:rustc-link-lib=dylib={}", "ffi");
    }

    // Build the extra wrapper functions.
    std::env::set_var("CFLAGS", get_llvm_cflags());
    gcc::Build::new()
        .file("wrappers/target.c")
        .compile("targetwrappers");
}
<|endoftext|>


<commit_before>// Copyright (C) 2016  ParadoxSpiral
//
// This file is part of mpv-rs.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#![feature(stmt_expr_attributes)]
#![allow(unused_imports)]

extern crate hyper;
extern crate git2 as git;

use git::Repository;

use std::env;
use std::io::{Read, Write};
use std::fs::File;
use std::path::Path;
use std::process::Command;

fn main() {
	#[cfg(feature="build_libmpv")] {
		let out_dir = env::var("OUT_DIR").unwrap();
		let target = env::var("TARGET").unwrap();

		if target.contains("windows") {
			let path = format!("{}/libmpv.7z", out_dir);
			let archive = Path::new(&path);

			if !archive.exists() {
				let mut buf = Vec::with_capacity(103063218);

				hyper::Client::new().get("https://mpv.srsfckn.biz/mpv-dev-20160826.7z")
									.send().expect("retrieving libmpv failed")
									.read_to_end(&mut buf).unwrap();

				File::create(archive)
					.expect("failed to open file!")
					.write_all(&buf)
					.unwrap();
			}

			Command::new("7z").arg("x").arg(&format!("-o{}", out_dir))
							  .arg(archive).output().expect("7z execution failed");

			if target.contains("x86_64") {
				println!("cargo:rustc-link-search=native={}/64/", out_dir);
			} else {
				println!("cargo:rustc-link-search=native={}/32/", out_dir);
			}
		} else {
		    // Assume unix like

		    // target doesn't really mean target. It means target(host) of build script, which is
		    // very confusing because it means the actual --target everywhere else.
		    #[cfg(target_pointer_width = "64")] {
		    	if (target.contains("x86") && ! target.contains("x86_64")) ||
		    	    target.contains("i686") {
		    		panic!("Cross-compiling to different arch not yet supported");
		    	}
		    }
		    #[cfg(target_pointer_width = "32")] {
		    	if target.contains("x86_64") {
		    		panic!("Cross-compiling to different arch not yet supported");
		    	}
		    }

		    let url = "https://github.com/mpv-player/mpv-build";
		    let path = format!("{}/libmpv", out_dir);
			let num_threads = env::var("NUM_JOBS").unwrap();

			if !Path::new(&path).exists() {
				Repository::clone(url, &path).expect("failed to mpv-build");
				Command::new("sh")
							  .arg("-c")
							  .arg(&format!("cd {} && ./update && echo --enable-libmpv-shared > \
							   				mpv_options && ./build -j{}",
							  				path, num_threads))
							  .output().expect("libmpv build failed");
			}
			println!("cargo:rustc-link-search=native={}/mpv/build/", path);
		}

	    println!("cargo:rustc-link-lib=mpv");
	}
}
<commit_msg>[build] fix build_libmpv for 64-bit unix (again)<commit_after>// Copyright (C) 2016  ParadoxSpiral
//
// This file is part of mpv-rs.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#![feature(stmt_expr_attributes)]
#![allow(unused_imports)]

extern crate hyper;
extern crate git2 as git;

use git::Repository;

use std::env;
use std::io::{Read, Write};
use std::fs::File;
use std::path::Path;
use std::process::Command;

fn main() {
	#[cfg(feature="build_libmpv")] {
		let out_dir = env::var("OUT_DIR").unwrap();
		let target = env::var("TARGET").unwrap();

		if target.contains("windows") {
			let path = format!("{}/libmpv.7z", out_dir);
			let archive = Path::new(&path);

			if !archive.exists() {
				let mut buf = Vec::with_capacity(103063218);

				hyper::Client::new().get("https://mpv.srsfckn.biz/mpv-dev-20160826.7z")
									.send().expect("retrieving libmpv failed")
									.read_to_end(&mut buf).unwrap();

				File::create(archive)
					.expect("failed to open file!")
					.write_all(&buf)
					.unwrap();
			}

			Command::new("7z").arg("x").arg(&format!("-o{}", out_dir))
							  .arg(archive).output().expect("7z execution failed");

			if target.contains("x86_64") {
				println!("cargo:rustc-link-search=native={}/64/", out_dir);
			} else {
				println!("cargo:rustc-link-search=native={}/32/", out_dir);
			}
		} else {
		    // Assume unix like

		    // target doesn't really mean target. It means target(host) of build script, which is
		    // very confusing because it means the actual --target everywhere else.
		    #[cfg(target_pointer_width = "64")] {
		    	if (target.contains("x86") && ! target.contains("x86_64")) ||
		    	    target.contains("i686") {
		    		panic!("Cross-compiling to different arch not yet supported");
		    	}
		    }
		    #[cfg(target_pointer_width = "32")] {
		    	if target.contains("x86_64") {
		    		panic!("Cross-compiling to different arch not yet supported");
		    	}
		    }

		    let url = "https://github.com/mpv-player/mpv-build";
		    let path = format!("{}/libmpv", out_dir);
			let num_threads = env::var("NUM_JOBS").unwrap();

			if !Path::new(&path).exists() {
				Repository::clone(url, &path).expect("failed to mpv-build");
				Command::new("sh")
							  .arg("-c")
							  .arg(&format!("cd {} && {0}/update && echo --enable-libmpv-shared > {0}/mpv_options && {0}/build -j{1}",
							  				path, num_threads))
							  .output().expect("libmpv build failed");
			}
			println!("cargo:rustc-link-search=native={}/mpv/build/", path);
		}

	    println!("cargo:rustc-link-lib=mpv");
	}
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate gcc;

use std::env;

fn main() {
    let target = env::var("TARGET").unwrap();
    if target.contains("msvc") {
        let mut config = gcc::Config::new();
        config.file("src/util_helpers.asm");
        config.file("src/aesni_helpers.asm");
        if target.contains("x86_64") {
            config.define("X64", None);
        }
        config.compile("lib_rust_crypto_helpers.a");
    } else {
        gcc::compile_library(
            "lib_rust_crypto_helpers.a",
            &["src/util_helpers.c", "src/aesni_helpers.c"]);
    }
}

<commit_msg>Use clang to build, unless the user explicitly specifies a different compiler.<commit_after>// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate gcc;

use std::env;
use std::path::Path;

fn main() {
    let target = env::var("TARGET").unwrap();
    if target.contains("msvc") {
        let mut config = gcc::Config::new();
        config.file("src/util_helpers.asm");
        config.file("src/aesni_helpers.asm");
        if target.contains("x86_64") {
            config.define("X64", None);
        }
        config.compile("lib_rust_crypto_helpers.a");
    }
    else {
        let mut cfg = gcc::Config::new();
        cfg.file("src/util_helpers.c");
        cfg.file("src/aesni_helpers.c");
        // gcc can't build this library so, unless the user has explicitly
        // specified a different C compiler, use clang.
        if env::var_os("CC").is_none() {
            cfg.compiler(Path::new("clang"));
        }
        cfg.compile("lib_rust_crypto_helpers.a");
    }
}

<|endoftext|>


<commit_before>use std::thread;
use std::io;
use std::io::Read;
use std::sync::mpsc;
use std::sync::mpsc::{SyncSender, Receiver};
use std::default::Default;
use std::marker::Send;
use std::option::Option::{None, Some};
use libc::types::common::c95::c_void;
use libc::types::common::c99::*;
use libc::types::os::arch::c95::*;

#[allow(unused, improper_ctypes)]
#[link(name = "mad")]
extern {
    fn mad_decoder_init(decoder: *mut MadDecoder,
                        message: *mut c_void,
                        input_callback: extern fn(message: *mut MadMessage,
                                                  stream: &MadStream) -> MadFlow,
                        header_callback: extern fn(),
                        filter_callback: extern fn(),
                        output_callback: extern fn(message: *mut MadMessage,
                                                   header: c_int,
                                                   pcm: &MadPcm) -> MadFlow,
                        error_callback: extern fn(message: *mut MadMessage,
                                                  stream: &MadStream,
                                                  frame: c_int) -> MadFlow,
                        message_callback: extern fn());
    #[allow(dead_code)]
    fn mad_decoder_run(input: &mut MadDecoder, mode: MadDecoderMode) -> c_int;
    fn mad_stream_buffer(stream: &MadStream, buf_start: *const u8, buf_length: size_t);
}

#[allow(unused)]
#[repr(C)]
enum MadFlow {
    Continue = 0x0000,    /* continue normally */
    Stop     = 0x0010,    /* stop decoding normally */
    Break    = 0x0011,    /* stop decoding and signal an error */
    Ignore   = 0x0020    /* ignore the current frame */
}

#[allow(unused)]
#[derive(Debug, Clone)]
#[repr(C)]
pub enum Error {
  None           = 0x0000,    /* no error */

  BufLen         = 0x0001,    /* input buffer too small (or eof) */
  BufPtr         = 0x0002,    /* invalid (null) buffer pointer */

  NoMem          = 0x0031,    /* not enough memory */

  LostSync       = 0x0101,    /* lost synchronization */
  BadLayer       = 0x0102,    /* reserved header layer value */
  BadBitRate     = 0x0103,    /* forbidden bitrate value */
  BadSampleRate  = 0x0104,    /* reserved sample frequency value */
  BadEmphasis    = 0x0105,    /* reserved emphasis value */

  BadCRC         = 0x0201,    /* crc check failed */
  BadBitAlloc    = 0x0211,    /* forbidden bit allocation value */
  BadScaleFactor = 0x0221,    /* bad scalefactor index */
  BadMode        = 0x0222,    /* bad bitrate/mode combination */
  BadFrameLen    = 0x0231,    /* bad frame length */
  BadBigValues   = 0x0232,    /* bad big_values count */
  BadBlockType   = 0x0233,    /* reserved block_type */
  BadScFSI       = 0x0234,    /* bad scalefactor selection info */
  BadDataPtr     = 0x0235,    /* bad main_data_begin pointer */
  BadPart3Len    = 0x0236,    /* bad audio data length */
  BadHuffTable   = 0x0237,    /* bad huffman table select */
  BadHuffData    = 0x0238,    /* huffman data overrun */
  BadStereo      = 0x0239,    /* incompatible block_type for js */
}

#[allow(unused)]
#[repr(C)]
struct MadBitPtr {
    byte: size_t,
    cache: uint16_t,
    left: uint16_t,
}

#[allow(unused)]
#[repr(C)]
struct MadStream {
    buffer: size_t,
    buff_end: size_t,
    skip_len: c_ulong,
    sync: c_int,
    free_rate: c_ulong,
    this_frame: size_t,
    next_frame: size_t,
    ptr: MadBitPtr,
    anc_ptr: MadBitPtr,
    anc_bitlen: c_uint,
    buffer_mdlen: size_t,
    md_len: c_uint,
    options: c_int,
    error: Error,
}

#[allow(unused)]
#[repr(C)]
struct MadPcm {
    sample_rate: c_uint,
    channels: uint16_t,
    length: uint16_t,
    samples: [[int32_t; 1152]; 2],
}

#[allow(unused)]
struct MadMessage<'a> {
    buffer: Box<[u8; 16384]>,
    reader: &'a mut (io::Read + 'a),
    sender: &'a SyncSender<Result<Frame, Error>>,
}

#[allow(unused)]
#[repr(C)]
enum MadDecoderMode {
    Sync = 0,
    Async
}

impl Default for MadDecoderMode {
    fn default() -> MadDecoderMode {MadDecoderMode::Sync}
}

#[derive(Default)]
#[repr(C)]
struct MadAsyncParameters {
    pid: c_long,
    ain: c_int,
    aout: c_int,
}

#[derive(Default)]
#[repr(C)]
struct MadDecoder {
    mode: MadDecoderMode,
    options: c_int,
    async: MadAsyncParameters,
    sync: size_t,
    cb_data: size_t,
    input_func: size_t,
    header_func: size_t,
    filter_func: size_t,
    output_func: size_t,
    error_func: size_t,
    message_func: size_t,
}

#[allow(unused)]
pub struct Frame {
    pub sample_rate: u32,
    pub channels: uint16_t,
    pub length: uint16_t,
    pub samples: [[i32; 1152]; 2],
}

#[allow(unused)]
extern fn empty_callback() {

}

#[allow(unused)]
extern fn input_callback (msg: *mut MadMessage, stream: &MadStream) -> MadFlow {
    unsafe {
        let buffer_size = (*msg).buffer.len();
        let next_frame_position = (stream.next_frame - stream.buffer) as usize;
        let unused_byte_count = buffer_size - next_frame_position;

        if unused_byte_count > 0 {
            for idx in 0 .. unused_byte_count {
                (*msg).buffer[idx] = (*msg).buffer[idx + next_frame_position];
            }
        }

        let mut bytes_read = None;

        if next_frame_position == 0 {
            bytes_read = Some((*msg).reader.read(&mut *(*msg).buffer).unwrap());
        } else {
            let slice = &mut (*msg).buffer[unused_byte_count .. buffer_size];
            bytes_read = Some((*msg).reader.read(slice).unwrap());
        }

        match bytes_read {
            None    => return MadFlow::Stop,
            Some(0) => return MadFlow::Stop,
            Some(n) => mad_stream_buffer(stream, (*msg).buffer.as_ptr(), n as u64),
        }
    }

    MadFlow::Continue
}

#[allow(unused)]
extern fn error_callback(msg: *mut MadMessage,
                         stream: &MadStream,
                         frame: c_int) -> MadFlow
{
    unsafe {
        let error_type = stream.error.clone();
        (*msg).sender.send(Err(error_type));
    }
    MadFlow::Continue
}

#[allow(unused)]
extern fn output_callback(msg: *mut MadMessage,
                          header: c_int,
                          pcm: &MadPcm) -> MadFlow
{
    unsafe {
        let frame = Ok(Frame {sample_rate: pcm.sample_rate,
                              channels: pcm.channels,
                              length: pcm.length,
                              samples: pcm.samples});
        (*msg).sender.send(frame);
    }
    MadFlow::Continue
}

#[allow(unused)]
pub fn decode<T>(mut reader: T) -> Receiver<Result<Frame, Error>>
    where T: io::Read + Send + 'static {
    let input_buffer = Box::new([0u8; 16384]);
    let (tx, rx) = mpsc::sync_channel::<Result<Frame, Error>>(2);
    thread::spawn(move || {
        unsafe {
            let mut message = MadMessage {
                buffer: input_buffer,
                reader: &mut reader,
                sender: &tx,
            };
            let message_ptr = &mut message as *mut _ as *mut c_void;
            let mut decoder: MadDecoder = Default::default();
            mad_decoder_init(&mut decoder,
                             message_ptr,
                             input_callback,
                             empty_callback,
                             empty_callback,
                             output_callback,
                             error_callback,
                             empty_callback);
            mad_decoder_run(&mut decoder, MadDecoderMode::Sync);
        }
    });
    rx
}

#[test]
fn data_sizes() {
    use std::mem::size_of;
    assert_eq!(size_of::<MadBitPtr>(), 16);
    assert_eq!(size_of::<Error>(), 4);
    assert_eq!(size_of::<MadDecoder>(), 88);
    assert_eq!(size_of::<MadPcm>(), 9224);
    assert_eq!(size_of::<MadStream>(), 120);
}

#[test]
fn test_short_file() {
    use std::path::Path;
    use std::fs::File;
    let path = Path::new("test_samples/tailtoddle_lo.mp3");
    let f = File::open(&path).unwrap();
    let decoder = decode(f);
    let mut frame_count = 0;
    let mut error_count = 0;
    for frame in decoder.iter() {
        match frame {
            Ok(_) => frame_count += 1,
            Err(_) => error_count += 1,
        }
    }
    assert_eq!(frame_count, 1656);
    assert_eq!(error_count, 1);
}

<commit_msg>Added decoder object<commit_after>use std::thread;
use std::io;
use std::io::Read;
use std::sync::mpsc;
use std::sync::mpsc::{SyncSender, Receiver, RecvError};
use std::default::Default;
use std::marker::Send;
use std::option::Option::{None, Some};
use libc::types::common::c95::c_void;
use libc::types::common::c99::*;
use libc::types::os::arch::c95::*;

enum Error {
    Mad(MadError),
    Recv(RecvError),
}

struct Decoder {
    rx: Box<Receiver<Result<Frame, MadError>>>,
}

impl Decoder {
    fn new(reader: Box<io::Read + Send + 'static>) -> Decoder {
        Decoder {
            rx: Box::new(decode(reader)),
        }
    }

    pub fn get_frame(&self) -> Result<Frame, Error> {
        match self.rx.recv() {
            Err(e) => Err(Error::Recv(e)),
            Ok(Err(e)) => Err(Error::Mad(e)),
            Ok(Ok(frame)) => Ok(frame),
        }
    }
}

#[allow(unused, improper_ctypes)]
#[link(name = "mad")]
extern {
    fn mad_decoder_init(decoder: *mut MadDecoder,
                        message: *mut c_void,
                        input_callback: extern fn(message: *mut MadMessage,
                                                  stream: &MadStream) -> MadFlow,
                        header_callback: extern fn(),
                        filter_callback: extern fn(),
                        output_callback: extern fn(message: *mut MadMessage,
                                                   header: c_int,
                                                   pcm: &MadPcm) -> MadFlow,
                        error_callback: extern fn(message: *mut MadMessage,
                                                  stream: &MadStream,
                                                  frame: c_int) -> MadFlow,
                        message_callback: extern fn());
    #[allow(dead_code)]
    fn mad_decoder_run(input: &mut MadDecoder, mode: MadDecoderMode) -> c_int;
    fn mad_stream_buffer(stream: &MadStream, buf_start: *const u8, buf_length: size_t);
}

#[allow(unused)]
#[repr(C)]
enum MadFlow {
    Continue = 0x0000,    /* continue normally */
    Stop     = 0x0010,    /* stop decoding normally */
    Break    = 0x0011,    /* stop decoding and signal an error */
    Ignore   = 0x0020    /* ignore the current frame */
}

#[allow(unused)]
#[derive(Debug, Clone)]
#[repr(C)]
pub enum MadError {
  None           = 0x0000,    /* no error */

  BufLen         = 0x0001,    /* input buffer too small (or eof) */
  BufPtr         = 0x0002,    /* invalid (null) buffer pointer */

  NoMem          = 0x0031,    /* not enough memory */

  LostSync       = 0x0101,    /* lost synchronization */
  BadLayer       = 0x0102,    /* reserved header layer value */
  BadBitRate     = 0x0103,    /* forbidden bitrate value */
  BadSampleRate  = 0x0104,    /* reserved sample frequency value */
  BadEmphasis    = 0x0105,    /* reserved emphasis value */

  BadCRC         = 0x0201,    /* crc check failed */
  BadBitAlloc    = 0x0211,    /* forbidden bit allocation value */
  BadScaleFactor = 0x0221,    /* bad scalefactor index */
  BadMode        = 0x0222,    /* bad bitrate/mode combination */
  BadFrameLen    = 0x0231,    /* bad frame length */
  BadBigValues   = 0x0232,    /* bad big_values count */
  BadBlockType   = 0x0233,    /* reserved block_type */
  BadScFSI       = 0x0234,    /* bad scalefactor selection info */
  BadDataPtr     = 0x0235,    /* bad main_data_begin pointer */
  BadPart3Len    = 0x0236,    /* bad audio data length */
  BadHuffTable   = 0x0237,    /* bad huffman table select */
  BadHuffData    = 0x0238,    /* huffman data overrun */
  BadStereo      = 0x0239,    /* incompatible block_type for js */
}

#[allow(unused)]
#[repr(C)]
struct MadBitPtr {
    byte: size_t,
    cache: uint16_t,
    left: uint16_t,
}

#[allow(unused)]
#[repr(C)]
struct MadStream {
    buffer: size_t,
    buff_end: size_t,
    skip_len: c_ulong,
    sync: c_int,
    free_rate: c_ulong,
    this_frame: size_t,
    next_frame: size_t,
    ptr: MadBitPtr,
    anc_ptr: MadBitPtr,
    anc_bitlen: c_uint,
    buffer_mdlen: size_t,
    md_len: c_uint,
    options: c_int,
    error: MadError,
}

#[allow(unused)]
#[repr(C)]
struct MadPcm {
    sample_rate: c_uint,
    channels: uint16_t,
    length: uint16_t,
    samples: [[int32_t; 1152]; 2],
}

#[allow(unused)]
struct MadMessage<'a> {
    buffer: Box<[u8; 16384]>,
    reader: &'a mut (io::Read + 'a),
    sender: &'a SyncSender<Result<Frame, MadError>>,
}

#[allow(unused)]
#[repr(C)]
enum MadDecoderMode {
    Sync = 0,
    Async
}

impl Default for MadDecoderMode {
    fn default() -> MadDecoderMode {MadDecoderMode::Sync}
}

#[derive(Default)]
#[repr(C)]
struct MadAsyncParameters {
    pid: c_long,
    ain: c_int,
    aout: c_int,
}

#[derive(Default)]
#[repr(C)]
struct MadDecoder {
    mode: MadDecoderMode,
    options: c_int,
    async: MadAsyncParameters,
    sync: size_t,
    cb_data: size_t,
    input_func: size_t,
    header_func: size_t,
    filter_func: size_t,
    output_func: size_t,
    error_func: size_t,
    message_func: size_t,
}

#[allow(unused)]
pub struct Frame {
    pub sample_rate: u32,
    pub channels: uint16_t,
    pub length: uint16_t,
    pub samples: [[i32; 1152]; 2],
}

#[allow(unused)]
extern fn empty_callback() {

}

#[allow(unused)]
extern fn input_callback (msg: *mut MadMessage, stream: &MadStream) -> MadFlow {
    unsafe {
        let buffer_size = (*msg).buffer.len();
        let next_frame_position = (stream.next_frame - stream.buffer) as usize;
        let unused_byte_count = buffer_size - next_frame_position;

        if unused_byte_count > 0 {
            for idx in 0 .. unused_byte_count {
                (*msg).buffer[idx] = (*msg).buffer[idx + next_frame_position];
            }
        }

        let mut bytes_read = None;

        if next_frame_position == 0 {
            bytes_read = Some((*msg).reader.read(&mut *(*msg).buffer).unwrap());
        } else {
            let slice = &mut (*msg).buffer[unused_byte_count .. buffer_size];
            bytes_read = Some((*msg).reader.read(slice).unwrap());
        }

        match bytes_read {
            None    => return MadFlow::Stop,
            Some(0) => return MadFlow::Stop,
            Some(n) => mad_stream_buffer(stream, (*msg).buffer.as_ptr(), n as u64),
        }
    }

    MadFlow::Continue
}

#[allow(unused)]
extern fn error_callback(msg: *mut MadMessage,
                         stream: &MadStream,
                         frame: c_int) -> MadFlow
{
    unsafe {
        let error_type = stream.error.clone();
        (*msg).sender.send(Err(error_type));
    }
    MadFlow::Continue
}

#[allow(unused)]
extern fn output_callback(msg: *mut MadMessage,
                          header: c_int,
                          pcm: &MadPcm) -> MadFlow
{
    unsafe {
        let frame = Ok(Frame {sample_rate: pcm.sample_rate,
                              channels: pcm.channels,
                              length: pcm.length,
                              samples: pcm.samples});
        (*msg).sender.send(frame);
    }
    MadFlow::Continue
}

#[allow(unused)]
pub fn decode<T>(mut reader: T) -> Receiver<Result<Frame, MadError>>
    where T: io::Read + Send + 'static {
    let input_buffer = Box::new([0u8; 16384]);
    let (tx, rx) = mpsc::sync_channel::<Result<Frame, MadError>>(2);
    thread::spawn(move || {
        unsafe {
            let mut message = MadMessage {
                buffer: input_buffer,
                reader: &mut reader,
                sender: &tx,
            };
            let message_ptr = &mut message as *mut _ as *mut c_void;
            let mut decoder: MadDecoder = Default::default();
            mad_decoder_init(&mut decoder,
                             message_ptr,
                             input_callback,
                             empty_callback,
                             empty_callback,
                             output_callback,
                             error_callback,
                             empty_callback);
            mad_decoder_run(&mut decoder, MadDecoderMode::Sync);
        }
    });
    rx
}

#[test]
fn data_sizes() {
    use std::mem::size_of;
    assert_eq!(size_of::<MadError>(), 4);
    assert_eq!(size_of::<MadBitPtr>(), 16);
    assert_eq!(size_of::<MadDecoder>(), 88);
    assert_eq!(size_of::<MadPcm>(), 9224);
    assert_eq!(size_of::<MadStream>(), 120);
}

#[test]
fn test_short_file() {
    use std::path::Path;
    use std::fs::File;
    let path = Path::new("test_samples/tailtoddle_lo.mp3");
    let f = File::open(&path).unwrap();
    let decoder = decode(f);
    let mut frame_count = 0;
    let mut error_count = 0;
    for frame in decoder.iter() {
        match frame {
            Ok(_) => frame_count += 1,
            Err(_) => error_count += 1,
        }
    }
    assert_eq!(frame_count, 1656);
    assert_eq!(error_count, 1);
}

<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#[macro_use] extern crate quote;
extern crate syn;

use std::env;
use std::path::Path;

#[cfg(feature = "dummy_match_byte")]
mod codegen {
    use std::path::Path;
    pub fn main(_: &Path) {}
}

#[cfg(not(feature = "dummy_match_byte"))]
#[path = "src/macros/mod.rs"]
mod macros;

#[cfg(not(feature = "dummy_match_byte"))]
mod codegen {
    use macros;
    use std::env;
    use std::path::Path;

    pub fn main(tokenizer_rs: &Path) {
        macros::match_byte::expand(tokenizer_rs,
                                   &Path::new(&env::var("OUT_DIR").unwrap()).join("tokenizer.rs"));

    }
}

fn main() {
    let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let tokenizer_rs = Path::new(&manifest_dir).join("src/tokenizer.rs");
    codegen::main(&tokenizer_rs);
    println!("cargo:rerun-if-changed={}", tokenizer_rs.display());
}
<commit_msg>Fix a warning.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern crate quote;
extern crate syn;

use std::env;
use std::path::Path;

#[cfg(feature = "dummy_match_byte")]
mod codegen {
    use std::path::Path;
    pub fn main(_: &Path) {}
}

#[cfg(not(feature = "dummy_match_byte"))]
#[path = "src/macros/mod.rs"]
mod macros;

#[cfg(not(feature = "dummy_match_byte"))]
mod codegen {
    use macros;
    use std::env;
    use std::path::Path;

    pub fn main(tokenizer_rs: &Path) {
        macros::match_byte::expand(tokenizer_rs,
                                   &Path::new(&env::var("OUT_DIR").unwrap()).join("tokenizer.rs"));

    }
}

fn main() {
    let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let tokenizer_rs = Path::new(&manifest_dir).join("src/tokenizer.rs");
    codegen::main(&tokenizer_rs);
    println!("cargo:rerun-if-changed={}", tokenizer_rs.display());
}
<|endoftext|>


<commit_before>// build.rs

extern crate pkg_config;
extern crate bindgen;
extern crate cmake;
extern crate cc;

use std::env;
use std::path::Path;

use std::fs::{self, File};
use std::io::Write;

fn format_write(builder: bindgen::Builder, output: &str) {
    let s = builder
        .generate()
        .unwrap()
        .to_string()
        .replace("/**", "/*")
        .replace("/*!", "/*");

    let mut file = File::create(output).unwrap();

    let _ = file.write(s.as_bytes());
}

fn main() {
    let cargo_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let build_path = Path::new(&cargo_dir).join("aom_build/aom");

    let dst = cmake::Config::new(build_path)
        .define("CONFIG_AV1_ENCODER", "0")
        .define("CONFIG_DEBUG", "1")
        .define("CONFIG_EXPERIMENTAL", "1")
        .define("CONFIG_UNIT_TESTS", "0")
        .define("CONFIG_EXT_PARTITION", "0")
        .define("CONFIG_EXT_PARTITION_TYPES", "0")
        .define("CONFIG_KF_CTX", "0")
        .define("CONFIG_OBU", "0")
        .define("CONFIG_FILTER_INTRA", "0")
        .define("CONFIG_EXT_SKIP", "0")
        .define("CONFIG_LV_MAP", "1")
        .define("CONFIG_TXK_SEL", "0")
        .define("CONFIG_ANALYZER", "0")
        .define("AOM_TARGET_CPU", "generic")
        .build();

    // Dirty hack to force a rebuild whenever the defaults are changed upstream
    fs::remove_file(dst.join("build/CMakeCache.txt"));

    env::set_var("PKG_CONFIG_PATH", dst.join("lib/pkgconfig"));

    let libs = pkg_config::Config::new().statik(true).probe("aom").unwrap();
    let headers = libs.include_paths.clone();

    let mut builder = bindgen::builder()
        .raw_line("#![allow(dead_code)]")
        .raw_line("#![allow(non_camel_case_types)]")
        .raw_line("#![allow(non_snake_case)]")
        .raw_line("#![allow(non_upper_case_globals)]")
        .blacklist_type("max_align_t")
        .rustfmt_bindings(false)
        .header("data/aom.h");

    for header in headers {
        builder = builder.clang_arg("-I").clang_arg(header.to_str().unwrap());
    }

    // Manually fix the comment so rustdoc won't try to pick them
    format_write(builder, "tests/aom.rs");

    println!("cargo:rerun-if-changed=aom_build/aom/");

    cc::Build::new()
        .file("aom_build/aom/aom_dsp/fwd_txfm.c")
        .file("aom_build/aom/av1/encoder/dct.c")
        .include("aom_build")
        .include("aom_build/aom")
        .flag("-std=c99")
        .compile("libntr.a");
}
<commit_msg>Drop a warning for fs::remove() return value<commit_after>// build.rs

extern crate pkg_config;
extern crate bindgen;
extern crate cmake;
extern crate cc;

use std::env;
use std::path::Path;

use std::fs::{self, File};
use std::io::Write;

fn format_write(builder: bindgen::Builder, output: &str) {
    let s = builder
        .generate()
        .unwrap()
        .to_string()
        .replace("/**", "/*")
        .replace("/*!", "/*");

    let mut file = File::create(output).unwrap();

    let _ = file.write(s.as_bytes());
}

fn main() {
    let cargo_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
    let build_path = Path::new(&cargo_dir).join("aom_build/aom");

    let dst = cmake::Config::new(build_path)
        .define("CONFIG_AV1_ENCODER", "0")
        .define("CONFIG_DEBUG", "1")
        .define("CONFIG_EXPERIMENTAL", "1")
        .define("CONFIG_UNIT_TESTS", "0")
        .define("CONFIG_EXT_PARTITION", "0")
        .define("CONFIG_EXT_PARTITION_TYPES", "0")
        .define("CONFIG_KF_CTX", "0")
        .define("CONFIG_OBU", "0")
        .define("CONFIG_FILTER_INTRA", "0")
        .define("CONFIG_EXT_SKIP", "0")
        .define("CONFIG_LV_MAP", "1")
        .define("CONFIG_TXK_SEL", "0")
        .define("CONFIG_ANALYZER", "0")
        .define("AOM_TARGET_CPU", "generic")
        .build();

    // Dirty hack to force a rebuild whenever the defaults are changed upstream
    let _ = fs::remove_file(dst.join("build/CMakeCache.txt"));

    env::set_var("PKG_CONFIG_PATH", dst.join("lib/pkgconfig"));

    let libs = pkg_config::Config::new().statik(true).probe("aom").unwrap();
    let headers = libs.include_paths.clone();

    let mut builder = bindgen::builder()
        .raw_line("#![allow(dead_code)]")
        .raw_line("#![allow(non_camel_case_types)]")
        .raw_line("#![allow(non_snake_case)]")
        .raw_line("#![allow(non_upper_case_globals)]")
        .blacklist_type("max_align_t")
        .rustfmt_bindings(false)
        .header("data/aom.h");

    for header in headers {
        builder = builder.clang_arg("-I").clang_arg(header.to_str().unwrap());
    }

    // Manually fix the comment so rustdoc won't try to pick them
    format_write(builder, "tests/aom.rs");

    println!("cargo:rerun-if-changed=aom_build/aom/");

    cc::Build::new()
        .file("aom_build/aom/aom_dsp/fwd_txfm.c")
        .file("aom_build/aom/av1/encoder/dct.c")
        .include("aom_build")
        .include("aom_build/aom")
        .flag("-std=c99")
        .compile("libntr.a");
}
<|endoftext|>


<commit_before>extern crate cc;
extern crate glob;

use std::env;

#[cfg(not(feature = "orangepi"))]
const TARGET: &'static str = "wiringPi";
#[cfg(feature = "orangepi")]
const TARGET: &'static str = "WiringOP";

fn main() {
    if cfg!(feature = "development") {
        return;
    }

    // only build wiringpi/wiringop for arm/armv7 platforms

    let target = std::env::var("TARGET").unwrap();
    if !(target.starts_with("arm-") || target.starts_with("armv7-")) {
        return;
    }

    cc::Build::new()
        .files(
            glob::glob(&format!("{}/wiringPi/*.c", TARGET))
                .unwrap()
                .filter_map(|x| x.ok()),
        )
        .include(format!("{}/", TARGET))
        .include(format!("{}/wiringPi/", TARGET))
        .static_flag(true)
        .compile("wiringpi");
}
<commit_msg>Fix warning<commit_after>extern crate cc;
extern crate glob;

#[cfg(not(feature = "orangepi"))]
const TARGET: &'static str = "wiringPi";
#[cfg(feature = "orangepi")]
const TARGET: &'static str = "WiringOP";

fn main() {
    if cfg!(feature = "development") {
        return;
    }

    // only build wiringpi/wiringop for arm/armv7 platforms

    let target = std::env::var("TARGET").unwrap();
    if !(target.starts_with("arm-") || target.starts_with("armv7-")) {
        return;
    }

    cc::Build::new()
        .files(
            glob::glob(&format!("{}/wiringPi/*.c", TARGET))
                .unwrap()
                .filter_map(|x| x.ok()),
        )
        .include(format!("{}/", TARGET))
        .include(format!("{}/wiringPi/", TARGET))
        .static_flag(true)
        .compile("wiringpi");
}
<|endoftext|>