<commit_before>use std::collections::HashMap;
use std::env;
use std::ffi::OsStr;
use std::fs;
use std::fs::File;
use std::io;
use std::io::BufReader;
use std::io::BufWriter;
use std::io::Read;
use std::io::Seek;
use std::io::Write;
use std::path::Path;

extern crate env_logger;
extern crate tools;
use crate::languages::FormatAs;
use tools::config;
use tools::describe;
use tools::file_format::analysis::{read_analysis, read_jumps, read_source};
use tools::find_source_file;
use tools::format::format_file_data;
use tools::git_ops;
use tools::languages;

use tools::output::{PanelItem, PanelSection};

extern crate rustc_serialize;
use rustc_serialize::json;

fn read_json_from_file(path: &str) -> Option<json::Object> {
    let components_file = File::open(path).ok()?;
    let mut reader = BufReader::new(&components_file);
    json::Json::from_reader(&mut reader).ok()?.into_object()
}

/// For a given path, looks up the bugzilla product and component and
/// returns it in a tuple if it could be found. The JSON data format
/// comes from https://searchfox.org/mozilla-central/rev/47edbd91c43db6229cf32d1fc4bae9b325b9e2d0/python/mozbuild/mozbuild/frontend/mach_commands.py#209-223,243
/// and is fairly straightforward.
fn get_bugzilla_component<'a>(
    bugzilla_data: &'a json::Object,
    path: &str,
) -> Option<(&'a str, &'a str)> {
    let mut path_obj = bugzilla_data.get("paths")?;
    for path_component in path.split('/') {
        path_obj = path_obj.as_object()?.get(path_component)?;
    }
    let component_id = path_obj.as_i64()?.to_string();
    let mut result_iter = bugzilla_data
        .get("components")?
        .as_object()?
        .get(&component_id)?
        .as_array()?
        .iter();
    let product = result_iter.next()?.as_string()?;
    let component = result_iter.next()?.as_string()?;
    Some((product, component))
}

fn main() {
    env_logger::init();

    let args: Vec<_> = env::args().collect();
    let (base_args, fname_args) = args.split_at(3);

    let cfg = config::load(&base_args[1], true);
    println!("Config file read");

    let tree_name = &base_args[2];
    let tree_config = cfg.trees.get(tree_name).unwrap();

    let jumps_fname = format!("{}/jumps", tree_config.paths.index_path);
    //let jumps : std::collections::HashMap<String, tools::analysis::Jump> = std::collections::HashMap::new();
    let jumps = read_jumps(&jumps_fname);
    println!("Jumps read");

    let bugzilla_fname = format!("{}/bugzilla-components.json", tree_config.paths.index_path);
    let bugzilla_data = read_json_from_file(&bugzilla_fname);
    if bugzilla_data.is_some() {
        println!("Bugzilla components read");
    } else {
        println!("No bugzilla-components.json file found");
    }

    let (blame_commit, head_oid) = match &tree_config.git {
        &Some(ref git) => {
            let head_oid = git.repo.refname_to_id("HEAD").unwrap();
            let blame_commit = if let Some(ref blame_repo) = git.blame_repo {
                let blame_oid = blame_repo.refname_to_id("HEAD").unwrap();
                Some(blame_repo.find_commit(blame_oid).unwrap())
            } else {
                None
            };
            (blame_commit, Some(head_oid))
        }
        &None => (None, None),
    };

    let head_commit =
        head_oid.and_then(|oid| tree_config.git.as_ref().unwrap().repo.find_commit(oid).ok());

    let mut extension_mapping = HashMap::new();
    extension_mapping.insert("cpp", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("cc", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("cxx", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("h", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hh", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hpp", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hxx", ("source", vec!["cpp", "cc", "cxx"]));

    let mut diff_cache = git_ops::TreeDiffCache::new();
    for path in fname_args {
        println!("File {}", path);

        let output_fname = format!("{}/file/{}", tree_config.paths.index_path, path);
        let source_fname = find_source_file(
            path,
            &tree_config.paths.files_path,
            &tree_config.paths.objdir_path,
        );

        let format = languages::select_formatting(path);

        let output_file = File::create(output_fname).unwrap();
        let mut writer = BufWriter::new(output_file);

        let source_file = match File::open(source_fname.clone()) {
            Ok(f) => f,
            Err(_) => {
                println!("Unable to open file");
                continue;
            }
        };

        let path_wrapper = Path::new(&source_fname);
        let metadata = fs::symlink_metadata(path_wrapper).unwrap();
        if metadata.file_type().is_symlink() {
            let dest = fs::read_link(path_wrapper).unwrap();
            write!(writer, "Symlink to {}", dest.to_str().unwrap()).unwrap();
            continue;
        }

        let mut reader = BufReader::new(&source_file);

        match format {
            FormatAs::Binary => {
                let _ = io::copy(&mut reader, &mut writer);
                continue;
            }
            _ => {}
        };

        let analysis_fname = format!("{}/analysis/{}", tree_config.paths.index_path, path);
        let analysis = read_analysis(&analysis_fname, &mut read_source);

        let mut input = String::new();
        match reader.read_to_string(&mut input) {
            Ok(_) => {}
            Err(_) => {
                let mut bytes = Vec::new();
                reader.seek(std::io::SeekFrom::Start(0)).unwrap();
                match reader.read_to_end(&mut bytes) {
                    Ok(_) => {
                        input.push_str(&bytes.iter().map(|c| *c as char).collect::<String>());
                    }
                    Err(e) => {
                        println!("Unable to read file: {:?}", e);
                        continue;
                    }
                }
            }
        }

        if let Some(file_description) = describe::describe_file(&input, &path_wrapper, &format) {
            let description_fname =
                format!("{}/description/{}", tree_config.paths.index_path, path);
            let description_file = File::create(description_fname).unwrap();
            let mut desc_writer = BufWriter::new(description_file);
            write!(desc_writer, "{}", file_description).unwrap();
        }

        let extension = path_wrapper
            .extension()
            .unwrap_or(&OsStr::new(""))
            .to_str()
            .unwrap();
        let show_header = match extension_mapping.get(extension) {
            Some(&(ref description, ref try_extensions)) => {
                let mut result = None;
                for try_ext in try_extensions {
                    let try_buf = path_wrapper.with_extension(try_ext);
                    let try_path = try_buf.as_path();
                    if try_path.exists() {
                        let (path_base, _) = path.split_at(path.len() - extension.len() - 1);
                        result = Some((
                            description.to_owned(),
                            format!("/{}/source/{}.{}", tree_name, path_base, try_ext),
                        ));
                        break;
                    }
                }
                result
            }
            None => None,
        };

        let mut panel = vec![];

        let mut source_panel_items = vec![];
        if let Some((other_desc, other_path)) = show_header {
            source_panel_items.push(PanelItem {
                title: format!("Go to {} file", other_desc),
                link: other_path,
                update_link_lineno: "",
                accel_key: None,
            });
        };
        if let Some(ref bugzilla) = bugzilla_data {
            if !path.contains("__GENERATED__") {
                if let Some((product, component)) = get_bugzilla_component(bugzilla, &path) {
                    source_panel_items.push(PanelItem {
                        title: format!("File a bug in {} :: {}", product, component),
                        link: format!(
                            "https://bugzilla.mozilla.org/enter_bug.cgi?product={}&component={}",
                            product.replace("&", "%26"),
                            component.replace("&", "%26")
                        ),
                        update_link_lineno: "",
                        accel_key: None,
                    });
                }
            }
        }

        if !source_panel_items.is_empty() {
            panel.push(PanelSection {
                name: "Source code".to_owned(),
                items: source_panel_items,
            });
        };

        if let Some(oid) = head_oid {
            if !path.contains("__GENERATED__") {
                let mut vcs_panel_items = vec![];
                vcs_panel_items.push(PanelItem {
                    title: "Permalink".to_owned(),
                    link: format!("/{}/rev/{}/{}", tree_name, oid, path),
                    update_link_lineno: "#{}",
                    accel_key: Some('Y'),
                });
                if let Some(ref hg_root) = tree_config.paths.hg_root {
                    vcs_panel_items.push(PanelItem {
                        title: "Log".to_owned(),
                        link: format!("{}/log/tip/{}", hg_root, path),
                        update_link_lineno: "",
                        accel_key: Some('L'),
                    });
                    vcs_panel_items.push(PanelItem {
                        title: "Raw".to_owned(),
                        link: format!("{}/raw-file/tip/{}", hg_root, path),
                        update_link_lineno: "",
                        accel_key: Some('R'),
                    });
                }
                if tree_config.paths.git_blame_path.is_some() {
                    vcs_panel_items.push(PanelItem {
                        title: "Blame".to_owned(),
                        link: "javascript:alert('Hover over the gray bar on the left to see blame information.')".to_owned(),
                        update_link_lineno: "",
                        accel_key: None,
                    });
                }
                panel.push(PanelSection {
                    name: "Revision control".to_owned(),
                    items: vcs_panel_items,
                });
            }
        }

        let mut tools_items = vec![];
        if let Some(ref hg_root) = tree_config.paths.hg_root {
            tools_items.push(PanelItem {
                title: "HG Web".to_owned(),
                link: format!("{}/file/tip/{}", hg_root, path),
                update_link_lineno: "#{}",
                accel_key: None,
            });
        }
        if let Some(ref ccov_root) = tree_config.paths.ccov_root {
            tools_items.push(PanelItem {
                title: "Code Coverage".to_owned(),
                link: format!("{}#revision=latest&path={}&view=file", ccov_root, path),
                update_link_lineno: "&line={}",
                accel_key: None,
            });
        }
        if let Some(ref dxr_root) = tree_config.paths.dxr_root {
            tools_items.push(PanelItem {
                title: "DXR".to_owned(),
                link: format!("{}/source/{}", dxr_root, path),
                update_link_lineno: "#{}",
                accel_key: None,
            });
        }
        if let Some(ref github) = tree_config.paths.github_repo {
            match Path::new(path).extension().and_then(OsStr::to_str) {
                Some("md") | Some("rst") => {
                    tools_items.push(PanelItem {
                        title: "Rendered view".to_owned(),
                        link: format!(
                            "{}/blob/{}/{}",
                            github,
                            head_oid.map_or("master".to_string(), |x| x.to_string()),
                            path
                        ),
                        update_link_lineno: "",
                        accel_key: None,
                    });
                }
                _ => (),
            };
        }
        if !tools_items.is_empty() {
            panel.push(PanelSection {
                name: "Other Tools".to_owned(),
                items: tools_items,
            });
        }

        format_file_data(
            &cfg,
            tree_name,
            &panel,
            &head_commit,
            &blame_commit,
            path,
            input,
            &jumps,
            &analysis,
            &mut writer,
            Some(&mut diff_cache),
        )
        .unwrap();
    }
}
<commit_msg>Bug 1605386 - Add 'l' prefix to HG Web fragment for line numbers<commit_after>use std::collections::HashMap;
use std::env;
use std::ffi::OsStr;
use std::fs;
use std::fs::File;
use std::io;
use std::io::BufReader;
use std::io::BufWriter;
use std::io::Read;
use std::io::Seek;
use std::io::Write;
use std::path::Path;

extern crate env_logger;
extern crate tools;
use crate::languages::FormatAs;
use tools::config;
use tools::describe;
use tools::file_format::analysis::{read_analysis, read_jumps, read_source};
use tools::find_source_file;
use tools::format::format_file_data;
use tools::git_ops;
use tools::languages;

use tools::output::{PanelItem, PanelSection};

extern crate rustc_serialize;
use rustc_serialize::json;

fn read_json_from_file(path: &str) -> Option<json::Object> {
    let components_file = File::open(path).ok()?;
    let mut reader = BufReader::new(&components_file);
    json::Json::from_reader(&mut reader).ok()?.into_object()
}

/// For a given path, looks up the bugzilla product and component and
/// returns it in a tuple if it could be found. The JSON data format
/// comes from https://searchfox.org/mozilla-central/rev/47edbd91c43db6229cf32d1fc4bae9b325b9e2d0/python/mozbuild/mozbuild/frontend/mach_commands.py#209-223,243
/// and is fairly straightforward.
fn get_bugzilla_component<'a>(
    bugzilla_data: &'a json::Object,
    path: &str,
) -> Option<(&'a str, &'a str)> {
    let mut path_obj = bugzilla_data.get("paths")?;
    for path_component in path.split('/') {
        path_obj = path_obj.as_object()?.get(path_component)?;
    }
    let component_id = path_obj.as_i64()?.to_string();
    let mut result_iter = bugzilla_data
        .get("components")?
        .as_object()?
        .get(&component_id)?
        .as_array()?
        .iter();
    let product = result_iter.next()?.as_string()?;
    let component = result_iter.next()?.as_string()?;
    Some((product, component))
}

fn main() {
    env_logger::init();

    let args: Vec<_> = env::args().collect();
    let (base_args, fname_args) = args.split_at(3);

    let cfg = config::load(&base_args[1], true);
    println!("Config file read");

    let tree_name = &base_args[2];
    let tree_config = cfg.trees.get(tree_name).unwrap();

    let jumps_fname = format!("{}/jumps", tree_config.paths.index_path);
    //let jumps : std::collections::HashMap<String, tools::analysis::Jump> = std::collections::HashMap::new();
    let jumps = read_jumps(&jumps_fname);
    println!("Jumps read");

    let bugzilla_fname = format!("{}/bugzilla-components.json", tree_config.paths.index_path);
    let bugzilla_data = read_json_from_file(&bugzilla_fname);
    if bugzilla_data.is_some() {
        println!("Bugzilla components read");
    } else {
        println!("No bugzilla-components.json file found");
    }

    let (blame_commit, head_oid) = match &tree_config.git {
        &Some(ref git) => {
            let head_oid = git.repo.refname_to_id("HEAD").unwrap();
            let blame_commit = if let Some(ref blame_repo) = git.blame_repo {
                let blame_oid = blame_repo.refname_to_id("HEAD").unwrap();
                Some(blame_repo.find_commit(blame_oid).unwrap())
            } else {
                None
            };
            (blame_commit, Some(head_oid))
        }
        &None => (None, None),
    };

    let head_commit =
        head_oid.and_then(|oid| tree_config.git.as_ref().unwrap().repo.find_commit(oid).ok());

    let mut extension_mapping = HashMap::new();
    extension_mapping.insert("cpp", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("cc", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("cxx", ("header", vec!["h", "hh", "hpp", "hxx"]));
    extension_mapping.insert("h", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hh", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hpp", ("source", vec!["cpp", "cc", "cxx"]));
    extension_mapping.insert("hxx", ("source", vec!["cpp", "cc", "cxx"]));

    let mut diff_cache = git_ops::TreeDiffCache::new();
    for path in fname_args {
        println!("File {}", path);

        let output_fname = format!("{}/file/{}", tree_config.paths.index_path, path);
        let source_fname = find_source_file(
            path,
            &tree_config.paths.files_path,
            &tree_config.paths.objdir_path,
        );

        let format = languages::select_formatting(path);

        let output_file = File::create(output_fname).unwrap();
        let mut writer = BufWriter::new(output_file);

        let source_file = match File::open(source_fname.clone()) {
            Ok(f) => f,
            Err(_) => {
                println!("Unable to open file");
                continue;
            }
        };

        let path_wrapper = Path::new(&source_fname);
        let metadata = fs::symlink_metadata(path_wrapper).unwrap();
        if metadata.file_type().is_symlink() {
            let dest = fs::read_link(path_wrapper).unwrap();
            write!(writer, "Symlink to {}", dest.to_str().unwrap()).unwrap();
            continue;
        }

        let mut reader = BufReader::new(&source_file);

        match format {
            FormatAs::Binary => {
                let _ = io::copy(&mut reader, &mut writer);
                continue;
            }
            _ => {}
        };

        let analysis_fname = format!("{}/analysis/{}", tree_config.paths.index_path, path);
        let analysis = read_analysis(&analysis_fname, &mut read_source);

        let mut input = String::new();
        match reader.read_to_string(&mut input) {
            Ok(_) => {}
            Err(_) => {
                let mut bytes = Vec::new();
                reader.seek(std::io::SeekFrom::Start(0)).unwrap();
                match reader.read_to_end(&mut bytes) {
                    Ok(_) => {
                        input.push_str(&bytes.iter().map(|c| *c as char).collect::<String>());
                    }
                    Err(e) => {
                        println!("Unable to read file: {:?}", e);
                        continue;
                    }
                }
            }
        }

        if let Some(file_description) = describe::describe_file(&input, &path_wrapper, &format) {
            let description_fname =
                format!("{}/description/{}", tree_config.paths.index_path, path);
            let description_file = File::create(description_fname).unwrap();
            let mut desc_writer = BufWriter::new(description_file);
            write!(desc_writer, "{}", file_description).unwrap();
        }

        let extension = path_wrapper
            .extension()
            .unwrap_or(&OsStr::new(""))
            .to_str()
            .unwrap();
        let show_header = match extension_mapping.get(extension) {
            Some(&(ref description, ref try_extensions)) => {
                let mut result = None;
                for try_ext in try_extensions {
                    let try_buf = path_wrapper.with_extension(try_ext);
                    let try_path = try_buf.as_path();
                    if try_path.exists() {
                        let (path_base, _) = path.split_at(path.len() - extension.len() - 1);
                        result = Some((
                            description.to_owned(),
                            format!("/{}/source/{}.{}", tree_name, path_base, try_ext),
                        ));
                        break;
                    }
                }
                result
            }
            None => None,
        };

        let mut panel = vec![];

        let mut source_panel_items = vec![];
        if let Some((other_desc, other_path)) = show_header {
            source_panel_items.push(PanelItem {
                title: format!("Go to {} file", other_desc),
                link: other_path,
                update_link_lineno: "",
                accel_key: None,
            });
        };
        if let Some(ref bugzilla) = bugzilla_data {
            if !path.contains("__GENERATED__") {
                if let Some((product, component)) = get_bugzilla_component(bugzilla, &path) {
                    source_panel_items.push(PanelItem {
                        title: format!("File a bug in {} :: {}", product, component),
                        link: format!(
                            "https://bugzilla.mozilla.org/enter_bug.cgi?product={}&component={}",
                            product.replace("&", "%26"),
                            component.replace("&", "%26")
                        ),
                        update_link_lineno: "",
                        accel_key: None,
                    });
                }
            }
        }

        if !source_panel_items.is_empty() {
            panel.push(PanelSection {
                name: "Source code".to_owned(),
                items: source_panel_items,
            });
        };

        if let Some(oid) = head_oid {
            if !path.contains("__GENERATED__") {
                let mut vcs_panel_items = vec![];
                vcs_panel_items.push(PanelItem {
                    title: "Permalink".to_owned(),
                    link: format!("/{}/rev/{}/{}", tree_name, oid, path),
                    update_link_lineno: "#{}",
                    accel_key: Some('Y'),
                });
                if let Some(ref hg_root) = tree_config.paths.hg_root {
                    vcs_panel_items.push(PanelItem {
                        title: "Log".to_owned(),
                        link: format!("{}/log/tip/{}", hg_root, path),
                        update_link_lineno: "",
                        accel_key: Some('L'),
                    });
                    vcs_panel_items.push(PanelItem {
                        title: "Raw".to_owned(),
                        link: format!("{}/raw-file/tip/{}", hg_root, path),
                        update_link_lineno: "",
                        accel_key: Some('R'),
                    });
                }
                if tree_config.paths.git_blame_path.is_some() {
                    vcs_panel_items.push(PanelItem {
                        title: "Blame".to_owned(),
                        link: "javascript:alert('Hover over the gray bar on the left to see blame information.')".to_owned(),
                        update_link_lineno: "",
                        accel_key: None,
                    });
                }
                panel.push(PanelSection {
                    name: "Revision control".to_owned(),
                    items: vcs_panel_items,
                });
            }
        }

        let mut tools_items = vec![];
        if let Some(ref hg_root) = tree_config.paths.hg_root {
            tools_items.push(PanelItem {
                title: "HG Web".to_owned(),
                link: format!("{}/file/tip/{}", hg_root, path),
                update_link_lineno: "#l{}",
                accel_key: None,
            });
        }
        if let Some(ref ccov_root) = tree_config.paths.ccov_root {
            tools_items.push(PanelItem {
                title: "Code Coverage".to_owned(),
                link: format!("{}#revision=latest&path={}&view=file", ccov_root, path),
                update_link_lineno: "&line={}",
                accel_key: None,
            });
        }
        if let Some(ref dxr_root) = tree_config.paths.dxr_root {
            tools_items.push(PanelItem {
                title: "DXR".to_owned(),
                link: format!("{}/source/{}", dxr_root, path),
                update_link_lineno: "#{}",
                accel_key: None,
            });
        }
        if let Some(ref github) = tree_config.paths.github_repo {
            match Path::new(path).extension().and_then(OsStr::to_str) {
                Some("md") | Some("rst") => {
                    tools_items.push(PanelItem {
                        title: "Rendered view".to_owned(),
                        link: format!(
                            "{}/blob/{}/{}",
                            github,
                            head_oid.map_or("master".to_string(), |x| x.to_string()),
                            path
                        ),
                        update_link_lineno: "",
                        accel_key: None,
                    });
                }
                _ => (),
            };
        }
        if !tools_items.is_empty() {
            panel.push(PanelSection {
                name: "Other Tools".to_owned(),
                items: tools_items,
            });
        }

        format_file_data(
            &cfg,
            tree_name,
            &panel,
            &head_commit,
            &blame_commit,
            path,
            input,
            &jumps,
            &analysis,
            &mut writer,
            Some(&mut diff_cache),
        )
        .unwrap();
    }
}
<|endoftext|>


<commit_before>use std::char;
use std::cell::Cell;

#[derive(Debug)]
pub enum Value {
    Int(String),
    Str(String),
}

#[derive(Debug, PartialEq)]
pub enum TokenType {
    ILLEGAL,
    EOF,
    IDENT,
    INT,
    ASSIGN,
    PLUS,
    MINUS,
    COMMA,
    SEMICOLON,
    LPAREN,
    RPAREN,
    LBRACE,
    RBRACE,
    FUNCTION,
    LET,
}

impl TokenType {
    pub fn from_string(s: String) -> TokenType {
        match s.trim().as_ref() {
            "\n" => TokenType::EOF,
            "=" => TokenType::ASSIGN,
            "(" => TokenType::LPAREN,
            ")" => TokenType::RPAREN,
            "let" => TokenType::LET,
            "+" => TokenType::PLUS,
            "-" => TokenType::MINUS,
            "," => TokenType::COMMA,
            ";" => TokenType::SEMICOLON,
            "{" => TokenType::LBRACE,
            "}" => TokenType::RBRACE,
            "func" => TokenType::FUNCTION,
            c => TokenType::ILLEGAL
        }
    }
}

#[derive(Debug)]
pub struct Token {
    pub token_type: TokenType,
    pub value: Option<Value>,
}

pub struct Lexer {
    input: String,
    position: Cell<usize>,
    read_position: Cell<usize>,
}

impl Lexer {
    pub fn new(input: String) -> Lexer {
        return Lexer {
            input: input,
            position: Cell::new(0),
            read_position: Cell::new(0),
        };
    }

    pub fn read_char(&self) -> char {
        if self.read_position.get() >= self.input.len() {
            return '0';
        } else {
            let c = self.input.chars().nth(self.read_position.get()).unwrap();
            self.position.set(self.read_position.get());
            self.read_position.set(self.read_position.get() + 1);
            return c;
        }
    }

    pub fn next_token(&self) -> Token {
        let c = self.read_char();
        let token_type = TokenType::from_string(c.to_string());
        return Token {
            token_type: token_type,
            value: Some(Value::Str(c.to_string()))
        };
    }

    pub fn peek_char(&self) -> char {
        if self.read_position.get() >= self.input.len() {
            return '0';
        } else {
            return self.input.chars().nth(self.read_position.get()).unwrap();
        }
    }
}

#[test]
fn read_char() {
    let input = String::from("=!");
    let l = &Lexer::new(input);
    assert_eq!('=', l.read_char());
    assert_eq!('!', l.read_char());
    assert_eq!('0', l.read_char());
}

#[test]
fn peek_char() {
    let input = String::from("!=");
    let l = &Lexer::new(input);
    assert_eq!('!', l.peek_char());
    assert_eq!('!', l.peek_char());
}

#[test]
fn next_token() {
    let input = String::from("=");
    let l = &Lexer::new(input);
    assert_eq!(l.next_token().token_type, TokenType::ASSIGN);
}

#[test]
fn from_string() {
    assert_eq!(TokenType::from_string(String::from("=")), TokenType::ASSIGN);
    assert_eq!(TokenType::from_string(String::from("(")), TokenType::LPAREN);
}
<commit_msg>add test<commit_after>use std::char;
use std::cell::Cell;

#[derive(Debug)]
pub enum Value {
    Int(String),
    Str(String),
}

#[derive(Debug, PartialEq)]
pub enum TokenType {
    ILLEGAL,
    EOF,
    IDENT,
    INT,
    ASSIGN,
    PLUS,
    MINUS,
    COMMA,
    SEMICOLON,
    LPAREN,
    RPAREN,
    LBRACE,
    RBRACE,
    FUNCTION,
    LET,
}

impl TokenType {
    pub fn from_string(s: String) -> TokenType {
        match s.trim().as_ref() {
            "\n" => TokenType::EOF,
            "=" => TokenType::ASSIGN,
            "(" => TokenType::LPAREN,
            ")" => TokenType::RPAREN,
            "let" => TokenType::LET,
            "+" => TokenType::PLUS,
            "-" => TokenType::MINUS,
            "," => TokenType::COMMA,
            ";" => TokenType::SEMICOLON,
            "{" => TokenType::LBRACE,
            "}" => TokenType::RBRACE,
            "func" => TokenType::FUNCTION,
            c => TokenType::ILLEGAL
        }
    }
}

#[derive(Debug)]
pub struct Token {
    pub token_type: TokenType,
    pub value: Option<Value>,
}

pub struct Lexer {
    input: String,
    position: Cell<usize>,
    read_position: Cell<usize>,
}

impl Lexer {
    pub fn new(input: String) -> Lexer {
        return Lexer {
            input: input,
            position: Cell::new(0),
            read_position: Cell::new(0),
        };
    }

    pub fn read_char(&self) -> char {
        if self.read_position.get() >= self.input.len() {
            return '0';
        } else {
            let c = self.input.chars().nth(self.read_position.get()).unwrap();
            self.position.set(self.read_position.get());
            self.read_position.set(self.read_position.get() + 1);
            return c;
        }
    }

    pub fn next_token(&self) -> Token {
        let c = self.read_char();
        let token_type = TokenType::from_string(c.to_string());
        return Token {
            token_type: token_type,
            value: Some(Value::Str(c.to_string()))
        };
    }

    pub fn peek_char(&self) -> char {
        if self.read_position.get() >= self.input.len() {
            return '0';
        } else {
            return self.input.chars().nth(self.read_position.get()).unwrap();
        }
    }
}

#[test]
fn read_char() {
    let input = String::from("=!");
    let l = &Lexer::new(input);
    assert_eq!('=', l.read_char());
    assert_eq!('!', l.read_char());
    assert_eq!('0', l.read_char());
}

#[test]
fn peek_char() {
    let input = String::from("!=");
    let l = &Lexer::new(input);
    assert_eq!('!', l.peek_char());
    assert_eq!('!', l.peek_char());
}

#[test]
fn next_token() {
    let input = String::from("=()");
    let l = &Lexer::new(input);
    assert_eq!(l.next_token().token_type, TokenType::ASSIGN);
    assert_eq!(l.next_token().token_type, TokenType::LPAREN);
    assert_eq!(l.next_token().token_type, TokenType::RPAREN);
}

#[test]
fn from_string() {
    assert_eq!(TokenType::from_string(String::from("=")), TokenType::ASSIGN);
    assert_eq!(TokenType::from_string(String::from("(")), TokenType::LPAREN);
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use std::sync::Arc;

use buffer::BufferSlice;
use buffer::sys::UnsafeBuffer;
use command_buffer::states_manager::StatesManager;
use device::Queue;
use memory::Content;

use sync::AccessFlagBits;
use sync::Fence;
use sync::PipelineStages;
use sync::Semaphore;

/// Trait for objects that represent either a buffer or a slice of a buffer.
pub unsafe trait Buffer {
    /// Returns the inner information about this buffer.
    fn inner(&self) -> BufferInner;

    #[inline]
    fn size(&self) -> usize {
        self.inner().buffer.size()
    }

    #[inline]
    fn len(&self) -> usize where Self: TypedBuffer, Self::Content: Content {
        self.size() / <Self::Content as Content>::indiv_size()
    }

    #[inline]
    fn as_buffer_slice(&self) -> BufferSlice<Self::Content, &Self> where Self: Sized + TypedBuffer {
        BufferSlice::from(self)
    }

    #[inline]
    fn into_buffer_slice(self) -> BufferSlice<Self::Content, Self> where Self: Sized + TypedBuffer {
        BufferSlice::from(self)
    }
}

#[derive(Copy, Clone, Debug)]
pub struct BufferInner<'a> {
    pub buffer: &'a UnsafeBuffer,
    pub offset: usize
}

unsafe impl<'a, B: ?Sized> Buffer for &'a B where B: Buffer + 'a {
    #[inline]
    fn inner(&self) -> BufferInner {
        (**self).inner()
    }

    #[inline]
    fn size(&self) -> usize {
        (**self).size()
    }
}

unsafe impl<B: ?Sized> Buffer for Arc<B> where B: Buffer {
    #[inline]
    fn inner(&self) -> BufferInner {
        (**self).inner()
    }

    #[inline]
    fn size(&self) -> usize {
        (**self).size()
    }
}

/// Extension trait for `Buffer`. Types that implement this can be used in a `StdCommandBuffer`.
///
/// Each buffer and image used in a `StdCommandBuffer` have an associated state which is
/// represented by the `CommandListState` associated type of this trait. You can make multiple
/// buffers or images share the same state by making `is_same` return true.
pub unsafe trait TrackedBuffer<States = StatesManager>: Buffer {
    /// Returns a new state that corresponds to the moment after a slice of the buffer has been
    /// used in the pipeline. The parameters indicate in which way it has been used.
    ///
    /// If the transition should result in a pipeline barrier, then it must be returned by this
    /// function.
    // TODO: what should be the behavior if `num_command` is equal to the `num_command` of a
    // previous transition?
    fn transition(&self, states: &mut States, num_command: usize, offset: usize, size: usize,
                  write: bool, stage: PipelineStages, access: AccessFlagBits)
                  -> Option<PipelineBarrierRequest>;

    /// Function called when the command buffer builder is turned into a real command buffer.
    ///
    /// This function can return an additional pipeline barrier that will be applied at the end
    /// of the command buffer.
    fn finish(&self, in_s: &mut States, out: &mut States) -> Option<PipelineBarrierRequest>;

    /// Called right before the command buffer is submitted.
    // TODO: function should be unsafe because it must be guaranteed that a cb is submitted
    fn on_submit<F>(&self, states: &States, queue: &Arc<Queue>, fence: F) -> SubmitInfos
        where F: FnOnce() -> Arc<Fence>;
}

/// Requests that a pipeline barrier is created.
pub struct PipelineBarrierRequest {
    /// The number of the command after which the barrier should be placed. Must usually match
    /// the number that was passed to the previous call to `transition`, or 0 if the buffer hasn't
    /// been used yet.
    pub after_command_num: usize,

    /// The source pipeline stages of the transition.
    pub source_stage: PipelineStages,

    /// The destination pipeline stages of the transition.
    pub destination_stages: PipelineStages,

    /// If true, the pipeliner barrier is by region. There is literaly no reason to pass `false`
    /// here, but it is included just in case.
    pub by_region: bool,

    /// An optional memory barrier. See the docs of `PipelineMemoryBarrierRequest`.
    pub memory_barrier: Option<PipelineMemoryBarrierRequest>,
}

/// Requests that a memory barrier is created as part of the pipeline barrier.
///
/// By default, a pipeline barrier only guarantees that the source operations are executed before
/// the destination operations, but it doesn't make memory writes made by source operations visible
/// to the destination operations. In order to make so, you have to add a memory barrier.
///
/// The memory barrier always concerns the buffer that is currently being processed. You can't add
/// a memory barrier that concerns another resource.
pub struct PipelineMemoryBarrierRequest {
    /// Offset of start of the range to flush.
    pub offset: isize,
    /// Size of the range to flush.
    pub size: usize,
    /// Source accesses.
    pub source_access: AccessFlagBits,
    /// Destination accesses.
    pub destination_access: AccessFlagBits,
}

pub struct SubmitInfos {
    pub pre_semaphore: Option<(Arc<Semaphore>, PipelineStages)>,
    pub post_semaphore: Option<Arc<Semaphore>>,
    pub pre_barrier: Option<PipelineBarrierRequest>,
    pub post_barrier: Option<PipelineBarrierRequest>,
}

unsafe impl<B:? Sized, S> TrackedBuffer<S> for Arc<B> where B: TrackedBuffer<S> {
    #[inline]
    fn transition(&self, states: &mut S, num_command: usize, offset: usize,
                  size: usize, write: bool, stage: PipelineStages, access: AccessFlagBits)
                  -> Option<PipelineBarrierRequest>
    {
        (**self).transition(states, num_command, offset, size, write, stage, access)
    }

    #[inline]
    fn finish(&self, i: &mut S, o: &mut S) -> Option<PipelineBarrierRequest> {
        (**self).finish(i, o)
    }

    #[inline]
    fn on_submit<F>(&self, states: &S, queue: &Arc<Queue>, fence: F) -> SubmitInfos
        where F: FnOnce() -> Arc<Fence>
    {
        (**self).on_submit(states, queue, fence)
    }
}

unsafe impl<'a, B: ?Sized, S> TrackedBuffer<S> for &'a B where B: TrackedBuffer<S> + 'a {
    #[inline]
    fn transition(&self, states: &mut S, num_command: usize, offset: usize,
                  size: usize, write: bool, stage: PipelineStages, access: AccessFlagBits)
                  -> Option<PipelineBarrierRequest>
    {
        (**self).transition(states, num_command, offset, size, write, stage, access)
    }

    #[inline]
    fn finish(&self, i: &mut S, o: &mut S) -> Option<PipelineBarrierRequest> {
        (**self).finish(i, o)
    }

    #[inline]
    fn on_submit<F>(&self, states: &S, queue: &Arc<Queue>, fence: F) -> SubmitInfos
        where F: FnOnce() -> Arc<Fence>
    {
        (**self).on_submit(states, queue, fence)
    }
}

pub unsafe trait TypedBuffer: Buffer {
    type Content: ?Sized + 'static;
}

unsafe impl<B: ?Sized> TypedBuffer for Arc<B> where B: TypedBuffer {
    type Content = B::Content;
}

unsafe impl<'a, B: ?Sized + 'a> TypedBuffer for &'a B where B: TypedBuffer {
    type Content = B::Content;
}
<commit_msg>Add Buffer::slice() method<commit_after>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use std::ops::Range;
use std::sync::Arc;

use buffer::BufferSlice;
use buffer::sys::UnsafeBuffer;
use command_buffer::states_manager::StatesManager;
use device::Queue;
use memory::Content;

use sync::AccessFlagBits;
use sync::Fence;
use sync::PipelineStages;
use sync::Semaphore;

/// Trait for objects that represent either a buffer or a slice of a buffer.
pub unsafe trait Buffer {
    /// Returns the inner information about this buffer.
    fn inner(&self) -> BufferInner;

    #[inline]
    fn size(&self) -> usize {
        self.inner().buffer.size()
    }

    #[inline]
    fn len(&self) -> usize where Self: TypedBuffer, Self::Content: Content {
        self.size() / <Self::Content as Content>::indiv_size()
    }

    #[inline]
    fn as_buffer_slice(&self) -> BufferSlice<Self::Content, &Self> where Self: Sized + TypedBuffer {
        BufferSlice::from(self)
    }

    ///
    /// Returns `None` if out of range.
    #[inline]
    fn slice<T>(&self, range: Range<usize>) -> Option<BufferSlice<[T], &Self>>
        where Self: Sized + TypedBuffer<Content = [T]>,
              T: 'static
    {
        BufferSlice::slice(self.as_buffer_slice(), range)
    }

    #[inline]
    fn into_buffer_slice(self) -> BufferSlice<Self::Content, Self> where Self: Sized + TypedBuffer {
        BufferSlice::from(self)
    }
}

#[derive(Copy, Clone, Debug)]
pub struct BufferInner<'a> {
    pub buffer: &'a UnsafeBuffer,
    pub offset: usize
}

unsafe impl<'a, B: ?Sized> Buffer for &'a B where B: Buffer + 'a {
    #[inline]
    fn inner(&self) -> BufferInner {
        (**self).inner()
    }

    #[inline]
    fn size(&self) -> usize {
        (**self).size()
    }
}

unsafe impl<B: ?Sized> Buffer for Arc<B> where B: Buffer {
    #[inline]
    fn inner(&self) -> BufferInner {
        (**self).inner()
    }

    #[inline]
    fn size(&self) -> usize {
        (**self).size()
    }
}

/// Extension trait for `Buffer`. Types that implement this can be used in a `StdCommandBuffer`.
///
/// Each buffer and image used in a `StdCommandBuffer` have an associated state which is
/// represented by the `CommandListState` associated type of this trait. You can make multiple
/// buffers or images share the same state by making `is_same` return true.
pub unsafe trait TrackedBuffer<States = StatesManager>: Buffer {
    /// Returns a new state that corresponds to the moment after a slice of the buffer has been
    /// used in the pipeline. The parameters indicate in which way it has been used.
    ///
    /// If the transition should result in a pipeline barrier, then it must be returned by this
    /// function.
    // TODO: what should be the behavior if `num_command` is equal to the `num_command` of a
    // previous transition?
    fn transition(&self, states: &mut States, num_command: usize, offset: usize, size: usize,
                  write: bool, stage: PipelineStages, access: AccessFlagBits)
                  -> Option<PipelineBarrierRequest>;

    /// Function called when the command buffer builder is turned into a real command buffer.
    ///
    /// This function can return an additional pipeline barrier that will be applied at the end
    /// of the command buffer.
    fn finish(&self, in_s: &mut States, out: &mut States) -> Option<PipelineBarrierRequest>;

    /// Called right before the command buffer is submitted.
    // TODO: function should be unsafe because it must be guaranteed that a cb is submitted
    fn on_submit<F>(&self, states: &States, queue: &Arc<Queue>, fence: F) -> SubmitInfos
        where F: FnOnce() -> Arc<Fence>;
}

/// Requests that a pipeline barrier is created.
pub struct PipelineBarrierRequest {
    /// The number of the command after which the barrier should be placed. Must usually match
    /// the number that was passed to the previous call to `transition`, or 0 if the buffer hasn't
    /// been used yet.
    pub after_command_num: usize,

    /// The source pipeline stages of the transition.
    pub source_stage: PipelineStages,

    /// The destination pipeline stages of the transition.
    pub destination_stages: PipelineStages,

    /// If true, the pipeliner barrier is by region. There is literaly no reason to pass `false`
    /// here, but it is included just in case.
    pub by_region: bool,

    /// An optional memory barrier. See the docs of `PipelineMemoryBarrierRequest`.
    pub memory_barrier: Option<PipelineMemoryBarrierRequest>,
}

/// Requests that a memory barrier is created as part of the pipeline barrier.
///
/// By default, a pipeline barrier only guarantees that the source operations are executed before
/// the destination operations, but it doesn't make memory writes made by source operations visible
/// to the destination operations. In order to make so, you have to add a memory barrier.
///
/// The memory barrier always concerns the buffer that is currently being processed. You can't add
/// a memory barrier that concerns another resource.
pub struct PipelineMemoryBarrierRequest {
    /// Offset of start of the range to flush.
    pub offset: isize,
    /// Size of the range to flush.
    pub size: usize,
    /// Source accesses.
    pub source_access: AccessFlagBits,
    /// Destination accesses.
    pub destination_access: AccessFlagBits,
}

pub struct SubmitInfos {
    pub pre_semaphore: Option<(Arc<Semaphore>, PipelineStages)>,
    pub post_semaphore: Option<Arc<Semaphore>>,
    pub pre_barrier: Option<PipelineBarrierRequest>,
    pub post_barrier: Option<PipelineBarrierRequest>,
}

unsafe impl<B:? Sized, S> TrackedBuffer<S> for Arc<B> where B: TrackedBuffer<S> {
    #[inline]
    fn transition(&self, states: &mut S, num_command: usize, offset: usize,
                  size: usize, write: bool, stage: PipelineStages, access: AccessFlagBits)
                  -> Option<PipelineBarrierRequest>
    {
        (**self).transition(states, num_command, offset, size, write, stage, access)
    }

    #[inline]
    fn finish(&self, i: &mut S, o: &mut S) -> Option<PipelineBarrierRequest> {
        (**self).finish(i, o)
    }

    #[inline]
    fn on_submit<F>(&self, states: &S, queue: &Arc<Queue>, fence: F) -> SubmitInfos
        where F: FnOnce() -> Arc<Fence>
    {
        (**self).on_submit(states, queue, fence)
    }
}

unsafe impl<'a, B: ?Sized, S> TrackedBuffer<S> for &'a B where B: TrackedBuffer<S> + 'a {
    #[inline]
    fn transition(&self, states: &mut S, num_command: usize, offset: usize,
                  size: usize, write: bool, stage: PipelineStages, access: AccessFlagBits)
                  -> Option<PipelineBarrierRequest>
    {
        (**self).transition(states, num_command, offset, size, write, stage, access)
    }

    #[inline]
    fn finish(&self, i: &mut S, o: &mut S) -> Option<PipelineBarrierRequest> {
        (**self).finish(i, o)
    }

    #[inline]
    fn on_submit<F>(&self, states: &S, queue: &Arc<Queue>, fence: F) -> SubmitInfos
        where F: FnOnce() -> Arc<Fence>
    {
        (**self).on_submit(states, queue, fence)
    }
}

pub unsafe trait TypedBuffer: Buffer {
    type Content: ?Sized + 'static;
}

unsafe impl<B: ?Sized> TypedBuffer for Arc<B> where B: TypedBuffer {
    type Content = B::Content;
}

unsafe impl<'a, B: ?Sized + 'a> TypedBuffer for &'a B where B: TypedBuffer {
    type Content = B::Content;
}
<|endoftext|>


<commit_before>// Copyright (c) 2020, The rav1e contributors. All rights reserved
//
// This source code is subject to the terms of the BSD 2 Clause License and
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
// was not distributed with this source code in the LICENSE file, you can
// obtain it at www.aomedia.org/license/software. If the Alliance for Open
// Media Patent License 1.0 was not distributed with this source code in the
// PATENTS file, you can obtain it at www.aomedia.org/license/patent.

use thiserror::Error;

use std::sync::Arc;

use crate::api::{ChromaSampling, Context, ContextInner, PixelRange};
use crate::cpu_features::CpuFeatureLevel;
use crate::rayon::{ThreadPool, ThreadPoolBuilder};
use crate::tiling::TilingInfo;
use crate::util::Pixel;

mod encoder;
pub use encoder::*;

mod rate;
pub use rate::Error as RateControlError;
pub use rate::{RateControlConfig, RateControlSummary};

mod speedsettings;
pub use speedsettings::*;

/// Enumeration of possible invalid configuration errors.
#[derive(Debug, Clone, Copy, Eq, PartialEq, Error)]
#[non_exhaustive]
pub enum InvalidConfig {
  /// The width is invalid.
  #[error("invalid width {0} (expected >= 16, <= 65535)")]
  InvalidWidth(usize),
  /// The height is invalid.
  #[error("invalid height {0} (expected >= 16, <= 65535)")]
  InvalidHeight(usize),
  /// Aspect ratio numerator is invalid.
  #[error("invalid aspect ratio numerator {0} (expected > 0)")]
  InvalidAspectRatioNum(usize),
  /// Aspect ratio denominator is invalid.
  #[error("invalid aspect ratio denominator {0} (expected > 0)")]
  InvalidAspectRatioDen(usize),
  /// The render width (width adjusted based on the aspect ratio) is invalid.
  #[error("invalid render width {0} (expected >= 1, <= 65535")]
  InvalidRenderWidth(usize),
  /// The render height (height adjusted based on the aspect ratio) is invalid.
  #[error("invalid render height {0} (expected >= 1, <= 65535")]
  InvalidRenderHeight(usize),
  /// RDO lookahead frame count is invalid.
  #[error(
    "invalid rdo lookahead frames {actual} (expected <= {max} and >= {min})"
  )]
  InvalidRdoLookaheadFrames {
    /// The actual value.
    actual: usize,
    /// The maximal supported value.
    max: usize,
    /// The minimal supported value.
    min: usize,
  },
  /// Maximal keyframe interval is invalid.
  #[error("invalid max keyframe interval {actual} (expected <= {max})")]
  InvalidMaxKeyFrameInterval {
    /// The actual value.
    actual: u64,
    /// The maximal supported value.
    max: u64,
  },
  /// Tile columns is invalid.
  #[error("invalid tile cols {0} (expected power of 2)")]
  InvalidTileCols(usize),
  /// Tile rows is invalid.
  #[error("invalid tile rows {0} (expected power of 2)")]
  InvalidTileRows(usize),
  /// Framerate numerator is invalid.
  #[error("invalid framerate numerator {actual} (expected > 0, <= {max})")]
  InvalidFrameRateNum {
    /// The actual value.
    actual: u64,
    /// The maximal supported value.
    max: u64,
  },
  /// Framerate denominator is invalid.
  #[error("invalid framerate denominator {actual} (expected > 0, <= {max})")]
  InvalidFrameRateDen {
    /// The actual value.
    actual: u64,
    /// The maximal supported value.
    max: u64,
  },
  /// Reservoir frame delay is invalid.
  #[error("invalid reservoir frame delay {0} (expected >= 12, <= 131072)")]
  InvalidReservoirFrameDelay(i32),
  /// Reservoir frame delay is invalid.
  #[error(
    "invalid switch frame interval {0} (must only be used with low latency mode)"
  )]
  InvalidSwitchFrameInterval(u64),

  /// An option unsupported in still picture mode was enabled along with it.
  #[error("invalid option {0} specified with still picture mode")]
  InvalidOptionWithStillPicture(&'static str),

  /// The rate control needs a target bitrate in order to produce results
  #[error("The rate control requires a target bitrate")]
  TargetBitrateNeeded,

  /// The configuration
  #[error("Mismatch in the rate control configuration")]
  RateControlConfigurationMismatch,

  /// The color configuration mismatches AV1 constraints.
  #[error("Mismatch in the color configuration")]
  ColorConfigurationMismatch,
}

/// Contains the encoder configuration.
#[derive(Clone, Debug, Default)]
pub struct Config {
  /// Settings which impact the produced bitstream.
  pub(crate) enc: EncoderConfig,
  /// Rate control configuration
  pub(crate) rate_control: RateControlConfig,
  /// The number of threads in the threadpool.
  pub(crate) threads: usize,
  /// Shared thread pool
  pub(crate) pool: Option<Arc<ThreadPool>>,
}

impl Config {
  /// Create a default configuration
  ///
  /// same as Default::default()
  pub fn new() -> Self {
    Config::default()
  }

  /// Set the encoder configuration
  ///
  /// EncoderConfig contains the settings impacting the
  /// codec features used in the produced bitstream.
  pub fn with_encoder_config(mut self, enc: EncoderConfig) -> Self {
    self.enc = enc;
    self
  }

  /// Set the number of workers in the threadpool
  ///
  /// The threadpool is shared across all the different parallel
  /// components in the encoder.
  ///
  /// If it is left unset `new_context()` will create a new default one
  /// `new_*_channel()` will use the default global threadpool.
  pub fn with_threads(mut self, threads: usize) -> Self {
    self.threads = threads;
    self
  }

  /// Set the rate control configuration
  ///
  /// The default configuration is single pass
  pub fn with_rate_control(mut self, rate_control: RateControlConfig) -> Self {
    self.rate_control = rate_control;
    self
  }

  #[cfg(feature = "unstable")]
  /// Use the provided threadpool
  ///
  /// It takes priority over `with_threads()`
  pub fn with_thread_pool(mut self, pool: Arc<ThreadPool>) -> Self {
    self.pool = Some(pool);
    self
  }
}

fn check_tile_log2(n: usize) -> bool {
  let tile_log2 = TilingInfo::tile_log2(1, n);
  if tile_log2.is_none() {
    return false;
  }
  let tile_log2 = tile_log2.unwrap();

  ((1 << tile_log2) - n) == 0 || n == 0
}

impl Config {
  pub(crate) fn new_inner<T: Pixel>(
    &self,
  ) -> Result<ContextInner<T>, InvalidConfig> {
    assert!(
      8 * std::mem::size_of::<T>() >= self.enc.bit_depth,
      "The Pixel u{} does not match the Config bit_depth {}",
      8 * std::mem::size_of::<T>(),
      self.enc.bit_depth
    );

    self.validate()?;

    // Because we don't have a FrameInvariants yet,
    // this is the only way to get the CpuFeatureLevel in use.
    // Since we only call this once, this shouldn't cause
    // performance issues.
    info!("CPU Feature Level: {}", CpuFeatureLevel::default());

    let mut config = self.enc;
    config.set_key_frame_interval(
      config.min_key_frame_interval,
      config.max_key_frame_interval,
    );

    // FIXME: inter unsupported with 4:2:2 and 4:4:4 chroma sampling
    let chroma_sampling = config.chroma_sampling;

    // FIXME: tx partition for intra not supported for chroma 422
    if chroma_sampling == ChromaSampling::Cs422 {
      config.speed_settings.rdo_tx_decision = false;
    }

    let mut inner = ContextInner::new(&config);

    if self.rate_control.emit_pass_data {
      let params = inner.rc_state.get_twopass_out_params(&inner, 0);
      inner.rc_state.init_first_pass(params.pass1_log_base_q);
    }

    if let Some(ref s) = self.rate_control.summary {
      inner.rc_state.init_second_pass();
      inner.rc_state.setup_second_pass(s);
    }

    Ok(inner)
  }

  /// Create a new threadpool with this configuration if set,
  /// or return `None` if global threadpool should be used instead.
  pub(crate) fn new_thread_pool(&self) -> Option<Arc<ThreadPool>> {
    if let Some(ref p) = self.pool {
      Some(p.clone())
    } else if self.threads != 0 {
      let pool =
        ThreadPoolBuilder::new().num_threads(self.threads).build().unwrap();
      Some(Arc::new(pool))
    } else {
      None
    }
  }

  /// Creates a [`Context`] with this configuration.
  ///
  /// # Examples
  ///
  /// ```
  /// use rav1e::prelude::*;
  ///
  /// # fn main() -> Result<(), InvalidConfig> {
  /// let cfg = Config::default();
  /// let ctx: Context<u8> = cfg.new_context()?;
  /// # Ok(())
  /// # }
  /// ```
  ///
  /// [`Context`]: struct.Context.html
  pub fn new_context<T: Pixel>(&self) -> Result<Context<T>, InvalidConfig> {
    let inner = self.new_inner()?;
    let config = *inner.config;
    let pool = self.new_thread_pool();

    Ok(Context { is_flushing: false, inner, pool, config })
  }

  /// Validates the configuration.
  pub fn validate(&self) -> Result<(), InvalidConfig> {
    use InvalidConfig::*;

    let config = &self.enc;

    if (config.still_picture && config.width < 1)
      || (!config.still_picture && config.width < 16)
      || config.width > u16::max_value() as usize
    {
      return Err(InvalidWidth(config.width));
    }
    if (config.still_picture && config.height < 1)
      || (!config.still_picture && config.height < 16)
      || config.height > u16::max_value() as usize
    {
      return Err(InvalidHeight(config.height));
    }

    if config.sample_aspect_ratio.num == 0 {
      return Err(InvalidAspectRatioNum(
        config.sample_aspect_ratio.num as usize,
      ));
    }
    if config.sample_aspect_ratio.den == 0 {
      return Err(InvalidAspectRatioDen(
        config.sample_aspect_ratio.den as usize,
      ));
    }

    let (render_width, render_height) = config.render_size();
    if render_width == 0 || render_width > u16::max_value() as usize {
      return Err(InvalidRenderWidth(render_width));
    }
    if render_height == 0 || render_height > u16::max_value() as usize {
      return Err(InvalidRenderHeight(render_height));
    }

    if config.rdo_lookahead_frames > MAX_RDO_LOOKAHEAD_FRAMES
      || config.rdo_lookahead_frames < 1
    {
      return Err(InvalidRdoLookaheadFrames {
        actual: config.rdo_lookahead_frames,
        max: MAX_RDO_LOOKAHEAD_FRAMES,
        min: 1,
      });
    }
    if config.max_key_frame_interval > MAX_MAX_KEY_FRAME_INTERVAL {
      return Err(InvalidMaxKeyFrameInterval {
        actual: config.max_key_frame_interval,
        max: MAX_MAX_KEY_FRAME_INTERVAL,
      });
    }

    if !check_tile_log2(config.tile_cols) {
      return Err(InvalidTileCols(config.tile_cols));
    }
    if !check_tile_log2(config.tile_rows) {
      return Err(InvalidTileRows(config.tile_rows));
    }

    if config.time_base.num == 0
      || config.time_base.num > u32::max_value() as u64
    {
      return Err(InvalidFrameRateNum {
        actual: config.time_base.num,
        max: u32::max_value() as u64,
      });
    }
    if config.time_base.den == 0
      || config.time_base.den > u32::max_value() as u64
    {
      return Err(InvalidFrameRateDen {
        actual: config.time_base.den,
        max: u32::max_value() as u64,
      });
    }

    if let Some(delay) = config.reservoir_frame_delay {
      if !(12..=131_072).contains(&delay) {
        return Err(InvalidReservoirFrameDelay(delay));
      }
    }

    if config.switch_frame_interval > 0 && !config.low_latency {
      return Err(InvalidSwitchFrameInterval(config.switch_frame_interval));
    }

    if config.enable_timing_info && config.still_picture {
      return Err(InvalidOptionWithStillPicture("enable_timing_info"));
    }

    // <https://aomediacodec.github.io/av1-spec/#color-config-syntax>
    if let Some(color_description) = config.color_description {
      if config.chroma_sampling != ChromaSampling::Cs400
        && color_description.is_srgb_triple()
      {
        if config.pixel_range != PixelRange::Full {
          return Err(ColorConfigurationMismatch);
        }
        if config.chroma_sampling != ChromaSampling::Cs444 {
          return Err(ColorConfigurationMismatch);
        }
      }
    }

    // TODO: add more validation
    let rc = &self.rate_control;

    if (rc.emit_pass_data || rc.summary.is_some()) && config.bitrate == 0 {
      return Err(TargetBitrateNeeded);
    }

    Ok(())
  }
}
<commit_msg>Reword doc comment<commit_after>// Copyright (c) 2020, The rav1e contributors. All rights reserved
//
// This source code is subject to the terms of the BSD 2 Clause License and
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
// was not distributed with this source code in the LICENSE file, you can
// obtain it at www.aomedia.org/license/software. If the Alliance for Open
// Media Patent License 1.0 was not distributed with this source code in the
// PATENTS file, you can obtain it at www.aomedia.org/license/patent.

use thiserror::Error;

use std::sync::Arc;

use crate::api::{ChromaSampling, Context, ContextInner, PixelRange};
use crate::cpu_features::CpuFeatureLevel;
use crate::rayon::{ThreadPool, ThreadPoolBuilder};
use crate::tiling::TilingInfo;
use crate::util::Pixel;

mod encoder;
pub use encoder::*;

mod rate;
pub use rate::Error as RateControlError;
pub use rate::{RateControlConfig, RateControlSummary};

mod speedsettings;
pub use speedsettings::*;

/// Enumeration of possible invalid configuration errors.
#[derive(Debug, Clone, Copy, Eq, PartialEq, Error)]
#[non_exhaustive]
pub enum InvalidConfig {
  /// The width is invalid.
  #[error("invalid width {0} (expected >= 16, <= 65535)")]
  InvalidWidth(usize),
  /// The height is invalid.
  #[error("invalid height {0} (expected >= 16, <= 65535)")]
  InvalidHeight(usize),
  /// Aspect ratio numerator is invalid.
  #[error("invalid aspect ratio numerator {0} (expected > 0)")]
  InvalidAspectRatioNum(usize),
  /// Aspect ratio denominator is invalid.
  #[error("invalid aspect ratio denominator {0} (expected > 0)")]
  InvalidAspectRatioDen(usize),
  /// The render width (width adjusted based on the aspect ratio) is invalid.
  #[error("invalid render width {0} (expected >= 1, <= 65535")]
  InvalidRenderWidth(usize),
  /// The render height (height adjusted based on the aspect ratio) is invalid.
  #[error("invalid render height {0} (expected >= 1, <= 65535")]
  InvalidRenderHeight(usize),
  /// RDO lookahead frame count is invalid.
  #[error(
    "invalid rdo lookahead frames {actual} (expected <= {max} and >= {min})"
  )]
  InvalidRdoLookaheadFrames {
    /// The actual value.
    actual: usize,
    /// The maximal supported value.
    max: usize,
    /// The minimal supported value.
    min: usize,
  },
  /// Maximal keyframe interval is invalid.
  #[error("invalid max keyframe interval {actual} (expected <= {max})")]
  InvalidMaxKeyFrameInterval {
    /// The actual value.
    actual: u64,
    /// The maximal supported value.
    max: u64,
  },
  /// Tile columns is invalid.
  #[error("invalid tile cols {0} (expected power of 2)")]
  InvalidTileCols(usize),
  /// Tile rows is invalid.
  #[error("invalid tile rows {0} (expected power of 2)")]
  InvalidTileRows(usize),
  /// Framerate numerator is invalid.
  #[error("invalid framerate numerator {actual} (expected > 0, <= {max})")]
  InvalidFrameRateNum {
    /// The actual value.
    actual: u64,
    /// The maximal supported value.
    max: u64,
  },
  /// Framerate denominator is invalid.
  #[error("invalid framerate denominator {actual} (expected > 0, <= {max})")]
  InvalidFrameRateDen {
    /// The actual value.
    actual: u64,
    /// The maximal supported value.
    max: u64,
  },
  /// Reservoir frame delay is invalid.
  #[error("invalid reservoir frame delay {0} (expected >= 12, <= 131072)")]
  InvalidReservoirFrameDelay(i32),
  /// Reservoir frame delay is invalid.
  #[error(
    "invalid switch frame interval {0} (must only be used with low latency mode)"
  )]
  InvalidSwitchFrameInterval(u64),

  /// An option unsupported in still picture mode was enabled along with it.
  #[error("invalid option {0} specified with still picture mode")]
  InvalidOptionWithStillPicture(&'static str),

  /// The rate control needs a target bitrate in order to produce results
  #[error("The rate control requires a target bitrate")]
  TargetBitrateNeeded,

  /// The configuration
  #[error("Mismatch in the rate control configuration")]
  RateControlConfigurationMismatch,

  /// The color configuration mismatches AV1 constraints.
  #[error("Mismatch in the color configuration")]
  ColorConfigurationMismatch,
}

/// Contains the encoder configuration.
#[derive(Clone, Debug, Default)]
pub struct Config {
  /// Settings which impact the produced bitstream.
  pub(crate) enc: EncoderConfig,
  /// Rate control configuration
  pub(crate) rate_control: RateControlConfig,
  /// The number of threads in the threadpool.
  pub(crate) threads: usize,
  /// Shared thread pool
  pub(crate) pool: Option<Arc<ThreadPool>>,
}

impl Config {
  /// Create a default configuration
  ///
  /// same as Default::default()
  pub fn new() -> Self {
    Config::default()
  }

  /// Set the encoder configuration
  ///
  /// EncoderConfig contains the settings impacting the
  /// codec features used in the produced bitstream.
  pub fn with_encoder_config(mut self, enc: EncoderConfig) -> Self {
    self.enc = enc;
    self
  }

  /// Set the number of workers in the threadpool
  ///
  /// The threadpool is shared across all the different parallel
  /// components in the encoder.
  ///
  /// If it is left unset, the encoder will use the default global
  /// threadpool provided by Rayon instead.
  pub fn with_threads(mut self, threads: usize) -> Self {
    self.threads = threads;
    self
  }

  /// Set the rate control configuration
  ///
  /// The default configuration is single pass
  pub fn with_rate_control(mut self, rate_control: RateControlConfig) -> Self {
    self.rate_control = rate_control;
    self
  }

  #[cfg(feature = "unstable")]
  /// Use the provided threadpool
  ///
  /// It takes priority over `with_threads()`
  pub fn with_thread_pool(mut self, pool: Arc<ThreadPool>) -> Self {
    self.pool = Some(pool);
    self
  }
}

fn check_tile_log2(n: usize) -> bool {
  let tile_log2 = TilingInfo::tile_log2(1, n);
  if tile_log2.is_none() {
    return false;
  }
  let tile_log2 = tile_log2.unwrap();

  ((1 << tile_log2) - n) == 0 || n == 0
}

impl Config {
  pub(crate) fn new_inner<T: Pixel>(
    &self,
  ) -> Result<ContextInner<T>, InvalidConfig> {
    assert!(
      8 * std::mem::size_of::<T>() >= self.enc.bit_depth,
      "The Pixel u{} does not match the Config bit_depth {}",
      8 * std::mem::size_of::<T>(),
      self.enc.bit_depth
    );

    self.validate()?;

    // Because we don't have a FrameInvariants yet,
    // this is the only way to get the CpuFeatureLevel in use.
    // Since we only call this once, this shouldn't cause
    // performance issues.
    info!("CPU Feature Level: {}", CpuFeatureLevel::default());

    let mut config = self.enc;
    config.set_key_frame_interval(
      config.min_key_frame_interval,
      config.max_key_frame_interval,
    );

    // FIXME: inter unsupported with 4:2:2 and 4:4:4 chroma sampling
    let chroma_sampling = config.chroma_sampling;

    // FIXME: tx partition for intra not supported for chroma 422
    if chroma_sampling == ChromaSampling::Cs422 {
      config.speed_settings.rdo_tx_decision = false;
    }

    let mut inner = ContextInner::new(&config);

    if self.rate_control.emit_pass_data {
      let params = inner.rc_state.get_twopass_out_params(&inner, 0);
      inner.rc_state.init_first_pass(params.pass1_log_base_q);
    }

    if let Some(ref s) = self.rate_control.summary {
      inner.rc_state.init_second_pass();
      inner.rc_state.setup_second_pass(s);
    }

    Ok(inner)
  }

  /// Create a new threadpool with this configuration if set,
  /// or return `None` if global threadpool should be used instead.
  pub(crate) fn new_thread_pool(&self) -> Option<Arc<ThreadPool>> {
    if let Some(ref p) = self.pool {
      Some(p.clone())
    } else if self.threads != 0 {
      let pool =
        ThreadPoolBuilder::new().num_threads(self.threads).build().unwrap();
      Some(Arc::new(pool))
    } else {
      None
    }
  }

  /// Creates a [`Context`] with this configuration.
  ///
  /// # Examples
  ///
  /// ```
  /// use rav1e::prelude::*;
  ///
  /// # fn main() -> Result<(), InvalidConfig> {
  /// let cfg = Config::default();
  /// let ctx: Context<u8> = cfg.new_context()?;
  /// # Ok(())
  /// # }
  /// ```
  ///
  /// [`Context`]: struct.Context.html
  pub fn new_context<T: Pixel>(&self) -> Result<Context<T>, InvalidConfig> {
    let inner = self.new_inner()?;
    let config = *inner.config;
    let pool = self.new_thread_pool();

    Ok(Context { is_flushing: false, inner, pool, config })
  }

  /// Validates the configuration.
  pub fn validate(&self) -> Result<(), InvalidConfig> {
    use InvalidConfig::*;

    let config = &self.enc;

    if (config.still_picture && config.width < 1)
      || (!config.still_picture && config.width < 16)
      || config.width > u16::max_value() as usize
    {
      return Err(InvalidWidth(config.width));
    }
    if (config.still_picture && config.height < 1)
      || (!config.still_picture && config.height < 16)
      || config.height > u16::max_value() as usize
    {
      return Err(InvalidHeight(config.height));
    }

    if config.sample_aspect_ratio.num == 0 {
      return Err(InvalidAspectRatioNum(
        config.sample_aspect_ratio.num as usize,
      ));
    }
    if config.sample_aspect_ratio.den == 0 {
      return Err(InvalidAspectRatioDen(
        config.sample_aspect_ratio.den as usize,
      ));
    }

    let (render_width, render_height) = config.render_size();
    if render_width == 0 || render_width > u16::max_value() as usize {
      return Err(InvalidRenderWidth(render_width));
    }
    if render_height == 0 || render_height > u16::max_value() as usize {
      return Err(InvalidRenderHeight(render_height));
    }

    if config.rdo_lookahead_frames > MAX_RDO_LOOKAHEAD_FRAMES
      || config.rdo_lookahead_frames < 1
    {
      return Err(InvalidRdoLookaheadFrames {
        actual: config.rdo_lookahead_frames,
        max: MAX_RDO_LOOKAHEAD_FRAMES,
        min: 1,
      });
    }
    if config.max_key_frame_interval > MAX_MAX_KEY_FRAME_INTERVAL {
      return Err(InvalidMaxKeyFrameInterval {
        actual: config.max_key_frame_interval,
        max: MAX_MAX_KEY_FRAME_INTERVAL,
      });
    }

    if !check_tile_log2(config.tile_cols) {
      return Err(InvalidTileCols(config.tile_cols));
    }
    if !check_tile_log2(config.tile_rows) {
      return Err(InvalidTileRows(config.tile_rows));
    }

    if config.time_base.num == 0
      || config.time_base.num > u32::max_value() as u64
    {
      return Err(InvalidFrameRateNum {
        actual: config.time_base.num,
        max: u32::max_value() as u64,
      });
    }
    if config.time_base.den == 0
      || config.time_base.den > u32::max_value() as u64
    {
      return Err(InvalidFrameRateDen {
        actual: config.time_base.den,
        max: u32::max_value() as u64,
      });
    }

    if let Some(delay) = config.reservoir_frame_delay {
      if !(12..=131_072).contains(&delay) {
        return Err(InvalidReservoirFrameDelay(delay));
      }
    }

    if config.switch_frame_interval > 0 && !config.low_latency {
      return Err(InvalidSwitchFrameInterval(config.switch_frame_interval));
    }

    if config.enable_timing_info && config.still_picture {
      return Err(InvalidOptionWithStillPicture("enable_timing_info"));
    }

    // <https://aomediacodec.github.io/av1-spec/#color-config-syntax>
    if let Some(color_description) = config.color_description {
      if config.chroma_sampling != ChromaSampling::Cs400
        && color_description.is_srgb_triple()
      {
        if config.pixel_range != PixelRange::Full {
          return Err(ColorConfigurationMismatch);
        }
        if config.chroma_sampling != ChromaSampling::Cs444 {
          return Err(ColorConfigurationMismatch);
        }
      }
    }

    // TODO: add more validation
    let rc = &self.rate_control;

    if (rc.emit_pass_data || rc.summary.is_some()) && config.bitrate == 0 {
      return Err(TargetBitrateNeeded);
    }

    Ok(())
  }
}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::cmp::{Less,Equal,Greater};

#[deriving(TotalEq,TotalOrd)]
struct A<'self> {
    x: &'self int
}
fn main() {
    let (a, b) = (A { x: &1 }, A { x: &2 });

    assert!(a.equals(&a));
    assert!(b.equals(&b));

    assert_eq!(a.cmp(&a), Equal);
    assert_eq!(b.cmp(&b), Equal);

    assert_eq!(a.cmp(&b), Less);
    assert_eq!(b.cmp(&a), Greater);
}
<commit_msg>Make fn pub for windows compatibility<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::cmp::{Less,Equal,Greater};

#[deriving(TotalEq,TotalOrd)]
struct A<'self> {
    x: &'self int
}
pub fn main() {
    let (a, b) = (A { x: &1 }, A { x: &2 });

    assert!(a.equals(&a));
    assert!(b.equals(&b));

    assert_eq!(a.cmp(&a), Equal);
    assert_eq!(b.cmp(&b), Equal);

    assert_eq!(a.cmp(&b), Less);
    assert_eq!(b.cmp(&a), Greater);
}
<|endoftext|>


<commit_before>// ignore-windows: Concurrency on Windows is not supported yet.

// Joining the same thread from multiple threads is undefined behavior.

#![feature(rustc_private)]

extern crate libc;

use std::thread;
use std::{mem, ptr};

extern "C" fn thread_start(_null: *mut libc::c_void) -> *mut libc::c_void {
    ptr::null_mut()
}

fn main() {
    unsafe {
        let mut native: libc::pthread_t = mem::zeroed();
        let attr: libc::pthread_attr_t = mem::zeroed();
        // assert_eq!(libc::pthread_attr_init(&mut attr), 0); FIXME: this function is not yet implemented.
        assert_eq!(libc::pthread_create(&mut native, &attr, thread_start, ptr::null_mut()), 0);
        let mut native_copy: libc::pthread_t = mem::zeroed();
        ptr::copy_nonoverlapping(&native, &mut native_copy, 1);
        let handle = thread::spawn(move || {
            assert_eq!(libc::pthread_join(native_copy, ptr::null_mut()), 0); //~ ERROR: Undefined Behavior: trying to join a detached or already joined thread
        });
        assert_eq!(libc::pthread_join(native, ptr::null_mut()), 0);
        handle.join().unwrap();
    }
}
<commit_msg>Make multiple threads to try to join a thread while it is still running.<commit_after>// ignore-windows: Concurrency on Windows is not supported yet.

// Joining the same thread from multiple threads is undefined behavior.

#![feature(rustc_private)]

extern crate libc;

use std::thread;
use std::{mem, ptr};

extern "C" fn thread_start(_null: *mut libc::c_void) -> *mut libc::c_void {
    thread::yield_now();
    thread::yield_now();
    ptr::null_mut()
}

fn main() {
    unsafe {
        let mut native: libc::pthread_t = mem::zeroed();
        let attr: libc::pthread_attr_t = mem::zeroed();
        // assert_eq!(libc::pthread_attr_init(&mut attr), 0); FIXME: this function is not yet implemented.
        assert_eq!(libc::pthread_create(&mut native, &attr, thread_start, ptr::null_mut()), 0);
        let mut native_copy: libc::pthread_t = mem::zeroed();
        ptr::copy_nonoverlapping(&native, &mut native_copy, 1);
        let handle = thread::spawn(move || {
            assert_eq!(libc::pthread_join(native_copy, ptr::null_mut()), 0); //~ ERROR: Undefined Behavior: trying to join a detached or already joined thread
        });
        assert_eq!(libc::pthread_join(native, ptr::null_mut()), 0);
        handle.join().unwrap();
    }
}
<|endoftext|>