<commit_before>use std;
use std::collections::hash_map::HashMap;
use yaml_rust::Yaml;

use super::env;
use super::yaml_util;
use super::remember;

const VALID_ITEM_NAME_CHARS: &'static str =
    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_";

#[derive(Debug, PartialEq)]
pub enum Token<'a> {
    Substitute(Vec<Token<'a>>),
    Text(&'a str),
    With(&'a str),
    Env(&'a str, Vec<&'a str>),
    Request(&'a str, Vec<String>),
    DefaultVal(&'a str),
}

/// Untemplate the given string
///
/// The strategy this uses is to a take a string like:
///
///     "aaaa{{thing, [env].thing.id, last.response.body.thing}}"
///
/// and convert that to tokens:
///
///     vec![
///         Text("aaaa"),
///         Substitute([
///             With("aaaa"),
///             Env("env", ["thing", "id"]),
///             Request("last", ["response", "body", "thing"]),
///         ])
///     ]
///
/// and then do substitutions and build the resulting string
pub fn untemplate(text: &str, withs: &HashMap<&str, &str>, shortcuts: bool
                  ) -> Result<String, String> {
    let tokens = try!(Tokenizer::new(text, shortcuts).tokenize());
    let mut result = String::new();
    for token in tokens {
        match token {
            Token::Text(text) => {
                result.push_str(text);
            },
            Token::Substitute(options) => {
                let text = try!(substitute(&options, &withs));
                result.push_str(&text);
            },
            // todo: this would be solved in the type system by breaking up the Token enum into
            //    enum Token { Substitute, Text }
            //    enum SubstituteItem { With, Env, Request, DefaultVal }
            _ => { return Err(format!("BUG: invalid top-level Token option")); }
        }
    }
    Ok(result)
}

fn substitute<'a>(options: &Vec<Token<'a>>, withs: &HashMap<&str, &str>
                  ) -> Result<String, String> {
    for option in options {
        match option {
            &Token::With(with) => {
                if let Some(val) = withs.get(with) {
                    return Ok(val.to_string());
                }
            },
            &Token::Env(name, ref key_path) => {
                if let Ok(y) = env::load_environment(name) {
                    if let Some(&Yaml::String(ref val)) = yaml_util::get_nested_value(&y, key_path) {
                        return Ok(val.to_string());
                    }
                }
            },
            &Token::Request(name, ref key_path) => {
                // return Err("substitution not implemented for requests".to_string());
                // println!("sub Request {}.{:?}", name, key_path);
                let key_path: Vec<&str> = key_path.iter().map(|k| k.as_str()).collect();
                let poo = remember::find_remembered_key(name, &key_path);
                // println!("{:?}", poo);
                if let Ok(s) = poo {
                  //  println!("got value '{}'", s);
                    return Ok(s.to_string());
                }
                //if let Ok(y) = remember::load_remembered_request(name) {
                //    if let Some(&Yaml::String(ref val)) = yaml_util::get_nested_value(&y, key_path) {
                //        return Ok(val.to_string());
                //    }
                //}
            },
            &Token::DefaultVal(val) => {
                return Ok(val.to_string());
            },
            _ => { return Err("BUG: Saw invalid enum option in substitute".to_string()); },
        }
    }
    let s = try!(options_to_string(options));
    Err(format!("Failed to substitute for {}", s))
}

fn options_to_string<'a>(options: &Vec<Token<'a>>) -> Result<String, String> {
    // build a sensible error message from the options
    let mut result = String::from("{{");
    for option in options {
        match option {
            &Token::With(with) => {
                result.push_str(&format!(" {},", with));
            },
            &Token::Env(name, ref key_path) => {
                result.push_str(&format!(" [{}]", name));
                for key in key_path {
                    result.push_str(&format!(".{}", key));
                }
                result.push_str(",");
            },
            &Token::Request(name, ref key_path) => {
                result.push_str(&format!(" {}", name));
                for key in key_path {
                    result.push_str(&format!(".{}", key));
                }
                result.push_str(",");
            },
            &Token::DefaultVal(val) => {
                if result.ends_with(',') { result.pop(); }
                result.push_str(&format!(": {}", val));
            },
            _ => { return Err("BUG: Saw invalid enum option in substitute".to_string()); },
        }
    }
    if result.ends_with(',') { result.pop(); }
    result.push_str(" }}");
    Ok(result)
}

pub struct Tokenizer<'a> {
    text: &'a str,
    char_indices: std::iter::Peekable<std::str::CharIndices<'a>>,
    shortcuts: bool,
}

impl<'a> Tokenizer<'a> {
    pub fn new(text: &'a str, shortcuts: bool) -> Tokenizer<'a> {
        Tokenizer {
            text: text,
            char_indices: text.char_indices().peekable(),
            shortcuts: shortcuts,
        }
    }

    pub fn tokenize(&mut self) -> Result<Vec<Token<'a>>, String> {
        // reset the iterator back to the start of the text
        self.char_indices = self.text.char_indices().peekable();
        let mut result: Vec<Token<'a>> = Vec::new();
        loop {
            if self.eof() {
                break;
            } else if self.has("{{") {
                result.push(try!(self.read_braces()));
            } else if self.shortcuts && self.has("@") {
                self.next();
                result.push(try!(self.read_shortcut_item()));
            } else {
                result.push(try!(self.read_text()));
            }
        }
        Ok(result)
    }

    /// Use this to read any text outside of {{...}} lists and the shortcut notation @<thing>
    fn read_text(&mut self) -> Result<Token<'a>, String> {
        if self.eof() { return Err("Expected text?".to_string()); }
        let &(start, _) = self.peek().unwrap();
        let mut end;
        loop {
            if self.eof() {
                end = self.text.len();
                break;
            }
            if self.has("{{") || (self.shortcuts && self.has("@")) {
                let &(offset, _) = self.peek().unwrap();
                end = offset;
                break;
            }
            self.next();
        }
        if start == end {
            Err("Failed to read text".to_string())
        } else {
            Ok(Token::Text(&self.text[start..end]))
        }
    }

    /// Read a list of items delimited by double braces, like {{<item>, <item>, ...}}
    fn read_braces(&mut self) -> Result<Token<'a>, String> {
        if !self.has("{{") { panic!("BUG: read_braces called when no braces found"); }
        self.next(); self.next();
        let mut result: Vec<Token<'a>> = Vec::new();
        loop {
            let token = try!(self.read_brace_item());
            result.push(token);
            self.skip_whitespace();
            if self.has("}}") {
                self.next(); self.next();
                break;
            } else if self.eof() {
                return Err("Unclosed braces".to_string());
            } else if self.has(",") {
                self.next();
            } else if self.has(":") {
                self.next();
                let default = try!(self.read_default_value()).trim();
                if !self.has("}}") {
                    return Err("Default value must be the last list item after the ':'.".to_string());
                }
                result.push(Token::DefaultVal(default));
                self.next(); self.next();
                break;
            }
        }
        if result.is_empty() {
            Err("Found empty braces".to_string())
        } else {
            Ok(Token::Substitute(result))
        }
    }

    /// Read one of:
    ///     - A --with key like <name>
    ///     - A request lookup like "last.response.body.id"
    ///     - An environment lookup like "[env].thing.id"
    fn read_brace_item(&mut self) -> Result<Token<'a>, String> {
        self.skip_whitespace();
        if self.eof() {
            return Err("Empty item".to_string());
        } else if self.has("[") {
            self.read_env_item()
        } else {
            self.read_with_or_request_item()
        }
    }

    /// @body.id is euivalent to {{ last.response.body.id }}
    /// @id is equivalent to {{ last.response.body.id }}
    fn read_shortcut_item(&mut self) -> Result<Token<'a>, String> {
        let token =
            // If we have just one key, like @id, we'll get back a Token::With
            // If we have many keys, we'll get back a Token::Request with the first key in the
            // name and the remaining keys in the key_path
            match try!(self.read_brace_item()) {
                Token::Request(name, key_path) => {
                    let mut keys = vec!["response".to_string(), name.to_string()];
                    for k in key_path {
                        keys.push(k.to_string())
                    }
                    Token::Request("last", keys)
                },
                Token::With(name) => {
                    let keys = vec!["response".to_string(), "body".to_string(), name.to_string()];
                    Token::Request("last", keys)
                },
                token => token,
            };
        Ok(Token::Substitute(vec![token]))
    }

    /// Read something like [<name>].<key>.<key>
    fn read_env_item(&mut self) -> Result<Token<'a>, String> {
        assert!(!self.eof());

        try!(self.expect_char('['));
        self.skip_whitespace();
        let env_name =
            // an empty environment name means use the active env
            if self.has("]") {
                self.next();
                ""
            } else {
                let name = try!(self.read_item_name());
                self.skip_whitespace();
                try!(self.expect_char(']'));
                name
            };

        let key_path = try!(self.read_key_path());

        if key_path.is_empty() {
            Err(format!("No key found after environment {}", env_name))
        } else {
            Ok(Token::Env(env_name, key_path))
        }
    }

    /// Read a --with key name that looks like <name> -- returns a Token::With
    /// Or read a request name + key_path, like "last.response.body.id" -- returns a Token::Request
    fn read_with_or_request_item(&mut self) -> Result<Token<'a>, String> {
        assert!(!self.eof());
        let name = try!(self.read_item_name());
        if self.has(".") {
            let key_path = try!(self.read_key_path());
            if key_path.is_empty() {
                Err(format!("Expected key after \"{}.\"", name))
            } else {
                let key_path: Vec<String> = key_path.iter().map(|k| k.to_string()).collect();
                Ok(Token::Request(name, key_path))
            }
        } else {
            Ok(Token::With(name))
        }
    }

    /// Read a key path, starting with a '.', like ".response.body.id"
    fn read_key_path(&mut self) -> Result<Vec<&'a str>, String> {
        let mut key_path: Vec<&'a str> = Vec::new();
        loop {
            if self.eof() { break; }
            if !self.has(".") {
                break;
            } else {
                self.next();
            }
            let part = try!(self.read_item_name());
            key_path.push(part);
        }
        Ok(key_path)
    }

    /// Return the text between the ':' and closing "}}" in a list.
    /// Assumes the ':' has already been consumed.
    fn read_default_value(&mut self) -> Result<&'a str, String> {
        self.skip_whitespace();
        if self.eof() {
            return Err("Found eof while reading default value. Unclosed braces".to_string());
        }
        let &(start, _) = self.peek().unwrap();
        let mut end;
        loop {
            if self.eof() {
                return Err("Found eof while reading default value. Unclosed braces".to_string());
            }
            if self.has("}}") {
                let &(offset, _) = self.peek().unwrap();
                end = offset;
                break;
            }
            self.next();
        }
        if start == end {
            Err("Found empty default value.".to_string())
        } else {
            Ok(&self.text[start..end])
        }
    }

    /// Read contiguous text containing characters in VALID_ITEM_NAME_CHARS
    fn read_item_name(&mut self) -> Result<&'a str, String> {
        if self.eof() {
            return Err("Expected a name but found eof".to_string());
        }
        let &(start, _) = self.peek().unwrap();
        let mut end;
        loop {
            if self.eof() {
                end = self.text.len();
                break;
            }
            let &(offset, c) = self.peek().unwrap();
            if !VALID_ITEM_NAME_CHARS.contains(c) {
                end = offset;
                break;
            }
            self.next();
        }
        if start == end {
            Err("No item found".to_string())
        } else {
            Ok(&self.text[start..end])
        }
    }

    /// Check if the current position starts with the given string
    fn has(&mut self, s: &'static str) -> bool {
        if let Some(&(offset, _)) = self.peek() {
            let end =
                if offset + s.len() > self.text.len() {
                    self.text.len()
                } else {
                    offset + s.len()
                };
            s == &self.text[offset .. end]
        } else {
            false
        }
    }

    /// Check for the end of the text
    fn eof(&mut self) -> bool {
        return self.peek().is_none()
    }

    /// Return but do not consume the char at the current position
    fn peek_char(&mut self) -> Option<char> {
        match self.peek() {
            Some(&(_, c)) => { Some(c) },
            None => { None },
        }
    }

    #[inline]
    fn peek(&mut self) -> Option<&(usize, char)> {
        self.char_indices.peek()
    }

    #[inline]
    fn next(&mut self) -> Option<(usize, char)> {
        self.char_indices.next()
    }

    /// Consume the char at the current position if it matches. Else error.
    fn expect_char(&mut self, expected: char) -> Result<char, String> {
        match self.peek() {
            Some(&(_, c)) => {
                if c == expected {
                    self.next();
                    Ok(c)
                } else {
                    Err(format!("Expected character {:?} but found {:?}", expected, c))
                }
            },
            _ => { Err(format!("Expected chararacter {:?} but found eof", expected)) },
        }
    }

    fn skip_whitespace(&mut self) {
        loop {
            match self.peek_char() {
                Some(c) if c.is_whitespace() => { self.next(); }
                _ => { break; }
            }
        }
    }
}
<commit_msg>eliminate unused_mut warnings on updated rust<commit_after>use std;
use std::collections::hash_map::HashMap;
use yaml_rust::Yaml;

use super::env;
use super::yaml_util;
use super::remember;

const VALID_ITEM_NAME_CHARS: &'static str =
    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_";

#[derive(Debug, PartialEq)]
pub enum Token<'a> {
    Substitute(Vec<Token<'a>>),
    Text(&'a str),
    With(&'a str),
    Env(&'a str, Vec<&'a str>),
    Request(&'a str, Vec<String>),
    DefaultVal(&'a str),
}

/// Untemplate the given string
///
/// The strategy this uses is to a take a string like:
///
///     "aaaa{{thing, [env].thing.id, last.response.body.thing}}"
///
/// and convert that to tokens:
///
///     vec![
///         Text("aaaa"),
///         Substitute([
///             With("aaaa"),
///             Env("env", ["thing", "id"]),
///             Request("last", ["response", "body", "thing"]),
///         ])
///     ]
///
/// and then do substitutions and build the resulting string
pub fn untemplate(text: &str, withs: &HashMap<&str, &str>, shortcuts: bool
                  ) -> Result<String, String> {
    let tokens = try!(Tokenizer::new(text, shortcuts).tokenize());
    let mut result = String::new();
    for token in tokens {
        match token {
            Token::Text(text) => {
                result.push_str(text);
            },
            Token::Substitute(options) => {
                let text = try!(substitute(&options, &withs));
                result.push_str(&text);
            },
            // todo: this would be solved in the type system by breaking up the Token enum into
            //    enum Token { Substitute, Text }
            //    enum SubstituteItem { With, Env, Request, DefaultVal }
            _ => { return Err(format!("BUG: invalid top-level Token option")); }
        }
    }
    Ok(result)
}

fn substitute<'a>(options: &Vec<Token<'a>>, withs: &HashMap<&str, &str>
                  ) -> Result<String, String> {
    for option in options {
        match option {
            &Token::With(with) => {
                if let Some(val) = withs.get(with) {
                    return Ok(val.to_string());
                }
            },
            &Token::Env(name, ref key_path) => {
                if let Ok(y) = env::load_environment(name) {
                    if let Some(&Yaml::String(ref val)) = yaml_util::get_nested_value(&y, key_path) {
                        return Ok(val.to_string());
                    }
                }
            },
            &Token::Request(name, ref key_path) => {
                // return Err("substitution not implemented for requests".to_string());
                // println!("sub Request {}.{:?}", name, key_path);
                let key_path: Vec<&str> = key_path.iter().map(|k| k.as_str()).collect();
                let poo = remember::find_remembered_key(name, &key_path);
                // println!("{:?}", poo);
                if let Ok(s) = poo {
                  //  println!("got value '{}'", s);
                    return Ok(s.to_string());
                }
                //if let Ok(y) = remember::load_remembered_request(name) {
                //    if let Some(&Yaml::String(ref val)) = yaml_util::get_nested_value(&y, key_path) {
                //        return Ok(val.to_string());
                //    }
                //}
            },
            &Token::DefaultVal(val) => {
                return Ok(val.to_string());
            },
            _ => { return Err("BUG: Saw invalid enum option in substitute".to_string()); },
        }
    }
    let s = try!(options_to_string(options));
    Err(format!("Failed to substitute for {}", s))
}

fn options_to_string<'a>(options: &Vec<Token<'a>>) -> Result<String, String> {
    // build a sensible error message from the options
    let mut result = String::from("{{");
    for option in options {
        match option {
            &Token::With(with) => {
                result.push_str(&format!(" {},", with));
            },
            &Token::Env(name, ref key_path) => {
                result.push_str(&format!(" [{}]", name));
                for key in key_path {
                    result.push_str(&format!(".{}", key));
                }
                result.push_str(",");
            },
            &Token::Request(name, ref key_path) => {
                result.push_str(&format!(" {}", name));
                for key in key_path {
                    result.push_str(&format!(".{}", key));
                }
                result.push_str(",");
            },
            &Token::DefaultVal(val) => {
                if result.ends_with(',') { result.pop(); }
                result.push_str(&format!(": {}", val));
            },
            _ => { return Err("BUG: Saw invalid enum option in substitute".to_string()); },
        }
    }
    if result.ends_with(',') { result.pop(); }
    result.push_str(" }}");
    Ok(result)
}

pub struct Tokenizer<'a> {
    text: &'a str,
    char_indices: std::iter::Peekable<std::str::CharIndices<'a>>,
    shortcuts: bool,
}

impl<'a> Tokenizer<'a> {
    pub fn new(text: &'a str, shortcuts: bool) -> Tokenizer<'a> {
        Tokenizer {
            text: text,
            char_indices: text.char_indices().peekable(),
            shortcuts: shortcuts,
        }
    }

    pub fn tokenize(&mut self) -> Result<Vec<Token<'a>>, String> {
        // reset the iterator back to the start of the text
        self.char_indices = self.text.char_indices().peekable();
        let mut result: Vec<Token<'a>> = Vec::new();
        loop {
            if self.eof() {
                break;
            } else if self.has("{{") {
                result.push(try!(self.read_braces()));
            } else if self.shortcuts && self.has("@") {
                self.next();
                result.push(try!(self.read_shortcut_item()));
            } else {
                result.push(try!(self.read_text()));
            }
        }
        Ok(result)
    }

    /// Use this to read any text outside of {{...}} lists and the shortcut notation @<thing>
    fn read_text(&mut self) -> Result<Token<'a>, String> {
        if self.eof() { return Err("Expected text?".to_string()); }
        let &(start, _) = self.peek().unwrap();
        let end;
        loop {
            if self.eof() {
                end = self.text.len();
                break;
            }
            if self.has("{{") || (self.shortcuts && self.has("@")) {
                let &(offset, _) = self.peek().unwrap();
                end = offset;
                break;
            }
            self.next();
        }
        if start == end {
            Err("Failed to read text".to_string())
        } else {
            Ok(Token::Text(&self.text[start..end]))
        }
    }

    /// Read a list of items delimited by double braces, like {{<item>, <item>, ...}}
    fn read_braces(&mut self) -> Result<Token<'a>, String> {
        if !self.has("{{") { panic!("BUG: read_braces called when no braces found"); }
        self.next(); self.next();
        let mut result: Vec<Token<'a>> = Vec::new();
        loop {
            let token = try!(self.read_brace_item());
            result.push(token);
            self.skip_whitespace();
            if self.has("}}") {
                self.next(); self.next();
                break;
            } else if self.eof() {
                return Err("Unclosed braces".to_string());
            } else if self.has(",") {
                self.next();
            } else if self.has(":") {
                self.next();
                let default = try!(self.read_default_value()).trim();
                if !self.has("}}") {
                    return Err("Default value must be the last list item after the ':'.".to_string());
                }
                result.push(Token::DefaultVal(default));
                self.next(); self.next();
                break;
            }
        }
        if result.is_empty() {
            Err("Found empty braces".to_string())
        } else {
            Ok(Token::Substitute(result))
        }
    }

    /// Read one of:
    ///     - A --with key like <name>
    ///     - A request lookup like "last.response.body.id"
    ///     - An environment lookup like "[env].thing.id"
    fn read_brace_item(&mut self) -> Result<Token<'a>, String> {
        self.skip_whitespace();
        if self.eof() {
            return Err("Empty item".to_string());
        } else if self.has("[") {
            self.read_env_item()
        } else {
            self.read_with_or_request_item()
        }
    }

    /// @body.id is euivalent to {{ last.response.body.id }}
    /// @id is equivalent to {{ last.response.body.id }}
    fn read_shortcut_item(&mut self) -> Result<Token<'a>, String> {
        let token =
            // If we have just one key, like @id, we'll get back a Token::With
            // If we have many keys, we'll get back a Token::Request with the first key in the
            // name and the remaining keys in the key_path
            match try!(self.read_brace_item()) {
                Token::Request(name, key_path) => {
                    let mut keys = vec!["response".to_string(), name.to_string()];
                    for k in key_path {
                        keys.push(k.to_string())
                    }
                    Token::Request("last", keys)
                },
                Token::With(name) => {
                    let keys = vec!["response".to_string(), "body".to_string(), name.to_string()];
                    Token::Request("last", keys)
                },
                token => token,
            };
        Ok(Token::Substitute(vec![token]))
    }

    /// Read something like [<name>].<key>.<key>
    fn read_env_item(&mut self) -> Result<Token<'a>, String> {
        assert!(!self.eof());

        try!(self.expect_char('['));
        self.skip_whitespace();
        let env_name =
            // an empty environment name means use the active env
            if self.has("]") {
                self.next();
                ""
            } else {
                let name = try!(self.read_item_name());
                self.skip_whitespace();
                try!(self.expect_char(']'));
                name
            };

        let key_path = try!(self.read_key_path());

        if key_path.is_empty() {
            Err(format!("No key found after environment {}", env_name))
        } else {
            Ok(Token::Env(env_name, key_path))
        }
    }

    /// Read a --with key name that looks like <name> -- returns a Token::With
    /// Or read a request name + key_path, like "last.response.body.id" -- returns a Token::Request
    fn read_with_or_request_item(&mut self) -> Result<Token<'a>, String> {
        assert!(!self.eof());
        let name = try!(self.read_item_name());
        if self.has(".") {
            let key_path = try!(self.read_key_path());
            if key_path.is_empty() {
                Err(format!("Expected key after \"{}.\"", name))
            } else {
                let key_path: Vec<String> = key_path.iter().map(|k| k.to_string()).collect();
                Ok(Token::Request(name, key_path))
            }
        } else {
            Ok(Token::With(name))
        }
    }

    /// Read a key path, starting with a '.', like ".response.body.id"
    fn read_key_path(&mut self) -> Result<Vec<&'a str>, String> {
        let mut key_path: Vec<&'a str> = Vec::new();
        loop {
            if self.eof() { break; }
            if !self.has(".") {
                break;
            } else {
                self.next();
            }
            let part = try!(self.read_item_name());
            key_path.push(part);
        }
        Ok(key_path)
    }

    /// Return the text between the ':' and closing "}}" in a list.
    /// Assumes the ':' has already been consumed.
    fn read_default_value(&mut self) -> Result<&'a str, String> {
        self.skip_whitespace();
        if self.eof() {
            return Err("Found eof while reading default value. Unclosed braces".to_string());
        }
        let &(start, _) = self.peek().unwrap();
        let end;
        loop {
            if self.eof() {
                return Err("Found eof while reading default value. Unclosed braces".to_string());
            }
            if self.has("}}") {
                let &(offset, _) = self.peek().unwrap();
                end = offset;
                break;
            }
            self.next();
        }
        if start == end {
            Err("Found empty default value.".to_string())
        } else {
            Ok(&self.text[start..end])
        }
    }

    /// Read contiguous text containing characters in VALID_ITEM_NAME_CHARS
    fn read_item_name(&mut self) -> Result<&'a str, String> {
        if self.eof() {
            return Err("Expected a name but found eof".to_string());
        }
        let &(start, _) = self.peek().unwrap();
        let end;
        loop {
            if self.eof() {
                end = self.text.len();
                break;
            }
            let &(offset, c) = self.peek().unwrap();
            if !VALID_ITEM_NAME_CHARS.contains(c) {
                end = offset;
                break;
            }
            self.next();
        }
        if start == end {
            Err("No item found".to_string())
        } else {
            Ok(&self.text[start..end])
        }
    }

    /// Check if the current position starts with the given string
    fn has(&mut self, s: &'static str) -> bool {
        if let Some(&(offset, _)) = self.peek() {
            let end =
                if offset + s.len() > self.text.len() {
                    self.text.len()
                } else {
                    offset + s.len()
                };
            s == &self.text[offset .. end]
        } else {
            false
        }
    }

    /// Check for the end of the text
    fn eof(&mut self) -> bool {
        return self.peek().is_none()
    }

    /// Return but do not consume the char at the current position
    fn peek_char(&mut self) -> Option<char> {
        match self.peek() {
            Some(&(_, c)) => { Some(c) },
            None => { None },
        }
    }

    #[inline]
    fn peek(&mut self) -> Option<&(usize, char)> {
        self.char_indices.peek()
    }

    #[inline]
    fn next(&mut self) -> Option<(usize, char)> {
        self.char_indices.next()
    }

    /// Consume the char at the current position if it matches. Else error.
    fn expect_char(&mut self, expected: char) -> Result<char, String> {
        match self.peek() {
            Some(&(_, c)) => {
                if c == expected {
                    self.next();
                    Ok(c)
                } else {
                    Err(format!("Expected character {:?} but found {:?}", expected, c))
                }
            },
            _ => { Err(format!("Expected chararacter {:?} but found eof", expected)) },
        }
    }

    fn skip_whitespace(&mut self) {
        loop {
            match self.peek_char() {
                Some(c) if c.is_whitespace() => { self.next(); }
                _ => { break; }
            }
        }
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern crate cc;
extern crate glsl_to_cxx;
extern crate webrender_build;

use std::collections::HashSet;
use std::fmt::Write;
use webrender_build::shader::{ShaderFeatureFlags, get_shader_features};

fn write_load_shader(shader_keys: &[String]) {
    let shaders: Vec<_> = shader_keys.iter().map(|s| s.replace(',', "_")).collect();
    let mut load_shader = String::new();
    for s in &shaders {
        let _ = write!(load_shader, "#include \"{}.h\"\n", s);
    }
    load_shader.push_str("ProgramLoader load_shader(const char* name) {\n");
    for s in &shaders {
        let _ = write!(load_shader, "  if (!strcmp(name, \"{0}\")) {{ return {0}_program::loader; }}\n", s);
    }
    load_shader.push_str("  return nullptr;\n}\n");
    std::fs::write(std::env::var("OUT_DIR").unwrap() + "/load_shader.h", load_shader).unwrap();
}

fn process_imports(shader_dir: &str, shader: &str, included: &mut HashSet<String>, output: &mut String) {
    if !included.insert(shader.into()) {
        return;
    }
    println!("cargo:rerun-if-changed={}/{}.glsl", shader_dir, shader);
    let source = std::fs::read_to_string(format!("{}/{}.glsl", shader_dir, shader)).unwrap();
    for line in source.lines() {
        if line.starts_with("#include ") {
            let imports = line["#include ".len() ..].split(',');
            for import in imports {
                process_imports(shader_dir, import, included, output);
            }
        } else if line.starts_with("#version ") || line.starts_with("#extension ") {
            // ignore
        } else {
            output.push_str(line);
            output.push('\n');
        }
    }
}

fn translate_shader(shader_key: &str, shader_dir: &str) {
    let mut imported = String::new();
    imported.push_str("#define SWGL 1\n");
    imported.push_str("#define WR_MAX_VERTEX_TEXTURE_WIDTH 1024U\n");

    let mut features = shader_key.split(',');
    let basename = features.next().unwrap();
    for feature in features {
        let _ = write!(imported, "#define WR_FEATURE_{}\n", feature);
    }

    process_imports(shader_dir, basename, &mut HashSet::new(), &mut imported);

    let shader = shader_key.replace(',', "_");

    let out_dir = std::env::var("OUT_DIR").unwrap();
    let imp_name = format!("{}/{}.c", out_dir, shader);
    std::fs::write(&imp_name, imported).unwrap();

    let mut build = cc::Build::new();
    if build.get_compiler().is_like_msvc() {
        build.flag("/EP");
    } else {
        build.flag("-xc").flag("-P");
    }
    build.file(&imp_name);
    let vs = build.clone()
        .define("WR_VERTEX_SHADER", Some("1"))
        .expand();
    let fs = build.clone()
        .define("WR_FRAGMENT_SHADER", Some("1"))
        .expand();
    let vs_name = format!("{}/{}.vert", out_dir, shader);
    let fs_name = format!("{}/{}.frag", out_dir, shader);
    std::fs::write(&vs_name, vs).unwrap();
    std::fs::write(&fs_name, fs).unwrap();

    let mut args = vec![
        "glsl_to_cxx".to_string(),
        vs_name,
        fs_name,
    ];
    let frag_include = format!("{}/{}.frag.h", shader_dir, shader);
    if std::path::Path::new(&frag_include).exists() {
        println!("cargo:rerun-if-changed={}/{}.frag.h", shader_dir, shader);
        args.push(frag_include);
    }
    let result = glsl_to_cxx::translate(&mut args.into_iter());
    std::fs::write(format!("{}/{}.h", out_dir, shader), result).unwrap();
}

fn main() {
    let shader_dir = match std::env::var("MOZ_SRC") {
        Ok(dir) => dir + "/gfx/wr/webrender/res",
        Err(_) => std::env::var("CARGO_MANIFEST_DIR").unwrap() + "/../webrender/res",
    };

    let shader_flags =
        ShaderFeatureFlags::GL |
        ShaderFeatureFlags::DUAL_SOURCE_BLENDING;
    let mut shaders: Vec<String> = Vec::new();
    for (name, features) in get_shader_features(shader_flags) {
        shaders.extend(features.iter().map(|f| {
            if f.is_empty() { name.to_owned() } else { format!("{},{}", name, f) }
        }));
    }

    for shader in &shaders {
        translate_shader(shader, &shader_dir);
    }

    write_load_shader(&shaders);

    println!("cargo:rerun-if-changed=src/gl_defs.h");
    println!("cargo:rerun-if-changed=src/glsl.h");
    println!("cargo:rerun-if-changed=src/program.h");
    println!("cargo:rerun-if-changed=src/texture.h");
    println!("cargo:rerun-if-changed=src/vector_type.h");
    println!("cargo:rerun-if-changed=src/gl.cc");
    cc::Build::new()
        .cpp(true)
        .file("src/gl.cc")
        .flag("-std=c++14")
        .flag("-UMOZILLA_CONFIG_H")
        .flag("-fno-exceptions")
        .flag("-fno-rtti")
        .flag("-fno-math-errno")
        .define("_GLIBCXX_USE_CXX11_ABI", Some("0"))
        .include(shader_dir)
        .include("src")
        .include(std::env::var("OUT_DIR").unwrap())
        .compile("gl_cc");
}
<commit_msg>Bug 1626188 - ensure shader keys are in sorted order. r=jimb<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern crate cc;
extern crate glsl_to_cxx;
extern crate webrender_build;

use std::collections::HashSet;
use std::fmt::Write;
use webrender_build::shader::{ShaderFeatureFlags, get_shader_features};

fn write_load_shader(shader_keys: &[String]) {
    let shaders: Vec<_> = shader_keys.iter().map(|s| s.replace(',', "_")).collect();
    let mut load_shader = String::new();
    for s in &shaders {
        let _ = write!(load_shader, "#include \"{}.h\"\n", s);
    }
    load_shader.push_str("ProgramLoader load_shader(const char* name) {\n");
    for s in &shaders {
        let _ = write!(load_shader, "  if (!strcmp(name, \"{0}\")) {{ return {0}_program::loader; }}\n", s);
    }
    load_shader.push_str("  return nullptr;\n}\n");
    std::fs::write(std::env::var("OUT_DIR").unwrap() + "/load_shader.h", load_shader).unwrap();
}

fn process_imports(shader_dir: &str, shader: &str, included: &mut HashSet<String>, output: &mut String) {
    if !included.insert(shader.into()) {
        return;
    }
    println!("cargo:rerun-if-changed={}/{}.glsl", shader_dir, shader);
    let source = std::fs::read_to_string(format!("{}/{}.glsl", shader_dir, shader)).unwrap();
    for line in source.lines() {
        if line.starts_with("#include ") {
            let imports = line["#include ".len() ..].split(',');
            for import in imports {
                process_imports(shader_dir, import, included, output);
            }
        } else if line.starts_with("#version ") || line.starts_with("#extension ") {
            // ignore
        } else {
            output.push_str(line);
            output.push('\n');
        }
    }
}

fn translate_shader(shader_key: &str, shader_dir: &str) {
    let mut imported = String::new();
    imported.push_str("#define SWGL 1\n");
    imported.push_str("#define WR_MAX_VERTEX_TEXTURE_WIDTH 1024U\n");

    let mut features = shader_key.split(',');
    let basename = features.next().unwrap();
    for feature in features {
        let _ = write!(imported, "#define WR_FEATURE_{}\n", feature);
    }

    process_imports(shader_dir, basename, &mut HashSet::new(), &mut imported);

    let shader = shader_key.replace(',', "_");

    let out_dir = std::env::var("OUT_DIR").unwrap();
    let imp_name = format!("{}/{}.c", out_dir, shader);
    std::fs::write(&imp_name, imported).unwrap();

    let mut build = cc::Build::new();
    if build.get_compiler().is_like_msvc() {
        build.flag("/EP");
    } else {
        build.flag("-xc").flag("-P");
    }
    build.file(&imp_name);
    let vs = build.clone()
        .define("WR_VERTEX_SHADER", Some("1"))
        .expand();
    let fs = build.clone()
        .define("WR_FRAGMENT_SHADER", Some("1"))
        .expand();
    let vs_name = format!("{}/{}.vert", out_dir, shader);
    let fs_name = format!("{}/{}.frag", out_dir, shader);
    std::fs::write(&vs_name, vs).unwrap();
    std::fs::write(&fs_name, fs).unwrap();

    let mut args = vec![
        "glsl_to_cxx".to_string(),
        vs_name,
        fs_name,
    ];
    let frag_include = format!("{}/{}.frag.h", shader_dir, shader);
    if std::path::Path::new(&frag_include).exists() {
        println!("cargo:rerun-if-changed={}/{}.frag.h", shader_dir, shader);
        args.push(frag_include);
    }
    let result = glsl_to_cxx::translate(&mut args.into_iter());
    std::fs::write(format!("{}/{}.h", out_dir, shader), result).unwrap();
}

fn main() {
    let shader_dir = match std::env::var("MOZ_SRC") {
        Ok(dir) => dir + "/gfx/wr/webrender/res",
        Err(_) => std::env::var("CARGO_MANIFEST_DIR").unwrap() + "/../webrender/res",
    };

    let shader_flags =
        ShaderFeatureFlags::GL |
        ShaderFeatureFlags::DUAL_SOURCE_BLENDING;
    let mut shaders: Vec<String> = Vec::new();
    for (name, features) in get_shader_features(shader_flags) {
        shaders.extend(features.iter().map(|f| {
            if f.is_empty() { name.to_owned() } else { format!("{},{}", name, f) }
        }));
    }

    shaders.sort();

    for shader in &shaders {
        translate_shader(shader, &shader_dir);
    }

    write_load_shader(&shaders);

    println!("cargo:rerun-if-changed=src/gl_defs.h");
    println!("cargo:rerun-if-changed=src/glsl.h");
    println!("cargo:rerun-if-changed=src/program.h");
    println!("cargo:rerun-if-changed=src/texture.h");
    println!("cargo:rerun-if-changed=src/vector_type.h");
    println!("cargo:rerun-if-changed=src/gl.cc");
    cc::Build::new()
        .cpp(true)
        .file("src/gl.cc")
        .flag("-std=c++14")
        .flag("-UMOZILLA_CONFIG_H")
        .flag("-fno-exceptions")
        .flag("-fno-rtti")
        .flag("-fno-math-errno")
        .define("_GLIBCXX_USE_CXX11_ABI", Some("0"))
        .include(shader_dir)
        .include("src")
        .include(std::env::var("OUT_DIR").unwrap())
        .compile("gl_cc");
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern crate cc;
extern crate glsl_to_cxx;
extern crate webrender_build;

use std::collections::HashSet;
use std::fmt::Write;
use webrender_build::shader::{ShaderFeatureFlags, get_shader_features};

// Shader key is in "name feature,feature" format.
// File name needs to be formatted as "name_feature_feature".
fn shader_file(shader_key: &str) -> String {
    shader_key.replace(' ', "_").replace(',', "_")
}

fn write_load_shader(shader_keys: &[String]) {
    let mut load_shader = String::new();
    for s in shader_keys {
        let _ = write!(load_shader, "#include \"{}.h\"\n", shader_file(s));
    }
    load_shader.push_str("ProgramLoader load_shader(const char* name) {\n");
    for s in shader_keys {
        let _ = write!(load_shader, "  if (!strcmp(name, \"{}\")) {{ return {}_program::loader; }}\n",
                       s, shader_file(s));
    }
    load_shader.push_str("  return nullptr;\n}\n");
    std::fs::write(std::env::var("OUT_DIR").unwrap() + "/load_shader.h", load_shader).unwrap();
}

fn process_imports(shader_dir: &str, shader: &str, included: &mut HashSet<String>, output: &mut String) {
    if !included.insert(shader.into()) {
        return;
    }
    println!("cargo:rerun-if-changed={}/{}.glsl", shader_dir, shader);
    let source = std::fs::read_to_string(format!("{}/{}.glsl", shader_dir, shader)).unwrap();
    for line in source.lines() {
        if line.starts_with("#include ") {
            let imports = line["#include ".len() ..].split(',');
            for import in imports {
                process_imports(shader_dir, import, included, output);
            }
        } else if line.starts_with("#version ") || line.starts_with("#extension ") {
            // ignore
        } else {
            output.push_str(line);
            output.push('\n');
        }
    }
}

fn translate_shader(shader_key: &str, shader_dir: &str) {
    let mut imported = String::from("#define SWGL 1\n#define __VERSION__ 150\n");
    let _ = write!(imported, "#define WR_MAX_VERTEX_TEXTURE_WIDTH {}U\n",
                   webrender_build::MAX_VERTEX_TEXTURE_WIDTH);

    let (basename, features) =
        shader_key.split_at(shader_key.find(' ').unwrap_or(shader_key.len()));
    if !features.is_empty() {
        for feature in features.trim().split(',') {
            let _ = write!(imported, "#define WR_FEATURE_{}\n", feature);
        }
    }

    process_imports(shader_dir, basename, &mut HashSet::new(), &mut imported);

    let shader = shader_file(shader_key);

    let out_dir = std::env::var("OUT_DIR").unwrap();
    let imp_name = format!("{}/{}.c", out_dir, shader);
    std::fs::write(&imp_name, imported).unwrap();

    let mut build = cc::Build::new();
    build.no_default_flags(true);
    if let Ok(tool) = build.try_get_compiler() {
        if tool.is_like_msvc() {
            build.flag("/EP");
            if tool.path().to_str().map_or(false, |p| p.contains("clang")) {
                build.flag("/clang:-undef");
            } else {
                build.flag("/u");
            }
        } else {
            build.flag("-xc").flag("-P").flag("-undef");
        }
    }
    // Use SWGLPP target to avoid pulling CFLAGS/CXXFLAGS.
    build.target("SWGLPP");
    build.file(&imp_name);
    let vs = build.clone()
        .define("WR_VERTEX_SHADER", Some("1"))
        .expand();
    let fs = build.clone()
        .define("WR_FRAGMENT_SHADER", Some("1"))
        .expand();
    let vs_name = format!("{}/{}.vert", out_dir, shader);
    let fs_name = format!("{}/{}.frag", out_dir, shader);
    std::fs::write(&vs_name, vs).unwrap();
    std::fs::write(&fs_name, fs).unwrap();

    let args = vec![
        "glsl_to_cxx".to_string(),
        vs_name,
        fs_name,
    ];
    let result = glsl_to_cxx::translate(&mut args.into_iter());
    std::fs::write(format!("{}/{}.h", out_dir, shader), result).unwrap();
}

fn main() {
    let shader_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap() + "/../webrender/res";

    let shader_flags =
        ShaderFeatureFlags::GL |
        ShaderFeatureFlags::DUAL_SOURCE_BLENDING |
        ShaderFeatureFlags::ADVANCED_BLEND_EQUATION |
        ShaderFeatureFlags::DEBUG;
    let mut shaders: Vec<String> = Vec::new();
    for (name, features) in get_shader_features(shader_flags) {
        shaders.extend(features.iter().map(|f| {
            if f.is_empty() { name.to_owned() } else { format!("{} {}", name, f) }
        }));
    }

    shaders.sort();

    // We need to ensure that the C preprocessor does not pull compiler flags from
    // the host or target environment. Set up a SWGLPP target with empty flags to
    // work around this.
    if let Ok(target) = std::env::var("TARGET") {
        if let Ok(cc) = std::env::var(format!("CC_{}", target))
                        .or(std::env::var(format!("CC_{}", target.replace("-", "_")))) {
            std::env::set_var("CC_SWGLPP", cc);
        }
    }
    std::env::set_var("CFLAGS_SWGLPP", "");

    for shader in &shaders {
        translate_shader(shader, &shader_dir);
    }

    write_load_shader(&shaders);

    println!("cargo:rerun-if-changed=src/blend.h");
    println!("cargo:rerun-if-changed=src/composite.h");
    println!("cargo:rerun-if-changed=src/gl_defs.h");
    println!("cargo:rerun-if-changed=src/glsl.h");
    println!("cargo:rerun-if-changed=src/program.h");
    println!("cargo:rerun-if-changed=src/rasterize.h");
    println!("cargo:rerun-if-changed=src/swgl_ext.h");
    println!("cargo:rerun-if-changed=src/texture.h");
    println!("cargo:rerun-if-changed=src/vector_type.h");
    println!("cargo:rerun-if-changed=src/gl.cc");
    let mut build = cc::Build::new();
    build.cpp(true);

    if let Ok(tool) = build.try_get_compiler() {
        if tool.is_like_msvc() {
            build.flag("/std:c++17")
                 .flag("/EHs-")
                 .flag("/GR-")
                 .flag("/UMOZILLA_CONFIG_H");
        } else {
            build.flag("-std=c++17")
                 .flag("-fno-exceptions")
                 .flag("-fno-rtti")
                 .flag("-fno-math-errno")
                 .flag("-UMOZILLA_CONFIG_H");
        }
        // SWGL relies heavily on inlining for performance so override -Oz with -O2
        if tool.args().contains(&"-Oz".into()) {
            build.flag("-O2");
        }

        // Most GLSL compilers assume something like fast-math so we turn it on.
        // However, reciprocal division makes it so 1/1 = 0.999994 which can produce a lot of fuzz
        // in reftests and the use of reciprocal instructions usually involves a refinement step
        // which bloats our already bloated code. Further, our shader code is sufficiently parallel
        // that we're more likely to be throughput bound vs latency bound. Having fewer
        // instructions makes things easier on the processor and in places where it matters we can
        // probably explicitly use reciprocal instructions and avoid the refinement step.
        if tool.is_like_msvc() {
            build.flag("/fp:fast")
                 .flag("-Xclang")
                 .flag("-mrecip=none");
        } else if tool.is_like_clang() {
            // gcc only supports -mrecip=none on some targets so to keep
            // things simple we don't use -ffast-math with gcc at all
            build.flag("-ffast-math")
                 .flag("-mrecip=none");
        }
    }

    build.file("src/gl.cc")
        .define("_GLIBCXX_USE_CXX11_ABI", Some("0"))
        .include(shader_dir)
        .include("src")
        .include(std::env::var("OUT_DIR").unwrap())
        .compile("gl_cc");
}
<commit_msg>Bug 1746545 - Use -fno-finite-math-only in SWGL. r=jrmuizel<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern crate cc;
extern crate glsl_to_cxx;
extern crate webrender_build;

use std::collections::HashSet;
use std::fmt::Write;
use webrender_build::shader::{ShaderFeatureFlags, get_shader_features};

// Shader key is in "name feature,feature" format.
// File name needs to be formatted as "name_feature_feature".
fn shader_file(shader_key: &str) -> String {
    shader_key.replace(' ', "_").replace(',', "_")
}

fn write_load_shader(shader_keys: &[String]) {
    let mut load_shader = String::new();
    for s in shader_keys {
        let _ = write!(load_shader, "#include \"{}.h\"\n", shader_file(s));
    }
    load_shader.push_str("ProgramLoader load_shader(const char* name) {\n");
    for s in shader_keys {
        let _ = write!(load_shader, "  if (!strcmp(name, \"{}\")) {{ return {}_program::loader; }}\n",
                       s, shader_file(s));
    }
    load_shader.push_str("  return nullptr;\n}\n");
    std::fs::write(std::env::var("OUT_DIR").unwrap() + "/load_shader.h", load_shader).unwrap();
}

fn process_imports(shader_dir: &str, shader: &str, included: &mut HashSet<String>, output: &mut String) {
    if !included.insert(shader.into()) {
        return;
    }
    println!("cargo:rerun-if-changed={}/{}.glsl", shader_dir, shader);
    let source = std::fs::read_to_string(format!("{}/{}.glsl", shader_dir, shader)).unwrap();
    for line in source.lines() {
        if line.starts_with("#include ") {
            let imports = line["#include ".len() ..].split(',');
            for import in imports {
                process_imports(shader_dir, import, included, output);
            }
        } else if line.starts_with("#version ") || line.starts_with("#extension ") {
            // ignore
        } else {
            output.push_str(line);
            output.push('\n');
        }
    }
}

fn translate_shader(shader_key: &str, shader_dir: &str) {
    let mut imported = String::from("#define SWGL 1\n#define __VERSION__ 150\n");
    let _ = write!(imported, "#define WR_MAX_VERTEX_TEXTURE_WIDTH {}U\n",
                   webrender_build::MAX_VERTEX_TEXTURE_WIDTH);

    let (basename, features) =
        shader_key.split_at(shader_key.find(' ').unwrap_or(shader_key.len()));
    if !features.is_empty() {
        for feature in features.trim().split(',') {
            let _ = write!(imported, "#define WR_FEATURE_{}\n", feature);
        }
    }

    process_imports(shader_dir, basename, &mut HashSet::new(), &mut imported);

    let shader = shader_file(shader_key);

    let out_dir = std::env::var("OUT_DIR").unwrap();
    let imp_name = format!("{}/{}.c", out_dir, shader);
    std::fs::write(&imp_name, imported).unwrap();

    let mut build = cc::Build::new();
    build.no_default_flags(true);
    if let Ok(tool) = build.try_get_compiler() {
        if tool.is_like_msvc() {
            build.flag("/EP");
            if tool.path().to_str().map_or(false, |p| p.contains("clang")) {
                build.flag("/clang:-undef");
            } else {
                build.flag("/u");
            }
        } else {
            build.flag("-xc").flag("-P").flag("-undef");
        }
    }
    // Use SWGLPP target to avoid pulling CFLAGS/CXXFLAGS.
    build.target("SWGLPP");
    build.file(&imp_name);
    let vs = build.clone()
        .define("WR_VERTEX_SHADER", Some("1"))
        .expand();
    let fs = build.clone()
        .define("WR_FRAGMENT_SHADER", Some("1"))
        .expand();
    let vs_name = format!("{}/{}.vert", out_dir, shader);
    let fs_name = format!("{}/{}.frag", out_dir, shader);
    std::fs::write(&vs_name, vs).unwrap();
    std::fs::write(&fs_name, fs).unwrap();

    let args = vec![
        "glsl_to_cxx".to_string(),
        vs_name,
        fs_name,
    ];
    let result = glsl_to_cxx::translate(&mut args.into_iter());
    std::fs::write(format!("{}/{}.h", out_dir, shader), result).unwrap();
}

fn main() {
    let shader_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap() + "/../webrender/res";

    let shader_flags =
        ShaderFeatureFlags::GL |
        ShaderFeatureFlags::DUAL_SOURCE_BLENDING |
        ShaderFeatureFlags::ADVANCED_BLEND_EQUATION |
        ShaderFeatureFlags::DEBUG;
    let mut shaders: Vec<String> = Vec::new();
    for (name, features) in get_shader_features(shader_flags) {
        shaders.extend(features.iter().map(|f| {
            if f.is_empty() { name.to_owned() } else { format!("{} {}", name, f) }
        }));
    }

    shaders.sort();

    // We need to ensure that the C preprocessor does not pull compiler flags from
    // the host or target environment. Set up a SWGLPP target with empty flags to
    // work around this.
    if let Ok(target) = std::env::var("TARGET") {
        if let Ok(cc) = std::env::var(format!("CC_{}", target))
                        .or(std::env::var(format!("CC_{}", target.replace("-", "_")))) {
            std::env::set_var("CC_SWGLPP", cc);
        }
    }
    std::env::set_var("CFLAGS_SWGLPP", "");

    for shader in &shaders {
        translate_shader(shader, &shader_dir);
    }

    write_load_shader(&shaders);

    println!("cargo:rerun-if-changed=src/blend.h");
    println!("cargo:rerun-if-changed=src/composite.h");
    println!("cargo:rerun-if-changed=src/gl_defs.h");
    println!("cargo:rerun-if-changed=src/glsl.h");
    println!("cargo:rerun-if-changed=src/program.h");
    println!("cargo:rerun-if-changed=src/rasterize.h");
    println!("cargo:rerun-if-changed=src/swgl_ext.h");
    println!("cargo:rerun-if-changed=src/texture.h");
    println!("cargo:rerun-if-changed=src/vector_type.h");
    println!("cargo:rerun-if-changed=src/gl.cc");
    let mut build = cc::Build::new();
    build.cpp(true);

    if let Ok(tool) = build.try_get_compiler() {
        if tool.is_like_msvc() {
            build.flag("/std:c++17")
                 .flag("/EHs-")
                 .flag("/GR-")
                 .flag("/UMOZILLA_CONFIG_H");
        } else {
            build.flag("-std=c++17")
                 .flag("-fno-exceptions")
                 .flag("-fno-rtti")
                 .flag("-fno-math-errno")
                 .flag("-UMOZILLA_CONFIG_H");
        }
        // SWGL relies heavily on inlining for performance so override -Oz with -O2
        if tool.args().contains(&"-Oz".into()) {
            build.flag("-O2");
        }

        // Most GLSL compilers assume something like fast-math so we turn it on.
        // However, reciprocal division makes it so 1/1 = 0.999994 which can produce a lot of fuzz
        // in reftests and the use of reciprocal instructions usually involves a refinement step
        // which bloats our already bloated code. Further, our shader code is sufficiently parallel
        // that we're more likely to be throughput bound vs latency bound. Having fewer
        // instructions makes things easier on the processor and in places where it matters we can
        // probably explicitly use reciprocal instructions and avoid the refinement step.
        // Also, allow checks for non-finite values which fast-math may disable.
        if tool.is_like_msvc() {
            build.flag("/fp:fast")
                 .flag("-Xclang")
                 .flag("-mrecip=none")
                 .flag("/clang:-fno-finite-math-only");
        } else if tool.is_like_clang() {
            // gcc only supports -mrecip=none on some targets so to keep
            // things simple we don't use -ffast-math with gcc at all
            build.flag("-ffast-math")
                 .flag("-mrecip=none")
                 .flag("-fno-finite-math-only");
        }
    }

    build.file("src/gl.cc")
        .define("_GLIBCXX_USE_CXX11_ABI", Some("0"))
        .include(shader_dir)
        .include("src")
        .include(std::env::var("OUT_DIR").unwrap())
        .compile("gl_cc");
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2019 Fabian Schuiki

//! The central data structure of the compiler.
//!
//! The main piece of infrastructure provided by this module is the [`Context`]
//! trait. It exposes all queries supported by the compiler, together with many
//! other basic operations. Most compiler tasks interact only with [`Context`].
//! The complementary piece of the puzzle is the [`GlobalContext`] which caches
//! all intermediate results and implements the [`Context`] trait. The global
//! context also holds a reference to the [`GlobalArenas`] which own everything
//! that is allocated or interned during query execution.
//!
//! # Example
//!
//! ```
//! # extern crate moore_common as common;
//! # extern crate moore_svlog as svlog;
//! # use common::Session;
//! # use svlog::{GlobalContext, GlobalArenas};
//! let sess = Session::new();
//! let arena = GlobalArenas::default();
//! let gcx = GlobalContext::new(&sess, &arena);
//! ```

use crate::{
    ast,
    ast_map::{AstMap, AstNode},
    common::{arenas::Alloc, arenas::TypedArena, Session},
    crate_prelude::*,
    hir::{self, HirNode},
    ty::{Type, TypeKind},
};
use llhd;
use std::cell::RefCell;
use std::collections::HashMap;

/// The central data structure of the compiler. It stores references to various
/// arenas and tables that store the results of the various computations that
/// have been performed.
pub struct GlobalContext<'gcx> {
    /// The global compiler session.
    pub sess: &'gcx Session,
    /// The arena that owns all references.
    pub arena: &'gcx GlobalArenas<'gcx>,
    /// The underlying runtime for the query system.
    runtime: salsa::Runtime<GlobalContext<'gcx>>,
    /// The mapping of node IDs to abstract syntax tree nodes.
    ast_map: AstMap<'gcx>,
    /// The modules in the AST.
    modules: RefCell<HashMap<Name, NodeId>>,
    /// A mapping from node ids to spans for diagnostics.
    node_id_to_span: RefCell<HashMap<NodeId, Span>>,
}

impl<'gcx> GlobalContext<'gcx> {
    /// Create a new global context.
    pub fn new(sess: &'gcx Session, arena: &'gcx GlobalArenas<'gcx>) -> Self {
        GlobalContext {
            sess,
            arena,
            runtime: Default::default(),
            ast_map: Default::default(),
            modules: Default::default(),
            node_id_to_span: Default::default(),
        }
    }

    /// Add root AST nodes to the context for processing.
    ///
    /// Use the `find_global_item` function afterwards to look up the id of
    /// modules that were added.
    pub fn add_root_nodes(&self, ast: impl Iterator<Item = &'gcx ast::Root>) {
        for root in ast {
            for item in &root.items {
                match *item {
                    ast::Item::Module(ref m) => {
                        let id = self.map_ast(AstNode::Module(m));
                        self.modules.borrow_mut().insert(m.name, id);
                    }
                    _ => {
                        let _: Result<()> = self.unimp(item);
                    }
                }
            }
        }
    }

    /// Find a module in the AST.
    pub fn find_module(&self, name: Name) -> Option<NodeId> {
        self.modules.borrow().get(&name).cloned()
    }
}

impl DiagEmitter for GlobalContext<'_> {
    fn emit(&self, diag: DiagBuilder2) {
        self.sess.emit(diag)
    }
}

impl<'gcx> salsa::Database for GlobalContext<'gcx> {
    fn salsa_runtime(&self) -> &salsa::Runtime<Self> {
        &self.runtime
    }
}

impl<'gcx> BaseContext<'gcx> for GlobalContext<'gcx> {
    fn gcx(&self) -> &GlobalContext<'gcx> {
        self
    }
}

/// The arenas that allocate things in the global context.
///
/// Use this struct whenever you want to allocate or internalize
/// something during the compilation procedure.
pub struct GlobalArenas<'t> {
    ids: TypedArena<NodeId>,
    hir: hir::Arena<'t>,
}

impl Default for GlobalArenas<'_> {
    fn default() -> Self {
        GlobalArenas {
            ids: TypedArena::new(),
            hir: Default::default(),
        }
    }
}

impl<'t> GlobalArenas<'t> {
    /// Allocate a list of node IDs.
    pub fn alloc_ids(&'t self, ids: impl IntoIterator<Item = NodeId>) -> &'t [NodeId] {
        self.ids.alloc_extend(ids)
    }

    /// Allocate an HIR node into the global context.
    pub fn alloc_hir<T>(&'t self, hir: T) -> &'t T
    where
        hir::Arena<'t>: Alloc<'t, 't, T>,
        T: 't,
    {
        self.hir.alloc(hir)
    }
}

/// The fundamental compiler context.
///
/// This trait represents the context within which most compiler operations take
/// place. It is implemented by [`GlobalContext`] and also provides access to
/// the global context via the `gcx()` method.
pub trait BaseContext<'gcx>: salsa::Database + DiagEmitter {
    /// Get the global context.
    #[inline(always)]
    fn gcx(&self) -> &GlobalContext<'gcx>;

    /// Get the compiler session.
    #[inline(always)]
    fn sess(&self) -> &'gcx Session {
        self.gcx().sess
    }

    /// Access the arena into which values are to be allocated.
    #[inline(always)]
    fn arena(&self) -> &'gcx GlobalArenas<'gcx> {
        self.gcx().arena
    }

    /// Emit an internal compiler error that a node is not implemented.
    fn unimp<T: HasSpan + HasDesc, R>(&self, node: &T) -> Result<R> {
        self.emit(
            DiagBuilder2::bug(format!("{} not implemented", node.desc_full()))
                .span(node.human_span()),
        );
        Err(())
    }

    /// Emit an internal compiler error and message that a node is not
    /// implemented. Same as [`unimp`], but the caller can provide a message
    /// prefix.
    fn unimp_msg<T: HasSpan + HasDesc, R>(&self, msg: impl Into<String>, node: &T) -> Result<R> {
        self.emit(
            DiagBuilder2::bug(format!(
                "{} {} not implemented",
                msg.into(),
                node.desc_full()
            ))
            .span(node.human_span()),
        );
        Err(())
    }

    /// Allocate a new node id.
    ///
    /// The provided span is used primarily for diagnostic messages and is
    /// supposed to easily identify the node to the user in case of an error.
    fn alloc_id(&self, span: Span) -> NodeId {
        let id = NodeId::alloc();
        self.gcx().node_id_to_span.borrow_mut().insert(id, span);
        id
    }

    /// Associate an AST node with a node id.
    fn set_ast(&self, node_id: NodeId, ast: AstNode<'gcx>) {
        self.gcx().ast_map.set(node_id, ast)
    }

    /// Allocate a node id for an AST node and associate that id with the node.
    fn map_ast(&self, ast: AstNode<'gcx>) -> NodeId {
        let id = self.alloc_id(ast.human_span());
        self.set_ast(id, ast);
        id
    }

    /// Obtain the AST node associated with a node id.
    fn ast_of(&self, node_id: NodeId) -> Result<AstNode<'gcx>> {
        match self.gcx().ast_map.get(node_id) {
            Some(node) => Ok(node),
            None => {
                let span = self.gcx().node_id_to_span.borrow()[&node_id];
                self.emit(
                    DiagBuilder2::bug(format!("no ast node for `{}` in the map", span.extract()))
                        .span(span),
                );
                Err(())
            }
        }
    }

    /// Make a void type.
    fn mkty_void(&self) -> Type<'gcx> {
        static STATIC: TypeKind<'static> = TypeKind::Void;
        &STATIC
    }

    /// Make a bit type.
    fn mkty_bit(&self) -> Type<'gcx> {
        static STATIC: TypeKind<'static> = TypeKind::Bit;
        &STATIC
    }
}

salsa::query_group! {
    /// A collection of compiler queries.
    pub trait Context<'a>: BaseContext<'a> {
        /// Lower an AST node to HIR.
        fn hir_of(node_id: NodeId) -> Result<HirNode<'a>> {
            type HirOf;
            use fn hir::hir_of;
        }

        /// Determine the type of a node.
        fn type_of(node_id: NodeId) -> Result<Type<'a>> {
            type TypeOf;
            use fn typeck::type_of;
        }
    }
}

salsa::database_storage! {
    /// The query result storage embedded in the global context.
    pub struct GlobalStorage<'gcx> for GlobalContext<'gcx> {
        impl Context<'gcx> {
            fn hir_of() for HirOf<'gcx>;
            fn type_of() for TypeOf<'gcx>;
        }
    }
}
<commit_msg>svlog: add iterator over modules in context<commit_after>// Copyright (c) 2016-2019 Fabian Schuiki

//! The central data structure of the compiler.
//!
//! The main piece of infrastructure provided by this module is the [`Context`]
//! trait. It exposes all queries supported by the compiler, together with many
//! other basic operations. Most compiler tasks interact only with [`Context`].
//! The complementary piece of the puzzle is the [`GlobalContext`] which caches
//! all intermediate results and implements the [`Context`] trait. The global
//! context also holds a reference to the [`GlobalArenas`] which own everything
//! that is allocated or interned during query execution.
//!
//! # Example
//!
//! ```
//! # extern crate moore_common as common;
//! # extern crate moore_svlog as svlog;
//! # use common::Session;
//! # use svlog::{GlobalContext, GlobalArenas};
//! let sess = Session::new();
//! let arena = GlobalArenas::default();
//! let gcx = GlobalContext::new(&sess, &arena);
//! ```

use crate::{
    ast,
    ast_map::{AstMap, AstNode},
    common::{arenas::Alloc, arenas::TypedArena, Session},
    crate_prelude::*,
    hir::{self, HirNode},
    ty::{Type, TypeKind},
};
use llhd;
use std::cell::RefCell;
use std::collections::HashMap;

/// The central data structure of the compiler. It stores references to various
/// arenas and tables that store the results of the various computations that
/// have been performed.
pub struct GlobalContext<'gcx> {
    /// The global compiler session.
    pub sess: &'gcx Session,
    /// The arena that owns all references.
    pub arena: &'gcx GlobalArenas<'gcx>,
    /// The underlying runtime for the query system.
    runtime: salsa::Runtime<GlobalContext<'gcx>>,
    /// The mapping of node IDs to abstract syntax tree nodes.
    ast_map: AstMap<'gcx>,
    /// The modules in the AST.
    modules: RefCell<HashMap<Name, NodeId>>,
    /// A mapping from node ids to spans for diagnostics.
    node_id_to_span: RefCell<HashMap<NodeId, Span>>,
}

impl<'gcx> GlobalContext<'gcx> {
    /// Create a new global context.
    pub fn new(sess: &'gcx Session, arena: &'gcx GlobalArenas<'gcx>) -> Self {
        GlobalContext {
            sess,
            arena,
            runtime: Default::default(),
            ast_map: Default::default(),
            modules: Default::default(),
            node_id_to_span: Default::default(),
        }
    }

    /// Add root AST nodes to the context for processing.
    ///
    /// Use the `find_global_item` function afterwards to look up the id of
    /// modules that were added.
    pub fn add_root_nodes(&self, ast: impl Iterator<Item = &'gcx ast::Root>) {
        for root in ast {
            for item in &root.items {
                match *item {
                    ast::Item::Module(ref m) => {
                        let id = self.map_ast(AstNode::Module(m));
                        self.modules.borrow_mut().insert(m.name, id);
                    }
                    _ => {
                        let _: Result<()> = self.unimp(item);
                    }
                }
            }
        }
    }

    /// Find a module in the AST.
    pub fn find_module(&self, name: Name) -> Option<NodeId> {
        self.modules.borrow().get(&name).cloned()
    }

    /// Get an iterator over all modules in the AST.
    pub fn modules(&self) -> impl Iterator<Item = (Name, NodeId)> {
        self.modules.borrow().clone().into_iter()
    }
}

impl DiagEmitter for GlobalContext<'_> {
    fn emit(&self, diag: DiagBuilder2) {
        self.sess.emit(diag)
    }
}

impl<'gcx> salsa::Database for GlobalContext<'gcx> {
    fn salsa_runtime(&self) -> &salsa::Runtime<Self> {
        &self.runtime
    }
}

impl<'gcx> BaseContext<'gcx> for GlobalContext<'gcx> {
    fn gcx(&self) -> &GlobalContext<'gcx> {
        self
    }
}

/// The arenas that allocate things in the global context.
///
/// Use this struct whenever you want to allocate or internalize
/// something during the compilation procedure.
pub struct GlobalArenas<'t> {
    ids: TypedArena<NodeId>,
    hir: hir::Arena<'t>,
}

impl Default for GlobalArenas<'_> {
    fn default() -> Self {
        GlobalArenas {
            ids: TypedArena::new(),
            hir: Default::default(),
        }
    }
}

impl<'t> GlobalArenas<'t> {
    /// Allocate a list of node IDs.
    pub fn alloc_ids(&'t self, ids: impl IntoIterator<Item = NodeId>) -> &'t [NodeId] {
        self.ids.alloc_extend(ids)
    }

    /// Allocate an HIR node into the global context.
    pub fn alloc_hir<T>(&'t self, hir: T) -> &'t T
    where
        hir::Arena<'t>: Alloc<'t, 't, T>,
        T: 't,
    {
        self.hir.alloc(hir)
    }
}

/// The fundamental compiler context.
///
/// This trait represents the context within which most compiler operations take
/// place. It is implemented by [`GlobalContext`] and also provides access to
/// the global context via the `gcx()` method.
pub trait BaseContext<'gcx>: salsa::Database + DiagEmitter {
    /// Get the global context.
    #[inline(always)]
    fn gcx(&self) -> &GlobalContext<'gcx>;

    /// Get the compiler session.
    #[inline(always)]
    fn sess(&self) -> &'gcx Session {
        self.gcx().sess
    }

    /// Access the arena into which values are to be allocated.
    #[inline(always)]
    fn arena(&self) -> &'gcx GlobalArenas<'gcx> {
        self.gcx().arena
    }

    /// Emit an internal compiler error that a node is not implemented.
    fn unimp<T: HasSpan + HasDesc, R>(&self, node: &T) -> Result<R> {
        self.emit(
            DiagBuilder2::bug(format!("{} not implemented", node.desc_full()))
                .span(node.human_span()),
        );
        Err(())
    }

    /// Emit an internal compiler error and message that a node is not
    /// implemented. Same as [`unimp`], but the caller can provide a message
    /// prefix.
    fn unimp_msg<T: HasSpan + HasDesc, R>(&self, msg: impl Into<String>, node: &T) -> Result<R> {
        self.emit(
            DiagBuilder2::bug(format!(
                "{} {} not implemented",
                msg.into(),
                node.desc_full()
            ))
            .span(node.human_span()),
        );
        Err(())
    }

    /// Allocate a new node id.
    ///
    /// The provided span is used primarily for diagnostic messages and is
    /// supposed to easily identify the node to the user in case of an error.
    fn alloc_id(&self, span: Span) -> NodeId {
        let id = NodeId::alloc();
        self.gcx().node_id_to_span.borrow_mut().insert(id, span);
        id
    }

    /// Associate an AST node with a node id.
    fn set_ast(&self, node_id: NodeId, ast: AstNode<'gcx>) {
        self.gcx().ast_map.set(node_id, ast)
    }

    /// Allocate a node id for an AST node and associate that id with the node.
    fn map_ast(&self, ast: AstNode<'gcx>) -> NodeId {
        let id = self.alloc_id(ast.human_span());
        self.set_ast(id, ast);
        id
    }

    /// Obtain the AST node associated with a node id.
    fn ast_of(&self, node_id: NodeId) -> Result<AstNode<'gcx>> {
        match self.gcx().ast_map.get(node_id) {
            Some(node) => Ok(node),
            None => {
                let span = self.gcx().node_id_to_span.borrow()[&node_id];
                self.emit(
                    DiagBuilder2::bug(format!("no ast node for `{}` in the map", span.extract()))
                        .span(span),
                );
                Err(())
            }
        }
    }

    /// Make a void type.
    fn mkty_void(&self) -> Type<'gcx> {
        static STATIC: TypeKind<'static> = TypeKind::Void;
        &STATIC
    }

    /// Make a bit type.
    fn mkty_bit(&self) -> Type<'gcx> {
        static STATIC: TypeKind<'static> = TypeKind::Bit;
        &STATIC
    }
}

salsa::query_group! {
    /// A collection of compiler queries.
    pub trait Context<'a>: BaseContext<'a> {
        /// Lower an AST node to HIR.
        fn hir_of(node_id: NodeId) -> Result<HirNode<'a>> {
            type HirOf;
            use fn hir::hir_of;
        }

        /// Determine the type of a node.
        fn type_of(node_id: NodeId) -> Result<Type<'a>> {
            type TypeOf;
            use fn typeck::type_of;
        }
    }
}

salsa::database_storage! {
    /// The query result storage embedded in the global context.
    pub struct GlobalStorage<'gcx> for GlobalContext<'gcx> {
        impl Context<'gcx> {
            fn hir_of() for HirOf<'gcx>;
            fn type_of() for TypeOf<'gcx>;
        }
    }
}
<|endoftext|>


<commit_before>use crate::bus::Bus;

pub struct AHBLite<'a, T: 'a + Bus, R: 'a + Bus> {
    code: &'a mut T,
    sram: &'a mut R,
}

impl<'a, T, R> AHBLite<'a, T, R>
where
    T: Bus,
    R: Bus,
{
    pub fn new(code: &'a mut T, sram: &'a mut R) -> AHBLite<'a, T, R> {
        AHBLite { code, sram }
    }
}

impl<'a, T, R> Bus for AHBLite<'a, T, R>
where
    T: Bus,
    R: Bus,
{
    fn read8(&self, addr: u32) -> u8 {
        if self.code.in_range(addr) {
            self.code.read8(addr)
        } else if self.sram.in_range(addr) {
            self.sram.read8(addr)
        } else {
            panic!("bus access fault read8 addr 0x{:x}", addr);
        }
    }

    fn read16(&self, addr: u32) -> u16 {
        if addr & 1 == 1 {
            panic!("unaliged read16 addr 0x{:x}", addr);
        }

        if self.code.in_range(addr) {
            self.code.read16(addr)
        } else if self.sram.in_range(addr) {
            self.sram.read16(addr)
        } else {
            panic!("bus access fault read16 addr 0x{:x}", addr);
        }
    }

    fn read32(&self, addr: u32) -> u32 {
        if addr & 3 != 0 {
            panic!("unaliged read32 addr 0x{:x}", addr);
        }
        if self.code.in_range(addr) {
            self.code.read32(addr)
        } else if self.sram.in_range(addr) {
            self.sram.read32(addr)
        } else {
            panic!("bus access fault read32 addr 0x{:x}", addr);
        }
    }

    fn write32(&mut self, addr: u32, value: u32) {
        if addr & 3 != 0 {
            panic!("unaliged write32 addr 0x{:x}", addr);
        }
        if self.code.in_range(addr) {
            self.code.write32(addr, value);
        } else if self.sram.in_range(addr) {
            self.sram.write32(addr, value);
        } else {
            panic!("bus access fault write addr 0x{:x}", addr);
        }
    }

    fn write16(&mut self, addr: u32, value: u16) {
        if addr & 1 != 0 {
            panic!("unaligned write16 address 0x{:x}", addr);
        }
        if self.code.in_range(addr) {
            self.code.write16(addr, value);
        } else if self.sram.in_range(addr) {
            self.sram.write16(addr, value);
        } else {
            panic!("bus access fault write addr 0x{:x}", addr);
        }
    }

    fn write8(&mut self, addr: u32, value: u8) {
        if self.code.in_range(addr) {
            self.code.write8(addr, value);
        } else if self.sram.in_range(addr) {
            self.sram.write8(addr, value);
        } else {
            panic!("bus access fault write addr 0x{:x}", addr);
        }
    }
    #[allow(unused)]
    fn in_range(&self, addr: u32) -> bool {
        self.code.in_range(addr) || self.sram.in_range(addr)
    }
}
<commit_msg>Temporary fix: do not check for unaligned access<commit_after>use crate::bus::Bus;

pub struct AHBLite<'a, T: 'a + Bus, R: 'a + Bus> {
    code: &'a mut T,
    sram: &'a mut R,
}

impl<'a, T, R> AHBLite<'a, T, R>
where
    T: Bus,
    R: Bus,
{
    pub fn new(code: &'a mut T, sram: &'a mut R) -> AHBLite<'a, T, R> {
        AHBLite { code, sram }
    }
}

impl<'a, T, R> Bus for AHBLite<'a, T, R>
where
    T: Bus,
    R: Bus,
{
    fn read8(&self, addr: u32) -> u8 {
        if self.code.in_range(addr) {
            self.code.read8(addr)
        } else if self.sram.in_range(addr) {
            self.sram.read8(addr)
        } else {
            panic!("bus access fault read8 addr 0x{:x}", addr);
        }
    }

    fn read16(&self, addr: u32) -> u16 {
        /*
        FIXME: LDR{S}H{T}, STRH{T} support non-halfword aligned access.
        FIXME: TBH support non-hw aligned access
        FIXME: LDR{T}, STR{T} support non-hw aligned access

        if addr & 1 == 1 {
            panic!("unaliged read16 addr 0x{:x}", addr);
        }*/

        if self.code.in_range(addr) {
            self.code.read16(addr)
        } else if self.sram.in_range(addr) {
            self.sram.read16(addr)
        } else {
            panic!("bus access fault read16 addr 0x{:x}", addr);
        }
    }

    fn read32(&self, addr: u32) -> u32 {
        /*
        FIXME: LDR{S}H{T}, STRH{T} support non-halfword aligned access.
        FIXME: TBH support non-hw aligned access
        FIXME: LDR{T}, STR{T} support non-hw aligned access
        if addr & 3 != 0 {
            panic!("unaliged read32 addr 0x{:x}", addr);
        }
        */
        if self.code.in_range(addr) {
            self.code.read32(addr)
        } else if self.sram.in_range(addr) {
            self.sram.read32(addr)
        } else {
            panic!("bus access fault read32 addr 0x{:x}", addr);
        }
    }

    fn write32(&mut self, addr: u32, value: u32) {
        /*
        if addr & 3 != 0 {
            panic!("unaliged write32 addr 0x{:x}", addr);
        }
        */
        if self.code.in_range(addr) {
            self.code.write32(addr, value);
        } else if self.sram.in_range(addr) {
            self.sram.write32(addr, value);
        } else {
            panic!("bus access fault write addr 0x{:x}", addr);
        }
    }

    fn write16(&mut self, addr: u32, value: u16) {
        /*        if addr & 1 != 0 {
            panic!("unaligned write16 address 0x{:x}", addr);
        }*/
        if self.code.in_range(addr) {
            self.code.write16(addr, value);
        } else if self.sram.in_range(addr) {
            self.sram.write16(addr, value);
        } else {
            panic!("bus access fault write addr 0x{:x}", addr);
        }
    }

    fn write8(&mut self, addr: u32, value: u8) {
        if self.code.in_range(addr) {
            self.code.write8(addr, value);
        } else if self.sram.in_range(addr) {
            self.sram.write8(addr, value);
        } else {
            panic!("bus access fault write addr 0x{:x}", addr);
        }
    }
    #[allow(unused)]
    fn in_range(&self, addr: u32) -> bool {
        self.code.in_range(addr) || self.sram.in_range(addr)
    }
}
<|endoftext|>


<commit_before><commit_msg>I'm smart.<commit_after><|endoftext|>


<commit_before>extern crate rocksdb;
extern crate tempdir;

mod test_iterator;
mod test_multithreaded;
mod test_column_family;
mod test_compaction_filter;
<commit_msg>add test<commit_after>extern crate rocksdb;
extern crate tempdir;

mod test_iterator;
mod test_multithreaded;
mod test_column_family;
mod test_compaction_filter;
mod test_compact_range;
<|endoftext|>


<commit_before>use nom::{IResult, Needed};

use std::io;
use std::io::Read;
use std::ptr;

pub enum ErrorKind {
  IO(io::Error),
  Incomplete(usize),
  EndOfInput,
  Unknown,
}

pub trait StreamProducer {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T>;
}

pub struct ByteStream<'a> {
  offset: usize,
  bytes: &'a [u8],
}

impl<'a> ByteStream<'a> {
  pub fn new(bytes: &'a [u8]) -> Self {
    ByteStream {
      offset: 0,
      bytes: bytes,
    }
  }

  #[inline]
  pub fn len(&self) -> usize {
    self.bytes.len() - self.offset
  }

  #[inline]
  pub fn is_empty(&self) -> bool {
    self.len() == 0
  }
}

impl<'a> StreamProducer for ByteStream<'a> {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T> {
    if self.is_empty() {
      return Err(ErrorKind::EndOfInput);
    }

    match f(&self.bytes[self.offset..]) {
      IResult::Done(i, o)    => {
        self.offset += self.len() - i.len();

        Ok(o)
      }
      IResult::Incomplete(n) => {
        let mut needed = self.len();

        if let Needed::Size(size) = n {
          needed = size;
        }

        Err(ErrorKind::Incomplete(needed))
      }
      IResult::Error(_)      => Err(ErrorKind::Unknown),
    }
  }
}

pub struct Buffer {
  data: Vec<u8>,
  filled: usize,
  offset: usize,
}

impl Buffer {
  pub fn new() -> Self {
    Self::with_capacity(1024)
  }

  pub fn with_capacity(capacity: usize) -> Self {
    let mut buffer = Vec::with_capacity(capacity);

    unsafe {
      buffer.set_len(capacity);
    }

    Buffer {
      data: buffer,
      filled: 0,
      offset: 0,
    }
  }

  #[inline]
  pub fn len(&self) -> usize {
    self.filled - self.offset
  }

  #[inline]
  pub fn is_empty(&self) -> bool {
    self.len() == 0
  }

  #[inline]
  pub fn capacity(&self) -> usize {
    self.data.len()
  }

  pub fn as_slice(&self) -> &[u8] {
    &self.data[self.offset..self.filled]
  }

  pub fn fill<R: Read>(&mut self, reader: &mut R) -> io::Result<usize> {
    reader.read(&mut self.data[self.filled..]).map(|consumed| {
      self.filled += consumed;

      consumed
    })
  }

  pub fn resize(&mut self, size: usize) {
    if size > self.data.capacity() {
      self.data.reserve(size);

      let capacity = self.data.capacity();

      unsafe {
        self.data.set_len(capacity);
      }
    }

    if self.data.len() - self.filled < size  {
      let length      = self.filled - self.offset;
      let mut mut_ptr = self.data.as_mut_ptr();

      unsafe {
        let offset_ptr  = self.data.as_ptr().offset(self.offset as isize);

        ptr::copy(offset_ptr, mut_ptr, length);
      }

      self.filled -= self.offset;
      self.offset  = 0;
    }
  }

  pub fn consume(&mut self, consumed: usize) {
    self.offset += consumed;
  }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ParserState {
  Incomplete,
  EndOfInput,
}

fn fill<R: Read>(buffer: &mut Buffer, reader: &mut R, needed: usize)
                 -> io::Result<usize> {
  let mut read = 0;

  if buffer.len() < needed {
    buffer.resize(needed);

    while buffer.len() < needed {
      let size_read = try!(buffer.fill(reader));

      if size_read > 0 {
        read += size_read;
      } else {
        break;
      }
    }
  }

  Ok(read)
}

pub struct ReadStream<R: Read> {
  reader: R,
  buffer: Buffer,
  needed: usize,
  state: ParserState,
}

impl<R> ReadStream<R> where R: Read {
  pub fn new(reader: R) -> Self {
    ReadStream {
      reader: reader,
      buffer: Buffer::new(),
      needed: 0,
      state: ParserState::Incomplete,
    }
  }

  fn fill(&mut self) -> io::Result<usize> {
    let needed = self.needed;

    fill(&mut self.buffer, &mut self.reader, needed).map(|consumed| {
      if self.buffer.len() < needed {
        self.state = ParserState::EndOfInput;
      }

      consumed
    })
  }
}

fn from_iresult<T>(buffer: &Buffer, result: IResult<&[u8], T>)
                   -> Result<(usize, T), ErrorKind> {
  match result {
    IResult::Done(i, o)    => Ok((buffer.len() - i.len(), o)),
    IResult::Incomplete(n) => {
      let mut needed = buffer.capacity();

      if let Needed::Size(size) = n {
        needed = size;
      }

      Err(ErrorKind::Incomplete(needed))
    }
    IResult::Error(_)      => Err(ErrorKind::Unknown),
  }
}

impl<R> StreamProducer for ReadStream<R> where R: Read {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T> {
    if self.state != ParserState::EndOfInput {
      try!(self.fill().map_err(ErrorKind::IO));
    }

    let mut buffer = &mut self.buffer;

    if buffer.is_empty() {
      self.state = ParserState::EndOfInput;

      return Err(ErrorKind::EndOfInput);
    }

    let result = {
      let iresult = f(buffer.as_slice());

      from_iresult(&buffer, iresult)
    };

    match result {
      Ok((consumed, o)) => {
        buffer.consume(consumed);

        Ok(o)
      }
      Err(kind)         => {
        if let ErrorKind::Incomplete(needed) = kind {
          self.needed = needed;
        }

        Err(kind)
      }
    }
  }
}
<commit_msg>Add a Consumed variant to ErrorKind enum<commit_after>use nom::{IResult, Needed};

use std::io;
use std::io::Read;
use std::ptr;

pub enum ErrorKind {
  IO(io::Error),
  Incomplete(usize),
  Consumed(usize),
  EndOfInput,
  Unknown,
}

pub trait StreamProducer {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T>;
}

pub struct ByteStream<'a> {
  offset: usize,
  bytes: &'a [u8],
}

impl<'a> ByteStream<'a> {
  pub fn new(bytes: &'a [u8]) -> Self {
    ByteStream {
      offset: 0,
      bytes: bytes,
    }
  }

  #[inline]
  pub fn len(&self) -> usize {
    self.bytes.len() - self.offset
  }

  #[inline]
  pub fn is_empty(&self) -> bool {
    self.len() == 0
  }
}

impl<'a> StreamProducer for ByteStream<'a> {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T> {
    if self.is_empty() {
      return Err(ErrorKind::EndOfInput);
    }

    match f(&self.bytes[self.offset..]) {
      IResult::Done(i, o)    => {
        self.offset += self.len() - i.len();

        Ok(o)
      }
      IResult::Incomplete(n) => {
        let mut needed = self.len();

        if let Needed::Size(size) = n {
          needed = size;
        }

        Err(ErrorKind::Incomplete(needed))
      }
      IResult::Error(_)      => Err(ErrorKind::Unknown),
    }
  }
}

pub struct Buffer {
  data: Vec<u8>,
  filled: usize,
  offset: usize,
}

impl Buffer {
  pub fn new() -> Self {
    Self::with_capacity(1024)
  }

  pub fn with_capacity(capacity: usize) -> Self {
    let mut buffer = Vec::with_capacity(capacity);

    unsafe {
      buffer.set_len(capacity);
    }

    Buffer {
      data: buffer,
      filled: 0,
      offset: 0,
    }
  }

  #[inline]
  pub fn len(&self) -> usize {
    self.filled - self.offset
  }

  #[inline]
  pub fn is_empty(&self) -> bool {
    self.len() == 0
  }

  #[inline]
  pub fn capacity(&self) -> usize {
    self.data.len()
  }

  pub fn as_slice(&self) -> &[u8] {
    &self.data[self.offset..self.filled]
  }

  pub fn fill<R: Read>(&mut self, reader: &mut R) -> io::Result<usize> {
    reader.read(&mut self.data[self.filled..]).map(|consumed| {
      self.filled += consumed;

      consumed
    })
  }

  pub fn resize(&mut self, size: usize) {
    if size > self.data.capacity() {
      self.data.reserve(size);

      let capacity = self.data.capacity();

      unsafe {
        self.data.set_len(capacity);
      }
    }

    if self.data.len() - self.filled < size  {
      let length      = self.filled - self.offset;
      let mut mut_ptr = self.data.as_mut_ptr();

      unsafe {
        let offset_ptr  = self.data.as_ptr().offset(self.offset as isize);

        ptr::copy(offset_ptr, mut_ptr, length);
      }

      self.filled -= self.offset;
      self.offset  = 0;
    }
  }

  pub fn consume(&mut self, consumed: usize) {
    self.offset += consumed;
  }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ParserState {
  Incomplete,
  EndOfInput,
}

fn fill<R: Read>(buffer: &mut Buffer, reader: &mut R, needed: usize)
                 -> io::Result<usize> {
  let mut read = 0;

  if buffer.len() < needed {
    buffer.resize(needed);

    while buffer.len() < needed {
      let size_read = try!(buffer.fill(reader));

      if size_read > 0 {
        read += size_read;
      } else {
        break;
      }
    }
  }

  Ok(read)
}

pub struct ReadStream<R: Read> {
  reader: R,
  buffer: Buffer,
  needed: usize,
  state: ParserState,
}

impl<R> ReadStream<R> where R: Read {
  pub fn new(reader: R) -> Self {
    ReadStream {
      reader: reader,
      buffer: Buffer::new(),
      needed: 0,
      state: ParserState::Incomplete,
    }
  }

  fn fill(&mut self) -> io::Result<usize> {
    let needed = self.needed;

    fill(&mut self.buffer, &mut self.reader, needed).map(|consumed| {
      if self.buffer.len() < needed {
        self.state = ParserState::EndOfInput;
      }

      consumed
    })
  }
}

fn from_iresult<T>(buffer: &Buffer, result: IResult<&[u8], T>)
                   -> Result<(usize, T), ErrorKind> {
  match result {
    IResult::Done(i, o)    => Ok((buffer.len() - i.len(), o)),
    IResult::Incomplete(n) => {
      let mut needed = buffer.capacity();

      if let Needed::Size(size) = n {
        needed = size;
      }

      Err(ErrorKind::Incomplete(needed))
    }
    IResult::Error(_)      => Err(ErrorKind::Unknown),
  }
}

impl<R> StreamProducer for ReadStream<R> where R: Read {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T> {
    if self.state != ParserState::EndOfInput {
      try!(self.fill().map_err(ErrorKind::IO));
    }

    let mut buffer = &mut self.buffer;

    if buffer.is_empty() {
      self.state = ParserState::EndOfInput;

      return Err(ErrorKind::EndOfInput);
    }

    let result = {
      let iresult = f(buffer.as_slice());

      from_iresult(&buffer, iresult)
    };

    match result {
      Ok((consumed, o)) => {
        buffer.consume(consumed);

        Ok(o)
      }
      Err(kind)         => {
        if let ErrorKind::Incomplete(needed) = kind {
          self.needed = needed;
        }

        Err(kind)
      }
    }
  }
}
<|endoftext|>


<commit_before>extern crate config;
extern crate serde;
extern crate float_cmp;

#[macro_use]
extern crate serde_derive;

use std::collections::HashMap;
use float_cmp::ApproxEqUlps;
use config::*;

#[derive(Debug, Deserialize)]
struct Place {
    name: String,
    longitude: f64,
    latitude: f64,
    favorite: bool,
    telephone: Option<String>,
    reviews: u64,
    rating: Option<f32>,
}

#[derive(Debug, Deserialize)]
struct Settings {
    debug: f64,
    production: Option<String>,
    place: Place,
}

fn make() -> Config {
    let mut c = Config::default();
    c.merge(File::new("tests/Settings", FileFormat::Toml))
        .unwrap();

    c
}

#[test]
fn test_scalar() {
    let c = make();

    assert_eq!(c.get("debug").ok(), Some(true));
    assert_eq!(c.get("production").ok(), Some(false));
}

#[test]
fn test_scalar_type_loose() {
    let c = make();

    assert_eq!(c.get("debug").ok(), Some(true));
    assert_eq!(c.get("debug").ok(), Some("true".to_string()));
    assert_eq!(c.get("debug").ok(), Some(1));
    assert_eq!(c.get("debug").ok(), Some(1.0));

    assert_eq!(c.get("debug_s").ok(), Some(true));
    assert_eq!(c.get("debug_s").ok(), Some("true".to_string()));
    assert_eq!(c.get("debug_s").ok(), Some(1));
    assert_eq!(c.get("debug_s").ok(), Some(1.0));

    assert_eq!(c.get("production").ok(), Some(false));
    assert_eq!(c.get("production").ok(), Some("false".to_string()));
    assert_eq!(c.get("production").ok(), Some(0));
    assert_eq!(c.get("production").ok(), Some(0.0));

    assert_eq!(c.get("production_s").ok(), Some(false));
    assert_eq!(c.get("production_s").ok(), Some("false".to_string()));
    assert_eq!(c.get("production_s").ok(), Some(0));
    assert_eq!(c.get("production_s").ok(), Some(0.0));
}

#[test]
fn test_get_scalar_path() {
    let c = make();

    assert_eq!(c.get("place.favorite").ok(), Some(false));
    assert_eq!(c.get("place.creator.name").ok(), Some("John Smith".to_string()));
}

#[test]
fn test_map() {
    let c = make();
    let m: HashMap<String, Value> = c.get("place").unwrap();

    assert_eq!(m.len(), 7);
    assert_eq!(m["name"].clone().into_str().unwrap(), "Torre di Pisa".to_string());
    assert_eq!(m["reviews"].clone().into_int().unwrap(), 3866);
}

#[test]
fn test_map_str() {
    let c = make();
    let m: HashMap<String, String> = c.get("place.creator").unwrap();

    assert_eq!(m.len(), 1);
    assert_eq!(m["name"], "John Smith".to_string());
}

#[test]
fn test_map_struct() {
    #[derive(Debug, Deserialize)]
    struct Settings {
        place: HashMap<String, Value>,
    }

    let c = make();
    let s: Settings = c.deserialize().unwrap();

    assert_eq!(s.place.len(), 7);
    assert_eq!(s.place["name"].clone().into_str().unwrap(), "Torre di Pisa".to_string());
    assert_eq!(s.place["reviews"].clone().into_int().unwrap(), 3866);
}

#[test]
fn test_file_struct() {
    let c = make();

    // Deserialize the entire file as single struct
    let s: Settings = c.deserialize().unwrap();

    assert!(s.debug.approx_eq_ulps(&1.0, 2));
    assert_eq!(s.production, Some("false".to_string()));
    assert_eq!(s.place.name, "Torre di Pisa");
    assert!(s.place.longitude.approx_eq_ulps(&43.7224985, 2));
    assert!(s.place.latitude.approx_eq_ulps(&10.3970522, 2));
    assert_eq!(s.place.favorite, false);
    assert_eq!(s.place.reviews, 3866);
    assert_eq!(s.place.rating, Some(4.5));
    assert_eq!(s.place.telephone, None);
}

#[test]
fn test_scalar_struct() {
    let c = make();

    // Deserialize a scalar struct that has lots of different
    // data types
    let p: Place = c.get("place").unwrap();

    assert_eq!(p.name, "Torre di Pisa");
    assert!(p.longitude.approx_eq_ulps(&43.7224985, 2));
    assert!(p.latitude.approx_eq_ulps(&10.3970522, 2));
    assert_eq!(p.favorite, false);
    assert_eq!(p.reviews, 3866);
    assert_eq!(p.rating, Some(4.5));
    assert_eq!(p.telephone, None);
}

#[test]
fn test_array_scalar() {
    let c = make();
    let arr: Vec<i64> = c.get("arr").unwrap();

    assert_eq!(arr.len(), 10);
    assert_eq!(arr[3], 4);
}

#[test]
fn test_struct_array() {
    #[derive(Debug, Deserialize)]
    struct Settings {
        #[serde(rename = "arr")]
        elements: Vec<String>,
    }

    let c = make();
    let s: Settings = c.deserialize().unwrap();

    assert_eq!(s.elements.len(), 10);
    assert_eq!(s.elements[3], "4".to_string());
}
<commit_msg>Add test for a missing key<commit_after>extern crate config;
extern crate serde;
extern crate float_cmp;

#[macro_use]
extern crate serde_derive;

use std::collections::HashMap;
use float_cmp::ApproxEqUlps;
use config::*;

#[derive(Debug, Deserialize)]
struct Place {
    name: String,
    longitude: f64,
    latitude: f64,
    favorite: bool,
    telephone: Option<String>,
    reviews: u64,
    rating: Option<f32>,
}

#[derive(Debug, Deserialize)]
struct Settings {
    debug: f64,
    production: Option<String>,
    place: Place,
}

fn make() -> Config {
    let mut c = Config::default();
    c.merge(File::new("tests/Settings", FileFormat::Toml))
        .unwrap();

    c
}

#[test]
fn test_not_found() {
    let c = make();
    let res = c.get::<bool>("not_found");

    assert!(res.is_err());
    assert_eq!(res.unwrap_err().to_string(),
               "configuration property \"not_found\" not found"
                   .to_string());
}

#[test]
fn test_scalar() {
    let c = make();

    assert_eq!(c.get("debug").ok(), Some(true));
    assert_eq!(c.get("production").ok(), Some(false));
}

#[test]
fn test_scalar_type_loose() {
    let c = make();

    assert_eq!(c.get("debug").ok(), Some(true));
    assert_eq!(c.get("debug").ok(), Some("true".to_string()));
    assert_eq!(c.get("debug").ok(), Some(1));
    assert_eq!(c.get("debug").ok(), Some(1.0));

    assert_eq!(c.get("debug_s").ok(), Some(true));
    assert_eq!(c.get("debug_s").ok(), Some("true".to_string()));
    assert_eq!(c.get("debug_s").ok(), Some(1));
    assert_eq!(c.get("debug_s").ok(), Some(1.0));

    assert_eq!(c.get("production").ok(), Some(false));
    assert_eq!(c.get("production").ok(), Some("false".to_string()));
    assert_eq!(c.get("production").ok(), Some(0));
    assert_eq!(c.get("production").ok(), Some(0.0));

    assert_eq!(c.get("production_s").ok(), Some(false));
    assert_eq!(c.get("production_s").ok(), Some("false".to_string()));
    assert_eq!(c.get("production_s").ok(), Some(0));
    assert_eq!(c.get("production_s").ok(), Some(0.0));
}

#[test]
fn test_get_scalar_path() {
    let c = make();

    assert_eq!(c.get("place.favorite").ok(), Some(false));
    assert_eq!(c.get("place.creator.name").ok(), Some("John Smith".to_string()));
}

#[test]
fn test_map() {
    let c = make();
    let m: HashMap<String, Value> = c.get("place").unwrap();

    assert_eq!(m.len(), 7);
    assert_eq!(m["name"].clone().into_str().unwrap(), "Torre di Pisa".to_string());
    assert_eq!(m["reviews"].clone().into_int().unwrap(), 3866);
}

#[test]
fn test_map_str() {
    let c = make();
    let m: HashMap<String, String> = c.get("place.creator").unwrap();

    assert_eq!(m.len(), 1);
    assert_eq!(m["name"], "John Smith".to_string());
}

#[test]
fn test_map_struct() {
    #[derive(Debug, Deserialize)]
    struct Settings {
        place: HashMap<String, Value>,
    }

    let c = make();
    let s: Settings = c.deserialize().unwrap();

    assert_eq!(s.place.len(), 7);
    assert_eq!(s.place["name"].clone().into_str().unwrap(), "Torre di Pisa".to_string());
    assert_eq!(s.place["reviews"].clone().into_int().unwrap(), 3866);
}

#[test]
fn test_file_struct() {
    let c = make();

    // Deserialize the entire file as single struct
    let s: Settings = c.deserialize().unwrap();

    assert!(s.debug.approx_eq_ulps(&1.0, 2));
    assert_eq!(s.production, Some("false".to_string()));
    assert_eq!(s.place.name, "Torre di Pisa");
    assert!(s.place.longitude.approx_eq_ulps(&43.7224985, 2));
    assert!(s.place.latitude.approx_eq_ulps(&10.3970522, 2));
    assert_eq!(s.place.favorite, false);
    assert_eq!(s.place.reviews, 3866);
    assert_eq!(s.place.rating, Some(4.5));
    assert_eq!(s.place.telephone, None);
}

#[test]
fn test_scalar_struct() {
    let c = make();

    // Deserialize a scalar struct that has lots of different
    // data types
    let p: Place = c.get("place").unwrap();

    assert_eq!(p.name, "Torre di Pisa");
    assert!(p.longitude.approx_eq_ulps(&43.7224985, 2));
    assert!(p.latitude.approx_eq_ulps(&10.3970522, 2));
    assert_eq!(p.favorite, false);
    assert_eq!(p.reviews, 3866);
    assert_eq!(p.rating, Some(4.5));
    assert_eq!(p.telephone, None);
}

#[test]
fn test_array_scalar() {
    let c = make();
    let arr: Vec<i64> = c.get("arr").unwrap();

    assert_eq!(arr.len(), 10);
    assert_eq!(arr[3], 4);
}

#[test]
fn test_struct_array() {
    #[derive(Debug, Deserialize)]
    struct Settings {
        #[serde(rename = "arr")]
        elements: Vec<String>,
    }

    let c = make();
    let s: Settings = c.deserialize().unwrap();

    assert_eq!(s.elements.len(), 10);
    assert_eq!(s.elements[3], "4".to_string());
}
<|endoftext|>