<commit_before>extern mod std;
use std::hashmap::HashMap;
use sexpr;

// A very simple LISP-like language
// Globally scoped, no closures

/// Our value types
#[deriving(Clone)]
pub enum LispValue {
	List(~[LispValue]),
	Atom(~str),
	Str(~str),
	Num(float),
	Fn(~[~str], ~LispValue), // args, body
	// XXX: we need the symbol table to be a @-ptr
	// otherwise we can't mutate it. &-ptrs give lifetime
	// headaches, and ~-ptrs give move headaches.
	BIF(~str, ~[~str], extern fn(@mut HashMap<~str, ~LispValue>, ~[~LispValue])->~LispValue) // built-in function (args, closure)
}

// XXX: this is ugly but it won't automatically derive Eq because of the extern fn
impl Eq for LispValue {
	fn eq(&self, other: &LispValue) -> bool {
		match (self.clone(), other.clone()) {
			(BIF(ref x, _, _), BIF(ref y, _, _)) if *x == *y => true,
			(Str(ref x), Str(ref y)) if *x == *y => true,
			(Num(ref x), Num(ref y)) if *x == *y => true,
			(Atom(ref x), Atom(ref y)) if *x == *y => true,
			(List(ref x), List(ref y)) if *x == *y => true,
			(Fn(ref x, ref x2), Fn(ref y, ref y2)) if *x == *y && *x2 == *y2 => true,
			_ => false
		}
	}
}

fn from_sexpr(sexpr: &sexpr::Value) -> ~LispValue {
	match *sexpr {
		sexpr::List(ref v) => ~List(v.map(|x| *from_sexpr(x))),
		sexpr::Num(v) => ~Num(v),
		sexpr::Str(ref v) => ~Str(v.clone()),
		sexpr::Atom(ref v) => ~Atom(v.clone())
	}
}

/// The type of the global symbol table (string to a value mapping).
type SymbolTable = HashMap<~str, ~LispValue>;

/// Creates a new symbol table and returns it
pub fn new_symt() -> SymbolTable {
	HashMap::new()
}

/// Binds a symbol in the symbol table. Replaces if it already exists.
pub fn bind(symt: @mut SymbolTable, name: ~str, value: ~LispValue) {
	symt.insert(name, value);
}

/// Look up a symbol in the symbol table. Fails if not found.
pub fn lookup(symt: @mut SymbolTable, name: ~str) -> ~LispValue {
	match symt.find(&name) {
		Some(v) => v.clone(),
		None => fail!("couldn't find symbol: %s", name)
	}
}

fn id_(symt: @mut SymbolTable, v: ~[~LispValue]) -> ~LispValue { symt.insert(~"hi", ~Num(6.0)); v[0] }

/// Initializes standard library functions
pub fn init_std(symt: @mut SymbolTable) {
	bind(symt, ~"id", ~BIF(~"id", ~[~"x"], id_));
}

fn apply(symt: @mut SymbolTable, f: ~LispValue, args: ~[~LispValue]) -> ~LispValue {
	match *f {
		BIF(name, fnargs, bif) => {
			// apply built-in function
			if fnargs.len() != args.len() {
				fail!("function '%s' requires %u arguments, but it received %u arguments",
					name, fnargs.len(), args.len())
			}

			bif(symt, args)
		}

		_ => fail!("apply: need function")
	}
}

/// Evaluates an s-expression and returns a value.
pub fn eval(symt: @mut SymbolTable, input: sexpr::Value) -> ~LispValue {
	match input {
		sexpr::List(v) => {
			if(v.len() == 0) {
				fail!("eval given empty list")
			}

			// XXX: If we don't clone, the `match` partially moves v,
			// so we can't use it in the match arms.
			let v_ = v.clone();

			// evaluate a list as a function call
			match v[0] {
				sexpr::Atom(sym) => {
					let f = lookup(symt, sym);
					let args = v_.slice(1, v_.len());
					let xargs = args.map(|x| eval(symt, x.clone())); // eval'd args
					apply(symt, f, xargs)
				}
				_ => fail!("function calls take an atom"),
			}

		}
		_ => from_sexpr(&input) // return non-list values as they are
	}
}

#[cfg(test)]
mod test {
	use super::*;
	use sexpr;
	use sexpr::from_str;

	fn read(input: &str) -> sexpr::Value {
		from_str(input).unwrap()
	}

	#[test]
	fn test_eval() {
		let symt = @mut new_symt();
		init_std(symt);
		assert_eq!(eval(symt, read("123")), ~Num(123.0));
		assert_eq!(eval(symt, read("(id 123)")), ~Num(123.0));
		// should fail: assert_eq!(eval(&mut symt, read("(1 2 3)")), ~List(~[Num(1.0), Num(2.0), Num(3.0)]));
	}
}<commit_msg>remove forgotten test code<commit_after>extern mod std;
use std::hashmap::HashMap;
use sexpr;

// A very simple LISP-like language
// Globally scoped, no closures

/// Our value types
#[deriving(Clone)]
pub enum LispValue {
	List(~[LispValue]),
	Atom(~str),
	Str(~str),
	Num(float),
	Fn(~[~str], ~LispValue), // args, body
	// XXX: we need the symbol table to be a @-ptr
	// otherwise we can't mutate it. &-ptrs give lifetime
	// headaches, and ~-ptrs give move headaches.
	BIF(~str, ~[~str], extern fn(@mut HashMap<~str, ~LispValue>, ~[~LispValue])->~LispValue) // built-in function (args, closure)
}

// XXX: this is ugly but it won't automatically derive Eq because of the extern fn
impl Eq for LispValue {
	fn eq(&self, other: &LispValue) -> bool {
		match (self.clone(), other.clone()) {
			(BIF(ref x, _, _), BIF(ref y, _, _)) if *x == *y => true,
			(Str(ref x), Str(ref y)) if *x == *y => true,
			(Num(ref x), Num(ref y)) if *x == *y => true,
			(Atom(ref x), Atom(ref y)) if *x == *y => true,
			(List(ref x), List(ref y)) if *x == *y => true,
			(Fn(ref x, ref x2), Fn(ref y, ref y2)) if *x == *y && *x2 == *y2 => true,
			_ => false
		}
	}
}

fn from_sexpr(sexpr: &sexpr::Value) -> ~LispValue {
	match *sexpr {
		sexpr::List(ref v) => ~List(v.map(|x| *from_sexpr(x))),
		sexpr::Num(v) => ~Num(v),
		sexpr::Str(ref v) => ~Str(v.clone()),
		sexpr::Atom(ref v) => ~Atom(v.clone())
	}
}

/// The type of the global symbol table (string to a value mapping).
type SymbolTable = HashMap<~str, ~LispValue>;

/// Creates a new symbol table and returns it
pub fn new_symt() -> SymbolTable {
	HashMap::new()
}

/// Binds a symbol in the symbol table. Replaces if it already exists.
pub fn bind(symt: @mut SymbolTable, name: ~str, value: ~LispValue) {
	symt.insert(name, value);
}

/// Look up a symbol in the symbol table. Fails if not found.
pub fn lookup(symt: @mut SymbolTable, name: ~str) -> ~LispValue {
	match symt.find(&name) {
		Some(v) => v.clone(),
		None => fail!("couldn't find symbol: %s", name)
	}
}

fn id_(_symt: @mut SymbolTable, v: ~[~LispValue]) -> ~LispValue { v[0] }

/// Initializes standard library functions
pub fn init_std(symt: @mut SymbolTable) {
	bind(symt, ~"id", ~BIF(~"id", ~[~"x"], id_));
}

fn apply(symt: @mut SymbolTable, f: ~LispValue, args: ~[~LispValue]) -> ~LispValue {
	match *f {
		BIF(name, fnargs, bif) => {
			// apply built-in function
			if fnargs.len() != args.len() {
				fail!("function '%s' requires %u arguments, but it received %u arguments",
					name, fnargs.len(), args.len())
			}

			bif(symt, args)
		}

		_ => fail!("apply: need function")
	}
}

/// Evaluates an s-expression and returns a value.
pub fn eval(symt: @mut SymbolTable, input: sexpr::Value) -> ~LispValue {
	match input {
		sexpr::List(v) => {
			if(v.len() == 0) {
				fail!("eval given empty list")
			}

			// XXX: If we don't clone, the `match` partially moves v,
			// so we can't use it in the match arms.
			let v_ = v.clone();

			// evaluate a list as a function call
			match v[0] {
				sexpr::Atom(sym) => {
					let f = lookup(symt, sym);
					let args = v_.slice(1, v_.len());
					let xargs = args.map(|x| eval(symt, x.clone())); // eval'd args
					apply(symt, f, xargs)
				}
				_ => fail!("function calls take an atom"),
			}

		}
		_ => from_sexpr(&input) // return non-list values as they are
	}
}

#[cfg(test)]
mod test {
	use super::*;
	use sexpr;
	use sexpr::from_str;

	fn read(input: &str) -> sexpr::Value {
		from_str(input).unwrap()
	}

	#[test]
	fn test_eval() {
		let symt = @mut new_symt();
		init_std(symt);
		assert_eq!(eval(symt, read("123")), ~Num(123.0));
		assert_eq!(eval(symt, read("(id 123)")), ~Num(123.0));
		// should fail: assert_eq!(eval(&mut symt, read("(1 2 3)")), ~List(~[Num(1.0), Num(2.0), Num(3.0)]));
	}
}<|endoftext|>


<commit_before>use libloading::{Library, Symbol};
use regex::{Regex, RegexBuilder};
use std::collections::HashMap;
use std::fs;
use std::io;
use std::mem;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::SystemTime;
use tree_sitter::{Language, PropertySheet};

#[cfg(unix)]
const DYLIB_EXTENSION: &'static str = "so";

#[cfg(windows)]
const DYLIB_EXTENSION: &'static str = "dll";

struct LanguageRepo {
    name: String,
    path: PathBuf,
    language: Option<Language>,
    configurations: Vec<LanguageConfiguration>,
}

pub struct LanguageConfiguration {
    _name: String,
    _content_regex: Option<Regex>,
    _first_line_regex: Option<Regex>,
    file_types: Vec<String>,
    _highlight_property_sheet: Option<Result<PropertySheet, PathBuf>>,
}

pub struct Loader {
    parser_lib_path: PathBuf,
    language_repos: Vec<LanguageRepo>,
    language_configuration_ids_by_file_type: HashMap<String, Vec<(usize, usize)>>,
}

unsafe impl Send for Loader {}
unsafe impl Sync for Loader {}

impl Loader {
    pub fn new(parser_lib_path: PathBuf) -> Self {
        Loader {
            parser_lib_path,
            language_repos: Vec::new(),
            language_configuration_ids_by_file_type: HashMap::new(),
        }
    }

    pub fn find_all_languages(&mut self, parser_src_paths: &Vec<PathBuf>) -> io::Result<()> {
        for parser_container_dir in parser_src_paths.iter() {
            for entry in fs::read_dir(parser_container_dir)? {
                let entry = entry?;
                if let Some(parser_dir_name) = entry.file_name().to_str() {
                    if parser_dir_name.starts_with("tree-sitter-") {
                        if self
                            .find_language_at_path(&parser_container_dir.join(parser_dir_name))
                            .is_err()
                        {
                            eprintln!("Error loading {}", parser_dir_name);
                        }
                    }
                }
            }
        }
        Ok(())
    }

    pub fn language_at_path(&mut self, path: &Path) -> io::Result<Option<Language>> {
        if let Ok(id) = self.find_language_at_path(path) {
            Ok(Some(self.language_configuration_for_id(id)?.0))
        } else {
            Ok(None)
        }
    }

    pub fn language_configuration_for_file_name(
        &mut self,
        path: &Path,
    ) -> io::Result<Option<(Language, &LanguageConfiguration)>> {
        let ids = path
            .file_name()
            .and_then(|n| n.to_str())
            .and_then(|file_name| self.language_configuration_ids_by_file_type.get(file_name))
            .or_else(|| {
                path.extension()
                    .and_then(|extension| extension.to_str())
                    .and_then(|extension| {
                        self.language_configuration_ids_by_file_type.get(extension)
                    })
            });
        if let Some(ids) = ids {
            // TODO use `content-regex` to pick one
            for (repo_id, configuration_id) in ids.iter().cloned() {
                let (language, configurations) = self.language_configuration_for_id(repo_id)?;
                return Ok(Some((language, &configurations[configuration_id])));
            }
        }
        Ok(None)
    }

    fn language_configuration_for_id(
        &mut self,
        id: usize,
    ) -> io::Result<(Language, &Vec<LanguageConfiguration>)> {
        let repo = &self.language_repos[id];
        let language = if let Some(language) = repo.language {
            language
        } else {
            let src_path = repo.path.join("src");
            let language = self.load_language_at_path(&repo.name, &src_path, &src_path)?;
            self.language_repos[id].language = Some(language);
            language
        };
        Ok((language, &self.language_repos[id].configurations))
    }

    pub fn load_language_at_path(
        &self,
        name: &str,
        src_path: &Path,
        header_path: &Path,
    ) -> io::Result<Language> {
        let parser_path = src_path.join("parser.c");

        let scanner_path;
        let scanner_c_path = src_path.join("scanner.c");
        if scanner_c_path.exists() {
            scanner_path = Some(scanner_c_path);
        } else {
            let scanner_cc_path = src_path.join("scanner.cc");
            if scanner_cc_path.exists() {
                scanner_path = Some(scanner_cc_path);
            } else {
                scanner_path = None;
            }
        }

        self.load_language_from_sources(name, &header_path, &parser_path, &scanner_path)
    }

    pub fn load_language_from_sources(
        &self,
        name: &str,
        header_path: &Path,
        parser_path: &Path,
        scanner_path: &Option<PathBuf>,
    ) -> io::Result<Language> {
        let mut library_path = self.parser_lib_path.join(name);
        library_path.set_extension(DYLIB_EXTENSION);

        if needs_recompile(&library_path, &parser_path, &scanner_path)? {
            let mut config = cc::Build::new();
            config
                .cpp(true)
                .opt_level(2)
                .cargo_metadata(false)
                .target(env!("BUILD_TARGET"))
                .host(env!("BUILD_TARGET"));
            let compiler = config.get_compiler();
            let mut command = Command::new(compiler.path());
            for (key, value) in compiler.env() {
                command.env(key, value);
            }

            if cfg!(windows) {
                command
                    .args(&["/nologo", "/LD", "/I"])
                    .arg(header_path)
                    .arg("/Od")
                    .arg(parser_path);
                if let Some(scanner_path) = scanner_path.as_ref() {
                    command.arg(scanner_path);
                }
                command
                    .arg("/link")
                    .arg(format!("/out:{}", library_path.to_str().unwrap()));
            } else {
                command
                    .arg("-shared")
                    .arg("-fPIC")
                    .arg("-I")
                    .arg(header_path)
                    .arg("-o")
                    .arg(&library_path)
                    .arg("-xc")
                    .arg(parser_path);
                if let Some(scanner_path) = scanner_path.as_ref() {
                    if scanner_path.extension() == Some("c".as_ref()) {
                        command.arg(scanner_path);
                    } else {
                        command.arg("-xc++").arg(scanner_path);
                    }
                }
            }

            let output = command.output()?;
            if !output.status.success() {
                return Err(io::Error::new(
                    io::ErrorKind::Other,
                    format!(
                        "Parser compilation failed.\nStdout: {}\nStderr: {}",
                        String::from_utf8_lossy(&output.stdout),
                        String::from_utf8_lossy(&output.stderr)
                    )
                    .as_str(),
                ));
            }
        }

        let library = Library::new(library_path)?;
        let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(name));
        let language = unsafe {
            let language_fn: Symbol<unsafe extern "C" fn() -> Language> =
                library.get(language_fn_name.as_bytes())?;
            language_fn()
        };
        mem::forget(library);
        Ok(language)
    }

    fn find_language_at_path<'a>(&'a mut self, parser_path: &Path) -> io::Result<usize> {
        #[derive(Deserialize)]
        struct LanguageConfigurationJSON {
            name: String,
            #[serde(rename = "file-types")]
            file_types: Option<Vec<String>>,
            #[serde(rename = "content-regex")]
            content_regex: Option<String>,
            #[serde(rename = "first-line-regex")]
            first_line_regex: Option<String>,
            highlights: Option<String>,
        }

        #[derive(Deserialize)]
        struct PackageJSON {
            name: String,
            #[serde(rename = "tree-sitter")]
            tree_sitter: Option<Vec<LanguageConfigurationJSON>>,
        }

        let package_json_contents = fs::read_to_string(&parser_path.join("package.json"))?;
        let package_json: PackageJSON = serde_json::from_str(&package_json_contents)?;
        let configurations = package_json
            .tree_sitter
            .map_or(Vec::new(), |configurations| {
                configurations
                    .into_iter()
                    .map(|conf| LanguageConfiguration {
                        _name: conf.name,
                        file_types: conf.file_types.unwrap_or(Vec::new()),
                        _content_regex: conf
                            .content_regex
                            .and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
                        _first_line_regex: conf
                            .first_line_regex
                            .and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
                        _highlight_property_sheet: conf.highlights.map(|d| Err(d.into())),
                    })
                    .collect()
            });

        for (i, configuration) in configurations.iter().enumerate() {
            for file_type in &configuration.file_types {
                self.language_configuration_ids_by_file_type
                    .entry(file_type.to_string())
                    .or_insert(Vec::new())
                    .push((self.language_repos.len(), i));
            }
        }

        self.language_repos.push(LanguageRepo {
            name: package_json.name.split_at("tree-sitter-".len()).1.to_string(),
            path: parser_path.to_owned(),
            language: None,
            configurations,
        });

        Ok(self.language_repos.len() - 1)
    }
}

fn needs_recompile(
    lib_path: &Path,
    parser_c_path: &Path,
    scanner_path: &Option<PathBuf>,
) -> io::Result<bool> {
    if !lib_path.exists() {
        return Ok(true);
    }
    let lib_mtime = mtime(lib_path)?;
    if mtime(parser_c_path)? > lib_mtime {
        return Ok(true);
    }
    if let Some(scanner_path) = scanner_path {
        if mtime(scanner_path)? > lib_mtime {
            return Ok(true);
        }
    }
    Ok(false)
}

fn mtime(path: &Path) -> io::Result<SystemTime> {
    Ok(fs::metadata(path)?.modified()?)
}

fn replace_dashes_with_underscores(name: &str) -> String {
    let mut result = String::with_capacity(name.len());
    for c in name.chars() {
        if c == '-' {
            result.push('_');
        } else {
            result.push(c);
        }
    }
    result
}
<commit_msg>Add -xc compiler flag for pure-C external scanners<commit_after>use libloading::{Library, Symbol};
use regex::{Regex, RegexBuilder};
use std::collections::HashMap;
use std::fs;
use std::io;
use std::mem;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::SystemTime;
use tree_sitter::{Language, PropertySheet};

#[cfg(unix)]
const DYLIB_EXTENSION: &'static str = "so";

#[cfg(windows)]
const DYLIB_EXTENSION: &'static str = "dll";

struct LanguageRepo {
    name: String,
    path: PathBuf,
    language: Option<Language>,
    configurations: Vec<LanguageConfiguration>,
}

pub struct LanguageConfiguration {
    _name: String,
    _content_regex: Option<Regex>,
    _first_line_regex: Option<Regex>,
    file_types: Vec<String>,
    _highlight_property_sheet: Option<Result<PropertySheet, PathBuf>>,
}

pub struct Loader {
    parser_lib_path: PathBuf,
    language_repos: Vec<LanguageRepo>,
    language_configuration_ids_by_file_type: HashMap<String, Vec<(usize, usize)>>,
}

unsafe impl Send for Loader {}
unsafe impl Sync for Loader {}

impl Loader {
    pub fn new(parser_lib_path: PathBuf) -> Self {
        Loader {
            parser_lib_path,
            language_repos: Vec::new(),
            language_configuration_ids_by_file_type: HashMap::new(),
        }
    }

    pub fn find_all_languages(&mut self, parser_src_paths: &Vec<PathBuf>) -> io::Result<()> {
        for parser_container_dir in parser_src_paths.iter() {
            for entry in fs::read_dir(parser_container_dir)? {
                let entry = entry?;
                if let Some(parser_dir_name) = entry.file_name().to_str() {
                    if parser_dir_name.starts_with("tree-sitter-") {
                        if self
                            .find_language_at_path(&parser_container_dir.join(parser_dir_name))
                            .is_err()
                        {
                            eprintln!("Error loading {}", parser_dir_name);
                        }
                    }
                }
            }
        }
        Ok(())
    }

    pub fn language_at_path(&mut self, path: &Path) -> io::Result<Option<Language>> {
        if let Ok(id) = self.find_language_at_path(path) {
            Ok(Some(self.language_configuration_for_id(id)?.0))
        } else {
            Ok(None)
        }
    }

    pub fn language_configuration_for_file_name(
        &mut self,
        path: &Path,
    ) -> io::Result<Option<(Language, &LanguageConfiguration)>> {
        let ids = path
            .file_name()
            .and_then(|n| n.to_str())
            .and_then(|file_name| self.language_configuration_ids_by_file_type.get(file_name))
            .or_else(|| {
                path.extension()
                    .and_then(|extension| extension.to_str())
                    .and_then(|extension| {
                        self.language_configuration_ids_by_file_type.get(extension)
                    })
            });
        if let Some(ids) = ids {
            // TODO use `content-regex` to pick one
            for (repo_id, configuration_id) in ids.iter().cloned() {
                let (language, configurations) = self.language_configuration_for_id(repo_id)?;
                return Ok(Some((language, &configurations[configuration_id])));
            }
        }
        Ok(None)
    }

    fn language_configuration_for_id(
        &mut self,
        id: usize,
    ) -> io::Result<(Language, &Vec<LanguageConfiguration>)> {
        let repo = &self.language_repos[id];
        let language = if let Some(language) = repo.language {
            language
        } else {
            let src_path = repo.path.join("src");
            let language = self.load_language_at_path(&repo.name, &src_path, &src_path)?;
            self.language_repos[id].language = Some(language);
            language
        };
        Ok((language, &self.language_repos[id].configurations))
    }

    pub fn load_language_at_path(
        &self,
        name: &str,
        src_path: &Path,
        header_path: &Path,
    ) -> io::Result<Language> {
        let parser_path = src_path.join("parser.c");

        let scanner_path;
        let scanner_c_path = src_path.join("scanner.c");
        if scanner_c_path.exists() {
            scanner_path = Some(scanner_c_path);
        } else {
            let scanner_cc_path = src_path.join("scanner.cc");
            if scanner_cc_path.exists() {
                scanner_path = Some(scanner_cc_path);
            } else {
                scanner_path = None;
            }
        }

        self.load_language_from_sources(name, &header_path, &parser_path, &scanner_path)
    }

    pub fn load_language_from_sources(
        &self,
        name: &str,
        header_path: &Path,
        parser_path: &Path,
        scanner_path: &Option<PathBuf>,
    ) -> io::Result<Language> {
        let mut library_path = self.parser_lib_path.join(name);
        library_path.set_extension(DYLIB_EXTENSION);

        if needs_recompile(&library_path, &parser_path, &scanner_path)? {
            let mut config = cc::Build::new();
            config
                .cpp(true)
                .opt_level(2)
                .cargo_metadata(false)
                .target(env!("BUILD_TARGET"))
                .host(env!("BUILD_TARGET"));
            let compiler = config.get_compiler();
            let mut command = Command::new(compiler.path());
            for (key, value) in compiler.env() {
                command.env(key, value);
            }

            if cfg!(windows) {
                command
                    .args(&["/nologo", "/LD", "/I"])
                    .arg(header_path)
                    .arg("/Od")
                    .arg(parser_path);
                if let Some(scanner_path) = scanner_path.as_ref() {
                    command.arg(scanner_path);
                }
                command
                    .arg("/link")
                    .arg(format!("/out:{}", library_path.to_str().unwrap()));
            } else {
                command
                    .arg("-shared")
                    .arg("-fPIC")
                    .arg("-I")
                    .arg(header_path)
                    .arg("-o")
                    .arg(&library_path)
                    .arg("-xc")
                    .arg(parser_path);
                if let Some(scanner_path) = scanner_path.as_ref() {
                    if scanner_path.extension() == Some("c".as_ref()) {
                        command.arg("-xc").arg("-std=c99").arg(scanner_path);
                    } else {
                        command.arg("-xc++").arg(scanner_path);
                    }
                }
            }

            let output = command.output()?;
            if !output.status.success() {
                return Err(io::Error::new(
                    io::ErrorKind::Other,
                    format!(
                        "Parser compilation failed.\nStdout: {}\nStderr: {}",
                        String::from_utf8_lossy(&output.stdout),
                        String::from_utf8_lossy(&output.stderr)
                    )
                    .as_str(),
                ));
            }
        }

        let library = Library::new(library_path)?;
        let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(name));
        let language = unsafe {
            let language_fn: Symbol<unsafe extern "C" fn() -> Language> =
                library.get(language_fn_name.as_bytes())?;
            language_fn()
        };
        mem::forget(library);
        Ok(language)
    }

    fn find_language_at_path<'a>(&'a mut self, parser_path: &Path) -> io::Result<usize> {
        #[derive(Deserialize)]
        struct LanguageConfigurationJSON {
            name: String,
            #[serde(rename = "file-types")]
            file_types: Option<Vec<String>>,
            #[serde(rename = "content-regex")]
            content_regex: Option<String>,
            #[serde(rename = "first-line-regex")]
            first_line_regex: Option<String>,
            highlights: Option<String>,
        }

        #[derive(Deserialize)]
        struct PackageJSON {
            name: String,
            #[serde(rename = "tree-sitter")]
            tree_sitter: Option<Vec<LanguageConfigurationJSON>>,
        }

        let package_json_contents = fs::read_to_string(&parser_path.join("package.json"))?;
        let package_json: PackageJSON = serde_json::from_str(&package_json_contents)?;
        let configurations = package_json
            .tree_sitter
            .map_or(Vec::new(), |configurations| {
                configurations
                    .into_iter()
                    .map(|conf| LanguageConfiguration {
                        _name: conf.name,
                        file_types: conf.file_types.unwrap_or(Vec::new()),
                        _content_regex: conf
                            .content_regex
                            .and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
                        _first_line_regex: conf
                            .first_line_regex
                            .and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()),
                        _highlight_property_sheet: conf.highlights.map(|d| Err(d.into())),
                    })
                    .collect()
            });

        for (i, configuration) in configurations.iter().enumerate() {
            for file_type in &configuration.file_types {
                self.language_configuration_ids_by_file_type
                    .entry(file_type.to_string())
                    .or_insert(Vec::new())
                    .push((self.language_repos.len(), i));
            }
        }

        self.language_repos.push(LanguageRepo {
            name: package_json
                .name
                .split_at("tree-sitter-".len())
                .1
                .to_string(),
            path: parser_path.to_owned(),
            language: None,
            configurations,
        });

        Ok(self.language_repos.len() - 1)
    }
}

fn needs_recompile(
    lib_path: &Path,
    parser_c_path: &Path,
    scanner_path: &Option<PathBuf>,
) -> io::Result<bool> {
    if !lib_path.exists() {
        return Ok(true);
    }
    let lib_mtime = mtime(lib_path)?;
    if mtime(parser_c_path)? > lib_mtime {
        return Ok(true);
    }
    if let Some(scanner_path) = scanner_path {
        if mtime(scanner_path)? > lib_mtime {
            return Ok(true);
        }
    }
    Ok(false)
}

fn mtime(path: &Path) -> io::Result<SystemTime> {
    Ok(fs::metadata(path)?.modified()?)
}

fn replace_dashes_with_underscores(name: &str) -> String {
    let mut result = String::with_capacity(name.len());
    for c in name.chars() {
        if c == '-' {
            result.push('_');
        } else {
            result.push(c);
        }
    }
    result
}
<|endoftext|>


<commit_before>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// This file contains various trait resolution methods used by trans.
// They all assume regions can be erased and monomorphic types.  It
// seems likely that they should eventually be merged into more
// general routines.

use dep_graph::{DepGraph, DepKind, DepTrackingMap, DepTrackingMapConfig};
use infer::TransNormalize;
use std::cell::RefCell;
use std::marker::PhantomData;
use syntax::ast;
use syntax_pos::Span;
use traits::{FulfillmentContext, Obligation, ObligationCause, Reveal, SelectionContext, Vtable};
use ty::{self, Ty, TyCtxt};
use ty::subst::{Subst, Substs};
use ty::fold::{TypeFoldable, TypeFolder};
use util::common::MemoizationMap;

impl<'a, 'tcx> TyCtxt<'a, 'tcx, 'tcx> {
    /// Attempts to resolve an obligation to a vtable.. The result is
    /// a shallow vtable resolution -- meaning that we do not
    /// (necessarily) resolve all nested obligations on the impl. Note
    /// that type check should guarantee to us that all nested
    /// obligations *could be* resolved if we wanted to.
    pub fn trans_fulfill_obligation(self,
                                    span: Span,
                                    trait_ref: ty::PolyTraitRef<'tcx>)
                                    -> Vtable<'tcx, ()>
    {
        // Remove any references to regions; this helps improve caching.
        let trait_ref = self.erase_regions(&trait_ref);

        self.trans_trait_caches.trait_cache.memoize(trait_ref, || {
            debug!("trans::fulfill_obligation(trait_ref={:?}, def_id={:?})",
                   trait_ref, trait_ref.def_id());

            // Do the initial selection for the obligation. This yields the
            // shallow result we are looking for -- that is, what specific impl.
            self.infer_ctxt().enter(|infcx| {
                let mut selcx = SelectionContext::new(&infcx);

                let param_env = ty::ParamEnv::empty(Reveal::All);
                let obligation_cause = ObligationCause::misc(span,
                                                             ast::DUMMY_NODE_ID);
                let obligation = Obligation::new(obligation_cause,
                                                 param_env,
                                                 trait_ref.to_poly_trait_predicate());

                let selection = match selcx.select(&obligation) {
                    Ok(Some(selection)) => selection,
                    Ok(None) => {
                        // Ambiguity can happen when monomorphizing during trans
                        // expands to some humongo type that never occurred
                        // statically -- this humongo type can then overflow,
                        // leading to an ambiguous result. So report this as an
                        // overflow bug, since I believe this is the only case
                        // where ambiguity can result.
                        debug!("Encountered ambiguity selecting `{:?}` during trans, \
                                presuming due to overflow",
                               trait_ref);
                        self.sess.span_fatal(span,
                                            "reached the recursion limit during monomorphization \
                                             (selection ambiguity)");
                    }
                    Err(e) => {
                        span_bug!(span, "Encountered error `{:?}` selecting `{:?}` during trans",
                                  e, trait_ref)
                    }
                };

                debug!("fulfill_obligation: selection={:?}", selection);

                // Currently, we use a fulfillment context to completely resolve
                // all nested obligations. This is because they can inform the
                // inference of the impl's type parameters.
                let mut fulfill_cx = FulfillmentContext::new();
                let vtable = selection.map(|predicate| {
                    debug!("fulfill_obligation: register_predicate_obligation {:?}", predicate);
                    fulfill_cx.register_predicate_obligation(&infcx, predicate);
                });
                let vtable = infcx.drain_fulfillment_cx_or_panic(span, &mut fulfill_cx, &vtable);

                info!("Cache miss: {:?} => {:?}", trait_ref, vtable);
                vtable
            })
        })
    }

    /// Monomorphizes a type from the AST by first applying the in-scope
    /// substitutions and then normalizing any associated types.
    pub fn trans_apply_param_substs<T>(self,
                                       param_substs: &Substs<'tcx>,
                                       value: &T)
                                       -> T
        where T: TransNormalize<'tcx>
    {
        debug!("apply_param_substs(param_substs={:?}, value={:?})", param_substs, value);
        let substituted = value.subst(self, param_substs);
        let substituted = self.erase_regions(&substituted);
        AssociatedTypeNormalizer::new(self).fold(&substituted)
    }
}

struct AssociatedTypeNormalizer<'a, 'gcx: 'a> {
    tcx: TyCtxt<'a, 'gcx, 'gcx>,
}

impl<'a, 'gcx> AssociatedTypeNormalizer<'a, 'gcx> {
    fn new(tcx: TyCtxt<'a, 'gcx, 'gcx>) -> Self {
        AssociatedTypeNormalizer { tcx }
    }

    fn fold<T:TypeFoldable<'gcx>>(&mut self, value: &T) -> T {
        if !value.has_projections() {
            value.clone()
        } else {
            value.fold_with(self)
        }
    }
}

impl<'a, 'gcx> TypeFolder<'gcx, 'gcx> for AssociatedTypeNormalizer<'a, 'gcx> {
    fn tcx<'c>(&'c self) -> TyCtxt<'c, 'gcx, 'gcx> {
        self.tcx
    }

    fn fold_ty(&mut self, ty: Ty<'gcx>) -> Ty<'gcx> {
        if !ty.has_projections() {
            ty
        } else {
            self.tcx.trans_trait_caches.project_cache.memoize(ty, || {
                debug!("AssociatedTypeNormalizer: ty={:?}", ty);
                self.tcx.normalize_associated_type(&ty)
            })
        }
    }
}

/// Specializes caches used in trans -- in particular, they assume all
/// types are fully monomorphized and that free regions can be erased.
pub struct TransTraitCaches<'tcx> {
    trait_cache: RefCell<DepTrackingMap<TraitSelectionCache<'tcx>>>,
    project_cache: RefCell<DepTrackingMap<ProjectionCache<'tcx>>>,
}

impl<'tcx> TransTraitCaches<'tcx> {
    pub fn new(graph: DepGraph) -> Self {
        TransTraitCaches {
            trait_cache: RefCell::new(DepTrackingMap::new(graph.clone())),
            project_cache: RefCell::new(DepTrackingMap::new(graph)),
        }
    }
}

// Implement DepTrackingMapConfig for `trait_cache`
pub struct TraitSelectionCache<'tcx> {
    data: PhantomData<&'tcx ()>
}

impl<'tcx> DepTrackingMapConfig for TraitSelectionCache<'tcx> {
    type Key = ty::PolyTraitRef<'tcx>;
    type Value = Vtable<'tcx, ()>;
    fn to_dep_kind() -> DepKind {
        DepKind::TraitSelect
    }
}

// # Global Cache

pub struct ProjectionCache<'gcx> {
    data: PhantomData<&'gcx ()>
}

impl<'gcx> DepTrackingMapConfig for ProjectionCache<'gcx> {
    type Key = Ty<'gcx>;
    type Value = Ty<'gcx>;
    fn to_dep_kind() -> DepKind {
        DepKind::TraitSelect
    }
}

<commit_msg>add ParamEnv to the trait_cache key<commit_after>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// This file contains various trait resolution methods used by trans.
// They all assume regions can be erased and monomorphic types.  It
// seems likely that they should eventually be merged into more
// general routines.

use dep_graph::{DepGraph, DepKind, DepTrackingMap, DepTrackingMapConfig};
use infer::TransNormalize;
use std::cell::RefCell;
use std::marker::PhantomData;
use syntax::ast;
use syntax_pos::Span;
use traits::{FulfillmentContext, Obligation, ObligationCause, Reveal, SelectionContext, Vtable};
use ty::{self, Ty, TyCtxt};
use ty::subst::{Subst, Substs};
use ty::fold::{TypeFoldable, TypeFolder};
use util::common::MemoizationMap;

impl<'a, 'tcx> TyCtxt<'a, 'tcx, 'tcx> {
    /// Attempts to resolve an obligation to a vtable.. The result is
    /// a shallow vtable resolution -- meaning that we do not
    /// (necessarily) resolve all nested obligations on the impl. Note
    /// that type check should guarantee to us that all nested
    /// obligations *could be* resolved if we wanted to.
    pub fn trans_fulfill_obligation(self,
                                    span: Span,
                                    trait_ref: ty::PolyTraitRef<'tcx>)
                                    -> Vtable<'tcx, ()>
    {
        // Remove any references to regions; this helps improve caching.
        let trait_ref = self.erase_regions(&trait_ref);
        let param_env = ty::ParamEnv::empty(Reveal::All);

        self.trans_trait_caches.trait_cache.memoize((param_env, trait_ref), || {
            debug!("trans::fulfill_obligation(trait_ref={:?}, def_id={:?})",
                   (param_env, trait_ref), trait_ref.def_id());

            // Do the initial selection for the obligation. This yields the
            // shallow result we are looking for -- that is, what specific impl.
            self.infer_ctxt().enter(|infcx| {
                let mut selcx = SelectionContext::new(&infcx);

                let obligation_cause = ObligationCause::misc(span,
                                                             ast::DUMMY_NODE_ID);
                let obligation = Obligation::new(obligation_cause,
                                                 param_env,
                                                 trait_ref.to_poly_trait_predicate());

                let selection = match selcx.select(&obligation) {
                    Ok(Some(selection)) => selection,
                    Ok(None) => {
                        // Ambiguity can happen when monomorphizing during trans
                        // expands to some humongo type that never occurred
                        // statically -- this humongo type can then overflow,
                        // leading to an ambiguous result. So report this as an
                        // overflow bug, since I believe this is the only case
                        // where ambiguity can result.
                        debug!("Encountered ambiguity selecting `{:?}` during trans, \
                                presuming due to overflow",
                               trait_ref);
                        self.sess.span_fatal(span,
                                            "reached the recursion limit during monomorphization \
                                             (selection ambiguity)");
                    }
                    Err(e) => {
                        span_bug!(span, "Encountered error `{:?}` selecting `{:?}` during trans",
                                  e, trait_ref)
                    }
                };

                debug!("fulfill_obligation: selection={:?}", selection);

                // Currently, we use a fulfillment context to completely resolve
                // all nested obligations. This is because they can inform the
                // inference of the impl's type parameters.
                let mut fulfill_cx = FulfillmentContext::new();
                let vtable = selection.map(|predicate| {
                    debug!("fulfill_obligation: register_predicate_obligation {:?}", predicate);
                    fulfill_cx.register_predicate_obligation(&infcx, predicate);
                });
                let vtable = infcx.drain_fulfillment_cx_or_panic(span, &mut fulfill_cx, &vtable);

                info!("Cache miss: {:?} => {:?}", trait_ref, vtable);
                vtable
            })
        })
    }

    /// Monomorphizes a type from the AST by first applying the in-scope
    /// substitutions and then normalizing any associated types.
    pub fn trans_apply_param_substs<T>(self,
                                       param_substs: &Substs<'tcx>,
                                       value: &T)
                                       -> T
        where T: TransNormalize<'tcx>
    {
        debug!("apply_param_substs(param_substs={:?}, value={:?})", param_substs, value);
        let substituted = value.subst(self, param_substs);
        let substituted = self.erase_regions(&substituted);
        AssociatedTypeNormalizer::new(self).fold(&substituted)
    }
}

struct AssociatedTypeNormalizer<'a, 'gcx: 'a> {
    tcx: TyCtxt<'a, 'gcx, 'gcx>,
}

impl<'a, 'gcx> AssociatedTypeNormalizer<'a, 'gcx> {
    fn new(tcx: TyCtxt<'a, 'gcx, 'gcx>) -> Self {
        AssociatedTypeNormalizer { tcx }
    }

    fn fold<T:TypeFoldable<'gcx>>(&mut self, value: &T) -> T {
        if !value.has_projections() {
            value.clone()
        } else {
            value.fold_with(self)
        }
    }
}

impl<'a, 'gcx> TypeFolder<'gcx, 'gcx> for AssociatedTypeNormalizer<'a, 'gcx> {
    fn tcx<'c>(&'c self) -> TyCtxt<'c, 'gcx, 'gcx> {
        self.tcx
    }

    fn fold_ty(&mut self, ty: Ty<'gcx>) -> Ty<'gcx> {
        if !ty.has_projections() {
            ty
        } else {
            self.tcx.trans_trait_caches.project_cache.memoize(ty, || {
                debug!("AssociatedTypeNormalizer: ty={:?}", ty);
                self.tcx.normalize_associated_type(&ty)
            })
        }
    }
}

/// Specializes caches used in trans -- in particular, they assume all
/// types are fully monomorphized and that free regions can be erased.
pub struct TransTraitCaches<'tcx> {
    trait_cache: RefCell<DepTrackingMap<TraitSelectionCache<'tcx>>>,
    project_cache: RefCell<DepTrackingMap<ProjectionCache<'tcx>>>,
}

impl<'tcx> TransTraitCaches<'tcx> {
    pub fn new(graph: DepGraph) -> Self {
        TransTraitCaches {
            trait_cache: RefCell::new(DepTrackingMap::new(graph.clone())),
            project_cache: RefCell::new(DepTrackingMap::new(graph)),
        }
    }
}

// Implement DepTrackingMapConfig for `trait_cache`
pub struct TraitSelectionCache<'tcx> {
    data: PhantomData<&'tcx ()>
}

impl<'tcx> DepTrackingMapConfig for TraitSelectionCache<'tcx> {
    type Key = (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>);
    type Value = Vtable<'tcx, ()>;
    fn to_dep_kind() -> DepKind {
        DepKind::TraitSelect
    }
}

// # Global Cache

pub struct ProjectionCache<'gcx> {
    data: PhantomData<&'gcx ()>
}

impl<'gcx> DepTrackingMapConfig for ProjectionCache<'gcx> {
    type Key = Ty<'gcx>;
    type Value = Ty<'gcx>;
    fn to_dep_kind() -> DepKind {
        DepKind::TraitSelect
    }
}

<|endoftext|>


<commit_before>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::collections::HashMap;
use std::marker::PhantomData;
use std::sync::{Arc, Mutex};
use std::time::Instant;
use std::io::prelude::*;
use std::fs::File;

const OUTPUT_WIDTH_IN_PX: u64 = 1000;
const TIME_LINE_HEIGHT_IN_PX: u64 = 7;
const TIME_LINE_HEIGHT_STRIDE_IN_PX: usize = 10;

#[derive(Clone)]
struct Timing {
    start: Instant,
    end: Instant,
    work_package_kind: WorkPackageKind,
}

#[derive(Clone, Copy, Hash, Eq, PartialEq, Debug)]
pub struct TimelineId(pub usize);

#[derive(Clone)]
struct PerThread {
    timings: Vec<Timing>,
    open_work_package: Option<(Instant, WorkPackageKind)>,
}

#[derive(Clone)]
pub struct TimeGraph {
    data: Arc<Mutex<HashMap<TimelineId, PerThread>>>,
}

#[derive(Clone, Copy)]
pub struct WorkPackageKind(pub &'static [&'static str]);

pub struct RaiiToken {
    graph: TimeGraph,
    timeline: TimelineId,
    // The token must not be Send:
    _marker: PhantomData<*const ()>
}

impl Drop for RaiiToken {
    fn drop(&mut self) {
        self.graph.end(self.timeline);
    }
}

impl TimeGraph {
    pub fn new() -> TimeGraph {
        TimeGraph {
            data: Arc::new(Mutex::new(HashMap::new()))
        }
    }

    pub fn start(&self,
                 timeline: TimelineId,
                 work_package_kind: WorkPackageKind) -> RaiiToken {
        {
            let mut table = self.data.lock().unwrap();

            let mut data = table.entry(timeline).or_insert(PerThread {
                timings: Vec::new(),
                open_work_package: None,
            });

            assert!(data.open_work_package.is_none());
            data.open_work_package = Some((Instant::now(), work_package_kind));
        }

        RaiiToken {
            graph: self.clone(),
            timeline,
            _marker: PhantomData,
        }
    }

    fn end(&self, timeline: TimelineId) {
        let end = Instant::now();

        let mut table = self.data.lock().unwrap();
        let mut data = table.get_mut(&timeline).unwrap();

        if let Some((start, work_package_kind)) = data.open_work_package {
            data.timings.push(Timing {
                start,
                end,
                work_package_kind,
            });
        } else {
            bug!("end timing without start?")
        }

        data.open_work_package = None;
    }

    pub fn dump(&self, output_filename: &str) {
        let table = self.data.lock().unwrap();

        for data in table.values() {
            assert!(data.open_work_package.is_none());
        }

        let mut timelines: Vec<PerThread> =
            table.values().map(|data| data.clone()).collect();

        timelines.sort_by_key(|timeline| timeline.timings[0].start);

        let earliest_instant = timelines[0].timings[0].start;
        let latest_instant = timelines.iter()
                                       .map(|timeline| timeline.timings
                                                               .last()
                                                               .unwrap()
                                                               .end)
                                       .max()
                                       .unwrap();
        let max_distance = distance(earliest_instant, latest_instant);

        let mut file = File::create(format!("{}.html", output_filename)).unwrap();

        writeln!(file, "").unwrap();
        writeln!(file, "").unwrap();
        writeln!(file, "").unwrap();

        let mut color = 0;

        for (line_index, timeline) in timelines.iter().enumerate() {
            let line_top = line_index * TIME_LINE_HEIGHT_STRIDE_IN_PX;

            for span in &timeline.timings {
                let start = distance(earliest_instant, span.start);
                let end = distance(earliest_instant, span.end);

                let start = normalize(start, max_distance, OUTPUT_WIDTH_IN_PX);
                let end = normalize(end, max_distance, OUTPUT_WIDTH_IN_PX);

                let colors = span.work_package_kind.0;

                writeln!(file, "

",
                    line_top,
                    start,
                    end - start,
                    TIME_LINE_HEIGHT_IN_PX,
                    colors[color % colors.len()]
                    ).unwrap();

                color += 1;
            }
        }

        writeln!(file, "").unwrap();
        writeln!(file, "").unwrap();
    }
}

fn distance(zero: Instant, x: Instant) -> u64 {

    let duration = x.duration_since(zero);
    (duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64) // / div
}

fn normalize(distance: u64, max: u64, max_pixels: u64) -> u64 {
    (max_pixels * distance) / max
}

<commit_msg>Fixed another problem.<commit_after>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::collections::HashMap;
use std::marker::PhantomData;
use std::sync::{Arc, Mutex};
use std::time::Instant;
use std::io::prelude::*;
use std::fs::File;

const OUTPUT_WIDTH_IN_PX: u64 = 1000;
const TIME_LINE_HEIGHT_IN_PX: u64 = 7;
const TIME_LINE_HEIGHT_STRIDE_IN_PX: usize = 10;

#[derive(Clone)]
struct Timing {
    start: Instant,
    end: Instant,
    work_package_kind: WorkPackageKind,
}

#[derive(Clone, Copy, Hash, Eq, PartialEq, Debug)]
pub struct TimelineId(pub usize);

#[derive(Clone)]
struct PerThread {
    timings: Vec<Timing>,
    open_work_package: Option<(Instant, WorkPackageKind)>,
}

#[derive(Clone)]
pub struct TimeGraph {
    data: Arc<Mutex<HashMap<TimelineId, PerThread>>>,
}

#[derive(Clone, Copy)]
pub struct WorkPackageKind(pub &'static [&'static str]);

pub struct RaiiToken {
    graph: TimeGraph,
    timeline: TimelineId,
    // The token must not be Send:
    _marker: PhantomData<*const ()>
}

impl Drop for RaiiToken {
    fn drop(&mut self) {
        self.graph.end(self.timeline);
    }
}

impl TimeGraph {
    pub fn new() -> TimeGraph {
        TimeGraph {
            data: Arc::new(Mutex::new(HashMap::new()))
        }
    }

    pub fn start(&self,
                 timeline: TimelineId,
                 work_package_kind: WorkPackageKind) -> RaiiToken {
        {
            let mut table = self.data.lock().unwrap();

            let data = table.entry(timeline).or_insert(PerThread {
                timings: Vec::new(),
                open_work_package: None,
            });

            assert!(data.open_work_package.is_none());
            data.open_work_package = Some((Instant::now(), work_package_kind));
        }

        RaiiToken {
            graph: self.clone(),
            timeline,
            _marker: PhantomData,
        }
    }

    fn end(&self, timeline: TimelineId) {
        let end = Instant::now();

        let mut table = self.data.lock().unwrap();
        let data = table.get_mut(&timeline).unwrap();

        if let Some((start, work_package_kind)) = data.open_work_package {
            data.timings.push(Timing {
                start,
                end,
                work_package_kind,
            });
        } else {
            bug!("end timing without start?")
        }

        data.open_work_package = None;
    }

    pub fn dump(&self, output_filename: &str) {
        let table = self.data.lock().unwrap();

        for data in table.values() {
            assert!(data.open_work_package.is_none());
        }

        let mut timelines: Vec<PerThread> =
            table.values().map(|data| data.clone()).collect();

        timelines.sort_by_key(|timeline| timeline.timings[0].start);

        let earliest_instant = timelines[0].timings[0].start;
        let latest_instant = timelines.iter()
                                       .map(|timeline| timeline.timings
                                                               .last()
                                                               .unwrap()
                                                               .end)
                                       .max()
                                       .unwrap();
        let max_distance = distance(earliest_instant, latest_instant);

        let mut file = File::create(format!("{}.html", output_filename)).unwrap();

        writeln!(file, "").unwrap();
        writeln!(file, "").unwrap();
        writeln!(file, "").unwrap();

        let mut color = 0;

        for (line_index, timeline) in timelines.iter().enumerate() {
            let line_top = line_index * TIME_LINE_HEIGHT_STRIDE_IN_PX;

            for span in &timeline.timings {
                let start = distance(earliest_instant, span.start);
                let end = distance(earliest_instant, span.end);

                let start = normalize(start, max_distance, OUTPUT_WIDTH_IN_PX);
                let end = normalize(end, max_distance, OUTPUT_WIDTH_IN_PX);

                let colors = span.work_package_kind.0;

                writeln!(file, "

",
                    line_top,
                    start,
                    end - start,
                    TIME_LINE_HEIGHT_IN_PX,
                    colors[color % colors.len()]
                    ).unwrap();

                color += 1;
            }
        }

        writeln!(file, "").unwrap();
        writeln!(file, "").unwrap();
    }
}

fn distance(zero: Instant, x: Instant) -> u64 {

    let duration = x.duration_since(zero);
    (duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64) // / div
}

fn normalize(distance: u64, max: u64, max_pixels: u64) -> u64 {
    (max_pixels * distance) / max
}

<|endoftext|>


<commit_before>//! Code to adapt between the outgoing `std::io` and the incoming `std::rt::io` traits.
//! This code will be rendered obsolete before Rust 1.0.

use std::comm::{Port, Chan, SharedChan};
use std::{io, rt, task, vec};
use extra::{net, uv_iotask};

static READ_SIZE : uint = 1024;

/// Adapter of extra::net stuff to std::rt::io::net stuff
pub struct ExtraNetTcpListener {
    accept_ch: Chan<()>,
    stream_po: Port<Option<ExtraNetTcpStream>>,
}

enum StreamOperationRequest { Read(uint), Write }

impl ExtraNetTcpListener {
    pub fn bind(addr: rt::io::net::ip::IpAddr) -> Option<ExtraNetTcpListener> {
        match addr {
            rt::io::net::ip::Ipv4(a, b, c, d, p) => {
                let (accept_po, accept_ch) = stream();
                let (stream_po, stream_ch) = stream();
                let stream_ch = SharedChan::new(stream_ch);
                do spawn {
                    net::tcp::listen(
                        net::ip::v4::parse_addr(fmt!("%?.%?.%?.%?", a, b, c, d)),
                        p as uint,
                        100,
                        &uv_iotask::spawn_iotask(task::task()),
                        |_kill_ch| { },
                        |new_conn, _kill_ch| {
                            // An accept request has been sent, try accepting it
                            accept_po.recv();
                            let child_stream_ch = stream_ch.clone();
                            do task::spawn_supervised {
                                let accept_result = net::tcp::accept(new_conn);
                                match accept_result {
                                    Ok(socket) => {
                                        let (req_po, req_ch) = stream();
                                        let (write_po, write_ch) = stream();
                                        let (read_po, read_ch) = stream();
                                        let sockbuf = net::tcp::socket_buf(socket);
                                        let stream = ExtraNetTcpStream {
                                            req_ch: req_ch,
                                            write_ch: write_ch,
                                            read_po: read_po,
                                        };
                                        child_stream_ch.send(Some(stream));
                                        loop {
                                            match req_po.try_recv() {
                                                Some(Read(1)) => {
                                                    match sockbuf.read_byte() {
                                                        v if v >= 0 && v <= 255 => {
                                                            read_ch.send((~[v as u8], 1u));
                                                        },
                                                        _ => {
                                                            read_ch.send((~[0u8], 0u));
                                                        }
                                                    }
                                                },
                                                Some(Read(len)) => {
                                                    //let mut buf = vec::with_capacity(len);
                                                    let mut buf = vec::from_elem(len, 0);
                                                    let size = sockbuf.read(buf, len);
                                                    //let mut buf = ~[0u8, ..READ_SIZE];
                                                    //let size = sockbuf.read(buf, READ_SIZE);
                                                    read_ch.send((buf, size))
                                                },
                                                Some(Write) => {
                                                    sockbuf.write(write_po.recv());
                                                }
                                                None => {
                                                    // Closed: request completed
                                                    break;
                                                }
                                            }
                                        }
                                    }
                                    Err(err) => {
                                        debug!("Error accepting, %?", err);
                                        child_stream_ch.send(None);
                                    }
                                }
                            }
                        }
                    );
                }

                Some(ExtraNetTcpListener {
                    accept_ch: accept_ch,
                    stream_po: stream_po,
                })
            },
            rt::io::net::ip::Ipv6(_, _, _, _, _, _, _, _, _port) => fail!("no IPv6 support")
        }
    }
}

impl rt::io::Listener<ExtraNetTcpStream> for ExtraNetTcpListener {
    pub fn accept(&mut self) -> Option<ExtraNetTcpStream> {
        self.accept_ch.send(());

        match self.stream_po.try_recv() {
            Some(optstream) => optstream,
            None => None,
        }
    }
}

pub struct ExtraNetTcpStream {
    req_ch: Chan<StreamOperationRequest>,
    write_ch: Chan<~[u8]>,
    read_po: Port<(~[u8], uint)>,
}

impl rt::io::Reader for ExtraNetTcpStream {
    //io::Reader has fn read(&self, buf: &mut [u8], len: uint) -> uint
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        self.req_ch.send(Read(buf.len()));
        let (read_buf, num_bytes_read) = self.read_po.recv();
        let mut i = 0;
        while i < num_bytes_read {
            buf[i] = read_buf[i];
            i += 1;
        }
        return Some(num_bytes_read);
    }

    fn eof(&mut self) -> bool { fail!("ExtraNetTcpStream.eof() is not supported") }
}

impl rt::io::Writer for ExtraNetTcpStream {
    fn write(&mut self, buf: &[u8]) {
        self.req_ch.send(Write);
        self.write_ch.send(buf.to_owned())
    }

    fn flush(&mut self) { fail!("ExtraNetTcpStream.flush() is not supported") }
}
<commit_msg>Remove unused import.<commit_after>//! Code to adapt between the outgoing `std::io` and the incoming `std::rt::io` traits.
//! This code will be rendered obsolete before Rust 1.0.

use std::comm::{Port, Chan, SharedChan};
use std::{rt, task, vec};
use extra::{net, uv_iotask};

static READ_SIZE : uint = 1024;

/// Adapter of extra::net stuff to std::rt::io::net stuff
pub struct ExtraNetTcpListener {
    accept_ch: Chan<()>,
    stream_po: Port<Option<ExtraNetTcpStream>>,
}

enum StreamOperationRequest { Read(uint), Write }

impl ExtraNetTcpListener {
    pub fn bind(addr: rt::io::net::ip::IpAddr) -> Option<ExtraNetTcpListener> {
        match addr {
            rt::io::net::ip::Ipv4(a, b, c, d, p) => {
                let (accept_po, accept_ch) = stream();
                let (stream_po, stream_ch) = stream();
                let stream_ch = SharedChan::new(stream_ch);
                do spawn {
                    net::tcp::listen(
                        net::ip::v4::parse_addr(fmt!("%?.%?.%?.%?", a, b, c, d)),
                        p as uint,
                        100,
                        &uv_iotask::spawn_iotask(task::task()),
                        |_kill_ch| { },
                        |new_conn, _kill_ch| {
                            // An accept request has been sent, try accepting it
                            accept_po.recv();
                            let child_stream_ch = stream_ch.clone();
                            do task::spawn_supervised {
                                let accept_result = net::tcp::accept(new_conn);
                                match accept_result {
                                    Ok(socket) => {
                                        let (req_po, req_ch) = stream();
                                        let (write_po, write_ch) = stream();
                                        let (read_po, read_ch) = stream();
                                        let sockbuf = net::tcp::socket_buf(socket);
                                        let stream = ExtraNetTcpStream {
                                            req_ch: req_ch,
                                            write_ch: write_ch,
                                            read_po: read_po,
                                        };
                                        child_stream_ch.send(Some(stream));
                                        loop {
                                            match req_po.try_recv() {
                                                Some(Read(1)) => {
                                                    match sockbuf.read_byte() {
                                                        v if v >= 0 && v <= 255 => {
                                                            read_ch.send((~[v as u8], 1u));
                                                        },
                                                        _ => {
                                                            read_ch.send((~[0u8], 0u));
                                                        }
                                                    }
                                                },
                                                Some(Read(len)) => {
                                                    //let mut buf = vec::with_capacity(len);
                                                    let mut buf = vec::from_elem(len, 0);
                                                    let size = sockbuf.read(buf, len);
                                                    //let mut buf = ~[0u8, ..READ_SIZE];
                                                    //let size = sockbuf.read(buf, READ_SIZE);
                                                    read_ch.send((buf, size))
                                                },
                                                Some(Write) => {
                                                    sockbuf.write(write_po.recv());
                                                }
                                                None => {
                                                    // Closed: request completed
                                                    break;
                                                }
                                            }
                                        }
                                    }
                                    Err(err) => {
                                        debug!("Error accepting, %?", err);
                                        child_stream_ch.send(None);
                                    }
                                }
                            }
                        }
                    );
                }

                Some(ExtraNetTcpListener {
                    accept_ch: accept_ch,
                    stream_po: stream_po,
                })
            },
            rt::io::net::ip::Ipv6(_, _, _, _, _, _, _, _, _port) => fail!("no IPv6 support")
        }
    }
}

impl rt::io::Listener<ExtraNetTcpStream> for ExtraNetTcpListener {
    pub fn accept(&mut self) -> Option<ExtraNetTcpStream> {
        self.accept_ch.send(());

        match self.stream_po.try_recv() {
            Some(optstream) => optstream,
            None => None,
        }
    }
}

pub struct ExtraNetTcpStream {
    req_ch: Chan<StreamOperationRequest>,
    write_ch: Chan<~[u8]>,
    read_po: Port<(~[u8], uint)>,
}

impl rt::io::Reader for ExtraNetTcpStream {
    //io::Reader has fn read(&self, buf: &mut [u8], len: uint) -> uint
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        self.req_ch.send(Read(buf.len()));
        let (read_buf, num_bytes_read) = self.read_po.recv();
        let mut i = 0;
        while i < num_bytes_read {
            buf[i] = read_buf[i];
            i += 1;
        }
        return Some(num_bytes_read);
    }

    fn eof(&mut self) -> bool { fail!("ExtraNetTcpStream.eof() is not supported") }
}

impl rt::io::Writer for ExtraNetTcpStream {
    fn write(&mut self, buf: &[u8]) {
        self.req_ch.send(Write);
        self.write_ch.send(buf.to_owned())
    }

    fn flush(&mut self) { fail!("ExtraNetTcpStream.flush() is not supported") }
}
<|endoftext|>


<commit_before>// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use std::mem;
use std::fs::File;
use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};

use libc::{read, write, eventfd, dup, c_void};

use {Result, errno_result};

/// A safe wrapper around a Linux eventfd (man 2 eventfd).
///
/// An eventfd is useful because it is sendable across processes and can be used for signaling in
/// and out of the KVM API. They can also be polled like any other file descriptor.
pub struct EventFd {
    eventfd: File,
}

impl EventFd {
    /// Creates a new blocking EventFd with an initial value of 0.
    pub fn new() -> Result<EventFd> {
        // This is safe because eventfd merely allocated an eventfd for our process and we handle
        // the error case.
        let ret = unsafe { eventfd(0, 0) };
        if ret < 0 {
            return errno_result();
        }
        // This is safe because we checked ret for success and know the kernel gave us an fd that we
        // own.
        Ok(EventFd { eventfd: unsafe { File::from_raw_fd(ret) } })
    }

    /// Adds `v` to the eventfd's count, blocking until this won't overflow the count.
    pub fn write(&self, v: u64) -> Result<()> {
        // This is safe because we made this fd and the pointer we pass can not overflow because we
        // give the syscall's size parameter properly.
        let ret = unsafe {
            write(self.as_raw_fd(),
                  &v as *const u64 as *const c_void,
                  mem::size_of::<u64>())
        };
        if ret <= 0 {
            return errno_result();
        }
        Ok(())
    }

    /// Blocks until the the eventfd's count is non-zero, then resets the count to zero.
    pub fn read(&self) -> Result<u64> {
        let mut buf: u64 = 0;
        let ret = unsafe {
            // This is safe because we made this fd and the pointer we pass can not overflow because
            // we give the syscall's size parameter properly.
            read(self.as_raw_fd(),
                 &mut buf as *mut u64 as *mut c_void,
                 mem::size_of::<u64>())
        };
        if ret <= 0 {
            return errno_result();
        }
        Ok(buf)
    }

    /// Clones this EventFd, internally creating a new file descriptor. The new EventFd will share
    /// the same underlying count within the kernel.
    pub fn try_clone(&self) -> Result<EventFd> {
        // This is safe because we made this fd and properly check that it returns without error.
        let ret = unsafe { dup(self.as_raw_fd()) };
        if ret < 0 {
            return errno_result();
        }
        // This is safe because we checked ret for success and know the kernel gave us an fd that we
        // own.
        Ok(EventFd { eventfd: unsafe { File::from_raw_fd(ret) } })
    }
}

impl AsRawFd for EventFd {
    fn as_raw_fd(&self) -> RawFd {
        self.eventfd.as_raw_fd()
    }
}

impl FromRawFd for EventFd {
    unsafe fn from_raw_fd(fd: RawFd) -> Self {
        EventFd {
            eventfd: File::from_raw_fd(fd)
        }
    }
}

unsafe impl ::Pollable for EventFd {
    fn pollable_fd(&self) -> RawFd {
        self.eventfd.as_raw_fd()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn new() {
        EventFd::new().unwrap();
    }

    #[test]
    fn read_write() {
        let evt = EventFd::new().unwrap();
        evt.write(55).unwrap();
        assert_eq!(evt.read(), Ok(55));
    }

    #[test]
    fn clone() {
        let evt = EventFd::new().unwrap();
        let evt_clone = evt.try_clone().unwrap();
        evt.write(923).unwrap();
        assert_eq!(evt_clone.read(), Ok(923));
    }
}
<commit_msg>sys_util: implement IntoRawFd for EventFd<commit_after>// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use std::mem;
use std::fs::File;
use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};

use libc::{read, write, eventfd, dup, c_void};

use {Result, errno_result};

/// A safe wrapper around a Linux eventfd (man 2 eventfd).
///
/// An eventfd is useful because it is sendable across processes and can be used for signaling in
/// and out of the KVM API. They can also be polled like any other file descriptor.
pub struct EventFd {
    eventfd: File,
}

impl EventFd {
    /// Creates a new blocking EventFd with an initial value of 0.
    pub fn new() -> Result<EventFd> {
        // This is safe because eventfd merely allocated an eventfd for our process and we handle
        // the error case.
        let ret = unsafe { eventfd(0, 0) };
        if ret < 0 {
            return errno_result();
        }
        // This is safe because we checked ret for success and know the kernel gave us an fd that we
        // own.
        Ok(EventFd { eventfd: unsafe { File::from_raw_fd(ret) } })
    }

    /// Adds `v` to the eventfd's count, blocking until this won't overflow the count.
    pub fn write(&self, v: u64) -> Result<()> {
        // This is safe because we made this fd and the pointer we pass can not overflow because we
        // give the syscall's size parameter properly.
        let ret = unsafe {
            write(self.as_raw_fd(),
                  &v as *const u64 as *const c_void,
                  mem::size_of::<u64>())
        };
        if ret <= 0 {
            return errno_result();
        }
        Ok(())
    }

    /// Blocks until the the eventfd's count is non-zero, then resets the count to zero.
    pub fn read(&self) -> Result<u64> {
        let mut buf: u64 = 0;
        let ret = unsafe {
            // This is safe because we made this fd and the pointer we pass can not overflow because
            // we give the syscall's size parameter properly.
            read(self.as_raw_fd(),
                 &mut buf as *mut u64 as *mut c_void,
                 mem::size_of::<u64>())
        };
        if ret <= 0 {
            return errno_result();
        }
        Ok(buf)
    }

    /// Clones this EventFd, internally creating a new file descriptor. The new EventFd will share
    /// the same underlying count within the kernel.
    pub fn try_clone(&self) -> Result<EventFd> {
        // This is safe because we made this fd and properly check that it returns without error.
        let ret = unsafe { dup(self.as_raw_fd()) };
        if ret < 0 {
            return errno_result();
        }
        // This is safe because we checked ret for success and know the kernel gave us an fd that we
        // own.
        Ok(EventFd { eventfd: unsafe { File::from_raw_fd(ret) } })
    }
}

impl AsRawFd for EventFd {
    fn as_raw_fd(&self) -> RawFd {
        self.eventfd.as_raw_fd()
    }
}

impl FromRawFd for EventFd {
    unsafe fn from_raw_fd(fd: RawFd) -> Self {
        EventFd {
            eventfd: File::from_raw_fd(fd)
        }
    }
}

impl IntoRawFd for EventFd {
    fn into_raw_fd(self) -> RawFd {
        self.eventfd.into_raw_fd()
    }
}

unsafe impl ::Pollable for EventFd {
    fn pollable_fd(&self) -> RawFd {
        self.eventfd.as_raw_fd()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn new() {
        EventFd::new().unwrap();
    }

    #[test]
    fn read_write() {
        let evt = EventFd::new().unwrap();
        evt.write(55).unwrap();
        assert_eq!(evt.read(), Ok(55));
    }

    #[test]
    fn clone() {
        let evt = EventFd::new().unwrap();
        let evt_clone = evt.try_clone().unwrap();
        evt.write(923).unwrap();
        assert_eq!(evt_clone.read(), Ok(923));
    }
}
<|endoftext|>


<commit_before>use lexer::Tokenizer;
use lexer::token::{Token, TokenType};

macro_rules! token {
    ($tokenizer:expr, $token_type:ident, $accum:expr) => {{
        token!($tokenizer , TokenType::$token_type, $accum)
    }};
    ($tokenizer:expr, $token_type:expr, $accum:expr) => {{
        let tokenizer = $tokenizer as &$crate::lexer::Tokenizer;
        let token_type = $token_type as $crate::lexer::token::TokenType;
        Some(Token::new(token_type, tokenizer.last_position(), $accum))
    }};
}

/// Matcher.
pub trait Matcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token>;
}

/// A matcher that only matches white-space.
pub struct WhitespaceMatcher {}

impl Matcher for WhitespaceMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        let mut found = false;
        while !tokenizer.end() && tokenizer.peek().unwrap().is_whitespace() {
            found = true;
            tokenizer.next();
        }
        if found {
            token!(tokenizer, Whitespace, String::new())
        } else {
            None
        }
    }
}

/// A matcher that matches base-10 integer literals.
pub struct IntLiteralMatcher {}

impl Matcher for IntLiteralMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        let mut accum = String::new();
        let base = match tokenizer.peek().unwrap() {
            &'0' => {
                match tokenizer.peek_n(1) {
                    Some(chr) => {
                        match chr {
                            &'x' => 16, // base 16 (hexadecimal)
                            &'b' => 2, // base 2 (binary)
                            _ => 10, // base 10 (decimal)
                        }
                    }
                    _ => 10, // base 10 (decimal)
                }
            }
            _ => 10, // base 10 (decimal)
        };
        if base != 10 {
            tokenizer.advance(2); // skip prefix
        }
        while !tokenizer.end() && tokenizer.peek().unwrap().is_digit(base) {
            accum.push(tokenizer.next().unwrap());
        }
        if !accum.is_empty() {
            // Produce token as base-10 string
            let literal: String = match u64::from_str_radix(accum.as_str(), base) {
                Ok(result) => result.to_string(),
                Err(error) => panic!("Unable to parse integer literal: {}", error)
            };
            token!(tokenizer, IntLiteral, literal)
        } else {
            None
        }
    }
}

/// A matcher that matches float literals.
pub struct FloatLiteralMatcher {}

impl Matcher for FloatLiteralMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        let mut accum = String::new();
        let curr = tokenizer.next().unwrap();
        if curr.is_digit(10) {
            accum.push(curr)
        } else if curr == '.' {
            accum.push_str("0.")
        } else {
            return None;
        }
        while !tokenizer.end() {
            let current = *tokenizer.peek().unwrap();
            if !current.is_whitespace() && current.is_digit(10) || current == '.' {
                if current == '.' && accum.contains('.') {
                    panic!("Unexpected decimal point")
                }

                accum.push(tokenizer.next().unwrap());
            } else {
                break
            }
        }
        if accum.contains('.') {
            token!(tokenizer, FloatLiteral, accum)
        } else {
            token!(tokenizer, IntLiteral, accum)
        }
    }
}

/// A matcher that matches string literals.
pub struct StringLiteralMatcher {}

impl Matcher for StringLiteralMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        let mut raw_marker = false;
        let delimeter  = match tokenizer.peek().unwrap() {
            &'"'  => Some('"'),
            &'\'' => Some('\''),
            &'r' if tokenizer.peek_n(1) == Some(&'"') => {
                raw_marker = true;
                tokenizer.advance(1); // Skips prefix

                Some('"')
            },
            _ => return None,
        };
        tokenizer.advance(1); // Skips the opening delimeter
        let mut string       = String::new();
        let mut found_escape = false;
        loop {
            if tokenizer.end() {
                break
            }
            if raw_marker {
                if tokenizer.peek().unwrap() == &'"' {
                    break
                }
                string.push(tokenizer.next().unwrap())
            } else {
                if found_escape {
                    string.push(
                        match tokenizer.next().unwrap() {
                            c @ '\\' | c @ '\'' | c @ '"' => c,
                            'n' => '\n',
                            'r' => '\r',
                            't' => '\t',
                            s => panic!("Invalid character escape: {}", s),
                        }
                    );
                    found_escape = false
                } else {
                    match tokenizer.peek().unwrap() {
                        &'\\' => {
                            tokenizer.next();
                            found_escape = true
                        },
                        &c if &c == &delimeter.unwrap() => break,
                        _ => string.push(tokenizer.next().unwrap()),
                    }
                }
            }
        }
        tokenizer.advance(1); // Skips the closing delimeter
        match delimeter.unwrap() {
            '"'  => {
                token!(tokenizer, StringLiteral, string)
            },
            _ => {
                if string.len() == 1 {
                    token!(tokenizer, CharLiteral, string)
                } else {
                    panic!("Invalid char literal")
                }
            },
        }
    }
}

/// A matcher that matches constant elements
/// of the specified token type.
pub struct ConstantMatcher {
    token_type: TokenType,
    constants: Vec<String>,
}

impl ConstantMatcher {
    pub fn new(token_type: TokenType, constants: Vec<String>) -> Self {
        ConstantMatcher {
            token_type: token_type,
            constants: constants,
        }
    }
}

impl Matcher for ConstantMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        for constant in self.constants.clone() {
            let dat = tokenizer.clone().take(constant.len());
            if dat.size_hint().1.unwrap() != constant.len() {
                return None;
            }
            if dat.collect::<String>() == constant {
                tokenizer.advance(constant.len());
                return token!(tokenizer, self.token_type.clone(), constant)
            }
        }
        None
    }
}

/// A matcher that matches identifiers.
pub struct IdentifierMatcher {}

impl Matcher for IdentifierMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        let mut identifier = String::new();
        let curr = tokenizer.next().unwrap();
        if curr.is_alphabetic() || curr == '_' {
            identifier.push(curr)
        } else {
            return None;
        }
        while !tokenizer.end() {
            let current = *tokenizer.peek().unwrap();
            if !current.is_whitespace() && ("_?!".contains(current) || current.is_alphanumeric()) {
                identifier.push(tokenizer.next().unwrap());
            } else {
                break;
            }
        }
        if !identifier.is_empty() {
            token!(tokenizer, Identifier, identifier)
        } else {
            None
        }
    }
}<commit_msg>fix Float/IntLiteralMatchers so that 0b0110 and 0x0a1f work<commit_after>use lexer::Tokenizer;
use lexer::token::{Token, TokenType};

macro_rules! token {
    ($tokenizer:expr, $token_type:ident, $accum:expr) => {{
        token!($tokenizer , TokenType::$token_type, $accum)
    }};
    ($tokenizer:expr, $token_type:expr, $accum:expr) => {{
        let tokenizer = $tokenizer as &$crate::lexer::Tokenizer;
        let token_type = $token_type as $crate::lexer::token::TokenType;
        Some(Token::new(token_type, tokenizer.last_position(), $accum))
    }};
}

/// Matcher.
pub trait Matcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token>;
}

/// A matcher that only matches white-space.
pub struct WhitespaceMatcher {}

impl Matcher for WhitespaceMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        let mut found = false;
        while !tokenizer.end() && tokenizer.peek().unwrap().is_whitespace() {
            found = true;
            tokenizer.next();
        }
        if found {
            token!(tokenizer, Whitespace, String::new())
        } else {
            None
        }
    }
}

/// A matcher that matches base-10 integer literals.
pub struct IntLiteralMatcher {}

impl Matcher for IntLiteralMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        let mut accum = String::new();
        let base = match tokenizer.peek().unwrap() {
            &'0' => {
                match tokenizer.peek_n(1) {
                    Some(chr) => {
                        match chr {
                            &'x' => 16, // base 16 (hexadecimal)
                            &'b' => 2, // base 2 (binary)
                            _ => 10, // base 10 (decimal)
                        }
                    }
                    _ => 10, // base 10 (decimal)
                }
            }
            _ => 10, // base 10 (decimal)
        };
        if base != 10 {
            tokenizer.advance(2); // skip prefix
        }
        while !tokenizer.end() && tokenizer.peek().unwrap().is_digit(base) {
            accum.push(tokenizer.next().unwrap());
        }
        if !accum.is_empty() {
            // Produce token as base-10 string
            let literal: String = match u64::from_str_radix(accum.as_str(), base) {
                Ok(result) => result.to_string(),
                Err(error) => panic!("Unable to parse integer literal: {}", error)
            };
            token!(tokenizer, IntLiteral, literal)
        } else {
            None
        }
    }
}

/// A matcher that matches float literals.
pub struct FloatLiteralMatcher {}

impl Matcher for FloatLiteralMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        let mut accum = String::new();
        let curr = tokenizer.next().unwrap();
        if curr.is_digit(10) {
            accum.push(curr)
        } else if curr == '.' {
            accum.push_str("0.")
        } else {
            return None;
        }
        while !tokenizer.end() {
            let current = *tokenizer.peek().unwrap();
            if !current.is_whitespace() && current.is_digit(10) || current == '.' {
                if current == '.' && accum.contains('.') {
                    panic!("Unexpected decimal point")
                }

                accum.push(tokenizer.next().unwrap());
            } else {
                break
            }
        }
        if accum.contains('.') {
            token!(tokenizer, FloatLiteral, accum)
        } else {
            None
        }
    }
}

/// A matcher that matches string literals.
pub struct StringLiteralMatcher {}

impl Matcher for StringLiteralMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        let mut raw_marker = false;
        let delimeter  = match tokenizer.peek().unwrap() {
            &'"'  => Some('"'),
            &'\'' => Some('\''),
            &'r' if tokenizer.peek_n(1) == Some(&'"') => {
                raw_marker = true;
                tokenizer.advance(1); // Skips prefix

                Some('"')
            },
            _ => return None,
        };
        tokenizer.advance(1); // Skips the opening delimeter
        let mut string       = String::new();
        let mut found_escape = false;
        loop {
            if tokenizer.end() {
                break
            }
            if raw_marker {
                if tokenizer.peek().unwrap() == &'"' {
                    break
                }
                string.push(tokenizer.next().unwrap())
            } else {
                if found_escape {
                    string.push(
                        match tokenizer.next().unwrap() {
                            c @ '\\' | c @ '\'' | c @ '"' => c,
                            'n' => '\n',
                            'r' => '\r',
                            't' => '\t',
                            s => panic!("Invalid character escape: {}", s),
                        }
                    );
                    found_escape = false
                } else {
                    match tokenizer.peek().unwrap() {
                        &'\\' => {
                            tokenizer.next();
                            found_escape = true
                        },
                        &c if &c == &delimeter.unwrap() => break,
                        _ => string.push(tokenizer.next().unwrap()),
                    }
                }
            }
        }
        tokenizer.advance(1); // Skips the closing delimeter
        match delimeter.unwrap() {
            '"'  => {
                token!(tokenizer, StringLiteral, string)
            },
            _ => {
                if string.len() == 1 {
                    token!(tokenizer, CharLiteral, string)
                } else {
                    panic!("Invalid char literal")
                }
            },
        }
    }
}

/// A matcher that matches constant elements
/// of the specified token type.
pub struct ConstantMatcher {
    token_type: TokenType,
    constants: Vec<String>,
}

impl ConstantMatcher {
    pub fn new(token_type: TokenType, constants: Vec<String>) -> Self {
        ConstantMatcher {
            token_type: token_type,
            constants: constants,
        }
    }
}

impl Matcher for ConstantMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        for constant in self.constants.clone() {
            let dat = tokenizer.clone().take(constant.len());
            if dat.size_hint().1.unwrap() != constant.len() {
                return None;
            }
            if dat.collect::<String>() == constant {
                tokenizer.advance(constant.len());
                return token!(tokenizer, self.token_type.clone(), constant)
            }
        }
        None
    }
}

/// A matcher that matches identifiers.
pub struct IdentifierMatcher {}

impl Matcher for IdentifierMatcher {
    fn try_match(&self, tokenizer: &mut Tokenizer) -> Option<Token> {
        let mut identifier = String::new();
        let curr = tokenizer.next().unwrap();
        if curr.is_alphabetic() || curr == '_' {
            identifier.push(curr)
        } else {
            return None;
        }
        while !tokenizer.end() {
            let current = *tokenizer.peek().unwrap();
            if !current.is_whitespace() && ("_?!".contains(current) || current.is_alphanumeric()) {
                identifier.push(tokenizer.next().unwrap());
            } else {
                break;
            }
        }
        if !identifier.is_empty() {
            token!(tokenizer, Identifier, identifier)
        } else {
            None
        }
    }
}
<|endoftext|>


<commit_before>#![feature(env)]

extern crate "pkg-config" as pkg_config;

fn main() {
    if std::env::var("CARGO_FEATURE_USE_PKGCONFIG").is_some() {
      if build_pkgconfig() { return; }
      panic!("Could not find SDL2 via pkgconfig");
    } else {
      println!("cargo:rustc-flags=-l SDL2");
    }
}

fn build_pkgconfig() -> bool {
    let opts = pkg_config::default_options("sdl2");
    pkg_config::find_library_opts("sdl2", &opts).is_ok()
}
<commit_msg>Fix for env::var fiasco.<commit_after>#![feature(env)]

extern crate "pkg-config" as pkg_config;

fn main() {
    if std::env::var("CARGO_FEATURE_USE_PKGCONFIG").is_ok() {
      if build_pkgconfig() { return; }
      panic!("Could not find SDL2 via pkgconfig");
    } else {
      println!("cargo:rustc-flags=-l SDL2");
    }
}

fn build_pkgconfig() -> bool {
    let opts = pkg_config::default_options("sdl2");
    pkg_config::find_library_opts("sdl2", &opts).is_ok()
}
<|endoftext|>