<commit_before>use std::path::{PathBuf, Path};
use std::env;
use std::str::FromStr;
use clap::ArgMatches;
use coveralls_api::CiService;
use regex::Regex;

arg_enum!{
    /// Enum to represent possible output formats.
    #[derive(Debug)]
    pub enum OutputFile {
        Json,
        Toml,
        Stdout,
        Xml
    }
}

struct Ci(CiService);

impl FromStr for Ci {
    type Err = ();
    /// This will never error so no need to implement the error type.
    fn from_str(s: &str) -> Result<Ci, ()> {
        match s {
            "travis-ci" => Ok(Ci(CiService::Travis)),
            "travis-pro" => Ok(Ci(CiService::TravisPro)),
            "circle-ci" => Ok(Ci(CiService::Circle)),
            "semaphore" => Ok(Ci(CiService::Semaphore)),
            "jenkins" => Ok(Ci(CiService::Jenkins)),
            "codeship" => Ok(Ci(CiService::Codeship)),
            other => Ok(Ci(CiService::Other(other.to_string()))),
        }
    }
}

/// Specifies the current configuration tarpaulin is using.
#[derive(Debug, Default)]
pub struct Config {
    /// Path to the projects cargo manifest
    pub manifest: PathBuf,
    /// Flag to also run tests with the ignored attribute
    pub run_ignored: bool,
    /// Flag to ignore test functions in coverage statistics
    pub ignore_tests: bool,
    /// Flag to skip the clean step when preparing the target project
    pub skip_clean: bool,
    /// Verbose flag for printing information to the user
    pub verbose: bool,
    /// Flag to disable counting line hits in line coverage mode
    pub no_count: bool,
    /// Flag specifying to run line coverage (default)
    pub line_coverage: bool,
    /// Flag specifying to run branch coverage
    pub branch_coverage: bool,
    /// Output files to generate
    pub generate: Vec<OutputFile>,
    /// Key relating to coveralls service or repo
    pub coveralls: Option<String>,
    /// Enum representing CI tool used.
    pub ci_tool: Option<CiService>,
    /// Forward unexpected signals back to the tracee. Used for tests which
    /// rely on signals to work.
    pub forward_signals: bool,
    /// Features to include in the target project build
    pub features: Vec<String>,
    /// Build all packages in the workspace
    pub all: bool,
    /// Packages to include when building the target project
    pub packages: Vec<String>,
    /// Packages to exclude from testing
    pub exclude: Vec<String>,
    /// Files to exclude from testing
    excluded_files: Vec<Regex>,
    /// Varargs to be forwarded to the test executables.
    pub varargs: Vec<String>,
}

impl Config {
    fn get_list_from_args(args: &ArgMatches, key: &str) -> Vec<String> {
        match args.values_of_lossy(key) {
            Some(v) => v,
            None => vec![],
        }
    }

    /// Create configuration from clap ArgMatches.
    pub fn from_args(args: &ArgMatches) -> Config {
        let mut line = args.is_present("line");
        let mut branch = args.is_present("branch");
        let verbose = args.is_present("verbose");
        let ignored = args.is_present("ignored");
        let forward = args.is_present("forward");
        let skip_clean = args.is_present("skip-clean");
        let no_count = args.is_present("no-count");
        let ignore_tests = args.is_present("ignore-tests");
        // If no coverage selected do everything!
        if !branch && !line {
            branch = true;
            line = true;
        }
        let mut root = env::current_dir().unwrap();
        if let Some(path) = args.value_of("root") {
            root.push(path);
        };
        root.push("Cargo.toml");
        if let Ok(cpath) = root.canonicalize() {
            root = cpath;
        }
        let ci_tool = match value_t!(args, "ciserver", Ci) {
            Ok(ci) => Some(ci.0),
            Err(_) => None,
        };
        let coveralls = if let Some(cio) = args.value_of("coveralls") {
            Some(cio.to_string())
        } else {
            None
        };
        let out:Vec<OutputFile> = values_t!(args.values_of("out"), OutputFile)
            .unwrap_or_default();
        let features: Vec<String> = Config::get_list_from_args(args, "features");
        let all = args.is_present("all");
        let packages: Vec<String> = Config::get_list_from_args(args, "packages");
        let exclude: Vec<String> = Config::get_list_from_args(args, "exclude");
        let varargs: Vec<String> = Config::get_list_from_args(args, "args");
        let mut ex_files:Vec<Regex> = vec![];
        for temp_str in &Config::get_list_from_args(args, "exclude-files") {
            let s =  &temp_str.replace(".", r"\.").replace("*", ".*");
            println!("Wildcard: {}", s);
            if let Ok(re) = Regex::new(s) {
                ex_files.push(re);
            } else if verbose {
                println!("Error in wildcard expression: {}", temp_str);
            }
        }
        Config{
            manifest: root,
            run_ignored: ignored,
            ignore_tests: ignore_tests,
            verbose: verbose,
            no_count: no_count,
            line_coverage: line,
            skip_clean: skip_clean,
            branch_coverage: branch,
            generate: out,
            coveralls: coveralls,
            ci_tool: ci_tool,
            forward_signals: forward,
            features: features,
            all: all,
            packages: packages,
            exclude: exclude,
            excluded_files: ex_files,
            varargs: varargs
        }
    }

    /// Determine whether to send data to coveralls
    pub fn is_coveralls(&self) -> bool {
        self.coveralls.is_some()
    }

    pub fn exclude_path(&self, path: &Path) -> bool {
        let path = self.strip_project_path(path);
        self.excluded_files.iter()
                           .any(|x| x.is_match(path.to_str().unwrap_or("")))
    }

    /// Strips the directory the project manifest is in from the path. Provides a
    /// nicer path for printing to the user.
    pub fn strip_project_path<'a>(&'a self, path: &'a Path) -> &'a Path {
        if let Some(root) = self.manifest.parent() {
            path.strip_prefix(root).unwrap_or(path)
        } else {
            path
        }
    }
}
<commit_msg>Added tests for file filtering<commit_after>use std::path::{PathBuf, Path};
use std::env;
use std::str::FromStr;
use clap::ArgMatches;
use coveralls_api::CiService;
use regex::Regex;

arg_enum!{
    /// Enum to represent possible output formats.
    #[derive(Debug)]
    pub enum OutputFile {
        Json,
        Toml,
        Stdout,
        Xml
    }
}

struct Ci(CiService);

impl FromStr for Ci {
    type Err = ();
    /// This will never error so no need to implement the error type.
    fn from_str(s: &str) -> Result<Ci, ()> {
        match s {
            "travis-ci" => Ok(Ci(CiService::Travis)),
            "travis-pro" => Ok(Ci(CiService::TravisPro)),
            "circle-ci" => Ok(Ci(CiService::Circle)),
            "semaphore" => Ok(Ci(CiService::Semaphore)),
            "jenkins" => Ok(Ci(CiService::Jenkins)),
            "codeship" => Ok(Ci(CiService::Codeship)),
            other => Ok(Ci(CiService::Other(other.to_string()))),
        }
    }
}

/// Specifies the current configuration tarpaulin is using.
#[derive(Debug, Default)]
pub struct Config {
    /// Path to the projects cargo manifest
    pub manifest: PathBuf,
    /// Flag to also run tests with the ignored attribute
    pub run_ignored: bool,
    /// Flag to ignore test functions in coverage statistics
    pub ignore_tests: bool,
    /// Flag to skip the clean step when preparing the target project
    pub skip_clean: bool,
    /// Verbose flag for printing information to the user
    pub verbose: bool,
    /// Flag to disable counting line hits in line coverage mode
    pub no_count: bool,
    /// Flag specifying to run line coverage (default)
    pub line_coverage: bool,
    /// Flag specifying to run branch coverage
    pub branch_coverage: bool,
    /// Output files to generate
    pub generate: Vec<OutputFile>,
    /// Key relating to coveralls service or repo
    pub coveralls: Option<String>,
    /// Enum representing CI tool used.
    pub ci_tool: Option<CiService>,
    /// Forward unexpected signals back to the tracee. Used for tests which
    /// rely on signals to work.
    pub forward_signals: bool,
    /// Features to include in the target project build
    pub features: Vec<String>,
    /// Build all packages in the workspace
    pub all: bool,
    /// Packages to include when building the target project
    pub packages: Vec<String>,
    /// Packages to exclude from testing
    pub exclude: Vec<String>,
    /// Files to exclude from testing
    excluded_files: Vec<Regex>,
    /// Varargs to be forwarded to the test executables.
    pub varargs: Vec<String>,
}

impl Config {
    fn get_list_from_args(args: &ArgMatches, key: &str) -> Vec<String> {
        match args.values_of_lossy(key) {
            Some(v) => v,
            None => vec![],
        }
    }

    /// Create configuration from clap ArgMatches.
    pub fn from_args(args: &ArgMatches) -> Config {
        let mut line = args.is_present("line");
        let mut branch = args.is_present("branch");
        let verbose = args.is_present("verbose");
        let ignored = args.is_present("ignored");
        let forward = args.is_present("forward");
        let skip_clean = args.is_present("skip-clean");
        let no_count = args.is_present("no-count");
        let ignore_tests = args.is_present("ignore-tests");
        // If no coverage selected do everything!
        if !branch && !line {
            branch = true;
            line = true;
        }
        let mut root = env::current_dir().unwrap();
        if let Some(path) = args.value_of("root") {
            root.push(path);
        };
        root.push("Cargo.toml");
        if let Ok(cpath) = root.canonicalize() {
            root = cpath;
        }
        let ci_tool = match value_t!(args, "ciserver", Ci) {
            Ok(ci) => Some(ci.0),
            Err(_) => None,
        };
        let coveralls = if let Some(cio) = args.value_of("coveralls") {
            Some(cio.to_string())
        } else {
            None
        };
        let out:Vec<OutputFile> = values_t!(args.values_of("out"), OutputFile)
            .unwrap_or_default();
        let features: Vec<String> = Config::get_list_from_args(args, "features");
        let all = args.is_present("all");
        let packages: Vec<String> = Config::get_list_from_args(args, "packages");
        let exclude: Vec<String> = Config::get_list_from_args(args, "exclude");
        let varargs: Vec<String> = Config::get_list_from_args(args, "args");
        let mut ex_files:Vec<Regex> = vec![];
        for temp_str in &Config::get_list_from_args(args, "exclude-files") {
            let s =  &temp_str.replace(".", r"\.").replace("*", ".*");
            if let Ok(re) = Regex::new(s) {
                ex_files.push(re);
            } else if verbose {
                println!("Error in wildcard expression: {}", temp_str);
            }
        }
        Config{
            manifest: root,
            run_ignored: ignored,
            ignore_tests: ignore_tests,
            verbose: verbose,
            no_count: no_count,
            line_coverage: line,
            skip_clean: skip_clean,
            branch_coverage: branch,
            generate: out,
            coveralls: coveralls,
            ci_tool: ci_tool,
            forward_signals: forward,
            features: features,
            all: all,
            packages: packages,
            exclude: exclude,
            excluded_files: ex_files,
            varargs: varargs
        }
    }

    /// Determine whether to send data to coveralls
    pub fn is_coveralls(&self) -> bool {
        self.coveralls.is_some()
    }

    pub fn exclude_path(&self, path: &Path) -> bool {
        let path = self.strip_project_path(path);
        self.excluded_files.iter()
                           .any(|x| x.is_match(path.to_str().unwrap_or("")))
    }

    /// Strips the directory the project manifest is in from the path. Provides a
    /// nicer path for printing to the user.
    pub fn strip_project_path<'a>(&'a self, path: &'a Path) -> &'a Path {
        if let Some(root) = self.manifest.parent() {
            path.strip_prefix(root).unwrap_or(path)
        } else {
            path
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use clap::App;

    #[test]
    fn exclude_paths() {
        let matches = App::new("tarpaulin")
            .args_from_usage("--exclude-files [FILE]... 'Exclude given files from coverage results has * wildcard'")
            .get_matches_from_safe(vec!["tarpaulin", "--exclude-files", "*module*"])
            .unwrap();
        let conf = Config::from_args(&matches);
        assert!(conf.exclude_path(Path::new("src/module/file.rs")));
        assert!(!conf.exclude_path(Path::new("src/mod.rs")));
        assert!(!conf.exclude_path(Path::new("unrelated.rs")));
        assert!(conf.exclude_path(Path::new("module.rs")));
    }

    #[test]
    fn no_exclusions() {
        let matches = App::new("tarpaulin")
            .args_from_usage("--exclude-files [FILE]... 'Exclude given files from coverage results has * wildcard'")
            .get_matches_from_safe(vec!["tarpaulin"])
            .unwrap();
        let conf = Config::from_args(&matches);
        assert!(!conf.exclude_path(Path::new("src/module/file.rs")));
        assert!(!conf.exclude_path(Path::new("src/mod.rs")));
        assert!(!conf.exclude_path(Path::new("unrelated.rs")));
        assert!(!conf.exclude_path(Path::new("module.rs")));
    }

    #[test]
    fn exclude_exact_file() {
        let matches = App::new("tarpaulin")
            .args_from_usage("--exclude-files [FILE]... 'Exclude given files from coverage results has * wildcard'")
            .get_matches_from_safe(vec!["tarpaulin", "--exclude-files", "*/lib.rs"])
            .unwrap();
        let conf = Config::from_args(&matches);
        assert!(conf.exclude_path(Path::new("src/lib.rs")));
        assert!(!conf.exclude_path(Path::new("src/mod.rs")));
        assert!(!conf.exclude_path(Path::new("src/notlib.rs")));
        assert!(!conf.exclude_path(Path::new("lib.rs")));
    }
}
<|endoftext|>


<commit_before>use optim::*;
use nn::Parameters;
use std::ops::Neg;
use num;

pub struct SGD {
    optimizer: Optimizer,
}

impl SGD {
    pub fn new(defaults: HashMap<&'static str, OptimVal>) -> Self {
        let mut sgd_defaults = map_opt!{"lr" => OptimVal::Required, "momentum" => 0.0, "dampening" => 0.0,
                 "weight_decay"=> 0.0, "nesterov"=>false};

        for (ref key, ref value) in defaults {
            let cloned = value.clone();
            sgd_defaults.insert(key, cloned);
        }
        SGD { optimizer: Optimizer::new(sgd_defaults) }
    }
}

impl<T: Copy + From<OptimVal> + num::Num + num::Float + Neg<Output = T>> OptIntf<T> for SGD {
    fn optimizer(&mut self) -> &mut Optimizer {
        &mut self.optimizer
    }
    fn step(&mut self, params: Parameters<T>) {
        let group = &self.optimizer.defaults;
        let weight_decay: T = group["weight_decay"].clone().into();
        let momentum: T = group["momentum"].clone().into();
        let dampening: T = group["dampening"].clone().into();
        let nesterov: bool = group["nesterov"].clone().into();
        let lr: T = group["lr"].clone().into();

        for mut p in params {
            let mut v = &mut p.v;
            let mut d_p = if let Some(ref mut grad) = *v.grad() {
                grad.data().clone() as ::tensor::Tensor<T>
            } else {
                continue;
            };
            if !weight_decay.is_zero() {
                d_p.addt_(weight_decay, v.data());
            }
            if !momentum.is_zero() {
                let mut state = &mut self.optimizer.state[v.id];
                let mut buf: Tensor<T>;
                if !state.contains_key("momentum_buffer") {
                    buf = d_p.copy();
                    state.insert("momentum_buffer", buf.clone().into());
                } else {
                    buf = state["momentum_buffer"].clone().into();
                    buf.mul_(momentum).addt_(T::one() - dampening, &d_p);
                }
                if nesterov {
                    d_p = d_p.addt(momentum, &buf);
                } else {
                    d_p = buf;
                }

            }
            v.data().addt_(-lr, &d_p);
        }
    }
}
<commit_msg>fix line length<commit_after>use optim::*;
use nn::Parameters;
use std::ops::Neg;
use num;

pub struct SGD {
    optimizer: Optimizer,
}

impl SGD {
    pub fn new(defaults: HashMap<&'static str, OptimVal>) -> Self {
        let mut sgd_defaults = map_opt!{"lr" => OptimVal::Required,
            "momentum" => 0.0, "dampening" => 0.0,
            "weight_decay"=> 0.0, "nesterov"=>false};

        for (ref key, ref value) in defaults {
            let cloned = value.clone();
            sgd_defaults.insert(key, cloned);
        }
        SGD { optimizer: Optimizer::new(sgd_defaults) }
    }
}

impl<T: Copy + From<OptimVal> + num::Num + num::Float + Neg<Output = T>> OptIntf<T> for SGD {
    fn optimizer(&mut self) -> &mut Optimizer {
        &mut self.optimizer
    }
    fn step(&mut self, params: Parameters<T>) {
        let group = &self.optimizer.defaults;
        let weight_decay: T = group["weight_decay"].clone().into();
        let momentum: T = group["momentum"].clone().into();
        let dampening: T = group["dampening"].clone().into();
        let nesterov: bool = group["nesterov"].clone().into();
        let lr: T = group["lr"].clone().into();

        for mut p in params {
            let mut v = &mut p.v;
            let mut d_p = if let Some(ref mut grad) = *v.grad() {
                grad.data().clone() as ::tensor::Tensor<T>
            } else {
                continue;
            };
            if !weight_decay.is_zero() {
                d_p.addt_(weight_decay, v.data());
            }
            if !momentum.is_zero() {
                let mut state = &mut self.optimizer.state[v.id];
                let mut buf: Tensor<T>;
                if !state.contains_key("momentum_buffer") {
                    buf = d_p.copy();
                    state.insert("momentum_buffer", buf.clone().into());
                } else {
                    buf = state["momentum_buffer"].clone().into();
                    buf.mul_(momentum).addt_(T::one() - dampening, &d_p);
                }
                if nesterov {
                    d_p = d_p.addt(momentum, &buf);
                } else {
                    d_p = buf;
                }

            }
            v.data().addt_(-lr, &d_p);
        }
    }
}
<|endoftext|>


<commit_before>//! Module with helpers for dealing with RFC 5322

use super::rfc2047::decode_rfc2047;

trait Rfc5322Character {
    /// Is considered a special character by RFC 5322 Section 3.2.3
    fn is_special(&self) -> bool;
    fn is_printable(&self) -> bool;

    fn is_atext(&self) -> bool {
        self.is_printable() && !self.is_special()
    }
}

impl Rfc5322Character for char {
    fn is_special(&self) -> bool {
        match *self {
            '(' | ')' | '<' | '>' | '[' | ']' | ':' | ';' | '@' | '\\' | ',' | '.' | '\"' | ' ' => true,
            _ => false
        }
    }

    fn is_printable(&self) -> bool {
        match *self {
            ' '...'~' => true,
            _ => false,
        }
    }
}

/// RFC 5322 base parser for parsing
///  `atom`, `dot-atom`, `quoted-string`, `phrase`
///
/// This should prove useful for parsing other things that appear in RFC 5322,
/// as most are based off these core items.
///
/// It also implements a stack for tracking the position.
/// This allows the simple implementation of backtracking, by pushing the position
/// before a test and popping it if the test should fail.
pub struct Rfc5322Parser<'s> {
    s: &'s str,
    pos: uint,
    pos_stack: Vec<uint>,
}

impl<'s> Rfc5322Parser<'s> {
    /// Make a new parser, initialized with the given string.
    pub fn new(source: &'s str) -> Rfc5322Parser<'s> {
        Rfc5322Parser {
            s: source,
            pos: 0u,
            pos_stack: Vec::new(),
        }
    }

    /// Push the current position onto the stack.
    pub fn push_position(&mut self) {
        self.pos_stack.push(self.pos);
    }

    /// Move the position back to the last entry pushed
    pub fn pop_position(&mut self) {
        match self.pos_stack.pop() {
            Some(pos) => { self.pos = pos; },
            None => fail!("Popped position stack too far"),
        }
    }

    /// Consume a phrase from the input.
    ///
    /// A phrase is defined as:
    ///
    /// `word = atom / quoted-string
    /// phrase = 1*word`
    ///
    /// If `allow_dot_atom` is true, then `atom` can be a `dot-atom` in this phrase.
    pub fn consume_phrase(&mut self, allow_dot_atom: bool) -> Option<String> {
        let mut phrase = String::new();

        while !self.eof() {
            self.consume_linear_whitespace();
            let word = if self.peek() == '"' {
                // Word is a quoted string
                self.consume_quoted_string()
            } else if self.peek().is_atext() {
                self.consume_atom(allow_dot_atom)
            } else {
                // If it's not a quoted string, or an atom, it's no longer
                // in a phrase, so stop.
                break
            };

            if word.is_some() {
                // Unwrap word so it lives long enough...
                // XXX: word in this scope is `String`, in the parent scope, is `Option<String>`
                let word = word.unwrap();
                let w_slice = word.as_slice();
                // RFC 2047 encoded words start with =?, end with ?=
                let decoded_word =
                    if w_slice.starts_with("=?") && w_slice.ends_with("?=") {
                        match decode_rfc2047(w_slice) {
                            Some(w) => w,
                            None => w_slice.to_string(),
                        }
                    } else {
                        w_slice.to_string()
                    };

                // Make sure we put a leading space on, if this isn't the first insertion
                if phrase.len() > 0 {
                    phrase.push_str(" ");
                }
                phrase.push_str(decoded_word.as_slice());
            } else {
                return None
            }
        }

        Some(phrase)
    }

    /// Consume a quoted string from the input
    pub fn consume_quoted_string(&mut self) -> Option<String> {
        if self.peek() != '"' {
            // Fail if we were called wrong
            None
        } else {
            let mut quoted_string = String::new();
            let mut inside_escape = false;
            let mut terminated = false;
            // Consume the leading "
            self.consume_char();
            while !terminated && !self.eof() {
                match self.peek() {
                    '\\' if !inside_escape => {
                        // If we were not already being escaped, consume the
                        // escape character and mark that we're being escaped.
                        self.consume_char();
                        inside_escape = true;
                    },
                    '"' if !inside_escape => {
                        // If this is a DQUOTE and we haven't seen an escape character,
                        // consume it and mark that we should break from the loop
                        self.consume_char();
                        terminated = true;
                    },
                    _ => {
                        // Any old character gets pushed in
                        quoted_string.push(self.consume_char());
                        // Clear any escape character state we have
                        inside_escape = false;
                    },
                }
            }

            if inside_escape || !terminated {
                // Return an error state if we're still expecting a character
                None
            } else {
                Some(quoted_string)
            }
        }
    }

    /// Consume an atom from the input.
    ///
    /// If `allow_dot` is true, then also allow '.' to be considered as an
    /// atext character.
    pub fn consume_atom(&mut self, allow_dot: bool) -> Option<String> {
        if !self.peek().is_atext() {
            None
        } else {
            Some(self.consume_while(|c| {
                c.is_atext() || (allow_dot && c == '.')
            }))
        }
    }

    /// Consume LWSP (Linear whitespace)
    pub fn consume_linear_whitespace(&mut self) {
        self.consume_while(|c| { c == '\t' || c == ' ' });
    }

    /// Consume a single character from the input.
    pub fn consume_char(&mut self) -> char {
        if self.eof() {
            fail!("Consuming beyond end of input");
        }
        let ch_range = self.s.char_range_at(self.pos);
        self.pos = ch_range.next;
        ch_range.ch
    }

    /// Consume a set of characters, each passed to `test` until this function
    /// returns false.
    ///
    /// The position after calling this function will be pointing to the character
    /// which caused a false result from `test`.
    ///
    /// Returns the string of characters that returned true for the test function.
    pub fn consume_while(&mut self, test: |char| -> bool) -> String {
        let mut res = String::new();
        while !self.eof() && test(self.peek()) {
            res.push(self.consume_char())
        }
        res
    }

    /// Peek at the current character.
    ///
    /// Note that this does not do any bounds checking.
    pub fn peek(&self) -> char {
        self.s.char_at(self.pos)
    }

    /// Returns true if we have reached the end of the input.
    #[inline]
    pub fn eof(&self) -> bool {
        self.pos >= self.s.len()
    }

}

#[cfg(test)]
mod tests {
    use super::*;

    struct PhraseTestCase<'s> {
        input: &'s str,
        output: &'s str,
        name: &'s str,
    }

    #[test]
    fn test_consume_phrase() {
        let tests = [
            PhraseTestCase {
                input: "\"test phrase\"", output: "test phrase",
                name: "Simple quoted-string"
            },
            PhraseTestCase {
                input: "\"test \\\"phrase\\\"\"", output: "test \"phrase\"",
                name: "quoted-string with escape character"
            },
            PhraseTestCase {
                input: "\"=?utf-8?q?encoded=20q-string?=\"", output: "encoded q-string",
                name: "Encoded quoted-string"
            },
            PhraseTestCase {
                input: "atom test", output: "atom test",
                name: "Collection of atoms"
            },
            PhraseTestCase {
                input: "=?utf-8?q?encoded=20atom?=", output: "encoded atom",
                name: "Encoded atom"
            },
            PhraseTestCase {
                input: "Mix of atoms \"and quoted strings\"", output: "Mix of atoms and quoted strings",
                name: "Mix of atoms and quoted strings"
            },
            PhraseTestCase {
                input: "=?utf-8?q?encoded=20atoms?= mixed with \"unencoded\" \"=?utf-8?b?YW5kIGVuY29kZWQgcS1zdHJpbmdz?=\"",
                output: "encoded atoms mixed with unencoded and encoded q-strings",
                name: "Mix of atoms, q-strings of differing encodings"
            },
            PhraseTestCase {
                input: "\"John Smith\" <test@example.org>", output: "John Smith",
                name: "Stop consuming phrase at \"special\" character",
            }
        ];

        for t in tests.iter() {
            let mut p = Rfc5322Parser::new(t.input);
            let phrase = p.consume_phrase(false);
            assert!(phrase.is_some(), format!("{} returned Some", t.name));
            let test_name = format!("{} == {} for {}", phrase.clone().unwrap(), t.output, t.name);
            assert!(phrase.unwrap() == t.output.to_string(), test_name);
        }
    }
}
<commit_msg>Fix issue where parse_phrase would return an empty string.<commit_after>//! Module with helpers for dealing with RFC 5322

use super::rfc2047::decode_rfc2047;

trait Rfc5322Character {
    /// Is considered a special character by RFC 5322 Section 3.2.3
    fn is_special(&self) -> bool;
    fn is_printable(&self) -> bool;

    fn is_atext(&self) -> bool {
        self.is_printable() && !self.is_special()
    }
}

impl Rfc5322Character for char {
    fn is_special(&self) -> bool {
        match *self {
            '(' | ')' | '<' | '>' | '[' | ']' | ':' | ';' | '@' | '\\' | ',' | '.' | '\"' | ' ' => true,
            _ => false
        }
    }

    fn is_printable(&self) -> bool {
        match *self {
            ' '...'~' => true,
            _ => false,
        }
    }
}

/// RFC 5322 base parser for parsing
///  `atom`, `dot-atom`, `quoted-string`, `phrase`
///
/// This should prove useful for parsing other things that appear in RFC 5322,
/// as most are based off these core items.
///
/// It also implements a stack for tracking the position.
/// This allows the simple implementation of backtracking, by pushing the position
/// before a test and popping it if the test should fail.
pub struct Rfc5322Parser<'s> {
    s: &'s str,
    pos: uint,
    pos_stack: Vec<uint>,
}

impl<'s> Rfc5322Parser<'s> {
    /// Make a new parser, initialized with the given string.
    pub fn new(source: &'s str) -> Rfc5322Parser<'s> {
        Rfc5322Parser {
            s: source,
            pos: 0u,
            pos_stack: Vec::new(),
        }
    }

    /// Push the current position onto the stack.
    pub fn push_position(&mut self) {
        self.pos_stack.push(self.pos);
    }

    /// Move the position back to the last entry pushed
    pub fn pop_position(&mut self) {
        match self.pos_stack.pop() {
            Some(pos) => { self.pos = pos; },
            None => fail!("Popped position stack too far"),
        }
    }

    /// Consume a phrase from the input.
    ///
    /// A phrase is defined as:
    ///
    /// `word = atom / quoted-string
    /// phrase = 1*word`
    ///
    /// If `allow_dot_atom` is true, then `atom` can be a `dot-atom` in this phrase.
    pub fn consume_phrase(&mut self, allow_dot_atom: bool) -> Option<String> {
        let mut phrase = String::new();

        while !self.eof() {
            self.consume_linear_whitespace();
            let word = if self.peek() == '"' {
                // Word is a quoted string
                self.consume_quoted_string()
            } else if self.peek().is_atext() {
                self.consume_atom(allow_dot_atom)
            } else {
                // If it's not a quoted string, or an atom, it's no longer
                // in a phrase, so stop.
                break
            };

            if word.is_some() {
                // Unwrap word so it lives long enough...
                // XXX: word in this scope is `String`, in the parent scope, is `Option<String>`
                let word = word.unwrap();
                let w_slice = word.as_slice();
                // RFC 2047 encoded words start with =?, end with ?=
                let decoded_word =
                    if w_slice.starts_with("=?") && w_slice.ends_with("?=") {
                        match decode_rfc2047(w_slice) {
                            Some(w) => w,
                            None => w_slice.to_string(),
                        }
                    } else {
                        w_slice.to_string()
                    };

                // Make sure we put a leading space on, if this isn't the first insertion
                if phrase.len() > 0 {
                    phrase.push_str(" ");
                }
                phrase.push_str(decoded_word.as_slice());
            } else {
                return None
            }
        }

        if phrase.len() > 0 {
            Some(phrase)
        } else {
            None
        }
    }

    /// Consume a quoted string from the input
    pub fn consume_quoted_string(&mut self) -> Option<String> {
        if self.peek() != '"' {
            // Fail if we were called wrong
            None
        } else {
            let mut quoted_string = String::new();
            let mut inside_escape = false;
            let mut terminated = false;
            // Consume the leading "
            self.consume_char();
            while !terminated && !self.eof() {
                match self.peek() {
                    '\\' if !inside_escape => {
                        // If we were not already being escaped, consume the
                        // escape character and mark that we're being escaped.
                        self.consume_char();
                        inside_escape = true;
                    },
                    '"' if !inside_escape => {
                        // If this is a DQUOTE and we haven't seen an escape character,
                        // consume it and mark that we should break from the loop
                        self.consume_char();
                        terminated = true;
                    },
                    _ => {
                        // Any old character gets pushed in
                        quoted_string.push(self.consume_char());
                        // Clear any escape character state we have
                        inside_escape = false;
                    },
                }
            }

            if inside_escape || !terminated {
                // Return an error state if we're still expecting a character
                None
            } else {
                Some(quoted_string)
            }
        }
    }

    /// Consume an atom from the input.
    ///
    /// If `allow_dot` is true, then also allow '.' to be considered as an
    /// atext character.
    pub fn consume_atom(&mut self, allow_dot: bool) -> Option<String> {
        if !self.peek().is_atext() {
            None
        } else {
            Some(self.consume_while(|c| {
                c.is_atext() || (allow_dot && c == '.')
            }))
        }
    }

    /// Consume LWSP (Linear whitespace)
    pub fn consume_linear_whitespace(&mut self) {
        self.consume_while(|c| { c == '\t' || c == ' ' });
    }

    /// Consume a single character from the input.
    pub fn consume_char(&mut self) -> char {
        if self.eof() {
            fail!("Consuming beyond end of input");
        }
        let ch_range = self.s.char_range_at(self.pos);
        self.pos = ch_range.next;
        ch_range.ch
    }

    /// Consume a set of characters, each passed to `test` until this function
    /// returns false.
    ///
    /// The position after calling this function will be pointing to the character
    /// which caused a false result from `test`.
    ///
    /// Returns the string of characters that returned true for the test function.
    pub fn consume_while(&mut self, test: |char| -> bool) -> String {
        let mut res = String::new();
        while !self.eof() && test(self.peek()) {
            res.push(self.consume_char())
        }
        res
    }

    /// Peek at the current character.
    ///
    /// Note that this does not do any bounds checking.
    pub fn peek(&self) -> char {
        self.s.char_at(self.pos)
    }

    /// Returns true if we have reached the end of the input.
    #[inline]
    pub fn eof(&self) -> bool {
        self.pos >= self.s.len()
    }

}

#[cfg(test)]
mod tests {
    use super::*;

    struct PhraseTestCase<'s> {
        input: &'s str,
        output: &'s str,
        name: &'s str,
    }

    #[test]
    fn test_consume_phrase() {
        let tests = [
            PhraseTestCase {
                input: "\"test phrase\"", output: "test phrase",
                name: "Simple quoted-string"
            },
            PhraseTestCase {
                input: "\"test \\\"phrase\\\"\"", output: "test \"phrase\"",
                name: "quoted-string with escape character"
            },
            PhraseTestCase {
                input: "\"=?utf-8?q?encoded=20q-string?=\"", output: "encoded q-string",
                name: "Encoded quoted-string"
            },
            PhraseTestCase {
                input: "atom test", output: "atom test",
                name: "Collection of atoms"
            },
            PhraseTestCase {
                input: "=?utf-8?q?encoded=20atom?=", output: "encoded atom",
                name: "Encoded atom"
            },
            PhraseTestCase {
                input: "Mix of atoms \"and quoted strings\"", output: "Mix of atoms and quoted strings",
                name: "Mix of atoms and quoted strings"
            },
            PhraseTestCase {
                input: "=?utf-8?q?encoded=20atoms?= mixed with \"unencoded\" \"=?utf-8?b?YW5kIGVuY29kZWQgcS1zdHJpbmdz?=\"",
                output: "encoded atoms mixed with unencoded and encoded q-strings",
                name: "Mix of atoms, q-strings of differing encodings"
            },
            PhraseTestCase {
                input: "\"John Smith\" <test@example.org>", output: "John Smith",
                name: "Stop consuming phrase at \"special\" character",
            }
        ];

        for t in tests.iter() {
            let mut p = Rfc5322Parser::new(t.input);
            let phrase = p.consume_phrase(false);
            assert!(phrase.is_some(), format!("{} returned Some", t.name));
            let test_name = format!("{} == {} for {}", phrase.clone().unwrap(), t.output, t.name);
            assert!(phrase.unwrap() == t.output.to_string(), test_name);
        }
    }
}
<|endoftext|>


<commit_before>use crate::{CompileError, Config, Syntax};

use std::path::{Path, PathBuf};
use std::str::FromStr;

use quote::ToTokens;

pub struct TemplateInput<'a> {
    pub ast: &'a syn::DeriveInput,
    pub config: &'a Config<'a>,
    pub syntax: &'a Syntax<'a>,
    pub source: Source,
    pub print: Print,
    pub escaper: &'a str,
    pub ext: Option<String>,
    pub parent: Option<&'a syn::Type>,
    pub path: PathBuf,
}

impl<'a> TemplateInput<'a> {
    /// Extract the template metadata from the `DeriveInput` structure. This
    /// mostly recovers the data for the `TemplateInput` fields from the
    /// `template()` attribute list fields; it also finds the of the `_parent`
    /// field, if any.
    pub fn new<'n>(
        ast: &'n syn::DeriveInput,
        config: &'n Config,
    ) -> Result<TemplateInput<'n>, CompileError> {
        // Check that an attribute called `template()` exists and that it is
        // the proper type (list).
        let meta = ast
            .attrs
            .iter()
            .find_map(|attr| match attr.parse_meta() {
                Ok(m) => {
                    if m.path().is_ident("template") {
                        Some(Ok(m))
                    } else {
                        None
                    }
                }
                Err(e) => Some(Err(format!("unable to parse attribute: {}", e).into())),
            })
            .unwrap_or(Err(CompileError::Static("no attribute 'template' found")))?;

        let meta_list = match meta {
            syn::Meta::List(inner) => inner,
            _ => return Err("attribute 'template' has incorrect type".into()),
        };

        // Loop over the meta attributes and find everything that we
        // understand. Raise panics if something is not right.
        // `source` contains an enum that can represent `path` or `source`.
        let mut source = None;
        let mut print = Print::None;
        let mut escaping = None;
        let mut ext = None;
        let mut syntax = None;
        for item in meta_list.nested {
            let pair = match item {
                syn::NestedMeta::Meta(syn::Meta::NameValue(ref pair)) => pair,
                _ => {
                    return Err(format!(
                        "unsupported attribute argument {:?}",
                        item.to_token_stream()
                    )
                    .into())
                }
            };

            if pair.path.is_ident("path") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    if source.is_some() {
                        return Err("must specify 'source' or 'path', not both".into());
                    }
                    source = Some(Source::Path(s.value()));
                } else {
                    return Err("template path must be string literal".into());
                }
            } else if pair.path.is_ident("source") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    if source.is_some() {
                        return Err("must specify 'source' or 'path', not both".into());
                    }
                    source = Some(Source::Source(s.value()));
                } else {
                    return Err("template source must be string literal".into());
                }
            } else if pair.path.is_ident("print") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    print = s.value().parse()?;
                } else {
                    return Err("print value must be string literal".into());
                }
            } else if pair.path.is_ident("escape") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    escaping = Some(s.value());
                } else {
                    return Err("escape value must be string literal".into());
                }
            } else if pair.path.is_ident("ext") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    ext = Some(s.value());
                } else {
                    return Err("ext value must be string literal".into());
                }
            } else if pair.path.is_ident("syntax") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    syntax = Some(s.value())
                } else {
                    return Err("syntax value must be string literal".into());
                }
            } else {
                return Err(format!(
                    "unsupported attribute key '{}' found",
                    pair.path.to_token_stream()
                )
                .into());
            }
        }

        // Validate the `source` and `ext` value together, since they are
        // related. In case `source` was used instead of `path`, the value
        // of `ext` is merged into a synthetic `path` value here.
        let source = source.expect("template path or source not found in attributes");
        let path = match (&source, &ext) {
            (&Source::Path(ref path), None) => config.find_template(path, None)?,
            (&Source::Source(_), Some(ext)) => PathBuf::from(format!("{}.{}", ast.ident, ext)),
            (&Source::Path(_), Some(_)) => {
                return Err("'ext' attribute cannot be used with 'path' attribute".into())
            }
            (&Source::Source(_), None) => {
                return Err("must include 'ext' attribute when using 'source' attribute".into())
            }
        };

        // Check to see if a `_parent` field was defined on the context
        // struct, and store the type for it for use in the code generator.
        let parent = match ast.data {
            syn::Data::Struct(syn::DataStruct {
                fields: syn::Fields::Named(ref fields),
                ..
            }) => fields
                .named
                .iter()
                .find(|f| f.ident.as_ref().filter(|name| *name == "_parent").is_some())
                .map(|f| &f.ty),
            _ => None,
        };

        if parent.is_some() {
            eprint!(
                "   --> in struct {}\n   = use of deprecated field '_parent'\n",
                ast.ident
            );
        }

        // Validate syntax
        let syntax = syntax.map_or_else(
            || Ok(config.syntaxes.get(config.default_syntax).unwrap()),
            |s| {
                config.syntaxes.get(&s).ok_or_else(|| {
                    CompileError::String(format!("attribute syntax {} not exist", s))
                })
            },
        )?;

        // Match extension against defined output formats

        let extension = escaping.unwrap_or_else(|| {
            path.extension()
                .map(|s| s.to_str().unwrap())
                .unwrap_or("")
                .to_string()
        });

        let mut escaper = None;
        for (extensions, path) in &config.escapers {
            if extensions.contains(&extension) {
                escaper = Some(path);
                break;
            }
        }

        let escaper = escaper.ok_or_else(|| {
            CompileError::String(format!("no escaper defined for extension '{}'", extension,))
        })?;

        Ok(TemplateInput {
            ast,
            config,
            syntax,
            source,
            print,
            escaper,
            ext,
            parent,
            path,
        })
    }

    pub fn extension(&self) -> Option<&str> {
        extension(&self.path)
    }
}

fn extension(path: &Path) -> Option<&str> {
    let ext = path.extension().map(|s| s.to_str().unwrap())?;

    const JINJA_EXTENSIONS: [&str; 3] = ["j2", "jinja", "jinja2"];
    if JINJA_EXTENSIONS.contains(&ext) {
        Path::new(path.file_stem().unwrap())
            .extension()
            .map(|s| s.to_str().unwrap())
            .or(Some(ext))
    } else {
        Some(ext)
    }
}

pub enum Source {
    Path(String),
    Source(String),
}

#[derive(PartialEq)]
pub enum Print {
    All,
    Ast,
    Code,
    None,
}

impl FromStr for Print {
    type Err = CompileError;

    fn from_str(s: &str) -> Result<Print, Self::Err> {
        use self::Print::*;
        Ok(match s {
            "all" => All,
            "ast" => Ast,
            "code" => Code,
            "none" => None,
            v => return Err(format!("invalid value for print option: {}", v,).into()),
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_ext() {
        assert_eq!(extension(Path::new("foo-bar.txt")), Some("txt"));
        assert_eq!(extension(Path::new("foo-bar.html")), Some("html"));
        assert_eq!(extension(Path::new("foo-bar.unknown")), Some("unknown"));

        assert_eq!(extension(Path::new("foo/bar/baz.txt")), Some("txt"));
        assert_eq!(extension(Path::new("foo/bar/baz.html")), Some("html"));
        assert_eq!(extension(Path::new("foo/bar/baz.unknown")), Some("unknown"));
    }

    #[test]
    fn test_double_ext() {
        assert_eq!(extension(Path::new("foo-bar.html.txt")), Some("txt"));
        assert_eq!(extension(Path::new("foo-bar.txt.html")), Some("html"));
        assert_eq!(extension(Path::new("foo-bar.txt.unknown")), Some("unknown"));

        assert_eq!(extension(Path::new("foo/bar/baz.html.txt")), Some("txt"));
        assert_eq!(extension(Path::new("foo/bar/baz.txt.html")), Some("html"));
        assert_eq!(
            extension(Path::new("foo/bar/baz.txt.unknown")),
            Some("unknown")
        );
    }

    #[test]
    fn test_skip_jinja_ext() {
        assert_eq!(extension(Path::new("foo-bar.html.j2")), Some("html"));
        assert_eq!(extension(Path::new("foo-bar.html.jinja")), Some("html"));
        assert_eq!(extension(Path::new("foo-bar.html.jinja2")), Some("html"));

        assert_eq!(extension(Path::new("foo/bar/baz.txt.j2")), Some("txt"));
        assert_eq!(extension(Path::new("foo/bar/baz.txt.jinja")), Some("txt"));
        assert_eq!(extension(Path::new("foo/bar/baz.txt.jinja2")), Some("txt"));
    }

    #[test]
    fn test_only_jinja_ext() {
        assert_eq!(extension(Path::new("foo-bar.j2")), Some("j2"));
        assert_eq!(extension(Path::new("foo-bar.jinja")), Some("jinja"));
        assert_eq!(extension(Path::new("foo-bar.jinja2")), Some("jinja2"));
    }
}
<commit_msg>Changed to allow both path and ext<commit_after>use crate::{CompileError, Config, Syntax};

use std::path::{Path, PathBuf};
use std::str::FromStr;

use quote::ToTokens;

pub struct TemplateInput<'a> {
    pub ast: &'a syn::DeriveInput,
    pub config: &'a Config<'a>,
    pub syntax: &'a Syntax<'a>,
    pub source: Source,
    pub print: Print,
    pub escaper: &'a str,
    pub ext: Option<String>,
    pub parent: Option<&'a syn::Type>,
    pub path: PathBuf,
}

impl<'a> TemplateInput<'a> {
    /// Extract the template metadata from the `DeriveInput` structure. This
    /// mostly recovers the data for the `TemplateInput` fields from the
    /// `template()` attribute list fields; it also finds the of the `_parent`
    /// field, if any.
    pub fn new<'n>(
        ast: &'n syn::DeriveInput,
        config: &'n Config,
    ) -> Result<TemplateInput<'n>, CompileError> {
        // Check that an attribute called `template()` exists and that it is
        // the proper type (list).
        let meta = ast
            .attrs
            .iter()
            .find_map(|attr| match attr.parse_meta() {
                Ok(m) => {
                    if m.path().is_ident("template") {
                        Some(Ok(m))
                    } else {
                        None
                    }
                }
                Err(e) => Some(Err(format!("unable to parse attribute: {}", e).into())),
            })
            .unwrap_or(Err(CompileError::Static("no attribute 'template' found")))?;

        let meta_list = match meta {
            syn::Meta::List(inner) => inner,
            _ => return Err("attribute 'template' has incorrect type".into()),
        };

        // Loop over the meta attributes and find everything that we
        // understand. Raise panics if something is not right.
        // `source` contains an enum that can represent `path` or `source`.
        let mut source = None;
        let mut print = Print::None;
        let mut escaping = None;
        let mut ext = None;
        let mut syntax = None;
        for item in meta_list.nested {
            let pair = match item {
                syn::NestedMeta::Meta(syn::Meta::NameValue(ref pair)) => pair,
                _ => {
                    return Err(format!(
                        "unsupported attribute argument {:?}",
                        item.to_token_stream()
                    )
                    .into())
                }
            };

            if pair.path.is_ident("path") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    if source.is_some() {
                        return Err("must specify 'source' or 'path', not both".into());
                    }
                    source = Some(Source::Path(s.value()));
                } else {
                    return Err("template path must be string literal".into());
                }
            } else if pair.path.is_ident("source") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    if source.is_some() {
                        return Err("must specify 'source' or 'path', not both".into());
                    }
                    source = Some(Source::Source(s.value()));
                } else {
                    return Err("template source must be string literal".into());
                }
            } else if pair.path.is_ident("print") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    print = s.value().parse()?;
                } else {
                    return Err("print value must be string literal".into());
                }
            } else if pair.path.is_ident("escape") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    escaping = Some(s.value());
                } else {
                    return Err("escape value must be string literal".into());
                }
            } else if pair.path.is_ident("ext") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    ext = Some(s.value());
                } else {
                    return Err("ext value must be string literal".into());
                }
            } else if pair.path.is_ident("syntax") {
                if let syn::Lit::Str(ref s) = pair.lit {
                    syntax = Some(s.value())
                } else {
                    return Err("syntax value must be string literal".into());
                }
            } else {
                return Err(format!(
                    "unsupported attribute key '{}' found",
                    pair.path.to_token_stream()
                )
                .into());
            }
        }

        // Validate the `source` and `ext` value together, since they are
        // related. In case `source` was used instead of `path`, the value
        // of `ext` is merged into a synthetic `path` value here.
        let source = source.expect("template path or source not found in attributes");
        let path = match (&source, &ext) {
            (&Source::Path(ref path), _) => config.find_template(path, None)?,
            (&Source::Source(_), Some(ext)) => PathBuf::from(format!("{}.{}", ast.ident, ext)),
            (&Source::Source(_), None) => {
                return Err("must include 'ext' attribute when using 'source' attribute".into())
            }
        };

        // Check to see if a `_parent` field was defined on the context
        // struct, and store the type for it for use in the code generator.
        let parent = match ast.data {
            syn::Data::Struct(syn::DataStruct {
                fields: syn::Fields::Named(ref fields),
                ..
            }) => fields
                .named
                .iter()
                .find(|f| f.ident.as_ref().filter(|name| *name == "_parent").is_some())
                .map(|f| &f.ty),
            _ => None,
        };

        if parent.is_some() {
            eprint!(
                "   --> in struct {}\n   = use of deprecated field '_parent'\n",
                ast.ident
            );
        }

        // Validate syntax
        let syntax = syntax.map_or_else(
            || Ok(config.syntaxes.get(config.default_syntax).unwrap()),
            |s| {
                config.syntaxes.get(&s).ok_or_else(|| {
                    CompileError::String(format!("attribute syntax {} not exist", s))
                })
            },
        )?;

        // Match extension against defined output formats

        let extension = escaping.unwrap_or_else(|| {
            path.extension()
                .map(|s| s.to_str().unwrap())
                .unwrap_or("")
                .to_string()
        });

        let mut escaper = None;
        for (extensions, path) in &config.escapers {
            if extensions.contains(&extension) {
                escaper = Some(path);
                break;
            }
        }

        let escaper = escaper.ok_or_else(|| {
            CompileError::String(format!("no escaper defined for extension '{}'", extension,))
        })?;

        Ok(TemplateInput {
            ast,
            config,
            syntax,
            source,
            print,
            escaper,
            ext,
            parent,
            path,
        })
    }

    pub fn extension(&self) -> Option<&str> {
        self.ext.as_deref().or_else(|| extension(&self.path))
    }
}

fn extension(path: &Path) -> Option<&str> {
    let ext = path.extension().map(|s| s.to_str().unwrap())?;

    const JINJA_EXTENSIONS: [&str; 3] = ["j2", "jinja", "jinja2"];
    if JINJA_EXTENSIONS.contains(&ext) {
        Path::new(path.file_stem().unwrap())
            .extension()
            .map(|s| s.to_str().unwrap())
            .or(Some(ext))
    } else {
        Some(ext)
    }
}

pub enum Source {
    Path(String),
    Source(String),
}

#[derive(PartialEq)]
pub enum Print {
    All,
    Ast,
    Code,
    None,
}

impl FromStr for Print {
    type Err = CompileError;

    fn from_str(s: &str) -> Result<Print, Self::Err> {
        use self::Print::*;
        Ok(match s {
            "all" => All,
            "ast" => Ast,
            "code" => Code,
            "none" => None,
            v => return Err(format!("invalid value for print option: {}", v,).into()),
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_ext() {
        assert_eq!(extension(Path::new("foo-bar.txt")), Some("txt"));
        assert_eq!(extension(Path::new("foo-bar.html")), Some("html"));
        assert_eq!(extension(Path::new("foo-bar.unknown")), Some("unknown"));

        assert_eq!(extension(Path::new("foo/bar/baz.txt")), Some("txt"));
        assert_eq!(extension(Path::new("foo/bar/baz.html")), Some("html"));
        assert_eq!(extension(Path::new("foo/bar/baz.unknown")), Some("unknown"));
    }

    #[test]
    fn test_double_ext() {
        assert_eq!(extension(Path::new("foo-bar.html.txt")), Some("txt"));
        assert_eq!(extension(Path::new("foo-bar.txt.html")), Some("html"));
        assert_eq!(extension(Path::new("foo-bar.txt.unknown")), Some("unknown"));

        assert_eq!(extension(Path::new("foo/bar/baz.html.txt")), Some("txt"));
        assert_eq!(extension(Path::new("foo/bar/baz.txt.html")), Some("html"));
        assert_eq!(
            extension(Path::new("foo/bar/baz.txt.unknown")),
            Some("unknown")
        );
    }

    #[test]
    fn test_skip_jinja_ext() {
        assert_eq!(extension(Path::new("foo-bar.html.j2")), Some("html"));
        assert_eq!(extension(Path::new("foo-bar.html.jinja")), Some("html"));
        assert_eq!(extension(Path::new("foo-bar.html.jinja2")), Some("html"));

        assert_eq!(extension(Path::new("foo/bar/baz.txt.j2")), Some("txt"));
        assert_eq!(extension(Path::new("foo/bar/baz.txt.jinja")), Some("txt"));
        assert_eq!(extension(Path::new("foo/bar/baz.txt.jinja2")), Some("txt"));
    }

    #[test]
    fn test_only_jinja_ext() {
        assert_eq!(extension(Path::new("foo-bar.j2")), Some("j2"));
        assert_eq!(extension(Path::new("foo-bar.jinja")), Some("jinja"));
        assert_eq!(extension(Path::new("foo-bar.jinja2")), Some("jinja2"));
    }
}
<|endoftext|>


<commit_before>use cli::Config;

use std::io::stderr;
use std::io::Write;

pub struct Runtime {
    config : Config,
}

impl Runtime {

    pub fn new(config : Config) -> Runtime {
        Runtime {
            config: config,
        }
    }

    pub fn debug(&self, string : String) {
        if self.config.is_debugging() {
            println!("{}", string);
        }
    }

    pub fn print(&self, string : String) {
        if self.config.is_verbose() || self.config.is_debugging() {
            println!("{}", string);
        }
    }

    pub fn trace(string : String) {
        // writeln!(&mut stderr, "{}", string);
    }

    pub fn is_verbose(&self) -> bool {
        self.config.is_verbose()
    }

    pub fn is_debugging(&self) -> bool {
        self.config.is_debugging()
    }

}
<commit_msg>We should use references here<commit_after>use cli::Config;

use std::io::stderr;
use std::io::Write;

pub struct Runtime {
    config : Config,
}

impl Runtime {

    pub fn new(config : Config) -> Runtime {
        Runtime {
            config: config,
        }
    }

    pub fn debug(&self, string : &String) {
        if self.config.is_debugging() {
            println!("{}", string);
        }
    }

    pub fn print(&self, string : &String) {
        if self.config.is_verbose() || self.config.is_debugging() {
            println!("{}", string);
        }
    }

    pub fn trace(string : &String) {
        // writeln!(&mut stderr, "{}", string);
    }

    pub fn is_verbose(&self) -> bool {
        self.config.is_verbose()
    }

    pub fn is_debugging(&self) -> bool {
        self.config.is_debugging()
    }

}
<|endoftext|>


<commit_before>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#[macro_use]
extern crate clap;
#[macro_use]
extern crate slog;
extern crate slog_term;
extern crate slog_async;
#[macro_use]
extern crate slog_scope;
extern crate users;

#[macro_use]
extern crate coreos_metadata;

use std::fs::File;
use std::io::prelude::*;
use clap::{Arg, App};
use slog::Drain;

use coreos_metadata::fetch_metadata;

const CMDLINE_PATH: &'static str = "/proc/cmdline";
const CMDLINE_OEM_FLAG:&'static str = "coreos.oem.id";

#[derive(Debug)]
struct Config {
    provider: String,
    attributes_file: Option<String>,
    ssh_keys_user: Option<String>,
    hostname_file: Option<String>,
    network_units_dir: Option<String>,
}

fn main() {
    // setup logging
    let decorator = slog_term::TermDecorator::new().build();
    let drain = slog_term::FullFormat::new(decorator).build().fuse();
    let drain = slog_async::Async::new(drain).build().fuse();
    let log = slog::Logger::root(drain, slog_o!());
    let _guard = slog_scope::set_global_logger(log);

    debug!("Logging initialized");

    // initialize program
    let config = init()
        .unwrap_or_else(log_and_die!("initialization"));

    trace!("cli configuration - {:?}", config);

    // fetch the metadata from the configured provider
    let metadata = fetch_metadata(&config.provider)
        .unwrap_or_else(log_and_die!("fetching metadata from provider"));

    // write attributes if configured to do so
    config.attributes_file
        .map_or(Ok(()), |x| metadata.write_attributes(x))
        .unwrap_or_else(log_and_die!("writing metadata attributes"));

    // write ssh keys if configured to do so
    config.ssh_keys_user
        .map_or(Ok(()), |x| metadata.write_ssh_keys(x))
        .unwrap_or_else(log_and_die!("writing metadata attributes"));

    // write hostname if configured to do so
    config.hostname_file
        .map_or(Ok(()), |x| metadata.write_hostname(x))
        .unwrap_or_else(log_and_die!("writing metadata attributes"));

    // write network units if configured to do so
    config.network_units_dir
        .map_or(Ok(()), |x| metadata.write_network_units(x))
        .unwrap_or_else(log_and_die!("writing metadata attributes"));

    debug!("Done!")
}

fn init() -> Result<Config, String> {
    // setup cli
    let matches = App::new("coreos-metadata")
        .version(crate_version!())
        .arg(Arg::with_name("attributes")
             .long("attributes")
             .help("The file into which the metadata attributes are written")
             .takes_value(true))
        .arg(Arg::with_name("cmdline")
             .long("cmdline")
             .help("Read the cloud provider from the kernel cmdline"))
        .arg(Arg::with_name("hostname")
             .long("hostname")
             .help("The file into which the hostname should be written")
             .takes_value(true))
        .arg(Arg::with_name("network-units")
             .long("network-units")
             .help("The directory into which network units are written")
             .takes_value(true))
        .arg(Arg::with_name("provider")
             .long("provider")
             .help("The name of the cloud provider")
             .takes_value(true))
        .arg(Arg::with_name("ssh-keys")
             .long("ssh-keys")
             .help("Update SSH keys for the given user")
             .takes_value(true))
        .get_matches();

    // return configuration
    Ok(Config {
        provider: match matches.value_of("provider") {
            Some(provider) => String::from(provider),
            None => if matches.is_present("cmdline") {
                get_oem()?
            } else {
                return Err("Must set either --provider or --cmdline".to_string());
            }
        },
        attributes_file: matches.value_of("attributes").map(String::from),
        ssh_keys_user: matches.value_of("ssh-keys").map(String::from),
        hostname_file: matches.value_of("hostname").map(String::from),
        network_units_dir: matches.value_of("network-units").map(String::from),
    })
}

fn get_oem() -> Result<String, String> {
    // open the cmdline file
    let mut file = File::open(CMDLINE_PATH)
        .map_err(wrap_error!("Failed to open cmdline file ({})", CMDLINE_PATH))?;

    // read the contents
    let mut contents = String::new();
    file.read_to_string(&mut contents)
        .map_err(wrap_error!("Failed to read cmdline file ({})", CMDLINE_PATH))?;

    // split the contents into elements
    let params: Vec<Vec<&str>> = contents.split(' ')
        .map(|s| s.split('=').collect())
        .collect();

    // find the oem flag
    for p in params {
        if p.len() > 1 && p[0] == CMDLINE_OEM_FLAG {
            return Ok(String::from(p[1]));
        }
    }

    Err(format!("Couldn't find '{}' flag in cmdline file ({})", CMDLINE_OEM_FLAG, CMDLINE_PATH))
}
<commit_msg>bin: fixed top-level logging output<commit_after>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#[macro_use]
extern crate clap;
#[macro_use]
extern crate slog;
extern crate slog_term;
extern crate slog_async;
#[macro_use]
extern crate slog_scope;
extern crate users;

#[macro_use]
extern crate coreos_metadata;

use std::fs::File;
use std::io::prelude::*;
use clap::{Arg, App};
use slog::Drain;

use coreos_metadata::fetch_metadata;

const CMDLINE_PATH: &'static str = "/proc/cmdline";
const CMDLINE_OEM_FLAG:&'static str = "coreos.oem.id";

#[derive(Debug)]
struct Config {
    provider: String,
    attributes_file: Option<String>,
    ssh_keys_user: Option<String>,
    hostname_file: Option<String>,
    network_units_dir: Option<String>,
}

fn main() {
    // setup logging
    let decorator = slog_term::TermDecorator::new().build();
    let drain = slog_term::FullFormat::new(decorator).build().fuse();
    let drain = slog_async::Async::new(drain).build().fuse();
    let log = slog::Logger::root(drain, slog_o!());
    let _guard = slog_scope::set_global_logger(log);

    debug!("Logging initialized");

    // initialize program
    let config = init()
        .unwrap_or_else(log_and_die!("initialization"));

    trace!("cli configuration - {:?}", config);

    // fetch the metadata from the configured provider
    let metadata = fetch_metadata(&config.provider)
        .unwrap_or_else(log_and_die!("fetching metadata from provider"));

    // write attributes if configured to do so
    config.attributes_file
        .map_or(Ok(()), |x| metadata.write_attributes(x))
        .unwrap_or_else(log_and_die!("writing metadata attributes"));

    // write ssh keys if configured to do so
    config.ssh_keys_user
        .map_or(Ok(()), |x| metadata.write_ssh_keys(x))
        .unwrap_or_else(log_and_die!("writing ssh keys"));

    // write hostname if configured to do so
    config.hostname_file
        .map_or(Ok(()), |x| metadata.write_hostname(x))
        .unwrap_or_else(log_and_die!("writing hostname"));

    // write network units if configured to do so
    config.network_units_dir
        .map_or(Ok(()), |x| metadata.write_network_units(x))
        .unwrap_or_else(log_and_die!("writing network units"));

    debug!("Done!")
}

fn init() -> Result<Config, String> {
    // setup cli
    let matches = App::new("coreos-metadata")
        .version(crate_version!())
        .arg(Arg::with_name("attributes")
             .long("attributes")
             .help("The file into which the metadata attributes are written")
             .takes_value(true))
        .arg(Arg::with_name("cmdline")
             .long("cmdline")
             .help("Read the cloud provider from the kernel cmdline"))
        .arg(Arg::with_name("hostname")
             .long("hostname")
             .help("The file into which the hostname should be written")
             .takes_value(true))
        .arg(Arg::with_name("network-units")
             .long("network-units")
             .help("The directory into which network units are written")
             .takes_value(true))
        .arg(Arg::with_name("provider")
             .long("provider")
             .help("The name of the cloud provider")
             .takes_value(true))
        .arg(Arg::with_name("ssh-keys")
             .long("ssh-keys")
             .help("Update SSH keys for the given user")
             .takes_value(true))
        .get_matches();

    // return configuration
    Ok(Config {
        provider: match matches.value_of("provider") {
            Some(provider) => String::from(provider),
            None => if matches.is_present("cmdline") {
                get_oem()?
            } else {
                return Err("Must set either --provider or --cmdline".to_string());
            }
        },
        attributes_file: matches.value_of("attributes").map(String::from),
        ssh_keys_user: matches.value_of("ssh-keys").map(String::from),
        hostname_file: matches.value_of("hostname").map(String::from),
        network_units_dir: matches.value_of("network-units").map(String::from),
    })
}

fn get_oem() -> Result<String, String> {
    // open the cmdline file
    let mut file = File::open(CMDLINE_PATH)
        .map_err(wrap_error!("Failed to open cmdline file ({})", CMDLINE_PATH))?;

    // read the contents
    let mut contents = String::new();
    file.read_to_string(&mut contents)
        .map_err(wrap_error!("Failed to read cmdline file ({})", CMDLINE_PATH))?;

    // split the contents into elements
    let params: Vec<Vec<&str>> = contents.split(' ')
        .map(|s| s.split('=').collect())
        .collect();

    // find the oem flag
    for p in params {
        if p.len() > 1 && p[0] == CMDLINE_OEM_FLAG {
            return Ok(String::from(p[1]));
        }
    }

    Err(format!("Couldn't find '{}' flag in cmdline file ({})", CMDLINE_OEM_FLAG, CMDLINE_PATH))
}
<|endoftext|>


<commit_before>use itertools::{multipeek, MultiPeek};
use std::str;

#[derive(Debug, PartialEq)]
pub enum Token {
    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    Comma,
    Dot,
    Minus,
    Plus,
    Semicolon,
    Slash,
    Star,
    Bang,
    BangEqual,
    Equal,
    EqualEqual,
    Greater,
    GreaterEqual,
    Less,
    LessEqual,
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(f64),
    // Keywords
    And,
    Class,
    Else,
    False,
    Fun,
    For,
    If,
    Nil,
    Or,
    Print,
    Return,
    Super,
    This,
    True,
    Var,
    While,

    Eof,
    // The book doesn't have tokens for comments and
    // whitespaces. Introducing them the scanner can
    // deal with them uniformly and in a more
    // functional way.
    Comment,
    Whitespace,
}

#[derive(Debug,Clone,Copy)]
/// Represents a position in the source file.
/// Both line and column are represented by a 1-based
/// index, since this struct exists only to provide
/// context to the user.
struct Position {
    line: usize,
    column: usize,
}

impl Position {
    fn initial() -> Position {
        Position{line:1, column: 1}
    }

    fn increment_column(&mut self) -> () {
        self.column += 1;
    }

    fn increment_line(&mut self) -> () {
        self.line += 1;
        self.column = 1;
    }
}

#[derive(Debug)]
pub struct TokenWithContext {
    token: Token,
    // Takes a copy. Tokens can outlive the file they came from
    lexeme: String,
    /// Position of the first character of the token
    position: Position,
}

struct Scanner<'a> {
    current_position: Position,
    current_lexeme: String,
    source: MultiPeek<str::Chars<'a>>,
}

fn is_digit(c: char) -> bool {
    c >= '0' && c <= '9'
}

fn is_alpha(c: char) -> bool {
    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}

fn is_alphanumeric(c: char) -> bool {
    is_digit(c) || is_alpha(c)
}

fn is_whitespace(c: char) -> bool {
    match c {
        ' ' => true,
        '\r' => true,
        '\t' => true,
        '\n' => true,
        _ => false,
    }
}

impl<'a> Scanner<'a> {
    fn initialize(source: &String) -> Scanner {
        Scanner {
            current_position: Position::initial(),
            current_lexeme: "".into(),
            source: multipeek(source.chars()),
        }
    }

    fn peek_check(&mut self, check: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => check(c),
            None => false,
        }
    }

    fn peek_check2(&mut self, check1: &Fn(char) -> bool, check2: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&p1) => {
                match self.source.peek() {
                    Some(&p2) => check1(p1) && check2(p2),
                    None => false,
                }
            }
            None => false,
        }
    }

    fn advance(&mut self) -> Option<char> {
        let next = self.source.next();
        match next {
            Some(c) => {
                self.current_lexeme.push(c);
                if c == '\n' {
                    self.current_position.increment_line();
                }
                else{
                    self.current_position.increment_column();;
                }
            }
            None => (),
        }
        next
    }

    fn advance_if_match(&mut self, expected: char) -> bool {
        if self.peek_check(&|c| c == expected) {
            let _ = self.advance();
            true
        } else {
            false
        }
    }

    fn add_context(&mut self, token: Token, initial_position: Position) -> TokenWithContext {
        let result = TokenWithContext {
            token: token,
            lexeme: self.current_lexeme.clone(),
            position: initial_position,
        };
        self.current_lexeme = "".into();
        result
    }

    fn string(&mut self) -> Result<Token, String> {
        let initial_line = self.current_position.line;
        self.source.reset_peek();
        while self.peek_check(&|c| c != '"') {
            self.advance();
        }
        if !self.advance_if_match('"') {
            return Err(format!("Unterminated string at line {}", initial_line));
        }
        let literal_length = self.current_lexeme.len() - 2;
        // Trims delimiters
        let literal = self.current_lexeme.chars().skip(1).take(literal_length).collect();
        Ok(Token::StringLiteral(literal))
    }

    fn number(&mut self) -> Token {
        while self.peek_check(&is_digit) {
            self.advance();
        }
        if self.peek_check2(&|p1| p1 == '.', &is_digit) {
            self.advance(); // Consume the .
            while self.peek_check(&is_digit) {
                self.advance();
            }
        }
        let value = self.current_lexeme.parse::<f64>().unwrap();
        Token::NumberLiteral(value)
    }

    fn identifier(&mut self) -> Token {
        while self.peek_check(&is_alphanumeric) {
            self.advance();
        }
        match self.current_lexeme.as_ref() {
            "and" => Token::And,
            "class" => Token::Class,
            "else" => Token::Else,
            "false" => Token::False,
            "for" => Token::For,
            "fun" => Token::Fun,
            "if" => Token::If,
            "nil" => Token::Nil,
            "or" => Token::Or,
            "print" => Token::Print,
            "return" => Token::Return,
            "super" => Token::Super,
            "this" => Token::This,
            "true" => Token::True,
            "var" => Token::Var,
            "while" => Token::While,
            identifier => Token::Identifier(identifier.into()),
        }

    }

    fn scan_next(&mut self) -> Result<TokenWithContext, String> {
        let initial_position = self.current_position;
        // Check if there is something left to iterate on.
        // Early return if we're done.
        let next_char = match self.advance() {
            Some(c) => c,
            None => {
                return Ok(TokenWithContext {
                    token: Token::Eof,
                    // Not very meaningful. Is Eof needed at all?
                    lexeme: "".into(),
                    position: initial_position
                });
            }
        };
        let token = match next_char {
            '(' => Token::LeftParen,
            ')' => Token::RightParen,
            '{' => Token::LeftBrace,
            '}' => Token::RightBrace,
            ',' => Token::Comma,
            '.' => Token::Dot,
            '-' => Token::Minus,
            '+' => Token::Plus,
            ';' => Token::Semicolon,
            '*' => Token::Star,
            '!' => {
                if self.advance_if_match('=') {
                    Token::BangEqual
                } else {
                    Token::Bang
                }
            }
            '=' => {
                if self.advance_if_match('=') {
                    Token::EqualEqual
                } else {
                    Token::Equal
                }
            }
            '<' => {
                if self.advance_if_match('=') {
                    Token::LessEqual
                } else {
                    Token::Less
                }
            }
            '>' => {
                if self.advance_if_match('=') {
                    Token::GreaterEqual
                } else {
                    Token::Greater
                }
            }
            '/' => {
                if self.advance_if_match('/') {
                    // Comments go on till the end of the line
                    while self.peek_check(&|c| c != '\n') {
                        self.advance();
                    }
                    Token::Comment
                } else {
                    Token::Slash
                }
            }
            '"' => try!(self.string()),
            c if is_whitespace(c) => Token::Whitespace,
            c if is_digit(c) => self.number(),
            c if is_alpha(c) => self.identifier(),
            c => {
                return Err(format!("Unexpected character {} at line {}, column {}",
                                   c,
                                   self.current_position.line,
                                   self.current_position.column - 1));
            }
        };
        Ok(self.add_context(token, initial_position))
    }
}

pub fn scan(source: &String) -> Result<Vec<TokenWithContext>, String> {
    let mut scanner = Scanner::initialize(source);
    let mut tokens = Vec::new();
    let mut eof = false;
    while !eof {
        let token_with_context = try!(scanner.scan_next());
        match token_with_context.token {
            // Ignoring tokens we don't care about
            Token::Comment => {}
            Token::Whitespace => {}
            Token::Eof => {
                eof = true;
                tokens.push(token_with_context);
            }
            _ => tokens.push(token_with_context),
        }
    }
    Ok(tokens)
}

#[cfg(test)]
mod tests {
    use scanner::*;

    #[test]
    fn single_token() {
        let tokens = scan(&"+".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Plus);
    }

    #[test]
    fn expression() {
        let tokens = scan(&"1+2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(tokens[3].token, Token::Eof);
    }

    #[test]
    fn expression_with_whitespaces() {
        let tokens = scan(&"1 + 2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(tokens[3].token, Token::Eof);
    }

    #[test]
    fn assignement_with_comment() {
        let tokens = scan(&"var a = 1.0; // A comment".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Eof);
    }

    #[test]
    fn multiline_statements() {
        let tokens = scan(&r#"var a = 1.0;
                              var b = "Hello";"#
                .into())
            .unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Var);
        assert_eq!(tokens[6].token, Token::Identifier("b".into()));
        assert_eq!(tokens[7].token, Token::Equal);
        assert_eq!(tokens[8].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[9].token, Token::Semicolon);
        assert_eq!(tokens[10].token, Token::Eof);
        assert_eq!(tokens[1].position.line, 1);
        assert_eq!(tokens[1].position.column, 5);
        assert_eq!(tokens[9].position.line, 2);
        assert_eq!(tokens[9].position.column, 46); // There are 30 spaces before the text!
    }
}
<commit_msg>cargo fmt<commit_after>use itertools::{multipeek, MultiPeek};
use std::str;

#[derive(Debug, PartialEq)]
pub enum Token {
    LeftParen,
    RightParen,
    LeftBrace,
    RightBrace,
    Comma,
    Dot,
    Minus,
    Plus,
    Semicolon,
    Slash,
    Star,
    Bang,
    BangEqual,
    Equal,
    EqualEqual,
    Greater,
    GreaterEqual,
    Less,
    LessEqual,
    Identifier(String),
    StringLiteral(String),
    NumberLiteral(f64),
    // Keywords
    And,
    Class,
    Else,
    False,
    Fun,
    For,
    If,
    Nil,
    Or,
    Print,
    Return,
    Super,
    This,
    True,
    Var,
    While,

    Eof,
    // The book doesn't have tokens for comments and
    // whitespaces. Introducing them the scanner can
    // deal with them uniformly and in a more
    // functional way.
    Comment,
    Whitespace,
}

#[derive(Debug,Clone,Copy)]
/// Represents a position in the source file.
/// Both line and column are represented by a 1-based
/// index, since this struct exists only to provide
/// context to the user.
struct Position {
    line: usize,
    column: usize,
}

impl Position {
    fn initial() -> Position {
        Position {
            line: 1,
            column: 1,
        }
    }

    fn increment_column(&mut self) -> () {
        self.column += 1;
    }

    fn increment_line(&mut self) -> () {
        self.line += 1;
        self.column = 1;
    }
}

#[derive(Debug)]
pub struct TokenWithContext {
    token: Token,
    // Takes a copy. Tokens can outlive the file they came from
    lexeme: String,
    /// Position of the first character of the token
    position: Position,
}

struct Scanner<'a> {
    current_position: Position,
    current_lexeme: String,
    source: MultiPeek<str::Chars<'a>>,
}

fn is_digit(c: char) -> bool {
    c >= '0' && c <= '9'
}

fn is_alpha(c: char) -> bool {
    (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}

fn is_alphanumeric(c: char) -> bool {
    is_digit(c) || is_alpha(c)
}

fn is_whitespace(c: char) -> bool {
    match c {
        ' ' => true,
        '\r' => true,
        '\t' => true,
        '\n' => true,
        _ => false,
    }
}

impl<'a> Scanner<'a> {
    fn initialize(source: &String) -> Scanner {
        Scanner {
            current_position: Position::initial(),
            current_lexeme: "".into(),
            source: multipeek(source.chars()),
        }
    }

    fn peek_check(&mut self, check: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&c) => check(c),
            None => false,
        }
    }

    fn peek_check2(&mut self, check1: &Fn(char) -> bool, check2: &Fn(char) -> bool) -> bool {
        self.source.reset_peek();
        match self.source.peek() {
            Some(&p1) => {
                match self.source.peek() {
                    Some(&p2) => check1(p1) && check2(p2),
                    None => false,
                }
            }
            None => false,
        }
    }

    fn advance(&mut self) -> Option<char> {
        let next = self.source.next();
        match next {
            Some(c) => {
                self.current_lexeme.push(c);
                if c == '\n' {
                    self.current_position.increment_line();
                } else {
                    self.current_position.increment_column();;
                }
            }
            None => (),
        }
        next
    }

    fn advance_if_match(&mut self, expected: char) -> bool {
        if self.peek_check(&|c| c == expected) {
            let _ = self.advance();
            true
        } else {
            false
        }
    }

    fn add_context(&mut self, token: Token, initial_position: Position) -> TokenWithContext {
        let result = TokenWithContext {
            token: token,
            lexeme: self.current_lexeme.clone(),
            position: initial_position,
        };
        self.current_lexeme = "".into();
        result
    }

    fn string(&mut self) -> Result<Token, String> {
        let initial_line = self.current_position.line;
        self.source.reset_peek();
        while self.peek_check(&|c| c != '"') {
            self.advance();
        }
        if !self.advance_if_match('"') {
            return Err(format!("Unterminated string at line {}", initial_line));
        }
        let literal_length = self.current_lexeme.len() - 2;
        // Trims delimiters
        let literal = self.current_lexeme.chars().skip(1).take(literal_length).collect();
        Ok(Token::StringLiteral(literal))
    }

    fn number(&mut self) -> Token {
        while self.peek_check(&is_digit) {
            self.advance();
        }
        if self.peek_check2(&|p1| p1 == '.', &is_digit) {
            self.advance(); // Consume the .
            while self.peek_check(&is_digit) {
                self.advance();
            }
        }
        let value = self.current_lexeme.parse::<f64>().unwrap();
        Token::NumberLiteral(value)
    }

    fn identifier(&mut self) -> Token {
        while self.peek_check(&is_alphanumeric) {
            self.advance();
        }
        match self.current_lexeme.as_ref() {
            "and" => Token::And,
            "class" => Token::Class,
            "else" => Token::Else,
            "false" => Token::False,
            "for" => Token::For,
            "fun" => Token::Fun,
            "if" => Token::If,
            "nil" => Token::Nil,
            "or" => Token::Or,
            "print" => Token::Print,
            "return" => Token::Return,
            "super" => Token::Super,
            "this" => Token::This,
            "true" => Token::True,
            "var" => Token::Var,
            "while" => Token::While,
            identifier => Token::Identifier(identifier.into()),
        }

    }

    fn scan_next(&mut self) -> Result<TokenWithContext, String> {
        let initial_position = self.current_position;
        // Check if there is something left to iterate on.
        // Early return if we're done.
        let next_char = match self.advance() {
            Some(c) => c,
            None => {
                return Ok(TokenWithContext {
                    token: Token::Eof,
                    // Not very meaningful. Is Eof needed at all?
                    lexeme: "".into(),
                    position: initial_position,
                });
            }
        };
        let token = match next_char {
            '(' => Token::LeftParen,
            ')' => Token::RightParen,
            '{' => Token::LeftBrace,
            '}' => Token::RightBrace,
            ',' => Token::Comma,
            '.' => Token::Dot,
            '-' => Token::Minus,
            '+' => Token::Plus,
            ';' => Token::Semicolon,
            '*' => Token::Star,
            '!' => {
                if self.advance_if_match('=') {
                    Token::BangEqual
                } else {
                    Token::Bang
                }
            }
            '=' => {
                if self.advance_if_match('=') {
                    Token::EqualEqual
                } else {
                    Token::Equal
                }
            }
            '<' => {
                if self.advance_if_match('=') {
                    Token::LessEqual
                } else {
                    Token::Less
                }
            }
            '>' => {
                if self.advance_if_match('=') {
                    Token::GreaterEqual
                } else {
                    Token::Greater
                }
            }
            '/' => {
                if self.advance_if_match('/') {
                    // Comments go on till the end of the line
                    while self.peek_check(&|c| c != '\n') {
                        self.advance();
                    }
                    Token::Comment
                } else {
                    Token::Slash
                }
            }
            '"' => try!(self.string()),
            c if is_whitespace(c) => Token::Whitespace,
            c if is_digit(c) => self.number(),
            c if is_alpha(c) => self.identifier(),
            c => {
                return Err(format!("Unexpected character {} at line {}, column {}",
                                   c,
                                   self.current_position.line,
                                   self.current_position.column - 1));
            }
        };
        Ok(self.add_context(token, initial_position))
    }
}

pub fn scan(source: &String) -> Result<Vec<TokenWithContext>, String> {
    let mut scanner = Scanner::initialize(source);
    let mut tokens = Vec::new();
    let mut eof = false;
    while !eof {
        let token_with_context = try!(scanner.scan_next());
        match token_with_context.token {
            // Ignoring tokens we don't care about
            Token::Comment => {}
            Token::Whitespace => {}
            Token::Eof => {
                eof = true;
                tokens.push(token_with_context);
            }
            _ => tokens.push(token_with_context),
        }
    }
    Ok(tokens)
}

#[cfg(test)]
mod tests {
    use scanner::*;

    #[test]
    fn single_token() {
        let tokens = scan(&"+".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Plus);
    }

    #[test]
    fn expression() {
        let tokens = scan(&"1+2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(tokens[3].token, Token::Eof);
    }

    #[test]
    fn expression_with_whitespaces() {
        let tokens = scan(&"1 + 2".into()).unwrap();
        assert_eq!(tokens[0].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[1].token, Token::Plus);
        assert_eq!(tokens[2].token, Token::NumberLiteral(2.0f64));
        assert_eq!(tokens[3].token, Token::Eof);
    }

    #[test]
    fn assignement_with_comment() {
        let tokens = scan(&"var a = 1.0; // A comment".into()).unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Eof);
    }

    #[test]
    fn multiline_statements() {
        let tokens = scan(&r#"var a = 1.0;
                              var b = "Hello";"#
                .into())
            .unwrap();
        assert_eq!(tokens[0].token, Token::Var);
        assert_eq!(tokens[1].token, Token::Identifier("a".into()));
        assert_eq!(tokens[2].token, Token::Equal);
        assert_eq!(tokens[3].token, Token::NumberLiteral(1.0f64));
        assert_eq!(tokens[4].token, Token::Semicolon);
        assert_eq!(tokens[5].token, Token::Var);
        assert_eq!(tokens[6].token, Token::Identifier("b".into()));
        assert_eq!(tokens[7].token, Token::Equal);
        assert_eq!(tokens[8].token, Token::StringLiteral("Hello".into()));
        assert_eq!(tokens[9].token, Token::Semicolon);
        assert_eq!(tokens[10].token, Token::Eof);
        assert_eq!(tokens[1].position.line, 1);
        assert_eq!(tokens[1].position.column, 5);
        assert_eq!(tokens[9].position.line, 2);
        assert_eq!(tokens[9].position.column, 46); // There are 30 spaces before the text!
    }
}
<|endoftext|>