<commit_before>// Copyright (c) 2017-2019, The rav1e contributors. All rights reserved
//
// This source code is subject to the terms of the BSD 2 Clause License and
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
// was not distributed with this source code in the LICENSE file, you can
// obtain it at www.aomedia.org/license/software. If the Alliance for Open
// Media Patent License 1.0 was not distributed with this source code in the
// PATENTS file, you can obtain it at www.aomedia.org/license/patent.

use rustc_version::{version, Version};
#[allow(unused_imports)]
use std::env;
use std::fs;
use std::path::Path;
use std::process::exit;

#[allow(dead_code)]
fn rerun_dir<P: AsRef<Path>>(dir: P) {
  for entry in fs::read_dir(dir).unwrap() {
    let entry = entry.unwrap();
    let path = entry.path();
    println!("cargo:rerun-if-changed={}", path.to_string_lossy());

    if path.is_dir() {
      rerun_dir(path);
    }
  }
}

#[cfg(feature = "asm")]
fn build_nasm_files() {
  use std::fs::File;
  use std::io::Write;
  let out_dir = env::var("OUT_DIR").unwrap();
  {
    let dest_path = Path::new(&out_dir).join("config.asm");
    let mut config_file = File::create(dest_path).unwrap();
    config_file.write(b"	%define private_prefix rav1e\n").unwrap();
    config_file.write(b"	%define ARCH_X86_32 0\n").unwrap();
    config_file.write(b" %define ARCH_X86_64 1\n").unwrap();
    config_file.write(b"	%define PIC 1\n").unwrap();
    config_file.write(b" %define STACK_ALIGNMENT 16\n").unwrap();
    if cfg!(target_os = "macos") {
      config_file.write(b" %define PREFIX 1\n").unwrap();
    }
  }
  let mut config_include_arg = String::from("-I");
  config_include_arg.push_str(&out_dir);
  config_include_arg.push('/');
  nasm_rs::compile_library_args(
    "rav1easm",
    &[
      "src/x86/ipred.asm",
      "src/x86/ipred_ssse3.asm",
      "src/x86/itx.asm",
      "src/x86/itx_ssse3.asm",
      "src/x86/mc.asm",
      "src/x86/mc_ssse3.asm",
      "src/x86/me.asm",
      "src/x86/sad_sse2.asm",
      "src/x86/sad_avx.asm",
      "src/x86/satd.asm",
      "src/x86/cdef.asm",
      "src/x86/tables.asm",
    ],
    &[&config_include_arg, "-Isrc/"],
  );
  println!("cargo:rustc-link-lib=static=rav1easm");
  rerun_dir("src/x86");
  rerun_dir("src/ext/x86");
}

#[cfg(feature = "asm")]
fn build_asm_files() {
  use std::fs::File;
  use std::io::Write;
  let out_dir = env::var("OUT_DIR").unwrap();
  {
    let dest_path = Path::new(&out_dir).join("config.h");
    let mut config_file = File::create(dest_path).unwrap();
    config_file.write(b" #define PRIVATE_PREFIX rav1e_\n").unwrap();
    config_file.write(b" #define ARCH_AARCH64 1\n").unwrap();
    config_file.write(b" #define ARCH_ARM 0\n").unwrap();
    config_file.write(b" #define CONFIG_LOG 1 \n").unwrap();
    config_file.write(b" #define HAVE_ASM 1\n").unwrap();
  }
  cc::Build::new()
    .files(&[
      "src/arm/64/mc.S",
      "src/arm/64/itx.S",
      "src/arm/64/ipred.S",
      "src/arm/tables.S",
    ])
    .include(".")
    .include(&out_dir)
    .compile("rav1e-aarch64");
  rerun_dir("src/arm");
}

fn rustc_version_check() {
  // This should match the version in .travis.yml
  const REQUIRED_VERSION: &str = "1.36.0";
  if version().unwrap() < Version::parse(REQUIRED_VERSION).unwrap() {
    eprintln!("rav1e requires rustc >= {}.", REQUIRED_VERSION);
    exit(1);
  }
}

#[allow(unused_variables)]
fn main() {
  rustc_version_check();

  let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
  let os = env::var("CARGO_CFG_TARGET_OS").unwrap();
  // let env = env::var("CARGO_CFG_TARGET_ENV").unwrap();

  #[cfg(feature = "asm")]
  {
    if arch == "x86_64" {
      println!("cargo:rustc-cfg={}", "nasm_x86_64");
      build_nasm_files()
    }
    if arch == "aarch64" {
      println!("cargo:rustc-cfg={}", "asm_neon");
      build_asm_files()
    }
  }

  if os == "windows" && cfg!(feature = "decode_test") {
    panic!("Unsupported feature on this platform!");
  }

  vergen::generate_cargo_keys(vergen::ConstantsFlags::all())
    .expect("Unable to generate the cargo keys!");
}
<commit_msg>Remove clippy lints from build.rs<commit_after>// Copyright (c) 2017-2019, The rav1e contributors. All rights reserved
//
// This source code is subject to the terms of the BSD 2 Clause License and
// the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
// was not distributed with this source code in the LICENSE file, you can
// obtain it at www.aomedia.org/license/software. If the Alliance for Open
// Media Patent License 1.0 was not distributed with this source code in the
// PATENTS file, you can obtain it at www.aomedia.org/license/patent.

#![allow(clippy::print_literal)]
#![allow(clippy::unused_io_amount)]

use rustc_version::{version, Version};
#[allow(unused_imports)]
use std::env;
use std::fs;
use std::path::Path;
use std::process::exit;

#[allow(dead_code)]
fn rerun_dir<P: AsRef<Path>>(dir: P) {
  for entry in fs::read_dir(dir).unwrap() {
    let entry = entry.unwrap();
    let path = entry.path();
    println!("cargo:rerun-if-changed={}", path.to_string_lossy());

    if path.is_dir() {
      rerun_dir(path);
    }
  }
}

#[cfg(feature = "asm")]
fn build_nasm_files() {
  use std::fs::File;
  use std::io::Write;
  let out_dir = env::var("OUT_DIR").unwrap();
  {
    let dest_path = Path::new(&out_dir).join("config.asm");
    let mut config_file = File::create(dest_path).unwrap();
    config_file.write(b"	%define private_prefix rav1e\n").unwrap();
    config_file.write(b"	%define ARCH_X86_32 0\n").unwrap();
    config_file.write(b" %define ARCH_X86_64 1\n").unwrap();
    config_file.write(b"	%define PIC 1\n").unwrap();
    config_file.write(b" %define STACK_ALIGNMENT 16\n").unwrap();
    if cfg!(target_os = "macos") {
      config_file.write(b" %define PREFIX 1\n").unwrap();
    }
  }
  let mut config_include_arg = String::from("-I");
  config_include_arg.push_str(&out_dir);
  config_include_arg.push('/');
  nasm_rs::compile_library_args(
    "rav1easm",
    &[
      "src/x86/ipred.asm",
      "src/x86/ipred_ssse3.asm",
      "src/x86/itx.asm",
      "src/x86/itx_ssse3.asm",
      "src/x86/mc.asm",
      "src/x86/mc_ssse3.asm",
      "src/x86/me.asm",
      "src/x86/sad_sse2.asm",
      "src/x86/sad_avx.asm",
      "src/x86/satd.asm",
      "src/x86/cdef.asm",
      "src/x86/tables.asm",
    ],
    &[&config_include_arg, "-Isrc/"],
  );
  println!("cargo:rustc-link-lib=static=rav1easm");
  rerun_dir("src/x86");
  rerun_dir("src/ext/x86");
}

#[cfg(feature = "asm")]
fn build_asm_files() {
  use std::fs::File;
  use std::io::Write;
  let out_dir = env::var("OUT_DIR").unwrap();
  {
    let dest_path = Path::new(&out_dir).join("config.h");
    let mut config_file = File::create(dest_path).unwrap();
    config_file.write(b" #define PRIVATE_PREFIX rav1e_\n").unwrap();
    config_file.write(b" #define ARCH_AARCH64 1\n").unwrap();
    config_file.write(b" #define ARCH_ARM 0\n").unwrap();
    config_file.write(b" #define CONFIG_LOG 1 \n").unwrap();
    config_file.write(b" #define HAVE_ASM 1\n").unwrap();
  }
  cc::Build::new()
    .files(&[
      "src/arm/64/mc.S",
      "src/arm/64/itx.S",
      "src/arm/64/ipred.S",
      "src/arm/tables.S",
    ])
    .include(".")
    .include(&out_dir)
    .compile("rav1e-aarch64");
  rerun_dir("src/arm");
}

fn rustc_version_check() {
  // This should match the version in .travis.yml
  const REQUIRED_VERSION: &str = "1.36.0";
  if version().unwrap() < Version::parse(REQUIRED_VERSION).unwrap() {
    eprintln!("rav1e requires rustc >= {}.", REQUIRED_VERSION);
    exit(1);
  }
}

#[allow(unused_variables)]
fn main() {
  rustc_version_check();

  let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
  let os = env::var("CARGO_CFG_TARGET_OS").unwrap();
  // let env = env::var("CARGO_CFG_TARGET_ENV").unwrap();

  #[cfg(feature = "asm")]
  {
    if arch == "x86_64" {
      println!("cargo:rustc-cfg={}", "nasm_x86_64");
      build_nasm_files()
    }
    if arch == "aarch64" {
      println!("cargo:rustc-cfg={}", "asm_neon");
      build_asm_files()
    }
  }

  if os == "windows" && cfg!(feature = "decode_test") {
    panic!("Unsupported feature on this platform!");
  }

  vergen::generate_cargo_keys(vergen::ConstantsFlags::all())
    .expect("Unable to generate the cargo keys!");
}
<|endoftext|>


<commit_before>#![feature(path)]

use std::path::Path;
use std::process::Command;
use std::env;

#[cfg(target_os="macos")]
fn print_lib_dir() {
    println!("cargo:rustc-flags=-L /usr/local/opt/gettext/lib/");
}

#[cfg(not(target_os="macos"))]
fn print_lib_dir() {
}

fn main() {
    let curr_dir_str = env::var("CARGO_MANIFEST_DIR").unwrap();
    let curr_dir = Path::new(&curr_dir_str);
    Command::new("git").arg("submodule").arg("update").arg("--init")
        .current_dir(&curr_dir).status().unwrap();

    let nvim_dir = curr_dir.join("neovim");
    Command::new("make").arg("deps").current_dir(&nvim_dir).status().unwrap();
    Command::new("make").arg("libnvim").current_dir(&nvim_dir).status().unwrap();

    let nvim_lib_dir = nvim_dir.join("build").join("lib");
    let deps_lib_dir = nvim_dir.join(".deps").join("usr").join("lib");
    println!("cargo:rustc-flags=-L {} -L {} -l nvim:static",
        nvim_lib_dir.to_str().unwrap(),
        deps_lib_dir.to_str().unwrap());

    print_lib_dir();
}
<commit_msg>Remove feature flag<commit_after>use std::path::Path;
use std::process::Command;
use std::env;

#[cfg(target_os="macos")]
fn print_lib_dir() {
    println!("cargo:rustc-flags=-L /usr/local/opt/gettext/lib/");
}

#[cfg(not(target_os="macos"))]
fn print_lib_dir() {
}

fn main() {
    let curr_dir_str = env::var("CARGO_MANIFEST_DIR").unwrap();
    let curr_dir = Path::new(&curr_dir_str);
    Command::new("git").arg("submodule").arg("update").arg("--init")
        .current_dir(&curr_dir).status().unwrap();

    let nvim_dir = curr_dir.join("neovim");
    Command::new("make").arg("deps").current_dir(&nvim_dir).status().unwrap();
    Command::new("make").arg("libnvim").current_dir(&nvim_dir).status().unwrap();

    let nvim_lib_dir = nvim_dir.join("build").join("lib");
    let deps_lib_dir = nvim_dir.join(".deps").join("usr").join("lib");
    println!("cargo:rustc-flags=-L {} -L {} -l nvim:static",
        nvim_lib_dir.to_str().unwrap(),
        deps_lib_dir.to_str().unwrap());

    print_lib_dir();
}
<|endoftext|>


<commit_before>use ast::*;
use scanner::{Token, TokenWithContext};
use std::iter::Peekable;

/// This behave exactly as try! but wraps the returned result in a Some.
/// It's useful to remove some boilerplate in the code introduced by
/// the use of Option<Result<T, E>>
macro_rules! try_wrap_err {
    ($e:expr) => (match $e {Ok(e) => e, Err(e) => return Some(Err(e))})
}

#[derive(Debug)]
pub struct ParseError {
    message: String,
}

impl ParseError {
    fn from_message(message: String) -> ParseError {
        ParseError { message: message }
    }
}

pub fn parse(tokens: Vec<TokenWithContext>) -> Result<Expr, Vec<ParseError>> {
    let mut iter = tokens.iter().peekable();
    if let Some(result) = parse_expression(&mut iter) {
        match result {
            Ok(expr) => {
                if let None = iter.next() {
                    Ok(expr)
                } else {
                    Err(vec![ParseError::from_message("Parser didn't consume all the tokens"
                                 .into())])
                }
            }
            Err(error) => {
                {
                    synchronise(&mut iter);
                }
                Err(vec![error])
            }
        }

    } else {
        Err(vec![ParseError::from_message("Parser didn't return anything".into())])
    }
}

fn synchronise<'a, I>(tokens: &mut Peekable<I>)
    where I: Iterator<Item = &'a TokenWithContext>
{
    enum TokenKind{
        StartOfConstruct,
        BodyOfConstruct,
        EndOfConstruct,
    };
    fn classify(token: &Token) -> TokenKind{
        match token {
            &Token::Semicolon => TokenKind::EndOfConstruct,
            &Token::Class => TokenKind::StartOfConstruct,
            &Token::Fun => TokenKind::StartOfConstruct,
            &Token::Var => TokenKind::StartOfConstruct,
            &Token::For => TokenKind::StartOfConstruct,
            &Token::If => TokenKind::StartOfConstruct,
            &Token::While => TokenKind::StartOfConstruct,
            &Token::Print => TokenKind::StartOfConstruct,
            &Token::Return => TokenKind::StartOfConstruct,
            _ => TokenKind::BodyOfConstruct
    }
    }
    while let Some(token_kind) = tokens.peek().map(|t|classify(&t.token)) {
        match token_kind {
            TokenKind::StartOfConstruct => {
                break;
            },
            TokenKind::BodyOfConstruct =>{
                let _ = tokens.next();
            }
            TokenKind::EndOfConstruct => {
                let _ = tokens.next();
                break;
            },
        }
    }
}

fn parse_expression<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_equality(tokens)
}

fn parse_binary<'a, I>(tokens: &mut Peekable<I>,
                       map_operator: &Fn(&Token) -> Option<Operator>,
                       parse_subexpression: &Fn(&mut Peekable<I>)
                                                -> Option<Result<Expr, ParseError>>)
                       -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let mut expr;
    {
        let result = parse_subexpression(tokens);
        if let Some(Ok(subexpression)) =  result{
            expr = subexpression;
        } else {
            return result;
        }
    };
    while let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // We know it's right, we read it just for error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right;
        {
            if let Some(result) = parse_subexpression(tokens) {
                right = try_wrap_err!(result);
            } else {
                return Some(Err(ParseError::from_message(format!("Expected subexpression \
                                                                  after {:?} at {:?}",
                                                                 operator.lexeme,
                                                                 operator.position))));
            }
        };
        let binary_expression = BinaryExpr {
            left: expr,
            operator: mapped_operator,
            right: right,
        };
        expr = Expr::Binary(Box::new(binary_expression));
    }
    Some(Ok(expr))
}

fn parse_equality<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::BangEqual => Some(Operator::NotEqual),
            &Token::EqualEqual => Some(Operator::Equal),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_comparison)
}

fn parse_comparison<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Greater => Some(Operator::Greater),
            &Token::GreaterEqual => Some(Operator::GreaterEqual),
            &Token::Less => Some(Operator::Less),
            &Token::LessEqual => Some(Operator::LessEqual),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_term)
}

fn parse_term<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Minus => Some(Operator::Minus),
            &Token::Plus => Some(Operator::Plus),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_factor)
}

fn parse_factor<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Slash => Some(Operator::Slash),
            &Token::Star => Some(Operator::Star),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_unary)
}

fn parse_unary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Minus => Some(Operator::Minus),
            &Token::Bang => Some(Operator::Bang),
            _ => None,
        }
    }
    if let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // For error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right;
        {
            if let Some(result) = parse_unary(tokens) {
                right = try_wrap_err!(result);
            } else {
                return Some(Err(ParseError::from_message(format!("Expected subexpression \
                                                                  after {:?} at {:?}",
                                                                 operator.lexeme,
                                                                 operator.position))));
            }
        };
        let unary_expression = UnaryExpr {
            operator: mapped_operator,
            right: right,
        };
        return Some(Ok(Expr::Unary(Box::new(unary_expression))));
    } else {
        parse_primary(tokens)
    }
}

fn parse_primary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let primary_token;
    {
        primary_token = tokens.next();
    };
    if let Some(primary_token) = primary_token {
        let parsed_expression = match primary_token.token {
            Token::False => Expr::Literal(Literal::BoolLiteral(false)),
            Token::True => Expr::Literal(Literal::BoolLiteral(true)),
            Token::Nil => Expr::Literal(Literal::NilLiteral),
            Token::NumberLiteral(n) => Expr::Literal(Literal::NumberLiteral(n)),
            Token::StringLiteral(ref s) => Expr::Literal(Literal::StringLiteral(s.clone())),
            Token::LeftParen => {
                let expr;
                {
                    if let Some(result) = parse_expression(tokens) {
                        expr = try_wrap_err!(result);
                    } else {
                        return Some(Err(ParseError::from_message(format!("Unfinished grouping \
                                                                     expression near {:?} at \
                                                                     {:?}",
                                                                         primary_token.lexeme,
                                                                         primary_token.position))));
                    }
                };
                {
                    if let Some(token) = tokens.next() {
                        if token.token == Token::RightParen {
                            let grouping_expression = Grouping { expr: expr };
                            return Some(Ok(Expr::Grouping(Box::new(grouping_expression))));
                        } else {
                            return Some(Err(ParseError::from_message(format!("Missing ) near \
                                                                              {:?} at {:?}",
                                                                             token.lexeme,
                                                                             token.position))));
                        }

                    }
                    return Some(Err(ParseError::from_message(format!("Missing ) before end of \
                                                                      file"))));
                }
            }
            _ => {
                return Some(Err(ParseError::from_message(format!("Unexpected token {:?} at \
                                                                  {:?}",
                                                                 primary_token.lexeme,
                                                                 primary_token.position))));
            }
        };
        Some(Ok(parsed_expression))
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use scanner::*;
    use parser::*;
    use pretty_printer::PrettyPrint;

    #[test]
    fn literal() {
        let (tokens, _) = scan(&"123".into());
        let expr = parse(tokens).unwrap();
        assert_eq!("123", &expr.pretty_print());
    }

    #[test]
    fn binary() {
        let (tokens, _) = scan(&"123+456".into());
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ 123 456)", &expr.pretty_print());
    }

    #[test]
    fn precedence_add_mul() {
        let (tokens, _) = scan(&"123+456*789".into());
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ 123 (* 456 789))", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add() {
        let (tokens, _) = scan(&"123*456+789".into());
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ (* 123 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add_unary() {
        let (tokens, _) = scan(&"-123*456+789".into());
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ (* (- 123) 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn unclosed_group() {
        let (tokens, _) = scan(&"(2".into());
        assert!(parse(tokens).is_err());
    }

    #[test]
    fn unopened_group() {
        let (tokens, _) = scan(&"2)".into());
        assert!(parse(tokens).is_err());
    }
}
<commit_msg>cargo fmt<commit_after>use ast::*;
use scanner::{Token, TokenWithContext};
use std::iter::Peekable;

/// This behave exactly as try! but wraps the returned result in a Some.
/// It's useful to remove some boilerplate in the code introduced by
/// the use of Option<Result<T, E>>
macro_rules! try_wrap_err {
    ($e:expr) => (match $e {Ok(e) => e, Err(e) => return Some(Err(e))})
}

#[derive(Debug)]
pub struct ParseError {
    message: String,
}

impl ParseError {
    fn from_message(message: String) -> ParseError {
        ParseError { message: message }
    }
}

pub fn parse(tokens: Vec<TokenWithContext>) -> Result<Expr, Vec<ParseError>> {
    let mut iter = tokens.iter().peekable();
    if let Some(result) = parse_expression(&mut iter) {
        match result {
            Ok(expr) => {
                if let None = iter.next() {
                    Ok(expr)
                } else {
                    Err(vec![ParseError::from_message("Parser didn't consume all the tokens"
                                 .into())])
                }
            }
            Err(error) => {
                {
                    synchronise(&mut iter);
                }
                Err(vec![error])
            }
        }

    } else {
        Err(vec![ParseError::from_message("Parser didn't return anything".into())])
    }
}

fn synchronise<'a, I>(tokens: &mut Peekable<I>)
    where I: Iterator<Item = &'a TokenWithContext>
{
    enum TokenKind {
        StartOfConstruct,
        BodyOfConstruct,
        EndOfConstruct,
    };
    fn classify(token: &Token) -> TokenKind {
        match token {
            &Token::Semicolon => TokenKind::EndOfConstruct,
            &Token::Class => TokenKind::StartOfConstruct,
            &Token::Fun => TokenKind::StartOfConstruct,
            &Token::Var => TokenKind::StartOfConstruct,
            &Token::For => TokenKind::StartOfConstruct,
            &Token::If => TokenKind::StartOfConstruct,
            &Token::While => TokenKind::StartOfConstruct,
            &Token::Print => TokenKind::StartOfConstruct,
            &Token::Return => TokenKind::StartOfConstruct,
            _ => TokenKind::BodyOfConstruct,
        }
    }
    while let Some(token_kind) = tokens.peek().map(|t| classify(&t.token)) {
        match token_kind {
            TokenKind::StartOfConstruct => {
                break;
            }
            TokenKind::BodyOfConstruct => {
                let _ = tokens.next();
            }
            TokenKind::EndOfConstruct => {
                let _ = tokens.next();
                break;
            }
        }
    }
}

fn parse_expression<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    parse_equality(tokens)
}

fn parse_binary<'a, I>(tokens: &mut Peekable<I>,
                       map_operator: &Fn(&Token) -> Option<Operator>,
                       parse_subexpression: &Fn(&mut Peekable<I>)
                                                -> Option<Result<Expr, ParseError>>)
                       -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let mut expr;
    {
        let result = parse_subexpression(tokens);
        if let Some(Ok(subexpression)) = result {
            expr = subexpression;
        } else {
            return result;
        }
    };
    while let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // We know it's right, we read it just for error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right;
        {
            if let Some(result) = parse_subexpression(tokens) {
                right = try_wrap_err!(result);
            } else {
                return Some(Err(ParseError::from_message(format!("Expected subexpression \
                                                                  after {:?} at {:?}",
                                                                 operator.lexeme,
                                                                 operator.position))));
            }
        };
        let binary_expression = BinaryExpr {
            left: expr,
            operator: mapped_operator,
            right: right,
        };
        expr = Expr::Binary(Box::new(binary_expression));
    }
    Some(Ok(expr))
}

fn parse_equality<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::BangEqual => Some(Operator::NotEqual),
            &Token::EqualEqual => Some(Operator::Equal),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_comparison)
}

fn parse_comparison<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Greater => Some(Operator::Greater),
            &Token::GreaterEqual => Some(Operator::GreaterEqual),
            &Token::Less => Some(Operator::Less),
            &Token::LessEqual => Some(Operator::LessEqual),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_term)
}

fn parse_term<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Minus => Some(Operator::Minus),
            &Token::Plus => Some(Operator::Plus),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_factor)
}

fn parse_factor<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Slash => Some(Operator::Slash),
            &Token::Star => Some(Operator::Star),
            _ => None,
        }
    }
    parse_binary(tokens, &map_operator, &parse_unary)
}

fn parse_unary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    fn map_operator(token: &Token) -> Option<Operator> {
        match token {
            &Token::Minus => Some(Operator::Minus),
            &Token::Bang => Some(Operator::Bang),
            _ => None,
        }
    }
    if let Some(Some(mapped_operator)) = tokens.peek().map(|pt| map_operator(&pt.token)) {
        let operator; // For error reporting
        {
            operator = tokens.next().unwrap();
        }
        let right;
        {
            if let Some(result) = parse_unary(tokens) {
                right = try_wrap_err!(result);
            } else {
                return Some(Err(ParseError::from_message(format!("Expected subexpression \
                                                                  after {:?} at {:?}",
                                                                 operator.lexeme,
                                                                 operator.position))));
            }
        };
        let unary_expression = UnaryExpr {
            operator: mapped_operator,
            right: right,
        };
        return Some(Ok(Expr::Unary(Box::new(unary_expression))));
    } else {
        parse_primary(tokens)
    }
}

fn parse_primary<'a, I>(tokens: &mut Peekable<I>) -> Option<Result<Expr, ParseError>>
    where I: Iterator<Item = &'a TokenWithContext>
{
    let primary_token;
    {
        primary_token = tokens.next();
    };
    if let Some(primary_token) = primary_token {
        let parsed_expression = match primary_token.token {
            Token::False => Expr::Literal(Literal::BoolLiteral(false)),
            Token::True => Expr::Literal(Literal::BoolLiteral(true)),
            Token::Nil => Expr::Literal(Literal::NilLiteral),
            Token::NumberLiteral(n) => Expr::Literal(Literal::NumberLiteral(n)),
            Token::StringLiteral(ref s) => Expr::Literal(Literal::StringLiteral(s.clone())),
            Token::LeftParen => {
                let expr;
                {
                    if let Some(result) = parse_expression(tokens) {
                        expr = try_wrap_err!(result);
                    } else {
                        return Some(Err(ParseError::from_message(format!("Unfinished grouping \
                                                                     expression near {:?} at \
                                                                     {:?}",
                                                                         primary_token.lexeme,
                                                                         primary_token.position))));
                    }
                };
                {
                    if let Some(token) = tokens.next() {
                        if token.token == Token::RightParen {
                            let grouping_expression = Grouping { expr: expr };
                            return Some(Ok(Expr::Grouping(Box::new(grouping_expression))));
                        } else {
                            return Some(Err(ParseError::from_message(format!("Missing ) near \
                                                                              {:?} at {:?}",
                                                                             token.lexeme,
                                                                             token.position))));
                        }

                    }
                    return Some(Err(ParseError::from_message(format!("Missing ) before end of \
                                                                      file"))));
                }
            }
            _ => {
                return Some(Err(ParseError::from_message(format!("Unexpected token {:?} at \
                                                                  {:?}",
                                                                 primary_token.lexeme,
                                                                 primary_token.position))));
            }
        };
        Some(Ok(parsed_expression))
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use scanner::*;
    use parser::*;
    use pretty_printer::PrettyPrint;

    #[test]
    fn literal() {
        let (tokens, _) = scan(&"123".into());
        let expr = parse(tokens).unwrap();
        assert_eq!("123", &expr.pretty_print());
    }

    #[test]
    fn binary() {
        let (tokens, _) = scan(&"123+456".into());
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ 123 456)", &expr.pretty_print());
    }

    #[test]
    fn precedence_add_mul() {
        let (tokens, _) = scan(&"123+456*789".into());
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ 123 (* 456 789))", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add() {
        let (tokens, _) = scan(&"123*456+789".into());
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ (* 123 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn precedence_mul_add_unary() {
        let (tokens, _) = scan(&"-123*456+789".into());
        let expr = parse(tokens).unwrap();
        assert_eq!("(+ (* (- 123) 456) 789)", &expr.pretty_print());
    }

    #[test]
    fn unclosed_group() {
        let (tokens, _) = scan(&"(2".into());
        assert!(parse(tokens).is_err());
    }

    #[test]
    fn unopened_group() {
        let (tokens, _) = scan(&"2)".into());
        assert!(parse(tokens).is_err());
    }
}
<|endoftext|>


<commit_before>extern crate hoedown;

use std::fs::File;
use std::io::prelude::*;
use std::io::Result;
use std::path::Path;
use hoedown::Markdown;
use hoedown::renderer::html::Html;
use hoedown::renderer::html;
use hoedown::renderer::Render;
use hoedown::Buffer;

pub trait Parser {
    fn parse<'a>(&self, markdown_path: &Path) -> Result<Buffer>;

    fn new(path: &'static str) -> Self;

    #[allow(unused_must_use)]
    fn run<'a>(&self);

    fn create_out_file<'a>(&self, path: &Path, new_extension: &'a str) -> Result<File> {
        let mut path_buf = path.to_path_buf();
        path_buf.set_extension(new_extension);
        let file = try!(File::create(&path_buf.as_path()));
        return Ok(file);
    }

    fn path(&self) -> &'static str;
    fn new_extension(&self) -> &'static str;
}

pub struct MarkdownParser { markdown_path: &'static str, new_extension: &'static str}

impl MarkdownParser {}

impl Parser for MarkdownParser {
    fn new(path: &'static str) -> MarkdownParser {
        MarkdownParser { markdown_path: path, new_extension: "html" }
    }

    fn run<'a>(&self) {
        let markdown_path_str = self.path();
        let output_file_extension = self.new_extension();
        let markdown_path = Path::new(markdown_path_str);
        let markdown_buffer = self.parse(markdown_path).unwrap();
        let mut out_file = self.create_out_file(markdown_path, output_file_extension).unwrap();
        out_file.write(markdown_buffer.as_ref());
        ()
    }

    fn path(&self) -> &'static str {
        self.markdown_path
    }

    fn new_extension(&self) -> &'static str {
        self.new_extension
    }

    fn parse<'a>(&self, markdown_path: &Path) -> Result<Buffer> {
        let mut markdown_file = try!(File::open(&markdown_path));

        let mut input_markdown = String::new();
        try!(markdown_file.read_to_string(&mut input_markdown));
        let markdown_document = Markdown::new(&input_markdown);
        let mut html = Html::new(html::Flags::empty(), 0);

        return Ok(html.render(&markdown_document));
    }
}
<commit_msg>fixing minor unnecessary changes and removing warn<commit_after>use std::fs::File;
use std::io::prelude::*;
use std::io::Result;
use std::path::Path;
use hoedown::Markdown;
use hoedown::renderer::html::Html;
use hoedown::renderer::html;
use hoedown::renderer::Render;
use hoedown::Buffer;

pub trait Parser {
    fn parse<'a>(&self, markdown_path: &Path) -> Result<Buffer>;

    fn new(path: &'static str) -> Self;

    #[allow(unused_must_use)]
    fn run<'a>(&self);

    fn create_out_file<'a>(&self, path: &Path, new_extension: &'a str) -> Result<File> {
        let mut path_buf = path.to_path_buf();
        path_buf.set_extension(new_extension);
        let file = try!(File::create(&path_buf.as_path()));
        return Ok(file);
    }

    fn path(&self) -> &'static str;
    fn new_extension(&self) -> &'static str;
}

pub struct MarkdownParser { markdown_path: &'static str, new_extension: &'static str}

impl MarkdownParser {}

impl Parser for MarkdownParser {
    fn new(path: &'static str) -> MarkdownParser {
        MarkdownParser { markdown_path: path, new_extension: "html" }
    }

    #[allow(unused_must_use)]
    fn run<'a>(&self) {
        let markdown_path_str = self.path();
        let output_file_extension = self.new_extension();
        let markdown_path = Path::new(markdown_path_str);
        let markdown_buffer = self.parse(markdown_path).unwrap();
        let mut out_file = self.create_out_file(markdown_path, output_file_extension).unwrap();
        out_file.write(markdown_buffer.as_ref());
    }

    fn path(&self) -> &'static str {
        self.markdown_path
    }

    fn new_extension(&self) -> &'static str {
        self.new_extension
    }

    fn parse<'a>(&self, markdown_path: &Path) -> Result<Buffer> {
        let mut markdown_file = try!(File::open(&markdown_path));

        let mut input_markdown = String::new();
        try!(markdown_file.read_to_string(&mut input_markdown));
        let markdown_document = Markdown::new(&input_markdown);
        let mut html = Html::new(html::Flags::empty(), 0);

        return Ok(html.render(&markdown_document));
    }
}
<|endoftext|>


<commit_before>extern crate byteorder;

use std::error;
use std::fmt;
use std::io;
use std::str;

use self::byteorder::{ByteOrder, BigEndian, LittleEndian};

use super::{Catalog, Message};

/// Represents an error encountered while parsing an MO file.
#[derive(Debug)]
pub enum Error {
    /// An incorrect magic number has been encountered
    BadMagic,
    /// An invalid byte sequence for the given encoding has been encountered
    DecodingError,
    /// An unexpected EOF occured
    Eof,
    /// An I/O error occured
    Io(io::Error),
}
// Can not use use `Error::*` as per this issue:
// (https://github.com/rust-lang/rust/issues/4865)
use Error::{BadMagic, DecodingError, Eof, Io};

impl error::Error for Error {
    fn description(&self) -> &str {
        match *self {
            BadMagic => "bad magic number",
            DecodingError => "invalid byte sequence in a string",
            Eof => "unxpected end of file",
            Io(ref err) => err.description(),
        }
    }
}

impl fmt::Display for Error {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        let self_err: &error::Error = self;
        write!(fmt, "{}", self_err.description())
    }
}

impl From<io::Error> for Error {
    fn from(inner: io::Error) -> Error {
        Io(inner)
    }
}

/// According to the given magic number of a MO file,
/// returns the function which reads a `u32` in the relevant endianness.
fn get_read_u32_fn(magic: &[u8]) -> Option<fn(&[u8]) -> u32> {
    if magic == [0xde, 0x12, 0x04, 0x95] {
        Some(LittleEndian::read_u32)
    } else if magic == [0x95, 0x04, 0x12, 0xde] {
        Some(BigEndian::read_u32)
    } else {
        None
    }
}

fn from_utf8(bytes: &[u8]) -> Result<&str, Error> {
    str::from_utf8(bytes).map_err(|_| DecodingError)
}

pub fn parse_catalog<R: io::Read>(mut file: R) -> Result<Catalog, Error> {
    let mut contents = vec![];
    let n = try!(file.read_to_end(&mut contents));
    if n < 28 {
        return Err(Eof);
    }

    let read_u32 = match get_read_u32_fn(&contents[0..4]) {
        Some(f) => f,
        None => return Err(BadMagic),
    };

    // ignore hashing tables (bytes at 20..28)
    let num_strings = read_u32(&contents[8..12]) as usize;
    let mut off_otable = read_u32(&contents[12..16]) as usize;
    let mut off_ttable = read_u32(&contents[16..20]) as usize;
    if n < off_otable || n < off_ttable {
        return Err(Eof);
    }

    let mut catalog = Catalog::new();
    for _ in 0..num_strings {
        let id;
        let context;
        let translated: Vec<&str>;
        // Parse the original string
        {
            if n < off_otable + 8 {
                return Err(Eof);
            }
            let len = read_u32(&contents[off_otable..off_otable + 4]) as usize;
            let off = read_u32(&contents[off_otable + 4..off_otable + 8]) as usize;
            // +1 compensates for the ending NUL byte which is not included in length
            if n < off + len + 1 {
                return Err(Eof);
            }
            let mut original = &contents[off..off + len + 1];
            // check for context
            context = match original.iter().position(|x| *x == 4) {
                Some(idx) => {
                    let ctx = &original[..idx];
                    original = &original[idx + 1..];
                    Some(try!(from_utf8(ctx)))
                }
                None => None,
            };
            // extract msg_id singular, ignoring the plural
            id = match original.iter().position(|x| *x == 0).map(|i| &original[..i]) {
                Some(b) => try!(from_utf8(b)),
                None => return Err(Eof),
            };
        }

        // Parse the translation strings
        {
            if n < off_ttable + 8 {
                return Err(Eof);
            }
            let len = read_u32(&contents[off_ttable..off_ttable + 4]) as usize;
            let off = read_u32(&contents[off_ttable + 4..off_ttable + 8]) as usize;
            // +1 compensates for the ending NUL byte which is not included in length
            if n < off + len + 1 {
                return Err(Eof);
            }
            let decode_results = (&contents[off..off + len])
                                     .split(|x| *x == 0)
                                     .map(from_utf8)
                                     .collect::<Vec<_>>();
            if decode_results.iter().any(|r| r.is_err()) {
                return Err(DecodingError);
            }
            translated = decode_results.into_iter().map(|r| r.unwrap()).collect();
        }

        catalog.insert(Message::new(id, context, translated));

        off_otable += 8;
        off_ttable += 8;
    }

    Ok(catalog)
}

#[test]
fn test_get_read_u32_fn() {
    use std::mem;

    assert!(get_read_u32_fn(&[]).is_none());
    assert!(get_read_u32_fn(&[0xde, 0x12, 0x04, 0x95, 0x00]).is_none());

    {
        let le_ptr: *const ();
        let ret_ptr;
        unsafe {
            le_ptr = mem::transmute(LittleEndian::read_u32);
            ret_ptr = mem::transmute(get_read_u32_fn(&[0xde, 0x12, 0x04, 0x95]).unwrap());
        }
        assert_eq!(le_ptr, ret_ptr);
    }

    {
        let be_ptr: *const ();
        let ret_ptr;
        unsafe {
            be_ptr = mem::transmute(BigEndian::read_u32);
            ret_ptr = mem::transmute(get_read_u32_fn(&[0x95, 0x04, 0x12, 0xde]).unwrap());
        }
        assert_eq!(be_ptr, ret_ptr);
    }
}

#[test]
fn test_parse_catalog() {
    macro_rules! assert_variant {
        ($value:expr, $variant:path) => {
            match $value {
                $variant => (),
                _ => panic!("Expected {:?}, got {:?}", $variant, $value),
            }
        };
    }

    let fluff = [0; 24]; // zeros to pad our magic test cases to satisfy the length requirements

    {
        let mut reader = vec![1u8, 2, 3];
        reader.extend(fluff.iter().cloned());
        let err = parse_catalog(&reader[..]).unwrap_err();
        assert_variant!(err, Eof);
    }

    {
        let mut reader = vec![1u8, 2, 3, 4];
        reader.extend(fluff.iter().cloned());
        let err = parse_catalog(&reader[..]).unwrap_err();
        assert_variant!(err, BadMagic);
    }

    {
        let mut reader = vec![0x95, 0x04, 0x12, 0xde];
        reader.extend(fluff.iter().cloned());
        assert!(parse_catalog(&reader[..]).is_ok());
    }

    {
        let mut reader = vec![0xde, 0x12, 0x04, 0x95];
        reader.extend(fluff.iter().cloned());
        assert!(parse_catalog(&reader[..]).is_ok());
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/1.mo");
        let catalog = parse_catalog(reader).unwrap();
        assert_eq!(catalog.strings.len(), 1);
        assert_eq!(catalog.strings["this is context\x04Text"],
                   Message::new("Text", Some("this is context"), vec!["Tekstas", "Tekstai"]));
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/2.mo");
        let catalog = parse_catalog(reader).unwrap();
        assert_eq!(catalog.strings.len(), 2);
        assert_eq!(catalog.strings["Image"],
                   Message::new("Image", None, vec!["Nuotrauka", "Nuotraukos"]));
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/invalid_utf8.mo");
        let err = parse_catalog(reader).unwrap_err();
        assert_variant!(err, DecodingError);
    }
}
<commit_msg>Remove an unnecessary collect()<commit_after>extern crate byteorder;

use std::error;
use std::fmt;
use std::io;
use std::str;

use self::byteorder::{ByteOrder, BigEndian, LittleEndian};

use super::{Catalog, Message};

/// Represents an error encountered while parsing an MO file.
#[derive(Debug)]
pub enum Error {
    /// An incorrect magic number has been encountered
    BadMagic,
    /// An invalid byte sequence for the given encoding has been encountered
    DecodingError,
    /// An unexpected EOF occured
    Eof,
    /// An I/O error occured
    Io(io::Error),
}
// Can not use use `Error::*` as per this issue:
// (https://github.com/rust-lang/rust/issues/4865)
use Error::{BadMagic, DecodingError, Eof, Io};

impl error::Error for Error {
    fn description(&self) -> &str {
        match *self {
            BadMagic => "bad magic number",
            DecodingError => "invalid byte sequence in a string",
            Eof => "unxpected end of file",
            Io(ref err) => err.description(),
        }
    }
}

impl fmt::Display for Error {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        let self_err: &error::Error = self;
        write!(fmt, "{}", self_err.description())
    }
}

impl From<io::Error> for Error {
    fn from(inner: io::Error) -> Error {
        Io(inner)
    }
}

/// According to the given magic number of a MO file,
/// returns the function which reads a `u32` in the relevant endianness.
fn get_read_u32_fn(magic: &[u8]) -> Option<fn(&[u8]) -> u32> {
    if magic == [0xde, 0x12, 0x04, 0x95] {
        Some(LittleEndian::read_u32)
    } else if magic == [0x95, 0x04, 0x12, 0xde] {
        Some(BigEndian::read_u32)
    } else {
        None
    }
}

fn from_utf8(bytes: &[u8]) -> Result<&str, Error> {
    str::from_utf8(bytes).map_err(|_| DecodingError)
}

pub fn parse_catalog<R: io::Read>(mut file: R) -> Result<Catalog, Error> {
    let mut contents = vec![];
    let n = try!(file.read_to_end(&mut contents));
    if n < 28 {
        return Err(Eof);
    }

    let read_u32 = match get_read_u32_fn(&contents[0..4]) {
        Some(f) => f,
        None => return Err(BadMagic),
    };

    // ignore hashing tables (bytes at 20..28)
    let num_strings = read_u32(&contents[8..12]) as usize;
    let mut off_otable = read_u32(&contents[12..16]) as usize;
    let mut off_ttable = read_u32(&contents[16..20]) as usize;
    if n < off_otable || n < off_ttable {
        return Err(Eof);
    }

    let mut catalog = Catalog::new();
    for _ in 0..num_strings {
        let id;
        let context;
        let translated: Vec<&str>;
        // Parse the original string
        {
            if n < off_otable + 8 {
                return Err(Eof);
            }
            let len = read_u32(&contents[off_otable..off_otable + 4]) as usize;
            let off = read_u32(&contents[off_otable + 4..off_otable + 8]) as usize;
            // +1 compensates for the ending NUL byte which is not included in length
            if n < off + len + 1 {
                return Err(Eof);
            }
            let mut original = &contents[off..off + len + 1];
            // check for context
            context = match original.iter().position(|x| *x == 4) {
                Some(idx) => {
                    let ctx = &original[..idx];
                    original = &original[idx + 1..];
                    Some(try!(from_utf8(ctx)))
                }
                None => None,
            };
            // extract msg_id singular, ignoring the plural
            id = match original.iter().position(|x| *x == 0).map(|i| &original[..i]) {
                Some(b) => try!(from_utf8(b)),
                None => return Err(Eof),
            };
        }

        // Parse the translation strings
        {
            if n < off_ttable + 8 {
                return Err(Eof);
            }
            let len = read_u32(&contents[off_ttable..off_ttable + 4]) as usize;
            let off = read_u32(&contents[off_ttable + 4..off_ttable + 8]) as usize;
            // +1 compensates for the ending NUL byte which is not included in length
            if n < off + len + 1 {
                return Err(Eof);
            }
            translated = try!((&contents[off..off + len])
                                  .split(|x| *x == 0)
                                  .map(from_utf8)
                                  .collect::<Result<Vec<_>, _>>());
        }

        catalog.insert(Message::new(id, context, translated));

        off_otable += 8;
        off_ttable += 8;
    }

    Ok(catalog)
}

#[test]
fn test_get_read_u32_fn() {
    use std::mem;

    assert!(get_read_u32_fn(&[]).is_none());
    assert!(get_read_u32_fn(&[0xde, 0x12, 0x04, 0x95, 0x00]).is_none());

    {
        let le_ptr: *const ();
        let ret_ptr;
        unsafe {
            le_ptr = mem::transmute(LittleEndian::read_u32);
            ret_ptr = mem::transmute(get_read_u32_fn(&[0xde, 0x12, 0x04, 0x95]).unwrap());
        }
        assert_eq!(le_ptr, ret_ptr);
    }

    {
        let be_ptr: *const ();
        let ret_ptr;
        unsafe {
            be_ptr = mem::transmute(BigEndian::read_u32);
            ret_ptr = mem::transmute(get_read_u32_fn(&[0x95, 0x04, 0x12, 0xde]).unwrap());
        }
        assert_eq!(be_ptr, ret_ptr);
    }
}

#[test]
fn test_parse_catalog() {
    macro_rules! assert_variant {
        ($value:expr, $variant:path) => {
            match $value {
                $variant => (),
                _ => panic!("Expected {:?}, got {:?}", $variant, $value),
            }
        };
    }

    let fluff = [0; 24]; // zeros to pad our magic test cases to satisfy the length requirements

    {
        let mut reader = vec![1u8, 2, 3];
        reader.extend(fluff.iter().cloned());
        let err = parse_catalog(&reader[..]).unwrap_err();
        assert_variant!(err, Eof);
    }

    {
        let mut reader = vec![1u8, 2, 3, 4];
        reader.extend(fluff.iter().cloned());
        let err = parse_catalog(&reader[..]).unwrap_err();
        assert_variant!(err, BadMagic);
    }

    {
        let mut reader = vec![0x95, 0x04, 0x12, 0xde];
        reader.extend(fluff.iter().cloned());
        assert!(parse_catalog(&reader[..]).is_ok());
    }

    {
        let mut reader = vec![0xde, 0x12, 0x04, 0x95];
        reader.extend(fluff.iter().cloned());
        assert!(parse_catalog(&reader[..]).is_ok());
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/1.mo");
        let catalog = parse_catalog(reader).unwrap();
        assert_eq!(catalog.strings.len(), 1);
        assert_eq!(catalog.strings["this is context\x04Text"],
                   Message::new("Text", Some("this is context"), vec!["Tekstas", "Tekstai"]));
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/2.mo");
        let catalog = parse_catalog(reader).unwrap();
        assert_eq!(catalog.strings.len(), 2);
        assert_eq!(catalog.strings["Image"],
                   Message::new("Image", None, vec!["Nuotrauka", "Nuotraukos"]));
    }

    {
        let reader: &[u8] = include_bytes!("../test_cases/invalid_utf8.mo");
        let err = parse_catalog(reader).unwrap_err();
        assert_variant!(err, DecodingError);
    }
}
<|endoftext|>


<commit_before>use std::str::from_utf8;
use nom::IResult;

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     & 'a str,
    pub mode:     & 'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   & 'a str,
    pub typeflag: char, /* TODO: enum */
    pub linkname: & 'a str,
    pub ustar:    Option<UStarHeader<'a>>
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    & 'a str,
    pub version:  & 'a str,
    pub uname:    & 'a str,
    pub gname:    & 'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub prefix:   & 'a str,
}

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: & 'a str
}

/* TODO: validation */
fn str_to_u64(s: &str, base: u64) -> u64 {
    let mut u = 0;
    let mut f = 1;

    for c in s.chars().rev().skip_while(|&c| c == '\0') {
        u += f * ((c as u64) - ('0' as u64));
        f *= base;
    }

    u
}

pub fn octal_to_u64(o: &str) -> u64 {
    str_to_u64(o, 8)
}

fn parse_ustar(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        tag!("ustar\0")          ~
        version:  take_str!(2)   ~
        uname:    take_str!(32)  ~
        gname:    take_str!(32)  ~
        devmajor: take_str!(8)   ~
        devminor: take_str!(8)   ~
        prefix:   take_str!(155) ~
        take!(12), /* padding to 512 */
        ||{
            Some(UStarHeader {
                magic:    "ustar\0",
                version:  version,
                uname:    uname,
                gname:    gname,
                devmajor: octal_to_u64(devmajor),
                devminor: octal_to_u64(devminor),
                prefix:   prefix
            })
        }
    )
}

fn parse_posix(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        take!(255), /* padding to 512 */
        ||{
            None
        }
    )
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     take_str!(100) ~
        mode:     take_str!(8)   ~
        uid:      take_str!(8)   ~
        gid:      take_str!(8)   ~
        size:     take_str!(12)  ~
        mtime:    take_str!(12)  ~
        chksum:   take_str!(8)   ~
        typeflag: take!(1)       ~
        linkname: take_str!(100) ~
        ustar:    alt!(parse_ustar | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      octal_to_u64(uid),
                gid:      octal_to_u64(gid),
                size:     octal_to_u64(size),
                mtime:    octal_to_u64(mtime),
                chksum:   chksum,
                typeflag: typeflag[0] as char,
                linkname: linkname,
                ustar:    ustar
            }
        }
    )
}

fn parse_contents(i: &[u8], size: u64) -> IResult<&[u8], &str> {
    let trailing = size % 512;
    let padding = match trailing {
        0 => 0,
        t => 512 - t
    };
    chain!(i,
        contents: take_str!(size as usize) ~
        take!(padding as usize),
        ||{
            contents
        }
    )
}

fn parse_entry(i: &[u8]) -> IResult<&[u8], TarEntry> {
    chain!(i,
        header:   parse_header ~
        contents: apply!(parse_contents, header.size),
        ||{
            TarEntry {
                header: header,
                contents: contents
            }
        }
    )
}

pub fn parse_tar(i: &[u8]) -> IResult<&[u8], Vec<TarEntry>> {
    many0!(i, parse_entry)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn octal_to_u64_test() {
        match octal_to_u64("756") {
            494 => {},
            o => panic!("octal_to_u64 failed, expected 494 but got {}", o)
        }
        match octal_to_u64("") {
            0 => {},
            o => panic!("octal_to_u64 failed, expected 0 but got {}", o)
        }
    }
}
<commit_msg>ensure ustar version is "00"<commit_after>use std::str::from_utf8;
use nom::IResult;

#[derive(Debug,PartialEq,Eq)]
pub struct PosixHeader<'a> {
    pub name:     & 'a str,
    pub mode:     & 'a str,
    pub uid:      u64,
    pub gid:      u64,
    pub size:     u64,
    pub mtime:    u64,
    pub chksum:   & 'a str,
    pub typeflag: char, /* TODO: enum */
    pub linkname: & 'a str,
    pub ustar:    Option<UStarHeader<'a>>
}

#[derive(Debug,PartialEq,Eq)]
pub struct UStarHeader<'a> {
    pub magic:    & 'a str,
    pub version:  & 'a str,
    pub uname:    & 'a str,
    pub gname:    & 'a str,
    pub devmajor: u64,
    pub devminor: u64,
    pub prefix:   & 'a str,
}

#[derive(Debug,PartialEq,Eq)]
pub struct TarEntry<'a> {
    pub header:   PosixHeader<'a>,
    pub contents: & 'a str
}

/* TODO: validation */
fn str_to_u64(s: &str, base: u64) -> u64 {
    let mut u = 0;
    let mut f = 1;

    for c in s.chars().rev().skip_while(|&c| c == '\0') {
        u += f * ((c as u64) - ('0' as u64));
        f *= base;
    }

    u
}

pub fn octal_to_u64(o: &str) -> u64 {
    str_to_u64(o, 8)
}

fn parse_ustar00(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        tag!("00")               ~
        uname:    take_str!(32)  ~
        gname:    take_str!(32)  ~
        devmajor: take_str!(8)   ~
        devminor: take_str!(8)   ~
        prefix:   take_str!(155) ~
        take!(12), /* padding to 512 */
        ||{
            Some(UStarHeader {
                magic:    "ustar\0",
                version:  "00",
                uname:    uname,
                gname:    gname,
                devmajor: octal_to_u64(devmajor),
                devminor: octal_to_u64(devminor),
                prefix:   prefix
            })
        }
    )
}

fn parse_ustar(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        tag!("ustar\0") ~
        ustar: parse_ustar00,
        ||{
            ustar
        }
    )
}

fn parse_posix(i: &[u8]) -> IResult<&[u8], Option<UStarHeader>> {
    chain!(i,
        take!(255), /* padding to 512 */
        ||{
            None
        }
    )
}

fn parse_header(i: &[u8]) -> IResult<&[u8], PosixHeader> {
    chain!(i,
        name:     take_str!(100) ~
        mode:     take_str!(8)   ~
        uid:      take_str!(8)   ~
        gid:      take_str!(8)   ~
        size:     take_str!(12)  ~
        mtime:    take_str!(12)  ~
        chksum:   take_str!(8)   ~
        typeflag: take!(1)       ~
        linkname: take_str!(100) ~
        ustar:    alt!(parse_ustar | parse_posix),
        ||{
            PosixHeader {
                name:     name,
                mode:     mode,
                uid:      octal_to_u64(uid),
                gid:      octal_to_u64(gid),
                size:     octal_to_u64(size),
                mtime:    octal_to_u64(mtime),
                chksum:   chksum,
                typeflag: typeflag[0] as char,
                linkname: linkname,
                ustar:    ustar
            }
        }
    )
}

fn parse_contents(i: &[u8], size: u64) -> IResult<&[u8], &str> {
    let trailing = size % 512;
    let padding = match trailing {
        0 => 0,
        t => 512 - t
    };
    chain!(i,
        contents: take_str!(size as usize) ~
        take!(padding as usize),
        ||{
            contents
        }
    )
}

fn parse_entry(i: &[u8]) -> IResult<&[u8], TarEntry> {
    chain!(i,
        header:   parse_header ~
        contents: apply!(parse_contents, header.size),
        ||{
            TarEntry {
                header: header,
                contents: contents
            }
        }
    )
}

pub fn parse_tar(i: &[u8]) -> IResult<&[u8], Vec<TarEntry>> {
    many0!(i, parse_entry)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn octal_to_u64_test() {
        match octal_to_u64("756") {
            494 => {},
            o => panic!("octal_to_u64 failed, expected 494 but got {}", o)
        }
        match octal_to_u64("") {
            0 => {},
            o => panic!("octal_to_u64 failed, expected 0 but got {}", o)
        }
    }
}
<|endoftext|>


<commit_before>//! The `parser` module contains structures and functions for parsing the VQA
//! (Vector Quantized Animation) format.

use nom::{be_u32, le_u8, le_u16, le_u32};

#[derive(Debug)]
pub struct FormChunk {
    pub size: u32
}

named!(pub form_chunk<FormChunk>,
    do_parse!(
        tag!("FORM") >>
        size: be_u32 >>
        (FormChunk {
            size: size
        })
    )
);

#[derive(Debug)]
pub enum VQAVersion {
    One,
    Two,
    Three
}

named!(pub vqa_version<VQAVersion>,
    switch!(le_u16,
        1 => value!(VQAVersion::One) |
        2 => value!(VQAVersion::Two) |
        3 => value!(VQAVersion::Three)
    )
);

bitflags! {
    pub struct VQAFlags: u16 {
        const HAS_SOUND = 0b00000001;
    }
}

#[derive(Debug)]
pub struct VQAHeader {
    pub version       : VQAVersion, // VQA version number
    pub flags         : VQAFlags,   // VQA flags
    pub num_frames    : u16,        // Number of frames
    pub width         : u16,        // Movie width (pixels)
    pub height        : u16,        // Movie height (pixels)
    pub block_width   : u8,         // Width of each image block (pixels)
    pub block_height  : u8,         // Height of each image block (pixels)
    pub frame_rate    : u8,         // Frame rate of the VQA
    pub cbparts       : u8,         // How many images use the same lookup table
    pub colors        : u16,        // Max number of colors used in VQA
    pub maxblocks     : u16,        // Max number of image blocks
    pub unk1          : u32,        // Always 0?
    pub unk2          : u16,        // Some kind of size?
    pub freq          : u16,        // Sound sampling frequency
    pub channels      : u8,         // Number of sound channels
    pub bits          : u8,         // Sound resolution
    pub unk3          : u32,        // Always 0?
    pub unk4          : u16,        // 0 in old VQAs, 4 in HiColor VQAs?
    pub max_cbfz_size : u32,        // 0 in old VQAs, CBFZ size in HiColor
    pub unk5          : u32,        // Always 0?
}

named!(pub vqa_header<VQAHeader>,
    do_parse!(
        tag!("VQHD")              >>
        tag!(b"\x00\x00\x00\x2a") >> // VQAHeader is always 42 bytes long
        version: vqa_version      >>
        flags: le_u16             >>
        num_frames: le_u16        >>
        width: le_u16             >>
        height: le_u16            >>
        block_width: le_u8        >>
        block_height: le_u8       >>
        frame_rate: le_u8         >>
        cbparts: le_u8            >>
        colors: le_u16            >>
        maxblocks: le_u16         >>
        unk1: le_u32              >>
        unk2: le_u16              >>
        freq: le_u16              >>
        channels: le_u8           >>
        bits: le_u8               >>
        unk3: le_u32              >>
        unk4: le_u16              >>
        max_cbfz_size: le_u32     >>
        unk5: le_u32              >>
        (VQAHeader {
            version: version,
            flags: VQAFlags::from_bits_truncate(flags),
            num_frames: num_frames,
            width: width,
            height: height,
            block_width: block_width,
            block_height: block_height,
            frame_rate: frame_rate,
            cbparts: cbparts,
            colors: colors,
            maxblocks: maxblocks,
            unk1: unk1,
            unk2: unk2,
            freq: freq,
            channels: channels,
            bits: bits,
            unk3: unk3,
            unk4: unk4,
            max_cbfz_size: max_cbfz_size,
            unk5: unk5
        })
    )
);

#[derive(Debug)]
pub struct FINFChunk {
    pub size: u32,
    pub offsets: Vec<u32>
}

named!(pub finf_chunk<FINFChunk>,
    do_parse!(
        tag!("FINF") >>
        size: be_u32 >>
        offsets: many_m_n!(size as usize / 4, size as usize / 4, le_u32) >>
        (FINFChunk {
            size: size,
            offsets: offsets
        })
    )
);

#[derive(Debug)]
pub struct SND2Chunk<'a> {
    pub size: u32,
    pub data: &'a [u8]
}

named!(pub snd2_chunk<SND2Chunk>,
    do_parse!(
        tag!("SND2")       >>
        size: be_u32       >>
        data: take!(size)  >>
        (SND2Chunk {
            size: size,
            data: data
        })
    )
);

#[derive(Debug)]
pub struct VQFRChunk<'a> {
    pub size: u32,
    pub data: &'a [u8]
}

named!(pub vqfr_chunk<VQFRChunk>,
    do_parse!(
        tag!("VQFR")      >>
        size: be_u32      >>
        data: take!(size) >>
        (VQFRChunk {
            size: size,
            data: data
        })
    )
);
<commit_msg>parse cbf chunks<commit_after>//! The `parser` module contains structures and functions for parsing the VQA
//! (Vector Quantized Animation) format.

use nom::{be_u32, le_u8, le_u16, le_u32};

#[derive(Debug)]
pub struct FormChunk {
    pub size: u32
}

named!(pub form_chunk<FormChunk>,
    do_parse!(
        tag!("FORM") >>
        size: be_u32 >>
        (FormChunk {
            size: size
        })
    )
);

#[derive(Debug)]
pub enum VQAVersion {
    One,
    Two,
    Three
}

named!(pub vqa_version<VQAVersion>,
    switch!(le_u16,
        1 => value!(VQAVersion::One) |
        2 => value!(VQAVersion::Two) |
        3 => value!(VQAVersion::Three)
    )
);

bitflags! {
    pub struct VQAFlags: u16 {
        const HAS_SOUND = 0b00000001;
    }
}

#[derive(Debug)]
pub struct VQAHeader {
    pub version       : VQAVersion, // VQA version number
    pub flags         : VQAFlags,   // VQA flags
    pub num_frames    : u16,        // Number of frames
    pub width         : u16,        // Movie width (pixels)
    pub height        : u16,        // Movie height (pixels)
    pub block_width   : u8,         // Width of each image block (pixels)
    pub block_height  : u8,         // Height of each image block (pixels)
    pub frame_rate    : u8,         // Frame rate of the VQA
    pub cbparts       : u8,         // How many images use the same lookup table
    pub colors        : u16,        // Max number of colors used in VQA
    pub maxblocks     : u16,        // Max number of image blocks
    pub unk1          : u32,        // Always 0?
    pub unk2          : u16,        // Some kind of size?
    pub freq          : u16,        // Sound sampling frequency
    pub channels      : u8,         // Number of sound channels
    pub bits          : u8,         // Sound resolution
    pub unk3          : u32,        // Always 0?
    pub unk4          : u16,        // 0 in old VQAs, 4 in HiColor VQAs?
    pub max_cbfz_size : u32,        // 0 in old VQAs, CBFZ size in HiColor
    pub unk5          : u32,        // Always 0?
}

named!(pub vqa_header<VQAHeader>,
    do_parse!(
        tag!("VQHD")              >>
        tag!(b"\x00\x00\x00\x2a") >> // VQAHeader is always 42 bytes long
        version: vqa_version      >>
        flags: le_u16             >>
        num_frames: le_u16        >>
        width: le_u16             >>
        height: le_u16            >>
        block_width: le_u8        >>
        block_height: le_u8       >>
        frame_rate: le_u8         >>
        cbparts: le_u8            >>
        colors: le_u16            >>
        maxblocks: le_u16         >>
        unk1: le_u32              >>
        unk2: le_u16              >>
        freq: le_u16              >>
        channels: le_u8           >>
        bits: le_u8               >>
        unk3: le_u32              >>
        unk4: le_u16              >>
        max_cbfz_size: le_u32     >>
        unk5: le_u32              >>
        (VQAHeader {
            version: version,
            flags: VQAFlags::from_bits_truncate(flags),
            num_frames: num_frames,
            width: width,
            height: height,
            block_width: block_width,
            block_height: block_height,
            frame_rate: frame_rate,
            cbparts: cbparts,
            colors: colors,
            maxblocks: maxblocks,
            unk1: unk1,
            unk2: unk2,
            freq: freq,
            channels: channels,
            bits: bits,
            unk3: unk3,
            unk4: unk4,
            max_cbfz_size: max_cbfz_size,
            unk5: unk5
        })
    )
);

#[derive(Debug)]
pub struct FINFChunk {
    pub size: u32,
    pub offsets: Vec<u32>
}

named!(pub finf_chunk<FINFChunk>,
    do_parse!(
        tag!("FINF") >>
        size: be_u32 >>
        offsets: many_m_n!(size as usize / 4, size as usize / 4, le_u32) >>
        (FINFChunk {
            size: size,
            offsets: offsets
        })
    )
);

#[derive(Debug)]
pub struct SND2Chunk<'a> {
    pub size: u32,
    pub data: &'a [u8]
}

named!(pub snd2_chunk<SND2Chunk>,
    do_parse!(
        tag!("SND2")       >>
        size: be_u32       >>
        data: take!(size)  >>
        (SND2Chunk {
            size: size,
            data: data
        })
    )
);

#[derive(Debug)]
pub struct VQFRChunk<'a> {
    pub size: u32,
    pub data: &'a [u8]
}

named!(pub vqfr_chunk<VQFRChunk>,
    do_parse!(
        tag!("VQFR")      >>
        size: be_u32      >>
        data: take!(size) >>
        (VQFRChunk {
            size: size,
            data: data
        })
    )
);

#[derive(Debug)]
pub struct CBFChunk<'a> {
    pub size: u32,
    pub compressed: bool,
    pub data: &'a [u8]
}

named!(pub cbf_chunk<CBFChunk>,
    do_parse!(
        tag!("CBF")                      >>
        compressed: alt!(
            tag!("0") => { |_| false } |
            tag!("Z") => { |_| true  }
        )                                >>
        size: be_u32                     >>
        data: take!(size)                >>
        (CBFChunk {
            size: size,
            compressed: compressed,
            data: data
        })
    )
);
<|endoftext|>


<commit_before>use token::Token;
use tokenizer::{Tokenizer, LexerError};

pub type Program = Vec<Statement>;

#[derive(PartialEq, Debug)]
pub enum Statement {
    NamedFunction {
        name: String,
        definition: Function,
    },
    AnonymousFunction(Function),
    Assignment {
        name: String,
        value: Expr,
    },
    Expression(Expr),
}

pub type Expr = Vec<ExprItem>;

#[derive(PartialEq, Debug)]
pub enum ExprItem {
    AnonymousFunction(Function),
    SingleTerm(Term),
    Group(Expr),
}

#[derive(PartialEq, Debug)]
pub struct Function {
    params: Vec<String>,
    body: Expr,
}

#[derive(PartialEq, Debug)]
pub enum Term {
    Symbol(String),
    Number(f64),
    StringLiteral(String),
}

pub struct Parser {
    lexer: Tokenizer,
}

// Private type just for internal functions since the public interface
// cannot return option
type ParseResult<T> = Result<Option<T>, ParseError>;

#[derive(PartialEq, Debug)]
pub enum ParseError {
    ExpectedToken(Token),
    SyntaxError(LexerError),
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> Result<Program, ParseError> {
        let mut statements: Program = Vec::new();

        loop {
            let statement = try!(self.statement());
            if statement.is_none() {
                break;
            }

            let statement = statement.unwrap();
            statements.push(statement);
        }

        Ok(statements)
    }

    fn statement(&mut self) -> ParseResult<Statement> {
        loop {
            let line = try!(self.line());
            if line.is_none() {
                return Ok(None);
            }

            let line = line.unwrap();
            if line.is_empty() {
                continue;
            }

            return self.dispatch_statement(line)
        }
    }

    fn line(&mut self) -> ParseResult<Vec<Token>> {
        let mut line_tokens: Vec<Token> = Vec::new();

        let mut reached_eof = true;
        for next_token in &mut self.lexer {
            let next_token = try!(
                next_token.map_err(|e| ParseError::SyntaxError(e))
            );

            if next_token == Token::EOL {
                reached_eof = false;
                break;
            }

            line_tokens.push(next_token);
        }

        if reached_eof && line_tokens.is_empty() {
            return Ok(None);
        }

        Ok(Some(line_tokens))
    }

    fn dispatch_statement(&mut self, statement_tokens: Vec<Token>) -> ParseResult<Statement> {
        assert!(statement_tokens.len() > 0, "Got zero tokens to dispatch");
        // looking for either assignment or a named function
        // both are a collection of only symbols followed by equals
        let mut equals = None;
        for (i, token) in statement_tokens.iter().enumerate() {
            if *token == Token::Equals && i > 0 {
                equals = Some(i);
                break;
            }
            else if let Token::Symbol(_) = *token {
                // Symbols are good, continue
                continue;
            }
            else if *token == Token::Backslash && i == 0 {
                // Anonymous functions have backslashes,
                // but only in the first position
                continue;
            }
            else {
                // Found something that is not a symbol, cannot be
                // assignment or named function, stop
                break;
            }
        }

        if equals.is_none() {
            self.expr(statement_tokens)
        }
        else {
            let equals = equals.unwrap();

            // The 1 or many assumption being assumed in this
            // match is only okay because of the i > 0 above
            debug_assert!(equals > 0);

            let lhs = statement_tokens[..equals].to_vec();
            let rhs = statement_tokens[(equals+1)..].to_vec();

            if lhs[0] == Token::Backslash {
                return self.anonymous_function(lhs, rhs);
            }

            match equals {
                // only one argument: name = ...
                1 => self.assignment(lhs, rhs),
                // more than one argument: name arg1 ... = ...
                _ => self.named_function(lhs, rhs),
            }
        }
    }

    fn anonymous_function(&mut self, lhs: Vec<Token>, rhs: Vec<Token>) -> ParseResult<Statement> {
        Ok(None)
    }

    fn assignment(&mut self, lhs: Vec<Token>, rhs: Vec<Token>) -> ParseResult<Statement> {
        Ok(None)
    }

    fn named_function(&mut self, lhs: Vec<Token>, rhs: Vec<Token>) -> ParseResult<Statement> {
        Ok(None)
    }

    fn expr(&mut self, tokens: Vec<Token>) -> ParseResult<Statement> {
        Ok(None)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        \q r = q / (r - 3)
        doubleMe = \e = e * 2
        operator INFIX 6 "$$" (\x y = x * y + x)
        "#);

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::AnonymousFunction(Function {
                params: vec!["q".to_owned(), "r".to_owned()],
                body: vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("q".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Symbol("/".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("r".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(3f64)
                        ),
                    ]),
                ],
            }),
            Statement::Assignment {
                name: "doubleMe".to_owned(),
                value: vec![
                    ExprItem::AnonymousFunction(Function {
                        params: vec!["e".to_owned()],
                        body: vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("e".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("*".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(2f64)
                            ),
                        ],
                    }),
                ],
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<commit_msg>Implementing anonymous function and named function + clarifying the ParseResult type<commit_after>use token::Token;
use tokenizer::{Tokenizer, LexerError};

pub type Program = Vec<Statement>;

#[derive(PartialEq, Debug)]
pub enum Statement {
    NamedFunction {
        name: String,
        definition: Function,
    },
    AnonymousFunction(Function),
    Assignment {
        name: String,
        value: Expr,
    },
    Expression(Expr),
}

pub type Expr = Vec<ExprItem>;

#[derive(PartialEq, Debug)]
pub enum ExprItem {
    AnonymousFunction(Function),
    SingleTerm(Term),
    Group(Expr),
}

#[derive(PartialEq, Debug)]
pub struct Function {
    params: Vec<String>,
    body: Expr,
}

#[derive(PartialEq, Debug)]
pub enum Term {
    Symbol(String),
    Number(f64),
    StringLiteral(String),
}

pub struct Parser {
    lexer: Tokenizer,
}

pub type ParseResult<T> = Result<T, ParseError>;

#[derive(PartialEq, Debug)]
pub enum ParseError {
    ExpectedToken(Token),
    SyntaxError(LexerError),
}

impl Parser {
    pub fn new(lexer: Tokenizer) -> Parser {
        Parser {
            lexer: lexer,
        }
    }

    pub fn parse(&mut self) -> ParseResult<Program> {
        let mut statements: Program = Vec::new();

        loop {
            let statement = try!(self.statement());
            if statement.is_none() {
                break;
            }

            let statement = statement.unwrap();
            statements.push(statement);
        }

        Ok(statements)
    }

    fn statement(&mut self) -> ParseResult<Option<Statement>> {
        loop {
            let line = try!(self.line());
            if line.is_none() {
                return Ok(None);
            }

            let line = line.unwrap();
            if line.is_empty() {
                continue;
            }

            return Ok(Some(try!(self.dispatch_statement(line))));
        }
    }

    fn line(&mut self) -> ParseResult<Option<Vec<Token>>> {
        let mut line_tokens: Vec<Token> = Vec::new();

        let mut reached_eof = true;
        for next_token in &mut self.lexer {
            let next_token = try!(
                next_token.map_err(|e| ParseError::SyntaxError(e))
            );

            if next_token == Token::EOL {
                reached_eof = false;
                break;
            }

            line_tokens.push(next_token);
        }

        if reached_eof && line_tokens.is_empty() {
            return Ok(None);
        }

        Ok(Some(line_tokens))
    }

    fn dispatch_statement(&self, statement_tokens: Vec<Token>) -> ParseResult<Statement> {
        assert!(statement_tokens.len() > 0, "Got zero tokens to dispatch");
        // looking for either assignment or a named function
        // both are a collection of only symbols followed by equals
        let mut equals = None;
        for (i, token) in statement_tokens.iter().enumerate() {
            if *token == Token::Equals && i > 0 {
                equals = Some(i);
                break;
            }
            else if let Token::Symbol(_) = *token {
                // Symbols are good, continue
                continue;
            }
            else if *token == Token::Backslash && i == 0 {
                // Anonymous functions have backslashes,
                // but only in the first position
                continue;
            }
            else {
                // Found something that is not a symbol, cannot be
                // assignment or named function, stop
                break;
            }
        }

        if equals.is_none() {
            Ok(Statement::Expression(try!(self.expr(&statement_tokens[..]))))
        }
        else {
            let equals = equals.unwrap();

            // The 1 or many assumption being assumed in this
            // match is only okay because of the i > 0 above
            debug_assert!(equals > 0);

            let lhs = &statement_tokens[..equals];
            let rhs = &statement_tokens[(equals+1)..];

            if lhs[0] == Token::Backslash {
                return self.anonymous_function(lhs, rhs);
            }

            match equals {
                // only one argument: name = ...
                1 => self.assignment(lhs, rhs),
                // more than one argument: name arg1 ... = ...
                _ => self.named_function(lhs, rhs),
            }
        }
    }

    fn anonymous_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        debug_assert!(lhs[0] == Token::Backslash);

        Ok(Statement::AnonymousFunction(try!(self.function(&lhs[0..], rhs))))
    }

    fn named_function(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        let name = match lhs[0] {
            Token::Symbol(ref x) => x,
            _ => panic!("Expected symbol as function name"),
        };

        Ok(Statement::NamedFunction {
            name: name.clone(),
            definition: try!(self.function(&lhs[0..], rhs)),
        })
    }

    fn function(&self, params: &[Token], rhs: &[Token]) -> Result<Function, ParseError> {
        debug_assert!(params.len() > 0);

        Ok(Function {
            params: params.into_iter().map(move |ref sym| match **sym {
                Token::Symbol(ref x) => x.clone(),
                _ => panic!("Expected parameters to all be symbols"),
            }).collect(),
            body: try!(self.expr(rhs)),
        })
    }

    fn assignment(&self, lhs: &[Token], rhs: &[Token]) -> ParseResult<Statement> {
        Err(ParseError::ExpectedToken(Token::EOL))
    }

    fn expr(&self, tokens: &[Token]) -> ParseResult<Expr> {
        Ok(Vec::new())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use scanner::Scanner;
    use tokenizer::Tokenizer;

    #[test]
    fn complete_program() {
        let mut parser = parser_for(r#"
        defineUnit "km/h"

        my_var = 30 km/h
        f x yy = x km/h * 2 + (my_var / -3.5) * (8 ** yy) - 1.3e-3
        print (f 30)

        \q r = q / (r - 3)
        doubleMe = \e = e * 2
        operator INFIX 6 "$$" (\x y = x * y + x)
        "#);

        let syntax_tree: Program = vec![
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("defineUnit".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::StringLiteral("km/h".to_owned())
                ),
            ]),
            Statement::Assignment {
                name: "my_var".to_owned(),
                value: vec![
                    ExprItem::SingleTerm(Term::Number(30f64)),
                    ExprItem::SingleTerm(Term::Symbol("km/h".to_owned())),
                ],
            },
            Statement::NamedFunction {
                name: "f".to_owned(),
                definition: Function {
                    params: vec![
                        "x".to_owned(),
                        "yy".to_owned()
                    ],
                    body: vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("x".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("km/h".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(2f64)
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("+".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("my_var".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("/".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(-3.5f64)
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("*".to_owned())
                        ),
                        ExprItem::Group(vec![
                            ExprItem::SingleTerm(
                                Term::Number(8f64)
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("**".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("yy".to_owned())
                            ),
                        ]),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(1.3e-3f64)
                        ),
                    ],
                },
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("print".to_owned())
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("f".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Number(30f64)
                    ),
                ]),
            ]),
            Statement::AnonymousFunction(Function {
                params: vec!["q".to_owned(), "r".to_owned()],
                body: vec![
                    ExprItem::SingleTerm(
                        Term::Symbol("q".to_owned())
                    ),
                    ExprItem::SingleTerm(
                        Term::Symbol("/".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::SingleTerm(
                            Term::Symbol("r".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Symbol("-".to_owned())
                        ),
                        ExprItem::SingleTerm(
                            Term::Number(3f64)
                        ),
                    ]),
                ],
            }),
            Statement::Assignment {
                name: "doubleMe".to_owned(),
                value: vec![
                    ExprItem::AnonymousFunction(Function {
                        params: vec!["e".to_owned()],
                        body: vec![
                            ExprItem::SingleTerm(
                                Term::Symbol("e".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Symbol("*".to_owned())
                            ),
                            ExprItem::SingleTerm(
                                Term::Number(2f64)
                            ),
                        ],
                    }),
                ],
            },
            Statement::Expression(vec![
                ExprItem::SingleTerm(
                    Term::Symbol("operator".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Symbol("INFIX".to_owned())
                ),
                ExprItem::SingleTerm(
                    Term::Number(6f64)
                ),
                ExprItem::Group(vec![
                    ExprItem::SingleTerm(
                        Term::StringLiteral("$$".to_owned())
                    ),
                    ExprItem::Group(vec![
                        ExprItem::AnonymousFunction(
                            Function {
                                params: vec![
                                    "x".to_owned(),
                                    "y".to_owned()
                                ],
                                body: vec![
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("*".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("y".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("+".to_owned())
                                    ),
                                    ExprItem::SingleTerm(
                                        Term::Symbol("x".to_owned())
                                    ),
                                ],
                            },
                        ),
                    ]),
                ]),
            ]),
        ];

        let parsed = parser.parse().unwrap();
        assert_eq!(parsed, syntax_tree);
    }

    fn parser_for(string: &str) -> Parser {
        Parser::new(Tokenizer::new(Scanner::from_str(string)))
    }
}

<|endoftext|>


<commit_before>
pub struct ParseError {
  /// The line where the error occurred.
  pub line: u8,
}

pub struct Parser {
  pos: u8,
  source: String
}

pub enum Entry {
  Entity(String)
}

pub enum Value {
  Str(String)
}

impl Parser {
  pub fn new(source: String) -> Parser {
    Parser {
      pos: 0,
      source: source
    }
  }

  pub fn parse(&self) -> Vec<Entry> {
    let mut entries: Vec<Entry> = Vec::new();
    loop {
      let ch = match self.source.chars().next() {
        Some(ch) => ch,
        None => { break; }
      };

      if ch == '<' {
        entries.push(self.parse_entry());
      }
    }
    entries
  }

  pub fn parse_entry(&self) -> Entry {
    let id = self.parse_identifier();
    self.parse_entity(id)
  }

  pub fn parse_entity(&self, id: String) -> Entry {
    Entry::Entity(id)
  }

  pub fn parse_identifier(&self) -> String {
    let mut id = String::new();
    id.push('a');
    id
  }
}

/*  ---------------------- */

fn read_file() -> String {
  let s = "<entity1 'Value'>".to_string();
  return s
}

fn main() {
  let source = read_file();
  let parser = Parser::new(source);
  let entries = parser.parse();

  let entry1 = entries.pop();

  let id = match entry1 {
    Entry::Entity(ref id) => id.clone(),
  };

  println!("The result is {}", id);
}
<commit_msg>more work<commit_after>pub struct ParseError {
  pub line: u8,
}

pub struct Parser {
  pos: u8,
  source: String,
  ch: Option<char>
}

pub enum Entry {
  Entity {id: String}
}

pub enum Value {
  Str(String)
}

impl Parser {
  pub fn new(source: String) -> Parser {
    Parser {
      pos: 0,
      source: source,
      ch: None
    }
  }

  fn bump(&mut self) {
    self.ch = self.source.chars().next();

    self.pos += 1;
  }

  pub fn parse(&mut self) -> Vec<Entry> {
    let mut entries: Vec<Entry> = Vec::new();
    loop {
      let ch = match self.source.chars().next() {
        Some(ch) => ch,
        None => { break; }
      };

      if ch == '<' {
        entries.push(self.parse_entry());
        break;
      }
    }
    entries
  }

  pub fn parse_entry(&mut self) -> Entry {
    self.bump();
    let id = self.parse_identifier();
    self.parse_entity(id)
  }

  pub fn parse_entity(&self, id: String) -> Entry {
    Entry::Entity{id: id}
  }

  pub fn parse_identifier(&mut self) -> String {
    let mut id = String::new();

    loop {
      self.bump();
      let ch = match self.ch {
        Some(c) => c,
        None => break,
      };

      id.push(ch);
      break;
    }

    id
  }
}

/*  ---------------------- */

fn read_file() -> String {
  let s = "<entity1>".to_string();
  return s
}

fn main() {
  let source = read_file();
  let mut parser = Parser::new(source);
  let mut entries = parser.parse();

  let entry1 = entries.pop();

  let id = match entry1 {
    Some(Entry::Entity{id}) => id.clone(),
    None => "".to_string()
  };

  println!("The result is {}", id);
}
<|endoftext|>