<commit_before>#![feature(plugin)]
#![plugin(rocket_codegen)]

extern crate getopts;
extern crate rocket;
#[macro_use] extern crate serde_derive;
extern crate serde_json;
#[macro_use] extern crate rocket_contrib;

extern crate pombase;

use std::sync::Mutex;
use std::process;
use std::env;

use getopts::Options;

use rocket_contrib::{Json, Value};

use pombase::api::query::Query;
use pombase::api::result::QueryAPIResult;
use pombase::api::search::Search;
use pombase::api::query_exec::QueryExec;
use pombase::api::server_data::ServerData;
use pombase::web::data::SolrTermSummary;

const PKG_NAME: &str = env!("CARGO_PKG_NAME");
const VERSION: &str = env!("CARGO_PKG_VERSION");

#[post("/query", data="<q>", format = "application/json")]
fn query_post(q: Json<Query>, state: rocket::State<Mutex<QueryExec>>)
              -> Option<Json<QueryAPIResult>>
{
    let query_exec = state.lock().expect("failed to lock");
    Some(Json(query_exec.exec(&q.into_inner())))
}

#[get ("/reload")]
fn reload(state: rocket::State<Mutex<QueryExec>>) {
    let mut query_exec = state.lock().expect("failed to lock");
    print!("reloading ...\n");
    query_exec.reload();
    print!("... done\n");
}

#[derive(Serialize, Debug)]
struct CompletionResponse {
    status: String,
    matches: Vec<SolrTermSummary>,
}

#[get ("/complete/<cv_name>/<q>")]
fn complete(cv_name: String, q: String, state: rocket::State<Mutex<Search>>)
              -> Option<Json<CompletionResponse>>
{
    let search = state.lock().expect("failed to lock");
    let res = search.term_complete(&cv_name, &q);

    let completion_response =
        match res {
            Ok(matches) => {
                CompletionResponse {
                    status: "Ok".to_owned(),
                    matches: matches,
                }
            },
            Err(err) => {
                println!("{:?}", err);
                CompletionResponse {
                    status: "Error".to_owned(),
                    matches: vec![],
                }
            },
        };

    Some(Json(completion_response))
}

#[get ("/ping")]
fn ping() -> Option<String> {
    Some(String::from("OK") + " " + PKG_NAME + " " + VERSION)
}

#[error(404)]
fn not_found() -> Json<Value> {
    Json(json!({
        "status": "error",
        "reason": "Resource was not found."
    }))
}

fn print_usage(program: &str, opts: Options) {
    let brief = format!("Usage: {} [options]", program);
    print!("{}", opts.usage(&brief));
}

fn main() {
    print!("{} v{}\n", PKG_NAME, VERSION);

    let args: Vec<String> = env::args().collect();
    let mut opts = Options::new();

    opts.optflag("h", "help", "print this help message");
    opts.optopt("c", "config-file", "Configuration file name", "CONFIG");
    opts.optopt("m", "search-maps", "Search data", "MAPS_JSON_FILE");
    opts.optopt("s", "gene-subsets", "Gene subset data", "SUBSETS_JSON_FILE");

    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => panic!("Invalid options\n{}", f)
    };

    let program = args[0].clone();

    if matches.opt_present("help") {
        print_usage(&program, opts);
        process::exit(0);
    }

    if !matches.opt_present("config-file") {
        print!("no -c|--config-file option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("search-maps") {
        print!("no --search-maps|-m option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("gene-subsets") {
        print!("no --gene-subsets|-s option\n");
        print_usage(&program, opts);
        process::exit(1);
    }

    let search_maps_filename = matches.opt_str("m").unwrap();
    let gene_subsets_filename = matches.opt_str("s").unwrap();
    println!("Reading data files ...");

    let config_file_name = matches.opt_str("c").unwrap();
    let server_data = ServerData::new(&config_file_name, &search_maps_filename,
                                      &gene_subsets_filename);
    let query_exec = QueryExec::new(server_data);
    let searcher = Search::new("http://localhost:8983/solr".to_owned());

    println!("Starting server ...");
    rocket::ignite()
        .mount("/", routes![query_post, reload, complete, ping])
        .catch(errors![not_found])
        .manage(Mutex::new(query_exec))
        .manage(Mutex::new(searcher))
        .launch();
}
<commit_msg>Add -w opt. and now serve all static files<commit_after>#![feature(plugin)]
#![plugin(rocket_codegen)]

extern crate getopts;
extern crate rocket;
#[macro_use] extern crate serde_derive;
extern crate serde_json;
#[macro_use] extern crate rocket_contrib;

extern crate pombase;

use std::sync::Mutex;
use std::process;
use std::env;
use std::path::{Path, PathBuf};

use getopts::Options;

use rocket_contrib::{Json, Value};

use rocket::response::NamedFile;

use pombase::api::query::Query;
use pombase::api::result::QueryAPIResult;
use pombase::api::search::Search;
use pombase::api::query_exec::QueryExec;
use pombase::api::server_data::ServerData;
use pombase::web::data::SolrTermSummary;

const PKG_NAME: &str = env!("CARGO_PKG_NAME");
const VERSION: &str = env!("CARGO_PKG_VERSION");

struct StaticFileState {
    web_root_dir: String,
}

// try the path, then try path + ".json", then default to loading the Angular app
// from /index.html
#[get("/<path..>", rank=3)]
fn get_misc(path: PathBuf, state: rocket::State<Mutex<StaticFileState>>) -> Option<NamedFile> {
    print!("full_path 1: {:?}\n", path);
    let web_root_dir = &state.lock().expect("failed to lock").web_root_dir;
    print!("web_root_dir: {:?}\n", web_root_dir);
    let root_dir_path = Path::new("/").join(web_root_dir);
    print!("root_dir_path: {:?}\n", root_dir_path);
    let full_path = root_dir_path.join(path);
    print!("full_path 1: {:?}\n", full_path);
    if full_path.exists() {
        NamedFile::open(full_path).ok()
    } else {
        print!("full_path: {:?}\n", full_path);
        let mut json_path_str = full_path.to_str().unwrap().to_owned();
        json_path_str += ".json";
        let json_path: PathBuf = json_path_str.into();

        if json_path.exists() {
            print!("exists: {:?}\n", json_path);
            NamedFile::open(json_path).ok()
        } else {
            print!("not exists: {:?}\n", json_path);
            NamedFile::open(root_dir_path.join("index.html")).ok()
        }
    }
}

#[get("/", rank=1)]
fn get_index(state: rocket::State<Mutex<StaticFileState>>) -> Option<NamedFile> {
    let web_root_dir = &state.lock().expect("failed to lock").web_root_dir;
    let root_dir_path = Path::new("/").join(web_root_dir);
    NamedFile::open(root_dir_path.join("index.html")).ok()
}

#[post("/api/v1/dataset/latest/query", rank=1, data="<q>", format = "application/json")]
fn query_post(q: Json<Query>, state: rocket::State<Mutex<QueryExec>>)
              -> Option<Json<QueryAPIResult>>
{
    let query_exec = state.lock().expect("failed to lock");
    Some(Json(query_exec.exec(&q.into_inner())))
}

#[get ("/reload")]
fn reload(state: rocket::State<Mutex<QueryExec>>) {
    let mut query_exec = state.lock().expect("failed to lock");
    print!("reloading ...\n");
    query_exec.reload();
    print!("... done\n");
}

#[derive(Serialize, Debug)]
struct CompletionResponse {
    status: String,
    matches: Vec<SolrTermSummary>,
}

#[get ("/api/v1/dataset/latest/complete/<cv_name>/<q>", rank=1)]
fn complete(cv_name: String, q: String, state: rocket::State<Mutex<Search>>)
              -> Option<Json<CompletionResponse>>
{
    let search = state.lock().expect("failed to lock");
    let res = search.term_complete(&cv_name, &q);

    let completion_response =
        match res {
            Ok(matches) => {
                CompletionResponse {
                    status: "Ok".to_owned(),
                    matches: matches,
                }
            },
            Err(err) => {
                println!("{:?}", err);
                CompletionResponse {
                    status: "Error".to_owned(),
                    matches: vec![],
                }
            },
        };

    Some(Json(completion_response))
}

#[get ("/ping", rank=1)]
fn ping() -> Option<String> {
    Some(String::from("OK") + " " + PKG_NAME + " " + VERSION)
}

#[error(404)]
fn not_found() -> Json<Value> {
    Json(json!({
        "status": "error",
        "reason": "Resource was not found."
    }))
}

fn print_usage(program: &str, opts: Options) {
    let brief = format!("Usage: {} [options]", program);
    print!("{}", opts.usage(&brief));
}

fn main() {
    print!("{} v{}\n", PKG_NAME, VERSION);

    let args: Vec<String> = env::args().collect();
    let mut opts = Options::new();

    opts.optflag("h", "help", "print this help message");
    opts.optopt("c", "config-file", "Configuration file name", "CONFIG");
    opts.optopt("m", "search-maps", "Search data", "MAPS_JSON_FILE");
    opts.optopt("s", "gene-subsets", "Gene subset data", "SUBSETS_JSON_FILE");
    opts.optopt("w", "web-root-dir", "Root web data directory", "WEB_ROOT_DIR");

    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => panic!("Invalid options\n{}", f)
    };

    let program = args[0].clone();

    if matches.opt_present("help") {
        print_usage(&program, opts);
        process::exit(0);
    }

    if !matches.opt_present("config-file") {
        print!("no -c|--config-file option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("search-maps") {
        print!("no --search-maps|-m option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("gene-subsets") {
        print!("no --gene-subsets|-s option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("web-root-dir") {
        print!("no --web-root-dir|-w option\n");
        print_usage(&program, opts);
        process::exit(1);
    }

    let search_maps_filename = matches.opt_str("m").unwrap();
    let gene_subsets_filename = matches.opt_str("s").unwrap();
    println!("Reading data files ...");

    let config_file_name = matches.opt_str("c").unwrap();
    let server_data = ServerData::new(&config_file_name, &search_maps_filename,
                                      &gene_subsets_filename);
    let query_exec = QueryExec::new(server_data);
    let searcher = Search::new("http://localhost:8983/solr".to_owned());

    let web_root_dir = matches.opt_str("w").unwrap();
    let static_file_state = StaticFileState {
        web_root_dir: web_root_dir,
    };

    println!("Starting server ...");
    rocket::ignite()
        .mount("/", routes![get_index, get_misc, query_post, reload, complete, ping])
        .catch(errors![not_found])
        .manage(Mutex::new(query_exec))
        .manage(Mutex::new(searcher))
        .manage(Mutex::new(static_file_state))
        .launch();
}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![plugin(rocket_codegen)]

extern crate getopts;
extern crate rocket;
extern crate serde_json;
#[macro_use] extern crate rocket_contrib;

extern crate pombase;

use std::sync::Mutex;
use std::process;
use std::env;

use getopts::Options;

use rocket_contrib::{Json, Value};

use pombase::api::query::Query;
use pombase::api::result::Result;
use pombase::api::query_exec::QueryExec;
use pombase::api::server_data::ServerData;

#[post("/query", data="<q>", format = "application/json")]
fn query_post(q: Json<Query>, state: rocket::State<Mutex<QueryExec>>) -> Option<Json<Result>> {
    let query_exec = state.lock().expect("failed to lock");
    Some(Json(query_exec.exec(&q.into_inner())))
}

#[get ("/reload")]
fn reload(state: rocket::State<Mutex<QueryExec>>) {
    let mut query_exec = state.lock().expect("failed to lock");
    print!("reloading ...\n");
    query_exec.reload();
}

#[error(404)]
fn not_found() -> Json<Value> {
    Json(json!({
        "status": "error",
        "reason": "Resource was not found."
    }))
}

const PKG_NAME: &str = env!("CARGO_PKG_NAME");
const VERSION: &str = env!("CARGO_PKG_VERSION");

fn print_usage(program: &str, opts: Options) {
    let brief = format!("Usage: {} [options]", program);
    print!("{}", opts.usage(&brief));
}

fn main() {
    print!("{} v{}\n", PKG_NAME, VERSION);

    let args: Vec<String> = env::args().collect();
    let mut opts = Options::new();

    opts.optflag("h", "help", "print this help message");
    opts.optopt("c", "config-file", "Configuration file name", "CONFIG");
    opts.optopt("m", "search-maps", "Search data", "MAPS_JSON_FILE");
    opts.optopt("s", "gene-subsets", "Gene subset data", "SUBSETS_JSON_FILE");

    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => panic!("Invalid options\n{}", f)
    };

    let program = args[0].clone();

    if matches.opt_present("help") {
        print_usage(&program, opts);
        process::exit(0);
    }

    if !matches.opt_present("config-file") {
        print!("no -c|--config-file option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("search-maps") {
        print!("no --search-maps|-m option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("gene-subsets") {
        print!("no --gene-subsets|-s option\n");
        print_usage(&program, opts);
        process::exit(1);
    }

    let search_maps_filename = matches.opt_str("m").unwrap();
    let gene_subsets_filename = matches.opt_str("s").unwrap();
    println!("Reading data files ...");

    let config_file_name = matches.opt_str("c").unwrap();
    let server_data = ServerData::new(&config_file_name, &search_maps_filename,
                                      &gene_subsets_filename);
    let query_exec = QueryExec::new(server_data);

    println!("Starting server ...");
    rocket::ignite()
        .mount("/", routes![query_post, reload])
        .catch(errors![not_found])
        .manage(Mutex::new(query_exec))
        .launch();
}
<commit_msg>Add a /ping route for monitoring<commit_after>#![feature(plugin)]
#![plugin(rocket_codegen)]

extern crate getopts;
extern crate rocket;
extern crate serde_json;
#[macro_use] extern crate rocket_contrib;

extern crate pombase;

use std::sync::Mutex;
use std::process;
use std::env;

use getopts::Options;

use rocket_contrib::{Json, Value};

use pombase::api::query::Query;
use pombase::api::result::Result;
use pombase::api::query_exec::QueryExec;
use pombase::api::server_data::ServerData;

#[post("/query", data="<q>", format = "application/json")]
fn query_post(q: Json<Query>, state: rocket::State<Mutex<QueryExec>>) -> Option<Json<Result>> {
    let query_exec = state.lock().expect("failed to lock");
    Some(Json(query_exec.exec(&q.into_inner())))
}

#[get ("/reload")]
fn reload(state: rocket::State<Mutex<QueryExec>>) {
    let mut query_exec = state.lock().expect("failed to lock");
    print!("reloading ...\n");
    query_exec.reload();
}

#[get ("/ping")]
fn ping() -> Option<String> {
    Some("OK".into())
}

#[error(404)]
fn not_found() -> Json<Value> {
    Json(json!({
        "status": "error",
        "reason": "Resource was not found."
    }))
}

const PKG_NAME: &str = env!("CARGO_PKG_NAME");
const VERSION: &str = env!("CARGO_PKG_VERSION");

fn print_usage(program: &str, opts: Options) {
    let brief = format!("Usage: {} [options]", program);
    print!("{}", opts.usage(&brief));
}

fn main() {
    print!("{} v{}\n", PKG_NAME, VERSION);

    let args: Vec<String> = env::args().collect();
    let mut opts = Options::new();

    opts.optflag("h", "help", "print this help message");
    opts.optopt("c", "config-file", "Configuration file name", "CONFIG");
    opts.optopt("m", "search-maps", "Search data", "MAPS_JSON_FILE");
    opts.optopt("s", "gene-subsets", "Gene subset data", "SUBSETS_JSON_FILE");

    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => panic!("Invalid options\n{}", f)
    };

    let program = args[0].clone();

    if matches.opt_present("help") {
        print_usage(&program, opts);
        process::exit(0);
    }

    if !matches.opt_present("config-file") {
        print!("no -c|--config-file option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("search-maps") {
        print!("no --search-maps|-m option\n");
        print_usage(&program, opts);
        process::exit(1);
    }
    if !matches.opt_present("gene-subsets") {
        print!("no --gene-subsets|-s option\n");
        print_usage(&program, opts);
        process::exit(1);
    }

    let search_maps_filename = matches.opt_str("m").unwrap();
    let gene_subsets_filename = matches.opt_str("s").unwrap();
    println!("Reading data files ...");

    let config_file_name = matches.opt_str("c").unwrap();
    let server_data = ServerData::new(&config_file_name, &search_maps_filename,
                                      &gene_subsets_filename);
    let query_exec = QueryExec::new(server_data);

    println!("Starting server ...");
    rocket::ignite()
        .mount("/", routes![query_post, reload, ping])
        .catch(errors![not_found])
        .manage(Mutex::new(query_exec))
        .launch();
}
<|endoftext|>


<commit_before>// Copyright 2015 Pierre Talbot (IRCAM)

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::cmp::{min, max};
use std::num::SignedInt;

#[derive(Eq, Show, Copy, Clone)]
pub struct Interval {
  lb: int,
  ub: int
}

impl PartialEq<Interval> for Interval
{
  fn eq(&self, other: &Interval) -> bool {
    if self.is_empty() && other.is_empty() { true }
    else { self.lb == other.lb && self.ub == other.ub }
  }
}

impl Interval
{
  pub fn new(lb: int, ub: int) -> Interval {
    Interval { lb: lb, ub: ub }
  }

  pub fn empty() -> Interval {
    Interval::new(1, 0)
  }

  pub fn singleton(x: int) -> Interval {
    Interval::new(x, x)
  }

  pub fn is_singleton(self) -> bool {
    self.lb == self.ub
  }

  pub fn size(self) -> uint {
    if self.is_empty() { 0 }
    else { (self.ub - self.lb + 1).abs() as uint }
  }

  pub fn is_empty(self) -> bool {
    self.lb > self.ub
  }

  pub fn has_member(self, x: int) -> bool {
    x >= self.lb && x <= self.ub
  }

  pub fn is_subset_of(self, i: Interval) -> bool {
    if self.is_empty() { true }
    else {
      self.lb >= i.lb && self.ub <= i.ub
    }
  }

  pub fn intersection(self, i: Interval) -> Interval {
    Interval::new(
      max(self.lb, i.lb),
      min(self.ub, i.ub)
    )
  }

  pub fn hull(self, i: Interval) -> Interval {
    if self.is_empty() { i }
    else if i.is_empty() { self }
    else {
      Interval::new(
        min(self.lb, i.lb),
        max(self.ub, i.ub)
      )
    }
  }

  fn disjoint(self, i: Interval) -> bool {
    self.lb > i.ub || i.lb > self.ub
  }
}

pub trait ToInterval {
  fn to_interval(self) -> Interval;
}

impl ToInterval for Interval {
  fn to_interval(self) -> Interval { self }
}

impl ToInterval for (int, int) {
  fn to_interval(self) -> Interval {
    let (a, b) = self;
    Interval::new(a, b)
  }
}

impl ToInterval for () {
  fn to_interval(self) -> Interval {
    Interval::empty()
  }
}

impl ToInterval for int {
  fn to_interval(self) -> Interval {
    Interval::singleton(self)
  }
}

#[allow(non_upper_case_globals)]
#[cfg(test)]
mod tests {
  use super::*;

  const empty: Interval = Interval {lb: 1, ub: 0};
  const invalid: Interval = Interval {lb: 10, ub: -10};
  const zero: Interval = Interval {lb: 0, ub: 0};
  const one: Interval = Interval {lb: 1, ub: 1};

  const i1_2: Interval = Interval {lb: 1, ub: 2};
  const i0_10: Interval = Interval {lb: 0, ub: 10};
  const i0_15: Interval = Interval {lb: 0, ub: 15};
  const im5_10: Interval = Interval {lb: -5, ub: 10};
  const i5_10: Interval = Interval {lb: 5, ub: 10};
  const i0_5: Interval = Interval {lb: 0, ub: 5};
  const i0_4: Interval = Interval {lb: 0, ub: 4};
  const im5_5: Interval = Interval {lb: -5, ub: 5};
  const i20_30: Interval = Interval {lb: 20, ub: 30};
  const im30_m20: Interval = Interval {lb: -30, ub: -20};

  #[test]
  fn to_interval_id_test() {
    let id = i1_2.clone().to_interval();
    assert!(i1_2 == id);
    assert!(i1_2 == Interval::new(1, 2));
  }

  #[test]
  fn equality_test() {
    assert!(empty == empty);
    assert!(empty == invalid);
    assert!(invalid == empty);
    assert!(i1_2 == i1_2);
  }

  #[test]
  fn size_test() {
    assert!(zero.size() == 1);
    assert!(one.size() == 1);
    assert!(empty.size() == 0);
    assert!(invalid.size() == 0);

    assert!(i1_2.size() == 2);
    assert!(i0_10.size() == 11);
    assert!(im30_m20.size() == 11);
  }

  #[test]
  fn member_test() {
    assert!(i1_2.has_member(1));
    assert!(i1_2.has_member(2));
    assert!(!i1_2.has_member(0));
    assert!(!i1_2.has_member(3));

    assert!(zero.has_member(0));
    assert!(!zero.has_member(1));

    assert!(!empty.has_member(0));
    assert!(!empty.has_member(1));
    assert!(!empty.has_member(5));
    assert!(!empty.has_member(-5));

    assert!(!invalid.has_member(0));
    assert!(!invalid.has_member(-11));
    assert!(!invalid.has_member(11));
  }

  #[test]
  fn is_subset_of_test() {
    let cases = vec![
      (zero, zero,          true),
      (i1_2, i1_2,          true),
      (empty, empty,        true),
      (invalid, invalid,    true)
    ];

    // For each cases (x, y, res)
    // * x and y are the values
    // * res is a tuple (r, sym) where
    //    * r is true if x is a subset of y
    //    * sym is true if y is a subset of x
    let sym_cases = vec![
      // ||
      // |-|
      (empty, zero,         (true, false)),
      (invalid, zero,       (true, false)),
      (empty, invalid,      (true, true)),
      // ||
      //|--|
      (empty, i1_2,         (true, false)),
      (invalid, i1_2,       (true, false)),
      //  |--|
      // |----|
      (i1_2, i0_10,         (true, false)),
      // |--|
      //     |--|
      (i0_4, i5_10,         (false, false)),
      // |--|
      //    |--|
      (i0_5, i5_10,         (false, false)),
      // |---|
      //   |---|
      (im5_5, i0_10,        (false, false)),
      // |--|
      //         |--|
      (i0_10, i20_30,       (false, false)),
      // |--|
      // |---|
      (i0_10, i0_15,        (true, false)),
      // |---|
      //  |--|
      (im5_10, i0_10,       (false, true))
    ];

    for (x,y,r) in cases.into_iter() {
      assert!(x.is_subset_of(y) == r, "{} is subset of {} is not equal to {}", x, y, r);
    }

    for (x,y,(r1,r2)) in sym_cases.into_iter() {
      assert!(x.is_subset_of(y) == r1, "{} is subset of {} is not equal to {}", x, y, r1);
      assert!(y.is_subset_of(x) == r2, "{} is subset of {} is not equal to {}", y, x, r2);
    }
  }

  #[test]
  fn intersection_test() {
    let cases = vec![
      (zero, zero,          zero),
      (i1_2, i1_2,          i1_2),
      (empty, empty,        empty),
      (invalid, invalid,    invalid)
    ];

    // For each cases (x, y, res)
    // * x and y are the values
    // * res is the expected result, which should be the same
    // for x intersect y and y intersect x since the intersection
    // is commutative.
    let sym_cases = vec![
      // ||
      // |-|
      (empty, zero,         empty),
      (invalid, zero,       empty),
      (empty, invalid,      empty),
      // ||
      //|--|
      (empty, i1_2,         empty),
      (invalid, i1_2,       empty),
      //  |--|
      // |----|
      (i1_2, i0_10,         i1_2),
      // |--|
      //     |--|
      (i0_4, i5_10,         empty),
      // |--|
      //    |--|
      (i0_5, i5_10,         5.to_interval()),
      // |---|
      //   |---|
      (im5_5, i0_10,        (0,5).to_interval()),
      // |--|
      //         |--|
      (i0_10, i20_30,       empty),
      // |--|
      // |---|
      (i0_10, i0_15,        i0_10),
      // |---|
      //  |--|
      (im5_10, i0_10,       i0_10)
    ];

    for (x,y,r) in cases.into_iter() {
      assert!(x.intersection(y) == r, "{} intersection {} is not equal to {}", x, y, r);
    }

    for (x,y,r) in sym_cases.into_iter() {
      assert!(x.intersection(y) == r, "{} intersection {} is not equal to {}", x, y, r);
      assert!(y.intersection(x) == r, "{} intersection {} is not equal to {}", y, x, r);
    }
  }

  #[test]
  fn hull_test() {
    let cases = vec![
      (zero, zero,          zero),
      (i1_2, i1_2,          i1_2),
      (empty, empty,        empty),
      (invalid, invalid,    invalid)
    ];

    // For each cases (x, y, res)
    // * x and y are the values
    // * res is the expected result, which should be the same
    // for x intersect y and y intersect x since the intersection
    // is commutative.
    let sym_cases = vec![
      // ||
      // |-|
      (empty, zero,         zero),
      (invalid, zero,       zero),
      (empty, invalid,      empty),
      // ||
      //|--|
      (empty, i1_2,         i1_2),
      (invalid, i1_2,       i1_2),
      //  |--|
      // |----|
      (i1_2, i0_10,         i0_10),
      // |--|
      //     |--|
      (i0_4, i5_10,         i0_10),
      // |--|
      //    |--|
      (i0_5, i5_10,         i0_10),
      // |---|
      //   |---|
      (im5_5, i0_10,        (-5,10).to_interval()),
      // |--|
      //         |--|
      (i0_10, i20_30,       (0,30).to_interval()),
      // |--|
      // |---|
      (i0_10, i0_15,        i0_15),
      // |---|
      //  |--|
      (im5_10, i0_10,       im5_10)
    ];

    for (x,y,r) in cases.into_iter() {
      assert!(x.hull(y) == r, "{} hull {} is not equal to {}", x, y, r);
    }

    for (x,y,r) in sym_cases.into_iter() {
      assert!(x.hull(y) == r, "{} hull {} is not equal to {}", x, y, r);
      assert!(y.hull(x) == r, "{} hull {} is not equal to {}", y, x, r);
    }
  }
}<commit_msg>Test the is_disjoint operation.<commit_after>// Copyright 2015 Pierre Talbot (IRCAM)

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::cmp::{min, max};
use std::num::SignedInt;

#[derive(Eq, Show, Copy, Clone)]
pub struct Interval {
  lb: int,
  ub: int
}

impl PartialEq<Interval> for Interval
{
  fn eq(&self, other: &Interval) -> bool {
    if self.is_empty() && other.is_empty() { true }
    else { self.lb == other.lb && self.ub == other.ub }
  }
}

impl Interval
{
  pub fn new(lb: int, ub: int) -> Interval {
    Interval { lb: lb, ub: ub }
  }

  pub fn empty() -> Interval {
    Interval::new(1, 0)
  }

  pub fn singleton(x: int) -> Interval {
    Interval::new(x, x)
  }

  pub fn is_singleton(self) -> bool {
    self.lb == self.ub
  }

  pub fn size(self) -> uint {
    if self.is_empty() { 0 }
    else { (self.ub - self.lb + 1).abs() as uint }
  }

  pub fn is_empty(self) -> bool {
    self.lb > self.ub
  }

  pub fn has_member(self, x: int) -> bool {
    x >= self.lb && x <= self.ub
  }

  pub fn is_subset_of(self, i: Interval) -> bool {
    if self.is_empty() { true }
    else {
      self.lb >= i.lb && self.ub <= i.ub
    }
  }

  pub fn intersection(self, i: Interval) -> Interval {
    Interval::new(
      max(self.lb, i.lb),
      min(self.ub, i.ub)
    )
  }

  pub fn hull(self, i: Interval) -> Interval {
    if self.is_empty() { i }
    else if i.is_empty() { self }
    else {
      Interval::new(
        min(self.lb, i.lb),
        max(self.ub, i.ub)
      )
    }
  }

  pub fn is_disjoint(self, i: Interval) -> bool {
    self.lb > i.ub || i.lb > self.ub
  }
}

pub trait ToInterval {
  fn to_interval(self) -> Interval;
}

impl ToInterval for Interval {
  fn to_interval(self) -> Interval { self }
}

impl ToInterval for (int, int) {
  fn to_interval(self) -> Interval {
    let (a, b) = self;
    Interval::new(a, b)
  }
}

impl ToInterval for () {
  fn to_interval(self) -> Interval {
    Interval::empty()
  }
}

impl ToInterval for int {
  fn to_interval(self) -> Interval {
    Interval::singleton(self)
  }
}

#[allow(non_upper_case_globals)]
#[cfg(test)]
mod tests {
  use super::*;

  const empty: Interval = Interval {lb: 1, ub: 0};
  const invalid: Interval = Interval {lb: 10, ub: -10};
  const zero: Interval = Interval {lb: 0, ub: 0};
  const one: Interval = Interval {lb: 1, ub: 1};

  const i1_2: Interval = Interval {lb: 1, ub: 2};
  const i0_10: Interval = Interval {lb: 0, ub: 10};
  const i0_15: Interval = Interval {lb: 0, ub: 15};
  const im5_10: Interval = Interval {lb: -5, ub: 10};
  const i5_10: Interval = Interval {lb: 5, ub: 10};
  const i0_5: Interval = Interval {lb: 0, ub: 5};
  const i0_4: Interval = Interval {lb: 0, ub: 4};
  const im5_5: Interval = Interval {lb: -5, ub: 5};
  const i20_30: Interval = Interval {lb: 20, ub: 30};
  const im30_m20: Interval = Interval {lb: -30, ub: -20};

  #[test]
  fn to_interval_id_test() {
    let id = i1_2.clone().to_interval();
    assert!(i1_2 == id);
    assert!(i1_2 == Interval::new(1, 2));
  }

  #[test]
  fn equality_test() {
    assert!(empty == empty);
    assert!(empty == invalid);
    assert!(invalid == empty);
    assert!(i1_2 == i1_2);
  }

  #[test]
  fn size_test() {
    assert!(zero.size() == 1);
    assert!(one.size() == 1);
    assert!(empty.size() == 0);
    assert!(invalid.size() == 0);

    assert!(i1_2.size() == 2);
    assert!(i0_10.size() == 11);
    assert!(im30_m20.size() == 11);
  }

  #[test]
  fn member_test() {
    assert!(i1_2.has_member(1));
    assert!(i1_2.has_member(2));
    assert!(!i1_2.has_member(0));
    assert!(!i1_2.has_member(3));

    assert!(zero.has_member(0));
    assert!(!zero.has_member(1));

    assert!(!empty.has_member(0));
    assert!(!empty.has_member(1));
    assert!(!empty.has_member(5));
    assert!(!empty.has_member(-5));

    assert!(!invalid.has_member(0));
    assert!(!invalid.has_member(-11));
    assert!(!invalid.has_member(11));
  }

  #[test]
  fn is_subset_of_test() {
    let cases = vec![
      (zero, zero,          true),
      (i1_2, i1_2,          true),
      (empty, empty,        true),
      (invalid, invalid,    true)
    ];

    // For each cases (x, y, res)
    // * x and y are the values
    // * res is a tuple (r, sym) where
    //    * r is true if x is a subset of y
    //    * sym is true if y is a subset of x
    let sym_cases = vec![
      // ||
      // |-|
      (empty, zero,         (true, false)),
      (invalid, zero,       (true, false)),
      (empty, invalid,      (true, true)),
      // ||
      //|--|
      (empty, i1_2,         (true, false)),
      (invalid, i1_2,       (true, false)),
      //  |--|
      // |----|
      (i1_2, i0_10,         (true, false)),
      // |--|
      //     |--|
      (i0_4, i5_10,         (false, false)),
      // |--|
      //    |--|
      (i0_5, i5_10,         (false, false)),
      // |---|
      //   |---|
      (im5_5, i0_10,        (false, false)),
      // |--|
      //         |--|
      (i0_10, i20_30,       (false, false)),
      // |--|
      // |---|
      (i0_10, i0_15,        (true, false)),
      // |---|
      //  |--|
      (im5_10, i0_10,       (false, true))
    ];

    for (x,y,r) in cases.into_iter() {
      assert!(x.is_subset_of(y) == r, "{} is subset of {} is not equal to {}", x, y, r);
    }

    for (x,y,(r1,r2)) in sym_cases.into_iter() {
      assert!(x.is_subset_of(y) == r1, "{} is subset of {} is not equal to {}", x, y, r1);
      assert!(y.is_subset_of(x) == r2, "{} is subset of {} is not equal to {}", y, x, r2);
    }
  }

  #[test]
  fn intersection_test() {
    let cases = vec![
      (zero, zero,          zero),
      (i1_2, i1_2,          i1_2),
      (empty, empty,        empty),
      (invalid, invalid,    invalid)
    ];

    // For each cases (x, y, res)
    // * x and y are the values
    // * res is the expected result, which should be the same
    // for x intersect y and y intersect x since the intersection
    // is commutative.
    let sym_cases = vec![
      // ||
      // |-|
      (empty, zero,         empty),
      (invalid, zero,       empty),
      (empty, invalid,      empty),
      // ||
      //|--|
      (empty, i1_2,         empty),
      (invalid, i1_2,       empty),
      //  |--|
      // |----|
      (i1_2, i0_10,         i1_2),
      // |--|
      //     |--|
      (i0_4, i5_10,         empty),
      // |--|
      //    |--|
      (i0_5, i5_10,         5.to_interval()),
      // |---|
      //   |---|
      (im5_5, i0_10,        (0,5).to_interval()),
      // |--|
      //         |--|
      (i0_10, i20_30,       empty),
      // |--|
      // |---|
      (i0_10, i0_15,        i0_10),
      // |---|
      //  |--|
      (im5_10, i0_10,       i0_10)
    ];

    for (x,y,r) in cases.into_iter() {
      assert!(x.intersection(y) == r, "{} intersection {} is not equal to {}", x, y, r);
    }

    for (x,y,r) in sym_cases.into_iter() {
      assert!(x.intersection(y) == r, "{} intersection {} is not equal to {}", x, y, r);
      assert!(y.intersection(x) == r, "{} intersection {} is not equal to {}", y, x, r);
    }
  }

  #[test]
  fn hull_test() {
    let cases = vec![
      (zero, zero,          zero),
      (i1_2, i1_2,          i1_2),
      (empty, empty,        empty),
      (invalid, invalid,    invalid)
    ];

    // For each cases (x, y, res)
    // * x and y are the values
    // * res is the expected result, which should be the same
    // for the union hull of (x,y) or (y,x) since the union hull
    // is commutative.
    let sym_cases = vec![
      // ||
      // |-|
      (empty, zero,         zero),
      (invalid, zero,       zero),
      (empty, invalid,      empty),
      // ||
      //|--|
      (empty, i1_2,         i1_2),
      (invalid, i1_2,       i1_2),
      //  |--|
      // |----|
      (i1_2, i0_10,         i0_10),
      // |--|
      //     |--|
      (i0_4, i5_10,         i0_10),
      // |--|
      //    |--|
      (i0_5, i5_10,         i0_10),
      // |---|
      //   |---|
      (im5_5, i0_10,        (-5,10).to_interval()),
      // |--|
      //         |--|
      (i0_10, i20_30,       (0,30).to_interval()),
      // |--|
      // |---|
      (i0_10, i0_15,        i0_15),
      // |---|
      //  |--|
      (im5_10, i0_10,       im5_10)
    ];

    for (x,y,r) in cases.into_iter() {
      assert!(x.hull(y) == r, "{} hull {} is not equal to {}", x, y, r);
    }

    for (x,y,r) in sym_cases.into_iter() {
      assert!(x.hull(y) == r, "{} hull {} is not equal to {}", x, y, r);
      assert!(y.hull(x) == r, "{} hull {} is not equal to {}", y, x, r);
    }
  }

  #[test]
  fn is_disjoint_test() {
    let cases = vec![
      (zero, zero,          false),
      (i1_2, i1_2,          false),
      (empty, empty,        true),
      (invalid, invalid,    true)
    ];

    // For each cases (x, y, res)
    // * x and y are the values
    // * res is the expected result, which should be the same
    // for x is disjoint of y and y is disjoint of x since the
    // disjoint operation is commutative.
    let sym_cases = vec![
      // ||
      // |-|
      (empty, zero,         true),
      (invalid, zero,       true),
      (empty, invalid,      true),
      // ||
      //|--|
      (empty, i1_2,         true),
      (invalid, i1_2,       true),
      //  |--|
      // |----|
      (i1_2, i0_10,         false),
      // |--|
      //     |--|
      (i0_4, i5_10,         true),
      // |--|
      //    |--|
      (i0_5, i5_10,         false),
      // |---|
      //   |---|
      (im5_5, i0_10,        false),
      // |--|
      //         |--|
      (i0_10, i20_30,       true),
      // |--|
      // |---|
      (i0_10, i0_15,        false),
      // |---|
      //  |--|
      (im5_10, i0_10,       false)
    ];

    for (x,y,r) in cases.into_iter() {
      assert!(x.is_disjoint(y) == r, "{} is disjoint of {} is not equal to {}", x, y, r);
    }

    for (x,y,r) in sym_cases.into_iter() {
      assert!(x.is_disjoint(y) == r, "{} is disjoint of {} is not equal to {}", x, y, r);
      assert!(y.is_disjoint(x) == r, "{} is disjoint of {} is not equal to {}", y, x, r);
    }
  }
}<|endoftext|>


<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::counters::{FETCHED_TRANSACTION, UNABLE_TO_FETCH_TRANSACTION};
use aptos_rest_client::{Client as RestClient, State, Transaction};
use futures::channel::mpsc;
use futures::{SinkExt, StreamExt};
use serde_json::Value;
use std::time::Duration;
use tokio::task::JoinHandle;
use url::Url;

// TODO: make this configurable
const RETRY_TIME_MILLIS: u64 = 1000;
const TRANSACTION_FETCH_BATCH_SIZE: u16 = 500;
const TRANSACTION_CHANNEL_SIZE: usize = 35;
const MAX_THREADS: usize = 10;
const MAX_RETRIES: usize = 5;

#[derive(Debug)]
pub struct Fetcher {
    client: RestClient,
    chain_id: u8,
    current_version: u64,
    highest_known_version: u64,
    transactions_sender: mpsc::Sender<Vec<Transaction>>,
}

impl Fetcher {
    pub fn new(
        client: RestClient,
        current_version: u64,
        transactions_sender: mpsc::Sender<Vec<Transaction>>,
    ) -> Self {
        Self {
            client,
            chain_id: 0,
            current_version,
            highest_known_version: current_version,
            transactions_sender,
        }
    }

    pub async fn set_highest_known_version(&mut self) {
        let info = self.client.get_ledger_information().await;
        let res = info.unwrap();
        let state = res.state();
        self.highest_known_version = state.version;
        self.chain_id = state.chain_id;
    }

    pub async fn run(&mut self) {
        loop {
            if self.current_version >= self.highest_known_version {
                tokio::time::sleep(Duration::from_millis(200)).await;
                self.set_highest_known_version().await;
            }

            let num_missing = self.highest_known_version - self.current_version;
            let num_batches = std::cmp::min(
                (num_missing as f64 / TRANSACTION_FETCH_BATCH_SIZE as f64).ceil() as u64,
                MAX_THREADS as u64,
            ) as usize;
            let mut futures = vec![];
            for i in 0..num_batches {
                futures.push(fetch_nexts(
                    self.client.clone(),
                    self.current_version + (i as u64 * TRANSACTION_FETCH_BATCH_SIZE as u64),
                ));
            }
            let mut res: Vec<Vec<Transaction>> = futures::future::join_all(futures).await;
            res.sort_by(|a, b| {
                a.first()
                    .unwrap()
                    .version()
                    .unwrap()
                    .cmp(&b.first().unwrap().version().unwrap())
            });

            for batch in res {
                self.current_version = batch.last().unwrap().version().unwrap();
                self.transactions_sender.send(batch).await.unwrap();
            }
        }
    }
}

/// Fetches the next version based on its internal version counter
/// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
/// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
async fn fetch_nexts(client: RestClient, starting_version: u64) -> Vec<Transaction> {
    let mut retries = 0;
    while retries < MAX_RETRIES {
        retries += 1;

        let res = client
            .get_transactions(Some(starting_version), Some(TRANSACTION_FETCH_BATCH_SIZE))
            .await;

        match res {
            Ok(response) => {
                FETCHED_TRANSACTION.inc();
                return remove_null_bytes_from_txns(response.into_inner());
            }
            Err(err) => {
                let err_str = err.to_string();
                // If it's a 404, then we're all caught up; no need to increment the `UNABLE_TO_FETCH_TRANSACTION` counter
                if err_str.contains("404") {
                    aptos_logger::debug!(
                            "Could not fetch {} transactions starting at {}: all caught up. Will check again in {}ms.",
                            TRANSACTION_FETCH_BATCH_SIZE,
                            starting_version,
                            RETRY_TIME_MILLIS,
                        );
                }
                UNABLE_TO_FETCH_TRANSACTION.inc();
                aptos_logger::error!(
                    "Could not fetch {} transactions starting at {}, will retry in {}ms. Err: {:?}",
                    TRANSACTION_FETCH_BATCH_SIZE,
                    starting_version,
                    RETRY_TIME_MILLIS,
                    err
                );
            }
        }
        tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
    }

    panic!(
        "Could not fetch {} transactions starting at {}!",
        TRANSACTION_FETCH_BATCH_SIZE, starting_version
    );
}

#[derive(Debug)]
pub struct TransactionFetcher {
    starting_version: u64,
    client: RestClient,
    fetcher_handle: Option<JoinHandle<()>>,
    transactions_sender: Option<mpsc::Sender<Vec<Transaction>>>,
    transaction_receiver: mpsc::Receiver<Vec<Transaction>>,
}

impl TransactionFetcher {
    pub fn new(node_url: Url, starting_version: Option<u64>) -> Self {
        let (transactions_sender, transaction_receiver) =
            mpsc::channel::<Vec<Transaction>>(TRANSACTION_CHANNEL_SIZE);

        let client = RestClient::new(node_url);

        Self {
            starting_version: starting_version.unwrap_or(0),
            client,
            fetcher_handle: None,
            transactions_sender: Some(transactions_sender),
            transaction_receiver,
        }
    }
}

#[async_trait::async_trait]
impl TransactionFetcherTrait for TransactionFetcher {
    /// Fetches the next batch based on its internal version counter
    /// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
    /// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_next_batch(&mut self) -> Vec<Transaction> {
        self.transaction_receiver.next().await.unwrap()
    }

    /// fetches one version; this used for error checking/repair/etc
    /// In the event it can't, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_version(&self, version: u64) -> Transaction {
        loop {
            let res = self.client.get_transaction_by_version(version).await;
            match res {
                Ok(response) => {
                    FETCHED_TRANSACTION.inc();
                    return response.into_inner();
                }
                Err(err) => {
                    UNABLE_TO_FETCH_TRANSACTION.inc();
                    aptos_logger::error!(
                        "Could not fetch version {}, will retry in {}ms. Err: {:?}",
                        version,
                        RETRY_TIME_MILLIS,
                        err
                    );
                    tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
                }
            };
        }
    }

    async fn fetch_ledger_info(&mut self) -> State {
        let mut retries = 0;
        while retries < MAX_RETRIES {
            retries += 1;
            match self.client.get_ledger_information().await {
                Ok(inner) => return inner.into_inner(),
                Err(err) => {
                    aptos_logger::error!(
                        "Failed to get ledger info, will retry in {}ms. Err: {:?}",
                        RETRY_TIME_MILLIS,
                        err
                    );
                    tokio::time::sleep(Duration::from_secs(RETRY_TIME_MILLIS)).await
                }
            }
        }
        panic!("Failed to get ledger info.");
    }

    async fn set_version(&mut self, version: u64) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        self.starting_version = version;
    }

    async fn start(&mut self) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        let client = self.client.clone();
        let transactions_sender = self.transactions_sender.take().unwrap();
        let starting_version = self.starting_version;
        let fetcher_handle = tokio::spawn(async move {
            let mut fetcher = Fetcher::new(client, starting_version, transactions_sender);
            fetcher.run().await;
        });
        self.fetcher_handle = Some(fetcher_handle);
    }
}

pub fn string_null_byte_replacement(value: &mut str) -> String {
    value.replace('\u{0000}', "").replace("\\u0000", "")
}

pub fn recurse_remove_null_bytes_from_json(sub_json: &mut Value) {
    match sub_json {
        Value::Array(array) => {
            for item in array {
                recurse_remove_null_bytes_from_json(item);
            }
        }
        Value::Object(object) => {
            for (_key, value) in object {
                recurse_remove_null_bytes_from_json(value);
            }
        }
        Value::String(str) => {
            if !str.is_empty() {
                let replacement = string_null_byte_replacement(str);
                *str = replacement;
            }
        }
        _ => {}
    }
}

pub fn remove_null_bytes_from_txns(txns: Vec<Transaction>) -> Vec<Transaction> {
    txns.iter()
        .map(|txn| {
            let mut txn_json = serde_json::to_value(txn).unwrap();
            recurse_remove_null_bytes_from_json(&mut txn_json);
            serde_json::from_value::<Transaction>(txn_json).unwrap()
        })
        .collect::<Vec<Transaction>>()
}

/// For mocking TransactionFetcher in tests
#[async_trait::async_trait]
pub trait TransactionFetcherTrait: Send + Sync {
    async fn fetch_next_batch(&mut self) -> Vec<Transaction>;

    async fn fetch_version(&self, version: u64) -> Transaction;

    async fn fetch_ledger_info(&mut self) -> State;

    async fn set_version(&mut self, version: u64);

    async fn start(&mut self);
}
<commit_msg>Add loop on getting set highest known version<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::counters::{FETCHED_TRANSACTION, UNABLE_TO_FETCH_TRANSACTION};
use aptos_rest_client::{Client as RestClient, State, Transaction};
use futures::channel::mpsc;
use futures::{SinkExt, StreamExt};
use serde_json::Value;
use std::time::Duration;
use tokio::task::JoinHandle;
use url::Url;

// TODO: make this configurable
const RETRY_TIME_MILLIS: u64 = 1000;
const TRANSACTION_FETCH_BATCH_SIZE: u16 = 500;
const TRANSACTION_CHANNEL_SIZE: usize = 35;
const MAX_THREADS: usize = 10;
const MAX_RETRIES: usize = 5;

#[derive(Debug)]
pub struct Fetcher {
    client: RestClient,
    chain_id: u8,
    current_version: u64,
    highest_known_version: u64,
    transactions_sender: mpsc::Sender<Vec<Transaction>>,
}

impl Fetcher {
    pub fn new(
        client: RestClient,
        current_version: u64,
        transactions_sender: mpsc::Sender<Vec<Transaction>>,
    ) -> Self {
        Self {
            client,
            chain_id: 0,
            current_version,
            highest_known_version: current_version,
            transactions_sender,
        }
    }

    pub async fn set_highest_known_version(&mut self) -> anyhow::Result<()> {
        let res = self.client.get_ledger_information().await?;
        let state = res.state();
        self.highest_known_version = state.version;
        self.chain_id = state.chain_id;
        Ok(())
    }

    pub async fn run(&mut self) {
        loop {
            if self.current_version >= self.highest_known_version {
                tokio::time::sleep(Duration::from_millis(200)).await;
                if let Err(err) = self.set_highest_known_version().await {
                    aptos_logger::error!("Failed to set highest known version. Err: {:?}", err);
                    continue;
                }
            }

            let num_missing = self.highest_known_version - self.current_version;
            let num_batches = std::cmp::min(
                (num_missing as f64 / TRANSACTION_FETCH_BATCH_SIZE as f64).ceil() as u64,
                MAX_THREADS as u64,
            ) as usize;
            let mut futures = vec![];
            for i in 0..num_batches {
                futures.push(fetch_nexts(
                    self.client.clone(),
                    self.current_version + (i as u64 * TRANSACTION_FETCH_BATCH_SIZE as u64),
                ));
            }
            let mut res: Vec<Vec<Transaction>> = futures::future::join_all(futures).await;
            res.sort_by(|a, b| {
                a.first()
                    .unwrap()
                    .version()
                    .unwrap()
                    .cmp(&b.first().unwrap().version().unwrap())
            });

            for batch in res {
                self.current_version = batch.last().unwrap().version().unwrap();
                self.transactions_sender.send(batch).await.unwrap();
            }
        }
    }
}

/// Fetches the next version based on its internal version counter
/// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
/// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
async fn fetch_nexts(client: RestClient, starting_version: u64) -> Vec<Transaction> {
    let mut retries = 0;
    while retries < MAX_RETRIES {
        retries += 1;

        let res = client
            .get_transactions(Some(starting_version), Some(TRANSACTION_FETCH_BATCH_SIZE))
            .await;

        match res {
            Ok(response) => {
                FETCHED_TRANSACTION.inc();
                return remove_null_bytes_from_txns(response.into_inner());
            }
            Err(err) => {
                let err_str = err.to_string();
                // If it's a 404, then we're all caught up; no need to increment the `UNABLE_TO_FETCH_TRANSACTION` counter
                if err_str.contains("404") {
                    aptos_logger::debug!(
                            "Could not fetch {} transactions starting at {}: all caught up. Will check again in {}ms.",
                            TRANSACTION_FETCH_BATCH_SIZE,
                            starting_version,
                            RETRY_TIME_MILLIS,
                        );
                }
                UNABLE_TO_FETCH_TRANSACTION.inc();
                aptos_logger::error!(
                    "Could not fetch {} transactions starting at {}, will retry in {}ms. Err: {:?}",
                    TRANSACTION_FETCH_BATCH_SIZE,
                    starting_version,
                    RETRY_TIME_MILLIS,
                    err
                );
            }
        }
        tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
    }

    panic!(
        "Could not fetch {} transactions starting at {}!",
        TRANSACTION_FETCH_BATCH_SIZE, starting_version
    );
}

#[derive(Debug)]
pub struct TransactionFetcher {
    starting_version: u64,
    client: RestClient,
    fetcher_handle: Option<JoinHandle<()>>,
    transactions_sender: Option<mpsc::Sender<Vec<Transaction>>>,
    transaction_receiver: mpsc::Receiver<Vec<Transaction>>,
}

impl TransactionFetcher {
    pub fn new(node_url: Url, starting_version: Option<u64>) -> Self {
        let (transactions_sender, transaction_receiver) =
            mpsc::channel::<Vec<Transaction>>(TRANSACTION_CHANNEL_SIZE);

        let client = RestClient::new(node_url);

        Self {
            starting_version: starting_version.unwrap_or(0),
            client,
            fetcher_handle: None,
            transactions_sender: Some(transactions_sender),
            transaction_receiver,
        }
    }
}

#[async_trait::async_trait]
impl TransactionFetcherTrait for TransactionFetcher {
    /// Fetches the next batch based on its internal version counter
    /// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
    /// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_next_batch(&mut self) -> Vec<Transaction> {
        self.transaction_receiver.next().await.unwrap()
    }

    /// fetches one version; this used for error checking/repair/etc
    /// In the event it can't, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_version(&self, version: u64) -> Transaction {
        loop {
            let res = self.client.get_transaction_by_version(version).await;
            match res {
                Ok(response) => {
                    FETCHED_TRANSACTION.inc();
                    return response.into_inner();
                }
                Err(err) => {
                    UNABLE_TO_FETCH_TRANSACTION.inc();
                    aptos_logger::error!(
                        "Could not fetch version {}, will retry in {}ms. Err: {:?}",
                        version,
                        RETRY_TIME_MILLIS,
                        err
                    );
                    tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
                }
            };
        }
    }

    async fn fetch_ledger_info(&mut self) -> State {
        let mut retries = 0;
        while retries < MAX_RETRIES {
            retries += 1;
            match self.client.get_ledger_information().await {
                Ok(inner) => return inner.into_inner(),
                Err(err) => {
                    aptos_logger::error!(
                        "Failed to get ledger info, will retry in {}ms. Err: {:?}",
                        RETRY_TIME_MILLIS,
                        err
                    );
                    tokio::time::sleep(Duration::from_secs(RETRY_TIME_MILLIS)).await
                }
            }
        }
        panic!("Failed to get ledger info.");
    }

    async fn set_version(&mut self, version: u64) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        self.starting_version = version;
    }

    async fn start(&mut self) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        let client = self.client.clone();
        let transactions_sender = self.transactions_sender.take().unwrap();
        let starting_version = self.starting_version;
        let fetcher_handle = tokio::spawn(async move {
            let mut fetcher = Fetcher::new(client, starting_version, transactions_sender);
            fetcher.run().await;
        });
        self.fetcher_handle = Some(fetcher_handle);
    }
}

pub fn string_null_byte_replacement(value: &mut str) -> String {
    value.replace('\u{0000}', "").replace("\\u0000", "")
}

pub fn recurse_remove_null_bytes_from_json(sub_json: &mut Value) {
    match sub_json {
        Value::Array(array) => {
            for item in array {
                recurse_remove_null_bytes_from_json(item);
            }
        }
        Value::Object(object) => {
            for (_key, value) in object {
                recurse_remove_null_bytes_from_json(value);
            }
        }
        Value::String(str) => {
            if !str.is_empty() {
                let replacement = string_null_byte_replacement(str);
                *str = replacement;
            }
        }
        _ => {}
    }
}

pub fn remove_null_bytes_from_txns(txns: Vec<Transaction>) -> Vec<Transaction> {
    txns.iter()
        .map(|txn| {
            let mut txn_json = serde_json::to_value(txn).unwrap();
            recurse_remove_null_bytes_from_json(&mut txn_json);
            serde_json::from_value::<Transaction>(txn_json).unwrap()
        })
        .collect::<Vec<Transaction>>()
}

/// For mocking TransactionFetcher in tests
#[async_trait::async_trait]
pub trait TransactionFetcherTrait: Send + Sync {
    async fn fetch_next_batch(&mut self) -> Vec<Transaction>;

    async fn fetch_version(&self, version: u64) -> Transaction;

    async fn fetch_ledger_info(&mut self) -> State;

    async fn set_version(&mut self, version: u64);

    async fn start(&mut self);
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::io::prelude::*;
use std::mem;
use std::path::{Path, PathBuf};
use std::str;
use regex::Regex;

use lex::{lex, Token};

lazy_static! {
    static ref PAT_IS_WHITESPACE: Regex = Regex::new(r#"^\s+$"#).expect("Failed to compile whitespace regex");
}

type FileID = u32;

#[derive(Debug, Clone)]
pub enum NodeValue {
    Owned(String),
    Children(Vec<Node>),
}

#[derive(Debug, Clone)]
pub struct Node {
    pub value: NodeValue,
    pub file_id: FileID,
    pub lineno: i32,
}

impl Node {
    pub fn new(value: NodeValue, file_id: FileID, lineno: i32) -> Self {
        Node {
            value,
            file_id,
            lineno,
        }
    }

    pub fn new_children(value: Vec<Node>, file_id: FileID, lineno: i32) -> Self {
        Node {
            value: NodeValue::Children(value),
            file_id,
            lineno,
        }
    }

    pub fn new_string<S: Into<String>>(value: S, file_id: FileID, lineno: i32) -> Self {
        Node {
            value: NodeValue::Owned(value.into()),
            file_id,
            lineno,
        }
    }

    #[allow(dead_code)]
    pub fn print(&self, indent: usize) {
        match self.value {
            NodeValue::Owned(ref s) => {
                println!("{:indent$}{:?}", "", s, indent = indent);
            }
            NodeValue::Children(ref children) => {
                println!("{:indent$}\\", "", indent = indent);
                for child in children {
                    child.print(indent + 2);
                }
            }
        }
    }
}

enum StackRequest {
    None,
    Pop(u8),
    Push(Box<TokenHandler>),
}

trait TokenHandler {
    fn handle_token(&mut self, token: &Token) -> StackRequest;
    fn finish(&mut self) -> Node;
    fn push(&mut self, node: Node);
    fn name(&self) -> &'static str;
}

struct StateRocket {
    root: Vec<Node>,
    buffer: Vec<String>,
    file_id: FileID,
    lineno: i32,
}

impl StateRocket {
    fn new(file_id: FileID, lineno: i32) -> Self {
        StateRocket {
            root: vec![Node::new_string("concat", file_id, lineno)],
            buffer: vec![],
            file_id: file_id,
            lineno,
        }
    }

    fn ensure_string(&mut self) -> &mut String {
        if self.buffer.is_empty() {
            self.buffer.push(String::with_capacity(2));
        }

        self.buffer.last_mut().unwrap()
    }
}

impl TokenHandler for StateRocket {
    fn handle_token(&mut self, token: &Token) -> StackRequest {
        match *token {
            Token::Text(_, s) => {
                self.buffer.push(s.to_owned());
            }
            Token::Quote(_) => {
                self.ensure_string().push('"');
            }
            Token::StartBlock(lineno) => {
                if !self.buffer.is_empty() {
                    self.root
                        .push(Node::new_string(self.buffer.concat(), self.file_id, lineno));
                    self.buffer.clear();
                }
                return StackRequest::Push(Box::new(StateExpression::new(self.file_id, lineno)));
            }
            Token::RightParen => {
                self.ensure_string().push(')');
            }
            Token::Rocket(_) => {
                self.ensure_string().push_str("=>");
            }
            Token::Dedent => {
                // We need to pop both the rocket and the expression that started the rocket
                return StackRequest::Pop(2);
            }
        }

        StackRequest::None
    }

    fn finish(&mut self) -> Node {
        if !self.buffer.is_empty() {
            let string = self.buffer.concat();
            self.root.push(Node::new_string(string, self.file_id, -1));
        }

        Node::new_children(
            mem::replace(&mut self.root, vec![]),
            self.file_id,
            self.lineno,
        )
    }

    fn push(&mut self, node: Node) {
        self.root.push(node);
    }

    fn name(&self) -> &'static str {
        "rocket"
    }
}

struct StateExpression {
    root: Vec<Node>,
    file_id: FileID,
    lineno: i32,

    quote: Vec<String>,
    in_quote: bool,
    new_node: bool,
}

impl StateExpression {
    fn new(file_id: FileID, lineno: i32) -> Self {
        StateExpression {
            root: vec![],
            file_id,
            lineno,
            quote: vec![],
            in_quote: false,
            new_node: true,
        }
    }
}

impl TokenHandler for StateExpression {
    fn handle_token(&mut self, token: &Token) -> StackRequest {
        if self.in_quote {
            match *token {
                Token::Text(_, s) => self.quote.push(s.to_owned()),
                Token::Quote(lineno) => {
                    self.root
                        .push(Node::new_string(self.quote.concat(), self.file_id, lineno));

                    self.in_quote = false;
                    self.quote.clear();
                }
                Token::StartBlock(_) => self.quote.push("(:".to_owned()),
                Token::RightParen => self.quote.push(")".to_owned()),
                Token::Rocket(_) => self.quote.push("=>".to_owned()),
                Token::Dedent => (),
            }
            return StackRequest::None;
        }

        match *token {
            Token::Text(lineno, s) => {
                // When in an expression, whitespace only serves to separate tokens.
                if PAT_IS_WHITESPACE.is_match(s) {
                    self.new_node = true;
                } else {
                    let mut new_node = self.new_node;

                    if !new_node {
                        if let Some(last) = self.root.last_mut() {
                            match last.value {
                                NodeValue::Owned(ref mut val) => val.push_str(s),
                                NodeValue::Children(_) => new_node = true,
                            }
                        } else {
                            new_node = true;
                        }
                    }

                    if new_node {
                        self.root
                            .push(Node::new_string(s.to_owned(), self.file_id, lineno));
                    }
                    self.new_node = false;
                }
            }
            Token::Quote(_) => {
                self.in_quote = true;
            }
            Token::StartBlock(lineno) => {
                return StackRequest::Push(Box::new(StateExpression::new(self.file_id, lineno)));
            }
            Token::Rocket(lineno) => {
                return StackRequest::Push(Box::new(StateRocket::new(self.file_id, lineno)));
            }
            Token::RightParen | Token::Dedent => {
                return StackRequest::Pop(1);
            }
        }

        StackRequest::None
    }

    fn finish(&mut self) -> Node {
        Node::new_children(
            mem::replace(&mut self.root, vec![]),
            self.file_id,
            self.lineno,
        )
    }

    fn push(&mut self, node: Node) {
        self.root.push(node);
    }

    fn name(&self) -> &'static str {
        "expression"
    }
}

struct ParseContextStack {
    stack: Vec<Box<TokenHandler>>,
}

impl ParseContextStack {
    fn new(file_id: FileID, lineno: i32) -> Self {
        ParseContextStack {
            stack: vec![
                Box::new(StateRocket {
                    root: vec![Node::new_string("concat", file_id, lineno)],
                    buffer: vec![],
                    file_id: file_id,
                    lineno: lineno,
                }),
            ],
        }
    }

    fn handle(&mut self, token: &Token) {
        match self.stack
            .last_mut()
            .expect("Empty parse stack")
            .handle_token(token)
        {
            StackRequest::Push(handler) => {
                self.stack.push(handler);
            }
            StackRequest::Pop(n) => for _ in 0..n {
                let mut handler = self.stack.pop().expect("Cannot pop last handler");
                (**self.stack.last_mut().expect("Empty parse stack")).push(handler.finish());
            },
            StackRequest::None => (),
        }
    }
}

pub struct Parser {
    file_ids: Vec<PathBuf>,
}

impl Parser {
    pub fn new() -> Self {
        Parser { file_ids: vec![] }
    }

    pub fn get_node_source_path(&self, node: &Node) -> Option<&Path> {
        match self.file_ids.get(node.file_id as usize) {
            Some(p) => Some(p),
            None => None,
        }
    }

    pub fn parse(&mut self, path: &Path) -> Result<Node, String> {
        debug!("Parsing {}", path.to_string_lossy());

        let id = self.file_ids.len() as FileID;
        self.file_ids.push(path.to_owned());

        let mut file = match File::open(path) {
            Ok(f) => f,
            Err(_) => {
                return Err(format!("Failed to open {}", path.to_string_lossy()));
            }
        };
        let mut data = String::new();
        file.read_to_string(&mut data)
            .expect("Failed to read input file");

        let mut stack = ParseContextStack::new(id, 0);
        for token in lex(&data) {
            stack.handle(&token);
        }

        let root = stack.stack.pop().expect("Empty state stack").finish();
        match stack.stack.pop() {
            Some(_) => Err(format!(
                "Unterminated block started on line {}",
                root.lineno
            )),
            None => Ok(root),
        }
    }
}
<commit_msg>parse: Add testing<commit_after>use std::fs::File;
use std::io::prelude::*;
use std::mem;
use std::path::{Path, PathBuf};
use std::str;
use regex::Regex;

use lex::{lex, Token};

lazy_static! {
    static ref PAT_IS_WHITESPACE: Regex = Regex::new(r#"^\s+$"#).expect("Failed to compile whitespace regex");
}

type FileID = u32;

#[derive(Debug, Clone, PartialEq)]
pub enum NodeValue {
    Owned(String),
    Children(Vec<Node>),
}

#[derive(Debug, Clone, PartialEq)]
pub struct Node {
    pub value: NodeValue,
    pub file_id: FileID,
    pub lineno: i32,
}

impl Node {
    pub fn new(value: NodeValue, file_id: FileID, lineno: i32) -> Self {
        Node {
            value,
            file_id,
            lineno,
        }
    }

    pub fn new_children(value: Vec<Node>, file_id: FileID, lineno: i32) -> Self {
        Node {
            value: NodeValue::Children(value),
            file_id,
            lineno,
        }
    }

    pub fn new_string<S: Into<String>>(value: S, file_id: FileID, lineno: i32) -> Self {
        Node {
            value: NodeValue::Owned(value.into()),
            file_id,
            lineno,
        }
    }

    #[allow(dead_code)]
    pub fn print(&self, indent: usize) {
        match self.value {
            NodeValue::Owned(ref s) => {
                println!("{:indent$}{:?}", "", s, indent = indent);
            }
            NodeValue::Children(ref children) => {
                println!("{:indent$}\\", "", indent = indent);
                for child in children {
                    child.print(indent + 2);
                }
            }
        }
    }
}

enum StackRequest {
    None,
    Pop(u8),
    Push(Box<TokenHandler>),
}

trait TokenHandler {
    fn handle_token(&mut self, token: &Token) -> StackRequest;
    fn finish(&mut self) -> Node;
    fn push(&mut self, node: Node);
    fn name(&self) -> &'static str;
}

struct StateRocket {
    root: Vec<Node>,
    buffer: Vec<String>,
    file_id: FileID,
    lineno: i32,
}

impl StateRocket {
    fn new(file_id: FileID, lineno: i32) -> Self {
        StateRocket {
            root: vec![Node::new_string("concat", file_id, lineno)],
            buffer: vec![],
            file_id: file_id,
            lineno,
        }
    }

    fn ensure_string(&mut self) -> &mut String {
        if self.buffer.is_empty() {
            self.buffer.push(String::with_capacity(2));
        }

        self.buffer.last_mut().unwrap()
    }
}

impl TokenHandler for StateRocket {
    fn handle_token(&mut self, token: &Token) -> StackRequest {
        match *token {
            Token::Text(_, s) => {
                self.buffer.push(s.to_owned());
            }
            Token::Quote(_) => {
                self.ensure_string().push('"');
            }
            Token::StartBlock(lineno) => {
                if !self.buffer.is_empty() {
                    self.root
                        .push(Node::new_string(self.buffer.concat(), self.file_id, lineno));
                    self.buffer.clear();
                }
                return StackRequest::Push(Box::new(StateExpression::new(self.file_id, lineno)));
            }
            Token::RightParen => {
                self.ensure_string().push(')');
            }
            Token::Rocket(_) => {
                self.ensure_string().push_str("=>");
            }
            Token::Dedent => {
                // We need to pop both the rocket and the expression that started the rocket
                return StackRequest::Pop(2);
            }
        }

        StackRequest::None
    }

    fn finish(&mut self) -> Node {
        if !self.buffer.is_empty() {
            let string = self.buffer.concat();
            self.root
                .push(Node::new_string(string, self.file_id, self.lineno));
        }

        Node::new_children(
            mem::replace(&mut self.root, vec![]),
            self.file_id,
            self.lineno,
        )
    }

    fn push(&mut self, node: Node) {
        self.root.push(node);
    }

    fn name(&self) -> &'static str {
        "rocket"
    }
}

struct StateExpression {
    root: Vec<Node>,
    file_id: FileID,
    lineno: i32,

    quote: Vec<String>,
    in_quote: bool,
    new_node: bool,
}

impl StateExpression {
    fn new(file_id: FileID, lineno: i32) -> Self {
        StateExpression {
            root: vec![],
            file_id,
            lineno,
            quote: vec![],
            in_quote: false,
            new_node: true,
        }
    }
}

impl TokenHandler for StateExpression {
    fn handle_token(&mut self, token: &Token) -> StackRequest {
        if self.in_quote {
            match *token {
                Token::Text(_, s) => self.quote.push(s.to_owned()),
                Token::Quote(lineno) => {
                    self.root
                        .push(Node::new_string(self.quote.concat(), self.file_id, lineno));

                    self.in_quote = false;
                    self.quote.clear();
                }
                Token::StartBlock(_) => self.quote.push("(:".to_owned()),
                Token::RightParen => self.quote.push(")".to_owned()),
                Token::Rocket(_) => self.quote.push("=>".to_owned()),
                Token::Dedent => (),
            }
            return StackRequest::None;
        }

        match *token {
            Token::Text(lineno, s) => {
                // When in an expression, whitespace only serves to separate tokens.
                if PAT_IS_WHITESPACE.is_match(s) {
                    self.new_node = true;
                } else {
                    let mut new_node = self.new_node;

                    if !new_node {
                        if let Some(last) = self.root.last_mut() {
                            match last.value {
                                NodeValue::Owned(ref mut val) => val.push_str(s),
                                NodeValue::Children(_) => new_node = true,
                            }
                        } else {
                            new_node = true;
                        }
                    }

                    if new_node {
                        self.root
                            .push(Node::new_string(s.to_owned(), self.file_id, lineno));
                    }
                    self.new_node = false;
                }
            }
            Token::Quote(_) => {
                self.in_quote = true;
            }
            Token::StartBlock(lineno) => {
                return StackRequest::Push(Box::new(StateExpression::new(self.file_id, lineno)));
            }
            Token::Rocket(lineno) => {
                return StackRequest::Push(Box::new(StateRocket::new(self.file_id, lineno)));
            }
            Token::RightParen | Token::Dedent => {
                return StackRequest::Pop(1);
            }
        }

        StackRequest::None
    }

    fn finish(&mut self) -> Node {
        Node::new_children(
            mem::replace(&mut self.root, vec![]),
            self.file_id,
            self.lineno,
        )
    }

    fn push(&mut self, node: Node) {
        self.root.push(node);
    }

    fn name(&self) -> &'static str {
        "expression"
    }
}

struct ParseContextStack {
    stack: Vec<Box<TokenHandler>>,
}

impl ParseContextStack {
    fn new(file_id: FileID, lineno: i32) -> Self {
        ParseContextStack {
            stack: vec![
                Box::new(StateRocket {
                    root: vec![Node::new_string("concat", file_id, lineno)],
                    buffer: vec![],
                    file_id: file_id,
                    lineno: lineno,
                }),
            ],
        }
    }

    fn handle(&mut self, token: &Token) {
        match self.stack
            .last_mut()
            .expect("Empty parse stack")
            .handle_token(token)
        {
            StackRequest::Push(handler) => {
                self.stack.push(handler);
            }
            StackRequest::Pop(n) => for _ in 0..n {
                let mut handler = self.stack.pop().expect("Cannot pop last handler");
                (**self.stack.last_mut().expect("Empty parse stack")).push(handler.finish());
            },
            StackRequest::None => (),
        }
    }
}

pub struct Parser {
    file_ids: Vec<PathBuf>,
}

impl Parser {
    pub fn new() -> Self {
        Parser { file_ids: vec![] }
    }

    pub fn get_node_source_path(&self, node: &Node) -> Option<&Path> {
        match self.file_ids.get(node.file_id as usize) {
            Some(p) => Some(p),
            None => None,
        }
    }

    fn parse_string(&mut self, id: FileID, data: String) -> Result<Node, String> {
        let mut stack = ParseContextStack::new(id, 0);
        for token in lex(&data) {
            stack.handle(&token);
        }

        let root = stack.stack.pop().expect("Empty state stack").finish();
        match stack.stack.pop() {
            Some(_) => Err(format!(
                "Unterminated block started on line {}",
                root.lineno
            )),
            None => Ok(root),
        }
    }

    pub fn parse(&mut self, path: &Path) -> Result<Node, String> {
        debug!("Parsing {}", path.to_string_lossy());

        let id = self.file_ids.len() as FileID;
        self.file_ids.push(path.to_owned());

        let mut file = match File::open(path) {
            Ok(f) => f,
            Err(_) => {
                return Err(format!("Failed to open {}", path.to_string_lossy()));
            }
        };
        let mut data = String::new();
        file.read_to_string(&mut data)
            .expect("Failed to read input file");

        self.parse_string(id, data)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn rocket(mut args: Vec<Node>, lineno: i32) -> Node {
        let mut children = vec![Node::new_string("concat", 0, lineno)];
        for arg in args.drain(..) {
            children.push(arg);
        }
        Node::new_children(children, 0, lineno)
    }

    #[test]
    fn test_empty() {
        let mut parser = Parser::new();
        assert_eq!(parser.parse_string(0, "".to_owned()), Ok(rocket(vec![], 0)));
    }

    #[test]
    fn test_complex() {
        let mut parser = Parser::new();
        let src = "(:h1 Rocket)

Rocket is a fast and powerful text markup format.

(:h2 (:ref writing-your-first-project \"Getting Started\"))
(:h2 =>Example)
(:code txt =>
    (:():h1 Rocket)

    Rocket is a fast and powerful text markup format.

    (:():h2 (:():ref writing-your-first-project \"Getting Started\"))

\x20\x20\x20\x20

(:toctree
  \"reference\"
  \"tutorials\")";

        let h1 = Node::new_children(
            vec![
                Node::new_string("h1", 0, 0),
                Node::new_string("Rocket", 0, 0),
            ],
            0,
            0,
        );
        let para1 = Node::new_string(
            "\n\nRocket is a fast and powerful text markup format.\n\n",
            0,
            4,
        );
        let h2_1 = Node::new_children(
            vec![
                Node::new_string("h2", 0, 4),
                Node::new_children(
                    vec![
                        Node::new_string("ref", 0, 4),
                        Node::new_string("writing-your-first-project", 0, 4),
                        Node::new_string("Getting Started", 0, 4),
                    ],
                    0,
                    4,
                ),
            ],
            0,
            4,
        );
        let h2_2 = Node::new_children(
            vec![
                Node::new_string("h2", 0, 5),
                Node::new_string("=>Example", 0, 5),
            ],
            0,
            5,
        );
        let code = Node::new_children(
            vec![
                Node::new_string("code", 0, 6),
                Node::new_string("txt", 0, 6),
                rocket(
                    vec![
                        Node::new_children(vec![Node::new_string("(", 0, 7)], 0, 7),
                        Node::new_string(
                            ":h1 Rocket)\n\nRocket is a fast and powerful text markup format.\n\n",
                            0,
                            11,
                        ),
                        Node::new_children(vec![Node::new_string("(", 0, 11)], 0, 11),
                        Node::new_string(":h2 ", 0, 11),
                        Node::new_children(vec![Node::new_string("(", 0, 11)], 0, 11),
                        Node::new_string(
                            ":ref writing-your-first-project \"Getting Started\"))\n\n",
                            0,
                            6,
                        ),
                    ],
                    6,
                ),
            ],
            0,
            6,
        );
        let toctree = Node::new_children(
            vec![
                Node::new_string("toctree", 0, 15),
                Node::new_string("reference", 0, 16),
                Node::new_string("tutorials", 0, 17),
            ],
            0,
            15,
        );
        let result = rocket(
            vec![
                h1,
                para1,
                h2_1,
                Node::new_string("\n", 0, 5),
                h2_2,
                Node::new_string("\n", 0, 6),
                code,
                Node::new_string("\n\n", 0, 15),
                toctree,
            ],
            0,
        );
        assert_eq!(parser.parse_string(0, src.to_owned()), Ok(result));
    }
}
<|endoftext|>