<commit_before>use std::fmt::Debug;
use std::hash::Hash;
use std::io;
//mod funlist;
//use funlist::*;
use std::rc::Rc;

use collections::*;

#[derive(Clone,Copy,Hash,Eq,PartialEq,Debug)]
enum Op { Plus, Minus, Times, Divide }

#[derive(Clone,Copy,Hash,Eq,PartialEq,Debug)]
enum Tok {
  Num(isize),
  Op(Op)
}

fn pop<X:Clone+Copy+Hash+Eq+PartialEq+Debug>
  (stack:List<X>) -> (X, List<X>) {
    List::elim_arg(stack, (),
                   |_,_|    panic!("cannot pop an empty stack"),
                   |h,t, _| (h, t),
                   |_,t, _| pop(t))
}

fn push<X:Clone+Copy+Hash+Eq+PartialEq+Debug>
  (stack:List<X>, elm:X) -> List<X>
{
  List::cons(elm, stack)
}

fn precedence_check (top_op:&Op, next_op:&Op) -> bool {
  match (*top_op, *next_op) {
    (Op::Plus,  _       ) => false,
    (Op::Minus, Op::Plus) => true, // Critical!
    (Op::Minus, _       ) => false,

    (Op::Times, Op::Plus)   => true,
    (Op::Times, Op::Minus)  => true,
    (Op::Times, Op::Divide) => true,

    (Op::Divide, Op::Plus)  => true,
    (Op::Divide, Op::Minus) => true,
    (Op::Divide, Op::Times) => true,
    _ => false
  }
}

fn postfix_of_infix(infix: Tree<Tok>) -> List<Tok> {
  let (ops, postfix) : (List<Op>, List<Tok>) =
    tree_fold_seq
    (infix, Dir2::Left, (List::Nil, List::Nil),
     Rc::new(|tok, (ops, postfix)| {
       match tok {
         Tok::Num(n) => (ops, push(postfix, Tok::Num(n))),
         Tok::Op(op) => {
           if List::is_empty(&ops) { (push(ops, op), postfix) }
           else {
             fn myloop (op:Op, ops:List<Op>, postfix:List<Tok>) -> (List<Op>, List<Tok>) {
               if List::is_empty(&ops) { (ops, postfix) } else {
                 let (top_op, popped_ops) = pop(ops);
                 if precedence_check (&top_op, &op) {
                   myloop (op, popped_ops, push(postfix, Tok::Op(top_op)))
                 }
                 else {
                   (push(popped_ops, top_op), postfix)
                 }
               }};
             let (ops, postfix) = myloop(op, ops, postfix) ;
             (push(ops, op), postfix)
           }
         }
       }}),
     Rc::new(|_, a|   a),
     Rc::new(|_,_, a| a),
     );
  fn myloop (ops:List<Op>, postfix:List<Tok>) -> (List<Tok>) {
    if List::is_empty(&ops) { postfix } else {
      let (top_op, popped_ops) = pop(ops);
      myloop (popped_ops, push(postfix, Tok::Op(top_op)))
    }};
  myloop(ops, postfix)
}

#[test]
fn test_postfix_of_infix () {

  // 1 * 2 + 3
  let input = vec![
    NameElse::Else( Tok::Num(1) ),
    NameElse::Else( Tok::Op(Op::Times) ),
    NameElse::Else( Tok::Num(2) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Num(3) ),
    ];
  let input : List<Tok> = list_of_vec(&input);
  let tree  : Tree<Tok> = tree_of_list(Dir2::Left, input);
  let ans = postfix_of_infix(tree);
  println!("{:?}", ans)
}

fn evaluate_postfix(input: Tree<Tok>) -> isize {
  let stack =
    tree_fold_seq
    (input, Dir2::Left, List::Nil,
     Rc::new(|tok, stack| {
       match tok {
         Tok::Op(op) => {
           let (x,stack) = pop(stack);
           let (y,stack) = pop(stack);
           let z = match op {
             Op::Plus   => y + x,
             Op::Minus  => y - x,
             Op::Times  => y * x,
             Op::Divide => y / x,
           };
           push(stack, z)
         },
         Tok::Num(n) => {
           push(stack, n)
         }
       }}),
     Rc::new(|_, stack|   stack),
     Rc::new(|_,_, stack| stack),
     );
  let (x, stack) = pop(stack);
  assert!(List::is_empty(&stack));
  x
}

#[test]
fn test_eval_postfix () {
  let input = vec![
    NameElse::Else( Tok::Num(1) ),
    NameElse::Else( Tok::Num(2) ),
    NameElse::Else( Tok::Num(3) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Num(22) ),
    NameElse::Else( Tok::Num(33) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Num(12) ),
    NameElse::Else( Tok::Num(13) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    ];
  let input : List<Tok> = list_of_vec(&input);
  let tree  : Tree<Tok> = tree_of_list(Dir2::Left, input);
  let ans = evaluate_postfix(tree);
  assert_eq!(ans, 86)
}

#[test]
fn test_arith_eval () {

  // 1 * 2 + 3
  let input = vec![
    NameElse::Else( Tok::Num(1) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Num(2) ),
    NameElse::Else( Tok::Op(Op::Minus) ),
    NameElse::Else( Tok::Num(3) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Num(1) ),
    // NameElse::Else( Tok::Op(Op::Plus) ),
    // NameElse::Else( Tok::Num(1) ),
    // NameElse::Else( Tok::Op(Op::Times) ),
    // NameElse::Else( Tok::Num(2) ),
    // NameElse::Else( Tok::Op(Op::Divide) ),
    // NameElse::Else( Tok::Num(2) ),
    // NameElse::Else( Tok::Op(Op::Minus) ),
    // NameElse::Else( Tok::Num(3) ),
    ];
  println!("{:?}", &input);
  let input : List<Tok> = list_of_vec(&input);
  let tree  : Tree<Tok> = tree_of_list(Dir2::Left, input);
  let list  : List<Tok> = postfix_of_infix(tree);
  println!("{:?}", &list);
  //let input : List<Tok> = list_of_vec(&input);
  let tree  : Tree<Tok> = tree_of_list(Dir2::Right, list);
  let ans = evaluate_postfix(tree);
  println!("{:?}", ans)
}

// fn infix_to_postfix(input: List<char>) -> List<List<char>> {
//   let mut postfix = List::Nil;
//   let mut operators = List::Nil;
//   let mut temp_operand = List::Nil;

//   //'''
//   // Match all chars in 'input'
//   //     If c is a digit, then push onto a Digit list temporarily until all digits in an operand are read (after which add the operand to the Postfix list)
//   //     If c is an operator, pop everything of higher predence and add it to postfix list, then add c to list of Operators
//   //'''
//   postfix
// }

// fn evaluate_postfix(input: List<List<char>>) -> i32 {

//   //'''
//   // Convert list to number and apply operation on two at a time
//   //'''

// }

// fn test_arith_parse() {
//   println!("Enter expression in infix format");
//   let mut input = String::new();
//   io::stdin().read_line(&mut input).ok().expect("Failed to parse input");

//   // '''
//   // Convert string to List<char> (reverse order)
//   // '''
//   let input_list = panic!("");

//   let postfix_expr = infix_to_postfix(input_list);
//   let result = evaluate_postfix(postfix_expr);
// }

<commit_msg>making arith_parse stuff public<commit_after>use std::fmt::Debug;
use std::hash::Hash;
use std::rc::Rc;

use collections::*;

#[derive(Clone,Copy,Hash,Eq,PartialEq,Debug)]
pub enum Op { Plus, Minus, Times, Divide }

#[derive(Clone,Copy,Hash,Eq,PartialEq,Debug)]
pub enum Tok {
  Num(isize),
  Op(Op)
}

fn pop<X:Clone+Copy+Hash+Eq+PartialEq+Debug>
  (stack:List<X>) -> (X, List<X>) {
    List::elim_arg(stack, (),
                   |_,_|    panic!("cannot pop an empty stack"),
                   |h,t, _| (h, t),
                   |_,t, _| pop(t))
}

fn push<X:Clone+Copy+Hash+Eq+PartialEq+Debug>
  (stack:List<X>, elm:X) -> List<X>
{
  List::cons(elm, stack)
}

fn precedence_check (top_op:&Op, next_op:&Op) -> bool {
  match (*top_op, *next_op) {
    (Op::Plus,  _       ) => false,
    (Op::Minus, Op::Plus) => true, // Critical!
    (Op::Minus, _       ) => false,

    (Op::Times, Op::Plus)   => true,
    (Op::Times, Op::Minus)  => true,
    (Op::Times, Op::Divide) => true,

    (Op::Divide, Op::Plus)  => true,
    (Op::Divide, Op::Minus) => true,
    (Op::Divide, Op::Times) => true,
    _ => false
  }
}

pub fn postfix_of_infix(infix: Tree<Tok>) -> List<Tok> {
  let (ops, postfix) : (List<Op>, List<Tok>) =
    tree_fold_seq
    (infix, Dir2::Left, (List::Nil, List::Nil),
     Rc::new(|tok, (ops, postfix)| {
       match tok {
         Tok::Num(n) => (ops, push(postfix, Tok::Num(n))),
         Tok::Op(op) => {
           if List::is_empty(&ops) { (push(ops, op), postfix) }
           else {
             fn myloop (op:Op, ops:List<Op>, postfix:List<Tok>) -> (List<Op>, List<Tok>) {
               if List::is_empty(&ops) { (ops, postfix) } else {
                 let (top_op, popped_ops) = pop(ops);
                 if precedence_check (&top_op, &op) {
                   myloop (op, popped_ops, push(postfix, Tok::Op(top_op)))
                 }
                 else {
                   (push(popped_ops, top_op), postfix)
                 }
               }};
             let (ops, postfix) = myloop(op, ops, postfix) ;
             (push(ops, op), postfix)
           }
         }
       }}),
     Rc::new(|_, a|   a),
     Rc::new(|_,_, a| a),
     );
  fn myloop (ops:List<Op>, postfix:List<Tok>) -> (List<Tok>) {
    if List::is_empty(&ops) { postfix } else {
      let (top_op, popped_ops) = pop(ops);
      myloop (popped_ops, push(postfix, Tok::Op(top_op)))
    }};
  myloop(ops, postfix)
}

#[test]
fn test_postfix_of_infix () {

  // 1 * 2 + 3
  let input = vec![
    NameElse::Else( Tok::Num(1) ),
    NameElse::Else( Tok::Op(Op::Times) ),
    NameElse::Else( Tok::Num(2) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Num(3) ),
    ];
  let input : List<Tok> = list_of_vec(&input);
  let tree  : Tree<Tok> = tree_of_list(Dir2::Left, input);
  let ans = postfix_of_infix(tree);
  println!("{:?}", ans)
}

pub fn evaluate_postfix(input: Tree<Tok>) -> isize {
  let stack =
    tree_fold_seq
    (input, Dir2::Left, List::Nil,
     Rc::new(|tok, stack| {
       match tok {
         Tok::Op(op) => {
           let (x,stack) = pop(stack);
           let (y,stack) = pop(stack);
           let z = match op {
             Op::Plus   => y + x,
             Op::Minus  => y - x,
             Op::Times  => y * x,
             Op::Divide => y / x,
           };
           push(stack, z)
         },
         Tok::Num(n) => {
           push(stack, n)
         }
       }}),
     Rc::new(|_, stack|   stack),
     Rc::new(|_,_, stack| stack),
     );
  let (x, stack) = pop(stack);
  assert!(List::is_empty(&stack));
  x
}

#[test]
fn test_eval_postfix () {
  let input = vec![
    NameElse::Else( Tok::Num(1) ),
    NameElse::Else( Tok::Num(2) ),
    NameElse::Else( Tok::Num(3) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Num(22) ),
    NameElse::Else( Tok::Num(33) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Num(12) ),
    NameElse::Else( Tok::Num(13) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    ];
  let input : List<Tok> = list_of_vec(&input);
  let tree  : Tree<Tok> = tree_of_list(Dir2::Left, input);
  let ans = evaluate_postfix(tree);
  assert_eq!(ans, 86)
}

#[test]
fn test_arith_eval () {

  // 1 * 2 + 3
  let input = vec![
    NameElse::Else( Tok::Num(1) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Num(2) ),
    NameElse::Else( Tok::Op(Op::Minus) ),
    NameElse::Else( Tok::Num(3) ),
    NameElse::Else( Tok::Op(Op::Plus) ),
    NameElse::Else( Tok::Num(1) ),
    // NameElse::Else( Tok::Op(Op::Plus) ),
    // NameElse::Else( Tok::Num(1) ),
    // NameElse::Else( Tok::Op(Op::Times) ),
    // NameElse::Else( Tok::Num(2) ),
    // NameElse::Else( Tok::Op(Op::Divide) ),
    // NameElse::Else( Tok::Num(2) ),
    // NameElse::Else( Tok::Op(Op::Minus) ),
    // NameElse::Else( Tok::Num(3) ),
    ];
  println!("{:?}", &input);
  let input : List<Tok> = list_of_vec(&input);
  let tree  : Tree<Tok> = tree_of_list(Dir2::Left, input);
  let list  : List<Tok> = postfix_of_infix(tree);
  println!("{:?}", &list);
  //let input : List<Tok> = list_of_vec(&input);
  let tree  : Tree<Tok> = tree_of_list(Dir2::Right, list);
  let ans = evaluate_postfix(tree);
  println!("{:?}", ans)
}

// fn infix_to_postfix(input: List<char>) -> List<List<char>> {
//   let mut postfix = List::Nil;
//   let mut operators = List::Nil;
//   let mut temp_operand = List::Nil;

//   //'''
//   // Match all chars in 'input'
//   //     If c is a digit, then push onto a Digit list temporarily until all digits in an operand are read (after which add the operand to the Postfix list)
//   //     If c is an operator, pop everything of higher predence and add it to postfix list, then add c to list of Operators
//   //'''
//   postfix
// }

// fn evaluate_postfix(input: List<List<char>>) -> i32 {

//   //'''
//   // Convert list to number and apply operation on two at a time
//   //'''

// }

// fn test_arith_parse() {
//   println!("Enter expression in infix format");
//   let mut input = String::new();
//   io::stdin().read_line(&mut input).ok().expect("Failed to parse input");

//   // '''
//   // Convert string to List<char> (reverse order)
//   // '''
//   let input_list = panic!("");

//   let postfix_expr = infix_to_postfix(input_list);
//   let result = evaluate_postfix(postfix_expr);
// }

<|endoftext|>


<commit_before>extern crate hyper;
extern crate hyper_rustls;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate serde_json;
extern crate scraper;
extern crate chrono;
extern crate dotenv;

use hyper::Client;

use scraper::{Html, Selector};

use chrono::{UTC, TimeZone};

use std::collections::HashMap;
use std::fs::{File, OpenOptions};
use std::io::Read;
use std::env;

const NEWS_URL: &'static str = "http://na.finalfantasyxiv.com/lodestone/news/";

struct NewsScraper {
  client: Client,
  database: NewsDatabase
}

#[derive(Debug, Default, Serialize, Deserialize)]
struct NewsDatabase {
  items: HashMap<String, NewsItem>
}

#[derive(Debug, Serialize, Deserialize)]
struct NewsItem {
  title: String,
  url: String,
  kind: NewsKind,
  time: i64,
  tag: Option<String>
}

#[derive(Debug, Serialize, Deserialize)]
enum NewsKind {
  SpecialNotice,
  News,
  Topic
}

impl ToString for NewsKind {
  fn to_string(&self) -> String {
    match *self {
      NewsKind::News => "News",
      NewsKind::Topic => "Topic",
      NewsKind::SpecialNotice => "Special notice"
    }.to_string()
  }
}

fn main() {
  dotenv::dotenv().ok();

  let webhook_url = match env::var("LB_NEWS_WEBHOOK") {
    Ok(u) => u,
    Err(_) => {
      println!("No webhook specified");
      return;
    }
  };

  let database = match NewsDatabase::load() {
    Some(db) => db,
    None => NewsDatabase::default()
  };
  let mut scraper = NewsScraper::new(database);

  let news = match scraper.download_news() {
    Some(n) => n,
    None => {
      println!("could not download news");
      return;
    }
  };

  let downloaded_news = scraper.parse_news(&news);
  for (id, item) in downloaded_news {
    if !scraper.database.items.contains_key(&id) {
      let mut embed = json!({
        "type": "rich",
        "timestamp": UTC.timestamp(item.time, 0).to_rfc3339(),
        // "url": item.url,
        "fields": [
          {
            "name": "Title",
            "value": item.title,
            "inline": false
          },
          {
            "name": "Link",
            "value": item.url,
            "inline": false
          },
          {
            "name": "Kind",
            "value": item.kind.to_string(),
            "inline": true
          }
        ]
      });
      if let Some(ref tag) = item.tag {
        embed["fields"].as_array_mut().unwrap().push(json!({
          "name": "Tag",
          "value": tag,
          "inline": true
        }));
      }
      let data = json!({
        "embeds": [embed]
      });
      println!("{:#?}", item);
      // scraper.database.items.insert(id, item);
      // let res = scraper.client.post(&webhook_url)
      //   .header(hyper::header::ContentType::json())
      //   .body(&data.to_string())
      //   .send();
      // let mut data = match res {
      //   Ok(r) => r,
      //   Err(e) => {
      //     println!("error sending webhook: {}", e);
      //     continue;
      //   }
      // };
      // let mut content = String::new();
      // if let Err(e) = data.read_to_string(&mut content) {
      //   println!("could not read webhook response: {}", e);
      //   continue;
      // }
      // if data.status.class() != hyper::status::StatusClass::Success {
      //   println!("discord says no");
      //   println!("{}", content);
      // } else {
      //   println!("webhook sent");
      // }
    }
  }
  if scraper.database.save().is_none() {
    println!("could not save database");
  }
}

impl NewsDatabase {
  fn load() -> Option<Self> {
    let news_file = match File::open("./lodestone_news.json") {
      Ok(n) => n,
      Err(_) => return None
    };
    serde_json::from_reader(news_file).ok()
  }

  fn save(&self) -> Option<()> {
    let news_file = match OpenOptions::new().create(true).write(true).open("./lodestone_news.json") {
      Ok(n) => n,
      Err(_) => return None
    };
    serde_json::to_writer(news_file, self).ok()
  }
}

impl NewsScraper {
  fn new(database: NewsDatabase) -> Self {
    NewsScraper {
      client: Client::with_connector(hyper::net::HttpsConnector::new(hyper_rustls::TlsClient::new())),
      database: database
    }
  }

  fn download_news(&self) -> Option<String> {
    let mut response = match self.client.get(NEWS_URL).send() {
      Ok(r) => r,
      Err(_) => return None
    };

    let mut content = String::new();
    if response.read_to_string(&mut content).is_err() {
      return None;
    }
    Some(content)
  }

  fn parse_news(&self, news: &str) -> HashMap<String, NewsItem> {
    let html = Html::parse_document(news);
    let news_selector = Selector::parse("div.news__content.parts__space--add > ul:nth-of-type(2) > li").unwrap();
    let topics_selector = Selector::parse("div.news__content.parts__space--add > ul:nth-of-type(3) > li").unwrap();
    let title_selector = Selector::parse("p.news__list--title").unwrap();
    let time_script_selector = Selector::parse("time.news__list--time > script").unwrap();
    let mut lis: Vec<_> = html.select(&news_selector).map(|x| (NewsKind::News, x)).collect();
    lis.append(&mut html.select(&topics_selector).map(|x| (NewsKind::Topic, x)).collect());
    let mut items = HashMap::with_capacity(lis.len());
    for (kind, li) in lis {
      let child = match kind {
        NewsKind::News => li.first_child().and_then(|v| v.value().as_element()),
        NewsKind::Topic => li.select(&title_selector).next().and_then(|v| v.first_child().and_then(|x| x.value().as_element())),
        _ => {
          println!("unsupported news kind");
          continue;
        }
      };

      let child = match child {
        Some(c) => c,
        None => {
          println!("could not get news item child");
          continue
        }
      };

      let href = match child.attr("href") {
        Some(h) => h,
        None => {
          println!("invalid link in news item");
          continue;
        }
      };

      let (title, tag) = match kind {
        NewsKind::News => {
          let title = match li.select(&title_selector).next() {
            Some(t) => t,
            None => {
              println!("missing title in news item");
              continue;
            }
          };

          let tag: Option<String> = title.first_child()
            .and_then(scraper::ElementRef::wrap)
            .map(|c| c.text().collect::<String>())
            .map(|tag| tag[1..tag.len() - 1].to_string());

          let text_iter = title.text();
          let title: String = if tag.is_some() {
            text_iter.skip(1).collect()
          } else {
            text_iter.collect()
          };

          (title, tag)
        },
        NewsKind::Topic => {
          let text = li.select(&title_selector).next()
            .and_then(|v| v.first_child())
            .and_then(scraper::ElementRef::wrap)
            .map(|v| v.text().collect());
          match text {
            Some(t) => (t, None),
            None => {
              println!("invalid topic: no title");
              continue;
            }
          }
        },
        _ => unreachable!()
      };

      let time_script = match li.select(&time_script_selector).next() {
        Some(ts) => ts,
        None => {
          println!("news item missing time script");
          continue
        }
      };

      let time_script: String = time_script.text().collect();
      let time_string = match time_script.split("strftime(").nth(1).and_then(|v| v.split(',').next()) {
        Some(time) => time,
        None => {
          println!("invalid script in news item");
          continue
        }
      };
      let time: i64 = match time_string.parse() {
        Ok(t) => t,
        Err(_) => {
          println!("invalid time in time script");
          continue
        }
      };

      let id = match href.split('/').last() {
        Some(i) => i,
        None => {
          println!("invalid href in news item");
          continue;
        }
      };

      let news_item = NewsItem {
        title: title,
        url: format!("http://na.finalfantasyxiv.com{}", href),
        kind: kind,
        time: time,
        tag: tag
      };
      items.insert(id.to_string(), news_item);
    }

    items
  }
}
<commit_msg>fix: uncomment accidental comments<commit_after>extern crate hyper;
extern crate hyper_rustls;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate serde_json;
extern crate scraper;
extern crate chrono;
extern crate dotenv;

use hyper::Client;

use scraper::{Html, Selector};

use chrono::{UTC, TimeZone};

use std::collections::HashMap;
use std::fs::{File, OpenOptions};
use std::io::Read;
use std::env;

const NEWS_URL: &'static str = "http://na.finalfantasyxiv.com/lodestone/news/";

struct NewsScraper {
  client: Client,
  database: NewsDatabase
}

#[derive(Debug, Default, Serialize, Deserialize)]
struct NewsDatabase {
  items: HashMap<String, NewsItem>
}

#[derive(Debug, Serialize, Deserialize)]
struct NewsItem {
  title: String,
  url: String,
  kind: NewsKind,
  time: i64,
  tag: Option<String>
}

#[derive(Debug, Serialize, Deserialize)]
enum NewsKind {
  SpecialNotice,
  News,
  Topic
}

impl ToString for NewsKind {
  fn to_string(&self) -> String {
    match *self {
      NewsKind::News => "News",
      NewsKind::Topic => "Topic",
      NewsKind::SpecialNotice => "Special notice"
    }.to_string()
  }
}

fn main() {
  dotenv::dotenv().ok();

  let webhook_url = match env::var("LB_NEWS_WEBHOOK") {
    Ok(u) => u,
    Err(_) => {
      println!("No webhook specified");
      return;
    }
  };

  let database = match NewsDatabase::load() {
    Some(db) => db,
    None => NewsDatabase::default()
  };
  let mut scraper = NewsScraper::new(database);

  let news = match scraper.download_news() {
    Some(n) => n,
    None => {
      println!("could not download news");
      return;
    }
  };

  let downloaded_news = scraper.parse_news(&news);
  for (id, item) in downloaded_news {
    if !scraper.database.items.contains_key(&id) {
      let mut embed = json!({
        "type": "rich",
        "timestamp": UTC.timestamp(item.time, 0).to_rfc3339(),
        // "url": item.url,
        "fields": [
          {
            "name": "Title",
            "value": item.title,
            "inline": false
          },
          {
            "name": "Link",
            "value": item.url,
            "inline": false
          },
          {
            "name": "Kind",
            "value": item.kind.to_string(),
            "inline": true
          }
        ]
      });
      if let Some(ref tag) = item.tag {
        embed["fields"].as_array_mut().unwrap().push(json!({
          "name": "Tag",
          "value": tag,
          "inline": true
        }));
      }
      let data = json!({
        "embeds": [embed]
      });
      println!("{:#?}", item);
      scraper.database.items.insert(id, item);
      let res = scraper.client.post(&webhook_url)
        .header(hyper::header::ContentType::json())
        .body(&data.to_string())
        .send();
      let mut data = match res {
        Ok(r) => r,
        Err(e) => {
          println!("error sending webhook: {}", e);
          continue;
        }
      };
      let mut content = String::new();
      if let Err(e) = data.read_to_string(&mut content) {
        println!("could not read webhook response: {}", e);
        continue;
      }
      if data.status.class() != hyper::status::StatusClass::Success {
        println!("discord says no");
        println!("{}", content);
      } else {
        println!("webhook sent");
      }
    }
  }
  if scraper.database.save().is_none() {
    println!("could not save database");
  }
}

impl NewsDatabase {
  fn load() -> Option<Self> {
    let news_file = match File::open("./lodestone_news.json") {
      Ok(n) => n,
      Err(_) => return None
    };
    serde_json::from_reader(news_file).ok()
  }

  fn save(&self) -> Option<()> {
    let news_file = match OpenOptions::new().create(true).write(true).open("./lodestone_news.json") {
      Ok(n) => n,
      Err(_) => return None
    };
    serde_json::to_writer(news_file, self).ok()
  }
}

impl NewsScraper {
  fn new(database: NewsDatabase) -> Self {
    NewsScraper {
      client: Client::with_connector(hyper::net::HttpsConnector::new(hyper_rustls::TlsClient::new())),
      database: database
    }
  }

  fn download_news(&self) -> Option<String> {
    let mut response = match self.client.get(NEWS_URL).send() {
      Ok(r) => r,
      Err(_) => return None
    };

    let mut content = String::new();
    if response.read_to_string(&mut content).is_err() {
      return None;
    }
    Some(content)
  }

  fn parse_news(&self, news: &str) -> HashMap<String, NewsItem> {
    let html = Html::parse_document(news);
    let news_selector = Selector::parse("div.news__content.parts__space--add > ul:nth-of-type(2) > li").unwrap();
    let topics_selector = Selector::parse("div.news__content.parts__space--add > ul:nth-of-type(3) > li").unwrap();
    let title_selector = Selector::parse("p.news__list--title").unwrap();
    let time_script_selector = Selector::parse("time.news__list--time > script").unwrap();
    let mut lis: Vec<_> = html.select(&news_selector).map(|x| (NewsKind::News, x)).collect();
    lis.append(&mut html.select(&topics_selector).map(|x| (NewsKind::Topic, x)).collect());
    let mut items = HashMap::with_capacity(lis.len());
    for (kind, li) in lis {
      let child = match kind {
        NewsKind::News => li.first_child().and_then(|v| v.value().as_element()),
        NewsKind::Topic => li.select(&title_selector).next().and_then(|v| v.first_child().and_then(|x| x.value().as_element())),
        _ => {
          println!("unsupported news kind");
          continue;
        }
      };

      let child = match child {
        Some(c) => c,
        None => {
          println!("could not get news item child");
          continue
        }
      };

      let href = match child.attr("href") {
        Some(h) => h,
        None => {
          println!("invalid link in news item");
          continue;
        }
      };

      let (title, tag) = match kind {
        NewsKind::News => {
          let title = match li.select(&title_selector).next() {
            Some(t) => t,
            None => {
              println!("missing title in news item");
              continue;
            }
          };

          let tag: Option<String> = title.first_child()
            .and_then(scraper::ElementRef::wrap)
            .map(|c| c.text().collect::<String>())
            .map(|tag| tag[1..tag.len() - 1].to_string());

          let text_iter = title.text();
          let title: String = if tag.is_some() {
            text_iter.skip(1).collect()
          } else {
            text_iter.collect()
          };

          (title, tag)
        },
        NewsKind::Topic => {
          let text = li.select(&title_selector).next()
            .and_then(|v| v.first_child())
            .and_then(scraper::ElementRef::wrap)
            .map(|v| v.text().collect());
          match text {
            Some(t) => (t, None),
            None => {
              println!("invalid topic: no title");
              continue;
            }
          }
        },
        _ => unreachable!()
      };

      let time_script = match li.select(&time_script_selector).next() {
        Some(ts) => ts,
        None => {
          println!("news item missing time script");
          continue
        }
      };

      let time_script: String = time_script.text().collect();
      let time_string = match time_script.split("strftime(").nth(1).and_then(|v| v.split(',').next()) {
        Some(time) => time,
        None => {
          println!("invalid script in news item");
          continue
        }
      };
      let time: i64 = match time_string.parse() {
        Ok(t) => t,
        Err(_) => {
          println!("invalid time in time script");
          continue
        }
      };

      let id = match href.split('/').last() {
        Some(i) => i,
        None => {
          println!("invalid href in news item");
          continue;
        }
      };

      let news_item = NewsItem {
        title: title,
        url: format!("http://na.finalfantasyxiv.com{}", href),
        kind: kind,
        time: time,
        tag: tag
      };
      items.insert(id.to_string(), news_item);
    }

    items
  }
}
<|endoftext|>


<commit_before>//! Provide access to locally cached memdb SDKs
//!
//! The `MemDbStash` pulls in remote SDKs from an S3 bucket and provides
//! access to it.  This is used by the symbol server to manage the local
//! cache and also to refer to memdb files that are mmap'ed in.
use std::fs;
use std::io;
use std::iter::FromIterator;
use std::path::{Path, PathBuf};
use std::collections::{HashMap, HashSet};
use std::collections::hash_map::Values as HashMapValuesIter;
use std::sync::{Arc, RwLock};

use serde_json;
use xz2::write::XzDecoder;

use super::config::Config;
use super::sdk::SdkInfo;
use super::s3::S3;
use super::memdb::MemDb;
use super::utils::{ProgressIndicator, copy_with_progress};
use super::{Result, ResultExt, ErrorKind};

/// The main memdb stash type
pub struct MemDbStash<'a> {
    path: &'a Path,
    s3: S3<'a>,
    local_state: RwLock<Option<Arc<SdkSyncState>>>,
    memdbs: RwLock<HashMap<SdkInfo, Arc<MemDb<'a>>>>,
}

/// Information about a remotely available SDK
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct RemoteSdk {
    filename: String,
    info: SdkInfo,
    size: u64,
    etag: String,
}

#[derive(Serialize, Deserialize, Default, Debug, Clone)]
struct SdkSyncState {
    sdks: HashMap<String, RemoteSdk>,
}

/// Information about the health of the stash sync
#[derive(Debug)]
pub struct SyncStatus {
    remote_total: u32,
    missing: u32,
    different: u32,
}

impl RemoteSdk {
    /// Creates a remote SDK object from some information
    pub fn new(filename: String, info: SdkInfo, etag: String, size: u64) -> RemoteSdk {
        RemoteSdk {
            filename: filename,
            info: info,
            etag: etag,
            size: size,
        }
    }

    /// The remotely visible filename for the SDK
    pub fn filename(&self) -> &str {
        &self.filename
    }

    /// The local filename the SDK has in the stash folder
    pub fn local_filename(&self) -> &str {
        self.filename.trim_right_matches('z')
    }

    /// The size of the SDK in bytes
    pub fn size(&self) -> u64 {
        self.size
    }

    /// Returns the SDK info
    pub fn info(&self) -> &SdkInfo {
        &self.info
    }
}

/// Iterator over the SDKs
pub type RemoteSdkIter<'a> = HashMapValuesIter<'a, String, RemoteSdk>;

impl SdkSyncState {
    pub fn get_sdk(&self, filename: &str) -> Option<&RemoteSdk> {
        self.sdks.get(filename)
    }

    pub fn update_sdk(&mut self, sdk: &RemoteSdk) {
        self.sdks.insert(sdk.local_filename().to_string(), sdk.clone());
    }

    pub fn sdks<'a>(&'a self) -> RemoteSdkIter<'a> {
        self.sdks.values()
    }
}

impl SyncStatus {

    /// Returns the lag (number of SDKs behind upstream)
    pub fn lag(&self) -> u32 {
        self.missing + self.different
    }

    /// Returns true if the local sync is still considered healthy
    pub fn is_healthy(&self) -> bool {
        let total = self.remote_total as f32;
        let lag = self.lag() as f32;
        lag / total < 0.10
    }
}

impl<'a> MemDbStash<'a> {
    /// Opens a stash for a given config.
    pub fn new(config: &'a Config) -> Result<MemDbStash<'a>> {
        Ok(MemDbStash {
            path: config.get_symbol_dir()?,
            s3: S3::from_config(config)?,
            local_state: RwLock::new(None),
            memdbs: RwLock::new(HashMap::new()),
        })
    }

    fn get_local_sync_state_filename(&self) -> PathBuf {
        self.path.join("sync.state")
    }

    fn load_state(&self, filename: &Path) -> Result<Option<SdkSyncState>> {
        match fs::File::open(filename) {
            Ok(f) => Ok(Some(serde_json::from_reader(f)
                .chain_err(|| "Parsing error on loading sync state")?)),
            Err(err) => {
                if err.kind() == io::ErrorKind::NotFound {
                    Ok(None)
                } else {
                    Err(err).chain_err(|| "Error loading sync state")
                }
            }
        }
    }

    fn save_state(&self, new_state: &SdkSyncState, filename: &Path) -> Result<()> {
        let mut tmp_filename = filename.to_path_buf();
        tmp_filename.set_extension("tempstate");
        {
            let mut f = fs::File::create(&tmp_filename)?;
            serde_json::to_writer(&mut f, new_state)
                .chain_err(|| "Could not update sync state")?;
        }
        fs::rename(&tmp_filename, &filename)?;
        Ok(())
    }

    fn read_local_state(&self) -> Result<SdkSyncState> {
        let rv = self.load_state(&self.get_local_sync_state_filename())?
            .unwrap_or_else(|| Default::default());
        let mut opt = self.local_state.write().unwrap();
        *opt = Some(Arc::new(rv.clone()));
        Ok(rv)
    }

    fn get_local_state(&self) -> Result<Arc<SdkSyncState>> {
        if let Some(ref arc) = *self.local_state.read().unwrap() {
            Ok(arc.clone())
        } else {
            self.read_local_state()?;
            Ok(self.local_state.read().unwrap().as_ref().unwrap().clone())
        }
    }

    fn save_local_state(&self, new_state: &SdkSyncState) -> Result<()> {
        self.save_state(new_state, &self.get_local_sync_state_filename())?;
        let mut opt = self.local_state.write().unwrap();
        *opt = Some(Arc::new(new_state.clone()));
        Ok(())
    }

    fn fetch_remote_state(&self) -> Result<SdkSyncState> {
        let mut sdks = HashMap::new();
        for remote_sdk in self.s3.list_upstream_sdks()? {
            sdks.insert(remote_sdk.local_filename().into(), remote_sdk);
        }
        Ok(SdkSyncState { sdks: sdks })
    }

    fn update_sdk(&self, sdk: &RemoteSdk) -> Result<()> {
        // XXX: the progress bar here can stall out because we currently
        // need to buffer the download into memory in the s3 code :(
        let progress = ProgressIndicator::new(sdk.size() as usize);
        progress.set_message(&format!("Synchronizing {}", sdk.info()));
        let mut src = self.s3.download_sdk(sdk)?;
        let dst = fs::File::create(self.path.join(sdk.local_filename()))?;
        let mut dst = XzDecoder::new(dst);
        copy_with_progress(&progress, &mut src, &mut dst)?;
        progress.finish(&format!("Synchronized {}", sdk.info()));
        Ok(())
    }

    fn remove_sdk(&self, sdk: &RemoteSdk) -> Result<()> {
        println!("Deleting {}", sdk.info());
        if let Err(err) = fs::remove_file(self.path.join(sdk.local_filename())) {
            if err.kind() != io::ErrorKind::NotFound {
                return Err(err.into());
            }
        }
        Ok(())
    }

    /// Checks the local stash against the server
    pub fn get_sync_status(&self) -> Result<SyncStatus> {
        let local_state = self.read_local_state()?;
        let remote_state = self.fetch_remote_state()?;

        let mut remote_total = 0;
        let mut missing = 0;
        let mut different = 0;

        for sdk in remote_state.sdks() {
            if let Some(local_sdk) = local_state.get_sdk(sdk.local_filename()) {
                if local_sdk != sdk {
                    different += 1;
                }
            } else {
                missing += 1;
            }
            remote_total += 1;
        }

        Ok(SyncStatus {
            remote_total: remote_total as u32,
            missing: missing as u32,
            different: different as u32,
        })
    }

    /// Synchronize the local stash with the server
    pub fn sync(&self) -> Result<()> {
        let mut local_state = self.read_local_state()?;
        let remote_state = self.fetch_remote_state()?;

        let mut to_delete : HashSet<_> = HashSet::from_iter(
            local_state.sdks().map(|x| x.local_filename().to_string()));

        for sdk in remote_state.sdks() {
            if let Some(local_sdk) = local_state.get_sdk(sdk.local_filename()) {
                if local_sdk != sdk {
                    self.update_sdk(&sdk)?;
                } else {
                    println!("  ⸰ Unchanged {}", sdk.info());
                }
            } else {
                self.update_sdk(&sdk)?;
            }
            to_delete.remove(sdk.local_filename());
            local_state.update_sdk(&sdk);
            self.save_local_state(&local_state)?;
        }

        for local_filename in to_delete.iter() {
            if let Some(sdk) = local_state.get_sdk(local_filename) {
                self.remove_sdk(sdk)?;
            }
        }

        println!("Done synching");

        // if we get this far, the remote state is indeed the local state.
        self.save_local_state(&remote_state)?;

        Ok(())
    }

    /// Looks up an memdb by an SDK info if it's available
    pub fn get_memdb(&'a self, info: &SdkInfo) -> Result<Arc<MemDb<'a>>> {
        let local_state = self.get_local_state()?;
        if let Some(sdk) = local_state.get_sdk(&info.memdb_filename()) {
            if let Some(arc) = self.memdbs.read().unwrap().get(info) {
                return Ok(arc.clone());
            }
            let memdb = MemDb::from_path(self.path.join(info.memdb_filename()))?;
            self.memdbs.write().unwrap().insert(info.clone(), Arc::new(memdb));
            if let Some(arc) = self.memdbs.read().unwrap().get(info) {
                return Ok(arc.clone());
            }
        }

        Err(ErrorKind::UnknownSdk.into())
    }
}
<commit_msg>Restructured memdb stash<commit_after>//! Provide access to locally cached memdb SDKs
//!
//! The `MemDbStash` pulls in remote SDKs from an S3 bucket and provides
//! access to it.  This is used by the symbol server to manage the local
//! cache and also to refer to memdb files that are mmap'ed in.
use std::fs;
use std::io;
use std::iter::FromIterator;
use std::path::{Path, PathBuf};
use std::collections::{HashMap, HashSet};
use std::collections::hash_map::Values as HashMapValuesIter;
use std::sync::{Arc, RwLock};

use serde_json;
use xz2::write::XzDecoder;

use super::config::Config;
use super::sdk::SdkInfo;
use super::s3::S3;
use super::memdb::MemDb;
use super::utils::{ProgressIndicator, copy_with_progress};
use super::{Result, ResultExt, ErrorKind};

/// The main memdb stash type
pub struct MemDbStash<'a> {
    path: &'a Path,
    s3: S3<'a>,
    local_state: RwLock<Option<Arc<SdkSyncState>>>,
    memdbs: RwLock<HashMap<SdkInfo, Arc<MemDb<'a>>>>,
}

/// Information about a remotely available SDK
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct RemoteSdk {
    filename: String,
    info: SdkInfo,
    size: u64,
    etag: String,
}

#[derive(Serialize, Deserialize, Default, Debug, Clone)]
struct SdkSyncState {
    sdks: HashMap<String, RemoteSdk>,
}

/// Information about the health of the stash sync
#[derive(Debug)]
pub struct SyncStatus {
    remote_total: u32,
    missing: u32,
    different: u32,
}

impl RemoteSdk {
    /// Creates a remote SDK object from some information
    pub fn new(filename: String, info: SdkInfo, etag: String, size: u64) -> RemoteSdk {
        RemoteSdk {
            filename: filename,
            info: info,
            etag: etag,
            size: size,
        }
    }

    /// The remotely visible filename for the SDK
    pub fn filename(&self) -> &str {
        &self.filename
    }

    /// The local filename the SDK has in the stash folder
    pub fn local_filename(&self) -> &str {
        self.filename.trim_right_matches('z')
    }

    /// The size of the SDK in bytes
    pub fn size(&self) -> u64 {
        self.size
    }

    /// Returns the SDK info
    pub fn info(&self) -> &SdkInfo {
        &self.info
    }
}

/// Iterator over the SDKs
pub type RemoteSdkIter<'a> = HashMapValuesIter<'a, String, RemoteSdk>;

impl SdkSyncState {
    pub fn get_sdk(&self, filename: &str) -> Option<&RemoteSdk> {
        self.sdks.get(filename)
    }

    pub fn update_sdk(&mut self, sdk: &RemoteSdk) {
        self.sdks.insert(sdk.local_filename().to_string(), sdk.clone());
    }

    pub fn sdks<'a>(&'a self) -> RemoteSdkIter<'a> {
        self.sdks.values()
    }
}

impl SyncStatus {

    /// Returns the lag (number of SDKs behind upstream)
    pub fn lag(&self) -> u32 {
        self.missing + self.different
    }

    /// Returns true if the local sync is still considered healthy
    pub fn is_healthy(&self) -> bool {
        let total = self.remote_total as f32;
        let lag = self.lag() as f32;
        lag / total < 0.10
    }
}

impl<'a> MemDbStash<'a> {
    /// Opens a stash for a given config.
    pub fn new(config: &'a Config) -> Result<MemDbStash<'a>> {
        Ok(MemDbStash {
            path: config.get_symbol_dir()?,
            s3: S3::from_config(config)?,
            local_state: RwLock::new(None),
            memdbs: RwLock::new(HashMap::new()),
        })
    }

    fn get_local_sync_state_filename(&self) -> PathBuf {
        self.path.join("sync.state")
    }

    fn save_state(&self, new_state: &SdkSyncState, filename: &Path) -> Result<()> {
        let mut tmp_filename = filename.to_path_buf();
        tmp_filename.set_extension("tempstate");
        {
            let mut f = fs::File::create(&tmp_filename)?;
            serde_json::to_writer(&mut f, new_state)
                .chain_err(|| "Could not update sync state")?;
        }
        fs::rename(&tmp_filename, &filename)?;
        Ok(())
    }

    fn read_local_state(&self) -> Result<SdkSyncState> {
        let rv : SdkSyncState = match fs::File::open(&self.get_local_sync_state_filename()) {
            Ok(f) => serde_json::from_reader(f)
                .chain_err(|| "Parsing error on loading sync state")?,
            Err(err) => {
                if err.kind() == io::ErrorKind::NotFound {
                    Default::default()
                } else {
                    return Err(err).chain_err(|| "Error loading sync state");
                }
            }
        };
        let mut opt = self.local_state.write().unwrap();
        *opt = Some(Arc::new(rv.clone()));
        Ok(rv)
    }

    fn get_local_state(&self) -> Result<Arc<SdkSyncState>> {
        if let Some(ref arc) = *self.local_state.read().unwrap() {
            Ok(arc.clone())
        } else {
            self.read_local_state()?;
            Ok(self.local_state.read().unwrap().as_ref().unwrap().clone())
        }
    }

    fn save_local_state(&self, new_state: &SdkSyncState) -> Result<()> {
        self.save_state(new_state, &self.get_local_sync_state_filename())?;
        let mut opt = self.local_state.write().unwrap();
        *opt = Some(Arc::new(new_state.clone()));
        Ok(())
    }

    fn fetch_remote_state(&self) -> Result<SdkSyncState> {
        let mut sdks = HashMap::new();
        for remote_sdk in self.s3.list_upstream_sdks()? {
            sdks.insert(remote_sdk.local_filename().into(), remote_sdk);
        }
        Ok(SdkSyncState { sdks: sdks })
    }

    fn update_sdk(&self, sdk: &RemoteSdk) -> Result<()> {
        // XXX: the progress bar here can stall out because we currently
        // need to buffer the download into memory in the s3 code :(
        let progress = ProgressIndicator::new(sdk.size() as usize);
        progress.set_message(&format!("Synchronizing {}", sdk.info()));
        let mut src = self.s3.download_sdk(sdk)?;
        let dst = fs::File::create(self.path.join(sdk.local_filename()))?;
        let mut dst = XzDecoder::new(dst);
        copy_with_progress(&progress, &mut src, &mut dst)?;
        progress.finish(&format!("Synchronized {}", sdk.info()));
        Ok(())
    }

    fn remove_sdk(&self, sdk: &RemoteSdk) -> Result<()> {
        println!("Deleting {}", sdk.info());
        if let Err(err) = fs::remove_file(self.path.join(sdk.local_filename())) {
            if err.kind() != io::ErrorKind::NotFound {
                return Err(err.into());
            }
        }
        Ok(())
    }

    /// Checks the local stash against the server
    pub fn get_sync_status(&self) -> Result<SyncStatus> {
        let local_state = self.read_local_state()?;
        let remote_state = self.fetch_remote_state()?;

        let mut remote_total = 0;
        let mut missing = 0;
        let mut different = 0;

        for sdk in remote_state.sdks() {
            if let Some(local_sdk) = local_state.get_sdk(sdk.local_filename()) {
                if local_sdk != sdk {
                    different += 1;
                }
            } else {
                missing += 1;
            }
            remote_total += 1;
        }

        Ok(SyncStatus {
            remote_total: remote_total as u32,
            missing: missing as u32,
            different: different as u32,
        })
    }

    /// Synchronize the local stash with the server
    pub fn sync(&self) -> Result<()> {
        let mut local_state = self.read_local_state()?;
        let remote_state = self.fetch_remote_state()?;

        let mut to_delete : HashSet<_> = HashSet::from_iter(
            local_state.sdks().map(|x| x.local_filename().to_string()));

        for sdk in remote_state.sdks() {
            if let Some(local_sdk) = local_state.get_sdk(sdk.local_filename()) {
                if local_sdk != sdk {
                    self.update_sdk(&sdk)?;
                } else {
                    println!("  ⸰ Unchanged {}", sdk.info());
                }
            } else {
                self.update_sdk(&sdk)?;
            }
            to_delete.remove(sdk.local_filename());
            local_state.update_sdk(&sdk);
            self.save_local_state(&local_state)?;
        }

        for local_filename in to_delete.iter() {
            if let Some(sdk) = local_state.get_sdk(local_filename) {
                self.remove_sdk(sdk)?;
            }
        }

        println!("Done synching");

        // if we get this far, the remote state is indeed the local state.
        self.save_local_state(&remote_state)?;

        Ok(())
    }

    /// Looks up an memdb by an SDK info if it's available.
    ///
    /// This returns a memdb wrapped in an arc as internally the system
    /// might try to unload the memdb if no longer needed.  If the MemDb
    /// does not exist, a `UnknownSdk` error is returned.
    pub fn get_memdb(&'a self, info: &SdkInfo) -> Result<Arc<MemDb<'a>>> {
        let local_state = self.get_local_state()?;

        // make sure we check in the local state first if the SDK exists.
        // if we go directly to the memdbs array or look at the file system
        // we might start to consider things that are not available yet or
        // not available any longer.
        if local_state.get_sdk(&info.memdb_filename()).is_some() {
            if let Some(arc) = self.memdbs.read().unwrap().get(info) {
                return Ok(arc.clone());
            }
            let memdb = MemDb::from_path(self.path.join(info.memdb_filename()))?;
            self.memdbs.write().unwrap().insert(info.clone(), Arc::new(memdb));
            if let Some(arc) = self.memdbs.read().unwrap().get(info) {
                return Ok(arc.clone());
            }
        }

        Err(ErrorKind::UnknownSdk.into())
    }
}
<|endoftext|>


<commit_before>pub use self::area_frame_allocator::AreaFrameAllocator;
pub use self::paging::remap_the_kernel;
use self::paging::PhysicalAddress;
use multiboot2::BootInformation;

mod area_frame_allocator;
mod paging;
mod stack_allocator;

/// Size of a page
pub const PAGE_SIZE: usize = 4096;

/// Initialize the memory system
pub fn init(boot_info: &BootInformation) {
    // insure that this function is only called once
    assert_has_not_been_called!("memory::init must be called only once");

    // get the bootloader memory tag
    let memory_map_tag = boot_info.memory_map_tag().expect("Memory map tag required");

    // get the elf sections bootloader tag
    let elf_sections_tag = boot_info.elf_sections_tag().expect("Elf sections tag required");

    // get the kernel start address
    let kernel_start = elf_sections_tag.sections().map(|s| s.addr).min().unwrap();

    // get the kernel end address
    let kernel_end = elf_sections_tag.sections().map(|s| s.addr + s.size).max().unwrap();

    // initialize the frame allocator
    let mut frame_allocator = AreaFrameAllocator::new(kernel_start as usize,
                                                      kernel_end as usize,
                                                      boot_info.start_address(),
                                                      boot_info.end_address(),
                                                      memory_map_tag.memory_areas());
    // remap the kernel
    let mut active_table = remap_the_kernel(&mut frame_allocator, boot_info);

    use self::paging::Page;
    use bump_allocator::{HEAP_START, HEAP_SIZE};

    let heap_start_page = Page::containing_address(HEAP_START);
    let heap_end_page = Page::containing_address(HEAP_START + HEAP_SIZE - 1);

    for page in Page::range_inclusive(heap_start_page, heap_end_page) {
        active_table.map(page, paging::WRITABLE, &mut frame_allocator);
    }
}

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct Frame {
    number: usize,
}

impl Frame {
    fn containing_address(address: usize) -> Frame {
        Frame { number: address / PAGE_SIZE }
    }

    fn start_address(&self) -> PhysicalAddress {
        self.number * PAGE_SIZE
    }

    fn range_inclusive(start: Frame, end: Frame) -> FrameIter {
        FrameIter {
            start: start,
            end: end
        }
    }

    fn clone(&self) -> Frame {
        Frame { number: self.number }
    }
}

struct FrameIter {
    start: Frame,
    end: Frame,
}

impl Iterator for FrameIter {
    type Item = Frame;

    fn next(&mut self) -> Option<Frame> {
        if self.start <= self.end {
            let frame = self.start.clone();
            self.start.number += 1;
            Some(frame)
        } else {
            None
        }
    }
}

pub trait FrameAllocator {
    fn allocate_frame(&mut self) -> Option<Frame>;
    fn deallocate_frame(&mut self, frame: Frame);
}
<commit_msg>Remap the kernel stack<commit_after>pub use self::area_frame_allocator::AreaFrameAllocator;
pub use self::paging::remap_the_kernel;
pub use self::stack_allocator::Stack;
use self::paging::PhysicalAddress;
use multiboot2::BootInformation;

mod area_frame_allocator;
mod paging;
mod stack_allocator;

/// Size of a page
pub const PAGE_SIZE: usize = 4096;

/// Initialize the memory system
///
/// ## Returns
///
/// The memory controller that servers a simple interface to manage the memory allocation.
pub fn init(boot_info: &BootInformation) -> MemoryController {
    // insure that this function is only called once
    assert_has_not_been_called!("memory::init must be called only once");

    // get the bootloader memory tag
    let memory_map_tag = boot_info.memory_map_tag().expect("Memory map tag required");

    // get the elf sections bootloader tag
    let elf_sections_tag = boot_info.elf_sections_tag().expect("Elf sections tag required");

    // get the kernel start address
    let kernel_start = elf_sections_tag.sections().map(|s| s.addr).min().unwrap();

    // get the kernel end address
    let kernel_end = elf_sections_tag.sections().map(|s| s.addr + s.size).max().unwrap();

    // initialize the frame allocator
    let mut frame_allocator = AreaFrameAllocator::new(kernel_start as usize,
                                                      kernel_end as usize,
                                                      boot_info.start_address(),
                                                      boot_info.end_address(),
                                                      memory_map_tag.memory_areas());
    // remap the kernel
    let mut active_table = remap_the_kernel(&mut frame_allocator, boot_info);

    // remap heap
    use self::paging::Page;
    use bump_allocator::{HEAP_START, HEAP_SIZE};

    let heap_start_page = Page::containing_address(HEAP_START);
    let heap_end_page = Page::containing_address(HEAP_START + HEAP_SIZE - 1);

    for page in Page::range_inclusive(heap_start_page, heap_end_page) {
        active_table.map(page, paging::WRITABLE, &mut frame_allocator);
    }

    // remap Stack
    let stack_allocator = {
        // calculate the start and end address of the stack
        let stack_alloc_start = heap_end_page + 1;
        let stack_alloc_end = stack_alloc_start + 100;

        // create a new page range with the stack start address and end address
        let stack_alloc_range = Page::range_inclusive(stack_alloc_start, stack_alloc_end);

        // create a StackAllocator instance
        stack_allocator::StackAllocator::new(stack_alloc_range)
    };

    MemoryController {
        active_table: active_table,
        frame_allocator: frame_allocator,
        stack_allocator: stack_allocator
    }
}

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct Frame {
    number: usize,
}

impl Frame {
    fn containing_address(address: usize) -> Frame {
        Frame { number: address / PAGE_SIZE }
    }

    fn start_address(&self) -> PhysicalAddress {
        self.number * PAGE_SIZE
    }

    fn range_inclusive(start: Frame, end: Frame) -> FrameIter {
        FrameIter {
            start: start,
            end: end
        }
    }

    fn clone(&self) -> Frame {
        Frame { number: self.number }
    }
}

struct FrameIter {
    start: Frame,
    end: Frame,
}

impl Iterator for FrameIter {
    type Item = Frame;

    fn next(&mut self) -> Option<Frame> {
        if self.start <= self.end {
            let frame = self.start.clone();
            self.start.number += 1;
            Some(frame)
        } else {
            None
        }
    }
}

pub trait FrameAllocator {
    fn allocate_frame(&mut self) -> Option<Frame>;
    fn deallocate_frame(&mut self, frame: Frame);
}

pub struct MemoryController {
    active_table: paging::ActivePageTable,
    frame_allocator: AreaFrameAllocator,
    stack_allocator: stack_allocator::StackAllocator,
}

impl MemoryController {
    /// Alloc a new stack with the given size.
    ///
    /// ## Params
    /// * `size_in_pages` - number of pages to alloc.
    ///
    /// ## Returns
    /// Stack instance.
    pub fn alloc_stack(&mut self, size_in_pages: usize) -> Option<Stack> {
        let &mut MemoryController {
            ref mut active_table,
            ref mut frame_allocator,
            ref mut stack_allocator } = self;

        stack_allocator.alloc_stack(active_table, frame_allocator, size_in_pages)
    }
}
<|endoftext|>


<commit_before>use std::{cmp::min, convert::TryFrom, error::Error as StdError, fmt, str::FromStr};

use chrono::{DateTime, Datelike as _, Utc};

#[derive(Debug)]
pub enum InvalidDate {
    InvalidYear,
    InvalidMonth,
}

impl fmt::Display for InvalidDate {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(match *self {
            InvalidDate::InvalidYear => "invalid year",
            InvalidDate::InvalidMonth => "invalid month",
        })
    }
}

impl StdError for InvalidDate {}

#[derive(Copy, Clone, Debug)]
pub struct LaxDate {
    year: Year,
    month: Option<u8>,
    day: Option<u8>,
}

impl LaxDate {
    pub fn year(self) -> Year {
        self.year
    }

    pub fn month(self) -> Option<Month> {
        self.month
            .map(|m| Month(self.year.0 * 12 + u16::from(m) - 1))
    }
}

impl FromStr for LaxDate {
    type Err = InvalidDate;

    fn from_str(s: &str) -> Result<LaxDate, InvalidDate> {
        let mut parts = s.splitn(3, s);
        let year_part = parts.next().expect("non-empty split");
        Ok(LaxDate {
            year: Year::try_from(
                year_part
                    .parse::<u16>()
                    .map_err(|_| InvalidDate::InvalidYear)?,
            )?,
            month: parts
                .next()
                .and_then(|m| m.parse().ok())
                .filter(|m| 1 <= *m && *m <= 12),
            day: parts.next().and_then(|d| d.parse().ok()),
        })
    }
}

impl fmt::Display for LaxDate {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{:04}.", self.year.0)?;
        match self.month {
            Some(month) => write!(f, "{:02}.", month)?,
            None => f.write_str("??.")?,
        }
        match self.day {
            Some(day) => write!(f, "{:02}", day),
            None => f.write_str("??"),
        }
    }
}

const MAX_YEAR: u16 = 3000; // MAX_YEAR * 12 + 12 < 2^16

#[derive(Debug, Copy, Clone, Default, Ord, PartialOrd, Eq, PartialEq)]
pub struct Year(u16);

impl Year {
    pub fn max_value() -> Year {
        Year(MAX_YEAR)
    }

    pub fn add_years_saturating(self, years: u16) -> Year {
        min(Year(self.0.saturating_add(years)), Year::max_value())
    }
}

impl From<Year> for u16 {
    fn from(Year(year): Year) -> u16 {
        year
    }
}

impl TryFrom<u16> for Year {
    type Error = InvalidDate;

    fn try_from(year: u16) -> Result<Year, InvalidDate> {
        if year <= Year::max_value().0 {
            Ok(Year(year))
        } else {
            Err(InvalidDate::InvalidYear)
        }
    }
}

#[derive(Debug, Copy, Clone, Default, Ord, PartialOrd, Eq, PartialEq)]
pub struct Month(u16);

impl Month {
    pub fn max_value() -> Month {
        Month(MAX_YEAR * 12 + 11)
    }

    pub fn from_time_saturating(time: DateTime<Utc>) -> Month {
        let year = match time.year_ce() {
            (true, ce) => min(u32::from(MAX_YEAR), ce) as u16,
            (false, _) => 0,
        };

        Month(year * 12 + time.month0() as u16)
    }

    pub fn add_months_saturating(self, months: u16) -> Month {
        min(Month(self.0.saturating_add(months)), Month::max_value())
    }
}

impl From<Month> for u16 {
    fn from(Month(month): Month) -> u16 {
        month
    }
}

impl TryFrom<u16> for Month {
    type Error = InvalidDate;

    fn try_from(month: u16) -> Result<Month, InvalidDate> {
        if month <= Month::max_value().0 {
            Ok(Month(month))
        } else {
            Err(InvalidDate::InvalidMonth)
        }
    }
}

impl fmt::Display for Month {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{:04}-{:02}", self.0 / 12, self.0 % 12 + 1)
    }
}

impl FromStr for Month {
    type Err = InvalidDate;

    fn from_str(s: &str) -> Result<Month, InvalidDate> {
        match s.split_once(|ch| ch == '-' || ch == '/') {
            Some((year_part, month_part)) => {
                let year: u16 = year_part.parse().map_err(|_| InvalidDate::InvalidMonth)?;
                let month_plus_one: u16 =
                    month_part.parse().map_err(|_| InvalidDate::InvalidMonth)?;

                if year <= MAX_YEAR && 1 <= month_plus_one && month_plus_one <= 12 {
                    Ok(Month(year * 12 + month_plus_one - 1))
                } else {
                    Err(InvalidDate::InvalidMonth)
                }
            }
            None => Err(InvalidDate::InvalidMonth),
        }
    }
}

#[cfg(test)]
mod tests {
    use quickcheck::{Arbitrary, Gen};

    use super::*;

    impl Arbitrary for Month {
        fn arbitrary(g: &mut Gen) -> Month {
            Month(u16::arbitrary(g) % (u16::from(Month::max_value()) + 1))
        }
    }
}
<commit_msg>fix date split<commit_after>use std::{cmp::min, convert::TryFrom, error::Error as StdError, fmt, str::FromStr};

use chrono::{DateTime, Datelike as _, Utc};

#[derive(Debug)]
pub enum InvalidDate {
    InvalidYear,
    InvalidMonth,
}

impl fmt::Display for InvalidDate {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(match *self {
            InvalidDate::InvalidYear => "invalid year",
            InvalidDate::InvalidMonth => "invalid month",
        })
    }
}

impl StdError for InvalidDate {}

#[derive(Copy, Clone, Debug)]
pub struct LaxDate {
    year: Year,
    month: Option<u8>,
    day: Option<u8>,
}

impl LaxDate {
    pub fn year(self) -> Year {
        self.year
    }

    pub fn month(self) -> Option<Month> {
        self.month
            .map(|m| Month(self.year.0 * 12 + u16::from(m) - 1))
    }
}

impl FromStr for LaxDate {
    type Err = InvalidDate;

    fn from_str(s: &str) -> Result<LaxDate, InvalidDate> {
        let mut parts = s.splitn(3, ".");
        let year_part = parts.next().expect("non-empty split");
        Ok(LaxDate {
            year: Year::try_from(
                year_part
                    .parse::<u16>()
                    .map_err(|_| InvalidDate::InvalidYear)?,
            )?,
            month: parts
                .next()
                .and_then(|m| m.parse().ok())
                .filter(|m| 1 <= *m && *m <= 12),
            day: parts.next().and_then(|d| d.parse().ok()),
        })
    }
}

impl fmt::Display for LaxDate {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{:04}.", self.year.0)?;
        match self.month {
            Some(month) => write!(f, "{:02}.", month)?,
            None => f.write_str("??.")?,
        }
        match self.day {
            Some(day) => write!(f, "{:02}", day),
            None => f.write_str("??"),
        }
    }
}

const MAX_YEAR: u16 = 3000; // MAX_YEAR * 12 + 12 < 2^16

#[derive(Debug, Copy, Clone, Default, Ord, PartialOrd, Eq, PartialEq)]
pub struct Year(u16);

impl Year {
    pub fn max_value() -> Year {
        Year(MAX_YEAR)
    }

    pub fn add_years_saturating(self, years: u16) -> Year {
        min(Year(self.0.saturating_add(years)), Year::max_value())
    }
}

impl From<Year> for u16 {
    fn from(Year(year): Year) -> u16 {
        year
    }
}

impl TryFrom<u16> for Year {
    type Error = InvalidDate;

    fn try_from(year: u16) -> Result<Year, InvalidDate> {
        if year <= Year::max_value().0 {
            Ok(Year(year))
        } else {
            Err(InvalidDate::InvalidYear)
        }
    }
}

#[derive(Debug, Copy, Clone, Default, Ord, PartialOrd, Eq, PartialEq)]
pub struct Month(u16);

impl Month {
    pub fn max_value() -> Month {
        Month(MAX_YEAR * 12 + 11)
    }

    pub fn from_time_saturating(time: DateTime<Utc>) -> Month {
        let year = match time.year_ce() {
            (true, ce) => min(u32::from(MAX_YEAR), ce) as u16,
            (false, _) => 0,
        };

        Month(year * 12 + time.month0() as u16)
    }

    pub fn add_months_saturating(self, months: u16) -> Month {
        min(Month(self.0.saturating_add(months)), Month::max_value())
    }
}

impl From<Month> for u16 {
    fn from(Month(month): Month) -> u16 {
        month
    }
}

impl TryFrom<u16> for Month {
    type Error = InvalidDate;

    fn try_from(month: u16) -> Result<Month, InvalidDate> {
        if month <= Month::max_value().0 {
            Ok(Month(month))
        } else {
            Err(InvalidDate::InvalidMonth)
        }
    }
}

impl fmt::Display for Month {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{:04}-{:02}", self.0 / 12, self.0 % 12 + 1)
    }
}

impl FromStr for Month {
    type Err = InvalidDate;

    fn from_str(s: &str) -> Result<Month, InvalidDate> {
        match s.split_once(|ch| ch == '-' || ch == '/') {
            Some((year_part, month_part)) => {
                let year: u16 = year_part.parse().map_err(|_| InvalidDate::InvalidMonth)?;
                let month_plus_one: u16 =
                    month_part.parse().map_err(|_| InvalidDate::InvalidMonth)?;

                if year <= MAX_YEAR && 1 <= month_plus_one && month_plus_one <= 12 {
                    Ok(Month(year * 12 + month_plus_one - 1))
                } else {
                    Err(InvalidDate::InvalidMonth)
                }
            }
            None => Err(InvalidDate::InvalidMonth),
        }
    }
}

#[cfg(test)]
mod tests {
    use quickcheck::{Arbitrary, Gen};

    use super::*;

    impl Arbitrary for Month {
        fn arbitrary(g: &mut Gen) -> Month {
            Month(u16::arbitrary(g) % (u16::from(Month::max_value()) + 1))
        }
    }
}
<|endoftext|>


<commit_before>pub trait ProgramFragment {
    fn run(&self) -> i32;
}

pub trait Configurable {
    fn set_flag(&self) -> bool;
    fn set_arg(&self, key:&str, value:&str) -> &str;
}
<commit_msg>ProgramFragments now need to return a result<commit_after>use std::result::Result;
pub trait ProgramFragment {
    fn run(&self) -> Result<(), &'static str>;
}

pub trait Configurable {
    fn set_flag(&self) -> bool;
    fn set_arg(&self, key:&str, value:&str) -> &str;
}
<|endoftext|>


<commit_before>use clap::{App, Arg, AppSettings, Format, ArgMatches};
use std::io::{self, Write};
use std::process::exit;
use std::fs;

/// Prints an error on stderr and exit the program
pub fn print_error(s: &str) -> ! {
    writeln!(&mut io::stderr(), "{} {}", Format::Error("error:"), s).unwrap();
    exit(0);
}

/// create a book file with the command line arguments
/// and exit the process at the end
pub fn create_book(matches: &ArgMatches) -> ! {
    if let Some(values) = matches.values_of("files") {
        let numbering = match matches.value_of("numbering") {
            Some("false") => false,
            _ => true,
        };

        let s = matches.value_of("BOOK").unwrap();
        if fs::metadata(s).is_ok() {
            print_error(&format!("Could not create file {}: it already exists!", s));
        }
        let mut f = fs::File::create(s).unwrap();
        f.write_all(b"author: Your name\n").unwrap();
        f.write_all(b"title: Your title\n").unwrap();
        f.write_all(b"lang: en\n\n").unwrap();
        f.write_all(b"# Uncomment and fill to generate files\n").unwrap();
        f.write_all(b"# output.html: some_file.html\n").unwrap();
        f.write_all(b"# output.epub: some_file.epub\n").unwrap();
        f.write_all(b"# output.pdf: some_file.pdf\n\n").unwrap();
        f.write_all(b"# Uncomment and fill to set cover image (for Epub)\n").unwrap();
        f.write_all(b"# cover: some_cover.png\n\n").unwrap();
        f.write_all(b"# List of chapters\n").unwrap();
        for file in values {
            f.write_all(&format!("{} {}\n", if numbering {"+"} else {"-"}, file).as_bytes()).unwrap();
        }
        println!("Created {}, now you'll have to complete it!", s);
        exit(0);
    } else {
        print_error("--create must be used with a list of additonal files");
    }
}

pub fn create_matches<'a>() -> ArgMatches<'a> {
    let app = App::new("crowbook")
        .setting(AppSettings::ArgRequiredElseHelp)
        .setting(AppSettings::UnifiedHelpMessage)
        //        .usage("crowbook [FLAGS] [OPTIONS] <BOOK> [FILE]")
        .version(env!("CARGO_PKG_VERSION"))
        .about("Render a markdown book in Epub, PDF or HTML.")
        .after_help("Command line options allow to override options defined in <BOOK> configuration file.
E.g., even if this file specifies 'verbose: false', calling 'crowbook --verbose <BOOK>'
will activate verbose mode.

Note that Crowbook generates output files relatively to the directory where <BOOK> is:
$ crowbook foo/bar.book --to pdf --output baz.pdf
will thus generate baz.pdf in directory foo and not in current directory.")
        .arg_from_usage("-v, --verbose 'Activate verbose mode'")
        .arg(Arg::with_name("files")
             .value_name("FILES")
             .help("Files to put in book when using --create")
             .takes_value(true)
             .multiple(true)
             .index(2))
        .arg(Arg::with_name("create")
             .long("--create")
             .help("Creates a new book file with existing markdown files"))
        .arg(Arg::with_name("output")
             .long("--output")
             .short("-o")
             .value_name("FILE")
             .requires("to")
             .help("Specify output file"))
        .arg(Arg::with_name("autoclean")
             .long("--autoclean")
             .value_name("BOOL")
             .takes_value(true)
             .help("Try to clean input markdown")
             .possible_values(&["true", "false"]))
        .arg(Arg::with_name("numbering")
             .long("--numbering")
             .value_name("BOOL")
             .takes_value(true)
             .help("Number chapters or not")
             .possible_values(&["true", "false"]))
        .arg(Arg::with_name("to")
             .long("--to")
             .short("t")
             .takes_value(true)
             .possible_values(&["epub", "pdf", "html", "tex", "odt"])
             .value_name("FORMAT")
             .help("Generate specific format"))
        .arg(Arg::with_name("BOOK")
             .index(1)
             .required(true)
             .help("A file containing the book configuration"));

    let matches = app.get_matches();

    pre_check(&matches);

    matches
}

/// Pre-check the matches to see if there isn't illegal options not detected by clap
fn pre_check(matches: &ArgMatches) {
    if matches.is_present("files") && !matches.is_present("create") {
        print_error("A list of additional files is only valid with the --create option.");
    }
}
<commit_msg>helpers.rs: use Arg::from_usage to make code more concise<commit_after>use clap::{App, Arg, AppSettings, Format, ArgMatches};
use std::io::{self, Write};
use std::process::exit;
use std::fs;

/// Prints an error on stderr and exit the program
pub fn print_error(s: &str) -> ! {
    writeln!(&mut io::stderr(), "{} {}", Format::Error("error:"), s).unwrap();
    exit(0);
}

/// create a book file with the command line arguments
/// and exit the process at the end
pub fn create_book(matches: &ArgMatches) -> ! {
    if let Some(values) = matches.values_of("FILES") {
        let numbering = match matches.value_of("numbering") {
            Some("false") => false,
            _ => true,
        };

        let s = matches.value_of("BOOK").unwrap();
        if fs::metadata(s).is_ok() {
            print_error(&format!("Could not create file {}: it already exists!", s));
        }
        let mut f = fs::File::create(s).unwrap();
        f.write_all(b"author: Your name\n").unwrap();
        f.write_all(b"title: Your title\n").unwrap();
        f.write_all(b"lang: en\n\n").unwrap();
        f.write_all(b"# Uncomment and fill to generate files\n").unwrap();
        f.write_all(b"# output.html: some_file.html\n").unwrap();
        f.write_all(b"# output.epub: some_file.epub\n").unwrap();
        f.write_all(b"# output.pdf: some_file.pdf\n\n").unwrap();
        f.write_all(b"# Uncomment and fill to set cover image (for Epub)\n").unwrap();
        f.write_all(b"# cover: some_cover.png\n\n").unwrap();
        f.write_all(b"# List of chapters\n").unwrap();
        for file in values {
            f.write_all(&format!("{} {}\n", if numbering {"+"} else {"-"}, file).as_bytes()).unwrap();
        }
        println!("Created {}, now you'll have to complete it!", s);
        exit(0);
    } else {
        print_error("--create must be used with a list of additonal files");
    }
}

pub fn create_matches<'a>() -> ArgMatches<'a> {
    let app = App::new("crowbook")
        .setting(AppSettings::ArgRequiredElseHelp)
        .setting(AppSettings::UnifiedHelpMessage)
        .version(env!("CARGO_PKG_VERSION"))
        .about("Render a markdown book in Epub, PDF or HTML.")
        .after_help("Command line options allow to override options defined in <BOOK> configuration file.
E.g., even if this file specifies 'verbose: false', calling 'crowbook --verbose <BOOK>'
will activate verbose mode.

Note that Crowbook generates output files relatively to the directory where <BOOK> is:
$ crowbook foo/bar.book --to pdf --output baz.pdf
will thus generate baz.pdf in directory foo and not in current directory.")
        .arg_from_usage("-v, --verbose 'Activate verbose mode'")
        .arg_from_usage("--create 'Creates a new book with existing markdown files.'")
        .arg(Arg::from_usage("-o, --output [FILE] 'Specifies output file.'")
             .requires("to"))
        .arg(Arg::from_usage("[FILES]... 'Files to list in book when using --create'")
             .index(2))
        .arg(Arg::from_usage("--autoclean [BOOL] 'Set/unset markdown cleaning'")
             .possible_values(&["true", "false"]))
        .arg(Arg::from_usage("--numbering [BOOL] 'De/activate chapter numbering'")
             .possible_values(&["true", "false"]))
        .arg(Arg::from_usage("-t, --to [FORMAT] 'Generate specific format'")
             .possible_values(&["epub", "pdf", "html", "tex", "odt"]))
        .arg(Arg::with_name("BOOK")
             .index(1)
             .required(true)
             .help("File containing the book configuration."));

    let matches = app.get_matches();

    pre_check(&matches);

    matches
}

/// Pre-check the matches to see if there isn't illegal options not detected by clap
fn pre_check(matches: &ArgMatches) {
    if matches.is_present("files") && !matches.is_present("create") {
        print_error("A list of additional files is only valid with the --create option.");
    }
}
<|endoftext|>


<commit_before>extern crate rand;
extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

use rand::{Rng, SeedableRng, StdRng};

use timely::progress::timestamp::RootTimestamp;
use timely::dataflow::*;
use timely::dataflow::operators::*;
use timely::dataflow::channels::pact::Exchange;
use timely::drain::DrainExt;

fn main () {

    let node_cnt = std::env::args().skip(1).next().unwrap().parse::<usize>().unwrap();
    let edge_cnt = std::env::args().skip(2).next().unwrap().parse::<usize>().unwrap();

    timely::execute_from_args(std::env::args(), move |root| {

        let index = root.index() as usize;
        let peers = root.peers() as usize;

        let start = time::precise_time_s();

        let mut edges = Vec::new();
        let mut ranks = vec![1.0; (node_cnt / peers) + 1];   // holds ranks
        if (node_cnt % peers) < index { ranks.push(1.0); }
        let mut degrs = vec![0; ranks.len()];

        let mut going = start;

        let mut input = root.scoped(|builder| {

            let (input, graph) = builder.new_input::<(u32, u32)>();
            let (cycle, loopz) = builder.loop_variable::<(u32, f32)>(20, 1);

            graph.binary_notify(&loopz,
                                Exchange::new(|x: &(u32,u32)| x.0 as u64),
                                Exchange::new(|x: &(u32,f32)| x.0 as u64),
                                "pagerank",
                                vec![RootTimestamp::new(0)],
                                move |input1, input2, output, notificator| {

                // receive incoming edges (should only be iter 0)
                while let Some((_index, data)) = input1.next() {
                    for (src,dst) in data.drain_temp() {
                        degrs[src as usize] += 1;
                        edges.push((src,dst));
                    }
                }

                // all inputs received for iter, commence multiplication
                while let Some((iter, _)) = notificator.next() {

                    // record some timings in order to estimate per-iteration times
                    if iter.inner == 0 { println!("src: {}, dst: {}, edges: {}", ranks.len(), node_cnt, edges.len()); }
                    if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                    if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                    // prepare src for transmitting to destinations
                    for s in 0..ranks.len() { ranks[s] = 0.15 + 0.85 * ranks[s] / degrs[s] as f32; }

                    // wander through destinations
                    let mut session = output.session(&iter);
                    for &(src,dst) in &edges {
                        unsafe {
                            session.give((dst, *ranks.get_unchecked(src as usize)));
                        }
                    }

                    for s in &mut ranks { *s = 0.0; }
                }

                // receive data from workers, accumulate in src
                while let Some((iter, data)) = input2.next() {
                    notificator.notify_at(&iter);
                    for &(node, rank) in data.iter() {
                        unsafe { *ranks.get_unchecked_mut(node as usize / peers) += rank; }
                    }
                }
            }).connect_loop(cycle);

            input
        });

        let seed: &[_] = &[1, 2, 3, 4];
        let mut rng: StdRng = SeedableRng::from_seed(seed);

        for _index in 0..(edge_cnt / peers) {
            input.send((rng.gen_range(0, node_cnt as u32), rng.gen_range(0, node_cnt as u32)));
        }
    });
}<commit_msg>reduced size of ranks vector<commit_after>extern crate rand;
extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

use rand::{Rng, SeedableRng, StdRng};

use timely::progress::timestamp::RootTimestamp;
use timely::dataflow::*;
use timely::dataflow::operators::*;
use timely::dataflow::channels::pact::Exchange;
use timely::drain::DrainExt;

fn main () {

    let node_cnt = std::env::args().skip(1).next().unwrap().parse::<usize>().unwrap();
    let edge_cnt = std::env::args().skip(2).next().unwrap().parse::<usize>().unwrap();

    timely::execute_from_args(std::env::args(), move |root| {

        let index = root.index() as usize;
        let peers = root.peers() as usize;

        let start = time::precise_time_s();

        let mut edges = Vec::new();
        let mut ranks = vec![1.0; (node_cnt / peers) + 1];   // holds ranks
        if (node_cnt % peers) < index { ranks.push(1.0); }
        let mut degrs = vec![0; ranks.len()];

        let mut going = start;

        let mut input = root.scoped(|builder| {

            let (input, graph) = builder.new_input::<(u32, u32)>();
            let (cycle, loopz) = builder.loop_variable::<(u32, f32)>(20, 1);

            graph.binary_notify(&loopz,
                                Exchange::new(|x: &(u32,u32)| x.0 as u64),
                                Exchange::new(|x: &(u32,f32)| x.0 as u64),
                                "pagerank",
                                vec![RootTimestamp::new(0)],
                                move |input1, input2, output, notificator| {

                // receive incoming edges (should only be iter 0)
                while let Some((_index, data)) = input1.next() {
                    for (src,dst) in data.drain_temp() {
                        degrs[src as usize] += 1;
                        edges.push((src / peers,dst));
                    }
                }

                // all inputs received for iter, commence multiplication
                while let Some((iter, _)) = notificator.next() {

                    // record some timings in order to estimate per-iteration times
                    if iter.inner == 0 { println!("src: {}, dst: {}, edges: {}", ranks.len(), node_cnt, edges.len()); }
                    if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                    if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                    // prepare src for transmitting to destinations
                    for s in 0..ranks.len() { ranks[s] = 0.15 + 0.85 * ranks[s] / degrs[s] as f32; }

                    // wander through destinations
                    let mut session = output.session(&iter);
                    for &(src,dst) in &edges {
                        unsafe {
                            session.give((dst, *ranks.get_unchecked(src as usize)));
                        }
                    }

                    for s in &mut ranks { *s = 0.0; }
                }

                // receive data from workers, accumulate in src
                while let Some((iter, data)) = input2.next() {
                    notificator.notify_at(&iter);
                    for &(node, rank) in data.iter() {
                        unsafe { *ranks.get_unchecked_mut(node as usize / peers) += rank; }
                    }
                }
            }).connect_loop(cycle);

            input
        });

        let seed: &[_] = &[1, 2, 3, 4];
        let mut rng: StdRng = SeedableRng::from_seed(seed);

        for _index in 0..(edge_cnt / peers) {
            input.send((rng.gen_range(0, node_cnt as u32), rng.gen_range(0, node_cnt as u32)));
        }
    });
}<|endoftext|>


<commit_before>use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};

use anyhow::{bail, Context, Result};
use assert_cmd::prelude::*;
use tempdir::TempDir;
use tokio::io::{self, AsyncWriteExt};

use pueue_daemon_lib::run;
use pueue_lib::settings::*;

use super::{get_pid, sleep_ms};

/// Spawn the daemon main logic in it's own async function.
/// It'll be executed by the tokio multi-threaded executor.
pub fn boot_daemon(pueue_dir: &Path) -> Result<i32> {
    let path = pueue_dir.clone().to_path_buf();
    // Start/spin off the daemon and get its PID
    tokio::spawn(run_and_handle_error(path, true));
    let pid = get_pid(pueue_dir)?;

    let tries = 20;
    let mut current_try = 0;

    // Wait up to 1s for the unix socket to pop up.
    let socket_path = pueue_dir.join("pueue.pid");
    while current_try < tries {
        sleep_ms(50);
        if socket_path.exists() {
            return Ok(pid);
        }

        current_try += 1;
    }

    bail!("Daemon didn't boot after 1sec")
}

/// Spawn the daemon by calling the actual pueued binary.
/// This function also checks for the pid file and the unix socket to pop-up.
pub fn boot_standalone_daemon(pueue_dir: &Path) -> Result<Child> {
    let child = Command::cargo_bin("pueued")?
        .arg("--config")
        .arg(pueue_dir.join("pueue.yml").to_str().unwrap())
        .arg("-vvv")
        .stdout(Stdio::piped())
        .spawn()?;

    let tries = 20;
    let mut current_try = 0;

    // Wait up to 1s for the unix socket to pop up.
    let socket_path = pueue_dir.join("pueue.pid");
    while current_try < tries {
        sleep_ms(50);
        if socket_path.exists() {
            return Ok(child);
        }

        current_try += 1;
    }

    bail!("Daemon didn't boot in stand-alone mode after 1sec")
}

/// Internal helper function, which wraps the daemon main logic and prints any errors.
async fn run_and_handle_error(pueue_dir: PathBuf, test: bool) -> Result<()> {
    if let Err(err) = run(Some(pueue_dir.join("pueue.yml")), test).await {
        let mut stdout = io::stdout();
        stdout
            .write_all(format!("Entcountered error: {:?}", err).as_bytes())
            .await
            .expect("Failed to write to stdout.");
        stdout.flush().await?;

        return Err(err);
    }

    Ok(())
}

pub fn base_setup() -> Result<(Settings, TempDir)> {
    // Create a temporary directory used for testing.
    let tempdir = TempDir::new("pueue_lib").unwrap();
    let tempdir_path = tempdir.path();

    std::fs::create_dir(tempdir_path.join("certs")).unwrap();

    let shared = Shared {
        pueue_directory: tempdir_path.clone().to_path_buf(),
        #[cfg(not(target_os = "windows"))]
        use_unix_socket: true,
        #[cfg(not(target_os = "windows"))]
        unix_socket_path: tempdir_path.join("test.socket"),
        host: "localhost".to_string(),
        port: "51230".to_string(),
        daemon_cert: tempdir_path.join("certs").join("daemon.cert"),
        daemon_key: tempdir_path.join("certs").join("daemon.key"),
        shared_secret_path: tempdir_path.join("secret"),
    };

    let client = Client {
        restart_in_place: false,
        read_local_logs: true,
        show_confirmation_questions: false,
        show_expanded_aliases: false,
        dark_mode: false,
        max_status_lines: Some(15),
        status_time_format: "%H:%M:%S".into(),
        status_datetime_format: "%Y-%m-%d\n%H:%M:%S".into(),
    };

    let mut groups = BTreeMap::new();
    groups.insert("default".to_string(), 1);
    groups.insert("test".to_string(), 3);

    let daemon = Daemon {
        default_parallel_tasks: 1,
        pause_group_on_failure: false,
        pause_all_on_failure: false,
        callback: None,
        callback_log_lines: 15,
        groups,
    };

    let settings = Settings {
        client,
        daemon,
        shared,
    };

    settings
        .save(&Some(tempdir_path.join("pueue.yml")))
        .context("Couldn't write pueue config to temporary directory")?;

    Ok((settings, tempdir))
}
<commit_msg>tests: Wait for unix socket instead of PID<commit_after>use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};

use anyhow::{bail, Context, Result};
use assert_cmd::prelude::*;
use tempdir::TempDir;
use tokio::io::{self, AsyncWriteExt};

use pueue_daemon_lib::run;
use pueue_lib::settings::*;

use super::{get_pid, sleep_ms};

/// Spawn the daemon main logic in it's own async function.
/// It'll be executed by the tokio multi-threaded executor.
pub fn boot_daemon(pueue_dir: &Path) -> Result<i32> {
    let path = pueue_dir.clone().to_path_buf();
    // Start/spin off the daemon and get its PID
    tokio::spawn(run_and_handle_error(path, true));
    let pid = get_pid(pueue_dir)?;

    let tries = 20;
    let mut current_try = 0;

    // Wait up to 1s for the unix socket to pop up.
    let socket_path = pueue_dir.join("test.socket");
    while current_try < tries {
        sleep_ms(50);
        if socket_path.exists() {
            return Ok(pid);
        }

        current_try += 1;
    }

    bail!("Daemon didn't boot after 1sec")
}

/// Spawn the daemon by calling the actual pueued binary.
/// This function also checks for the pid file and the unix socket to pop-up.
pub fn boot_standalone_daemon(pueue_dir: &Path) -> Result<Child> {
    let child = Command::cargo_bin("pueued")?
        .arg("--config")
        .arg(pueue_dir.join("pueue.yml").to_str().unwrap())
        .arg("-vvv")
        .stdout(Stdio::piped())
        .spawn()?;

    let tries = 20;
    let mut current_try = 0;

    // Wait up to 1s for the unix socket to pop up.
    let socket_path = pueue_dir.join("test.socket");
    while current_try < tries {
        sleep_ms(50);
        if socket_path.exists() {
            return Ok(child);
        }

        current_try += 1;
    }

    bail!("Daemon didn't boot in stand-alone mode after 1sec")
}

/// Internal helper function, which wraps the daemon main logic and prints any errors.
async fn run_and_handle_error(pueue_dir: PathBuf, test: bool) -> Result<()> {
    if let Err(err) = run(Some(pueue_dir.join("pueue.yml")), test).await {
        let mut stdout = io::stdout();
        stdout
            .write_all(format!("Entcountered error: {:?}", err).as_bytes())
            .await
            .expect("Failed to write to stdout.");
        stdout.flush().await?;

        return Err(err);
    }

    Ok(())
}

pub fn base_setup() -> Result<(Settings, TempDir)> {
    // Create a temporary directory used for testing.
    let tempdir = TempDir::new("pueue_lib").unwrap();
    let tempdir_path = tempdir.path();

    std::fs::create_dir(tempdir_path.join("certs")).unwrap();

    let shared = Shared {
        pueue_directory: tempdir_path.clone().to_path_buf(),
        #[cfg(not(target_os = "windows"))]
        use_unix_socket: true,
        #[cfg(not(target_os = "windows"))]
        unix_socket_path: tempdir_path.join("test.socket"),
        host: "localhost".to_string(),
        port: "51230".to_string(),
        daemon_cert: tempdir_path.join("certs").join("daemon.cert"),
        daemon_key: tempdir_path.join("certs").join("daemon.key"),
        shared_secret_path: tempdir_path.join("secret"),
    };

    let client = Client {
        restart_in_place: false,
        read_local_logs: true,
        show_confirmation_questions: false,
        show_expanded_aliases: false,
        dark_mode: false,
        max_status_lines: Some(15),
        status_time_format: "%H:%M:%S".into(),
        status_datetime_format: "%Y-%m-%d\n%H:%M:%S".into(),
    };

    let mut groups = BTreeMap::new();
    groups.insert("default".to_string(), 1);
    groups.insert("test".to_string(), 3);

    let daemon = Daemon {
        default_parallel_tasks: 1,
        pause_group_on_failure: false,
        pause_all_on_failure: false,
        callback: None,
        callback_log_lines: 15,
        groups,
    };

    let settings = Settings {
        client,
        daemon,
        shared,
    };

    settings
        .save(&Some(tempdir_path.join("pueue.yml")))
        .context("Couldn't write pueue config to temporary directory")?;

    Ok((settings, tempdir))
}
<|endoftext|>