<commit_before>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use token::Token;
use error::{Result, Error, Source};
use logger::Logger;

use std::mem;
use std::fs::File;
use std::path::Path;
use std::convert::AsRef;
use std::io::Read;
use std::collections::HashMap;
use std::ops::BitOr;

use cmark::{Parser as CMParser, Event, Tag, Options, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES};

#[derive(Debug, Copy, Clone, PartialEq)]
/// The list of features used in a document.
pub struct Features {
    pub image: bool,
    pub blockquote: bool,
    pub codeblock: bool,
    pub ordered_list: bool,
    pub footnote: bool,
    pub table: bool,
    pub url: bool,
}

impl Features {
    /// Creates a new set of features where all are set to false
    pub fn new() -> Features {
        Features {
            image: false,
            blockquote: false,
            codeblock: false,
            ordered_list: false,
            footnote: false,
            table: false,
            url: false,
        }
    }
}

impl BitOr for Features {
    type Output = Self;

    fn bitor(self, rhs: Self) -> Self {
        Features {
            image: self.image | rhs.image,
            blockquote: self.blockquote | rhs.blockquote,
            codeblock: self.codeblock | rhs.codeblock,
            ordered_list: self.ordered_list | rhs.ordered_list,
            footnote: self.footnote | rhs.footnote,
            table: self.table | rhs.table,
            url: self.url | rhs.url,
        }
    }
}

/// A parser that reads markdown and convert it to AST (a vector of `Token`s)
///
/// This AST can then be used by various renderes.
///
/// As this Parser uses Pulldown-cmark's one, it should be able to parse most
/// *valid* CommonMark variant of Markdown.
///
/// Compared to other Markdown parser, it might fail more often on invalid code, e.g.
/// footnotes references that are not defined anywhere.
///
/// # Examples
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some *valid* Markdown[^1]\n\n[^1]: with a valid footnote");
/// assert!(result.is_ok());
/// ```
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some footnote pointing to nothing[^1] ");
/// assert!(result.is_err());
/// ```
pub struct Parser {
    footnotes: HashMap<String, Vec<Token>>,
    source: Source,
    features: Features,
}

impl Parser {
    /// Creates a parser
    pub fn new() -> Parser {
        Parser {
            footnotes: HashMap::new(),
            source: Source::empty(),
            features: Features::new(),
        }
    }

    /// Sets a parser's source file
    pub fn set_source_file(&mut self, s: &str) {
        self.source = Source::new(s);
    }

    /// Parse a file and returns an AST or  an error
    pub fn parse_file<P: AsRef<Path>>(&mut self, filename: P) -> Result<Vec<Token>> {
        let path: &Path = filename.as_ref();
        let mut f = File::open(path)
            .map_err(|_| {
                Error::file_not_found(&self.source,
                                      lformat!("markdown file"),
                                      format!("{}", path.display()))
            })?;
        let mut s = String::new();

        f.read_to_string(&mut s)
            .map_err(|_| {
                Error::parser(&self.source,
                              lformat!("file {file} contains invalid UTF-8, could not parse it",
                                       file = path.display()))
            })?;
        self.parse(&s)
    }

    /// Parse a string and returns an AST  an Error.
    pub fn parse(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut opts = Options::empty();
        opts.insert(OPTION_ENABLE_TABLES);
        opts.insert(OPTION_ENABLE_FOOTNOTES);
        let mut p = CMParser::new_ext(s, opts);

        let mut res = vec![];
        self.parse_events(&mut p, &mut res, None)?;

        self.parse_footnotes(&mut res)?;

        collapse(&mut res);

        find_standalone(&mut res);
        Ok(res)
    }

    /// Parse an inline string and returns a list of `Token`.
    ///
    /// This function removes the outermost `Paragraph` in most of the
    /// cases, as it is meant to be used for an inline string (e.g. metadata)
    pub fn parse_inline(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut tokens = self.parse(s)?;
        // Unfortunately, parser will put all this in a paragraph, so we might need to remove it.
        if tokens.len() == 1 {
            let res = match tokens[0] {
                Token::Paragraph(ref mut v) => Some(mem::replace(v, vec![])),
                _ => None,
            };
            match res {
                Some(tokens) => Ok(tokens),
                _ => Ok(tokens),
            }
        } else {
            Ok(tokens)
        }
    }

    /// Returns the list of features used by this parser
    pub fn features(&self) -> Features {
        self.features
    }

    /// Replace footnote reference with their definition
    fn parse_footnotes(&mut self, v: &mut Vec<Token>) -> Result<()> {
        for token in v {
            match *token {
                Token::Footnote(ref mut content) => {
                    let reference = if let Token::Str(ref text) = content[0] {
                        text.clone()
                    } else {
                        panic!("Reference is not a vector of a single Token::Str");
                    };
                    if let Some(in_vec) = self.footnotes.get(&reference) {
                        *content = in_vec.clone();
                    } else {
                        return Err(Error::parser(&self.source,
                                                 lformat!("footnote reference {reference} does \
                                                           not have a matching definition",
                                                          reference = &reference)));
                    }
                }
                Token::Paragraph(ref mut vec) |
                Token::Header(_, ref mut vec) |
                Token::Emphasis(ref mut vec) |
                Token::Strong(ref mut vec) |
                Token::Code(ref mut vec) |
                Token::BlockQuote(ref mut vec) |
                Token::CodeBlock(_, ref mut vec) |
                Token::List(ref mut vec) |
                Token::OrderedList(_, ref mut vec) |
                Token::Item(ref mut vec) |
                Token::Table(_, ref mut vec) |
                Token::TableHead(ref mut vec) |
                Token::TableRow(ref mut vec) |
                Token::TableCell(ref mut vec) |
                Token::Link(_, _, ref mut vec) |
                Token::Image(_, _, ref mut vec) => self.parse_footnotes(vec)?,
                _ => (),
            }
        }
        Ok(())
    }

    fn parse_events<'a>(&mut self,
                        p: &mut CMParser<'a>,
                        v: &mut Vec<Token>,
                        current_tag: Option<&Tag>)
                        -> Result<()> {
        while let Some(event) = p.next() {
            match event {
                Event::Html(text) |
                Event::InlineHtml(text) |
                Event::Text(text) => {
                    v.push(Token::Str(text.into_owned()));
                }
                Event::Start(tag) => self.parse_tag(p, v, tag)?,
                Event::End(tag) => {
                    debug_assert!(format!("{:?}", Some(&tag)) == format!("{:?}", current_tag),
                                  format!("Error: opening and closing tags mismatch!\n{:?} ≠ \
                                           {:?}",
                                          tag,
                                          current_tag));
                    break;
                }
                Event::SoftBreak => v.push(Token::SoftBreak),
                Event::HardBreak => v.push(Token::HardBreak),
                Event::FootnoteReference(text) => {
                    v.push(Token::Footnote(vec![Token::Str(text.into_owned())]))
                }
            }
        }
        Ok(())
    }

    fn parse_tag<'a>(&mut self,
                     p: &mut CMParser<'a>,
                     v: &mut Vec<Token>,
                     tag: Tag<'a>)
                     -> Result<()> {
        let mut res = vec![];

        self.parse_events(p, &mut res, Some(&tag))?;

        let token = match tag {
            Tag::Paragraph => Token::Paragraph(res),
            Tag::Emphasis => Token::Emphasis(res),
            Tag::Strong => Token::Strong(res),
            Tag::Code => Token::Code(res),
            Tag::Header(x) => Token::Header(x, res),
            Tag::Link(url, title) => {
                self.features.url = true;
                Token::Link(url.into_owned(), title.into_owned(), res)
            },
            Tag::Image(url, title) => {
                self.features.image = true;
                Token::Image(url.into_owned(), title.into_owned(), res)
            },
            Tag::Rule => Token::Rule,
            Tag::List(opt) => {
                if let Some(n) = opt {
                    self.features.ordered_list = true;
                    Token::OrderedList(n, res)
                } else {
                    Token::List(res)
                }
            }
            Tag::Item => Token::Item(res),
            Tag::BlockQuote => {
                self.features.blockquote = true;
                Token::BlockQuote(res)
            },
            Tag::CodeBlock(language) => {
                self.features.codeblock = true;
                Token::CodeBlock(language.into_owned(), res)
            },
            Tag::Table(v) => {
                self.features.table = true;
                // TODO: actually use v's alignments
                Token::Table(v.len() as i32, res)
            },
            Tag::TableHead => Token::TableHead(res),
            Tag::TableRow => Token::TableRow(res),
            Tag::TableCell => Token::TableCell(res),
            Tag::FootnoteDefinition(reference) => {
                if self.footnotes.contains_key(reference.as_ref()) {
                    Logger::display_warning(lformat!("in {file}, found footnote definition for \
                                                      note '{reference}' but previous \
                                                      definition already exist, overriding it",
                                                     file = self.source,
                                                     reference = reference));
                }
                self.footnotes.insert(reference.into_owned(), res);
                Token::SoftBreak
            }
        };
        v.push(token);
        Ok(())
    }
}

/// Replace consecutives Strs by a Str of both, collapse soft breaks to previous std and so on
fn collapse(ast: &mut Vec<Token>) {
    if ast.len() < 2 {
        return;
    }

    let mut i = 0;
    while i < ast.len() {
        if ast[i].is_str() && i < ast.len() - 1 {
            if ast[i + 1].is_str() {
                // Two consecutives Str, concatenate them
                let token = ast.remove(i + 1);
                if let (&mut Token::Str(ref mut dest), Token::Str(ref source)) = (&mut ast[i],
                                                                                token) {
                    //                        dest.push(' ');
                    dest.push_str(source);
                    continue;
                } else {
                    unreachable!();
                }
            } else if ast[i + 1] == Token::SoftBreak {
                ast.remove(i + 1);
                if let &mut Token::Str(ref mut dest) = &mut ast[i] {
                    dest.push(' ');
                    continue;
                } else {
                    unreachable!();
                }
            }
        }

        // If token is containing others, recurse into them
        if let Some(ref mut inner) = ast[i].inner_mut() {
            collapse(inner);
        }
        i += 1;
    }
}

/// Replace images which are alone in a paragraph by standalone images
fn find_standalone(ast: &mut Vec<Token>) {
    for token in ast {
        let res = if let &mut Token::Paragraph(ref mut inner) = token {
            if inner.len() == 1 {
                if inner[0].is_image() {
                    if let Token::Image(source, title, inner) = mem::replace(&mut inner[0],
                                                                             Token::Rule) {
                        Token::StandaloneImage(source, title, inner)
                    } else {
                        unreachable!();
                    }
                } else {
                    // If paragraph only contains a link only containing an image, ok too
                    if let Token::Link(ref url, ref alt, ref mut inner) = inner[0] {
                        if inner[0].is_image() {
                            if let Token::Image(source, title, inner) = mem::replace(&mut inner[0],
                                                                                Token::Rule) {
                                Token::Link(url.clone(), alt.clone(), vec![Token::StandaloneImage(source, title, inner)])
                            } else {
                                unreachable!();
                            }
                        } else {
                            continue;
                        }
                    } else {
                        continue;
                    }
                }
            } else {
                continue;
            }
        } else {
            continue;
        };

        *token = res;
    }
}
<commit_msg>Add guilty fixme comment for code smell<commit_after>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use token::Token;
use error::{Result, Error, Source};
use logger::Logger;

use std::mem;
use std::fs::File;
use std::path::Path;
use std::convert::AsRef;
use std::io::Read;
use std::collections::HashMap;
use std::ops::BitOr;

use cmark::{Parser as CMParser, Event, Tag, Options, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES};

#[derive(Debug, Copy, Clone, PartialEq)]
/// The list of features used in a document.
pub struct Features {
    pub image: bool,
    pub blockquote: bool,
    pub codeblock: bool,
    pub ordered_list: bool,
    pub footnote: bool,
    pub table: bool,
    pub url: bool,
}

impl Features {
    /// Creates a new set of features where all are set to false
    pub fn new() -> Features {
        Features {
            image: false,
            blockquote: false,
            codeblock: false,
            ordered_list: false,
            footnote: false,
            table: false,
            url: false,
        }
    }
}

impl BitOr for Features {
    type Output = Self;

    fn bitor(self, rhs: Self) -> Self {
        Features {
            image: self.image | rhs.image,
            blockquote: self.blockquote | rhs.blockquote,
            codeblock: self.codeblock | rhs.codeblock,
            ordered_list: self.ordered_list | rhs.ordered_list,
            footnote: self.footnote | rhs.footnote,
            table: self.table | rhs.table,
            url: self.url | rhs.url,
        }
    }
}

/// A parser that reads markdown and convert it to AST (a vector of `Token`s)
///
/// This AST can then be used by various renderes.
///
/// As this Parser uses Pulldown-cmark's one, it should be able to parse most
/// *valid* CommonMark variant of Markdown.
///
/// Compared to other Markdown parser, it might fail more often on invalid code, e.g.
/// footnotes references that are not defined anywhere.
///
/// # Examples
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some *valid* Markdown[^1]\n\n[^1]: with a valid footnote");
/// assert!(result.is_ok());
/// ```
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some footnote pointing to nothing[^1] ");
/// assert!(result.is_err());
/// ```
pub struct Parser {
    footnotes: HashMap<String, Vec<Token>>,
    source: Source,
    features: Features,
}

impl Parser {
    /// Creates a parser
    pub fn new() -> Parser {
        Parser {
            footnotes: HashMap::new(),
            source: Source::empty(),
            features: Features::new(),
        }
    }

    /// Sets a parser's source file
    pub fn set_source_file(&mut self, s: &str) {
        self.source = Source::new(s);
    }

    /// Parse a file and returns an AST or  an error
    pub fn parse_file<P: AsRef<Path>>(&mut self, filename: P) -> Result<Vec<Token>> {
        let path: &Path = filename.as_ref();
        let mut f = File::open(path)
            .map_err(|_| {
                Error::file_not_found(&self.source,
                                      lformat!("markdown file"),
                                      format!("{}", path.display()))
            })?;
        let mut s = String::new();

        f.read_to_string(&mut s)
            .map_err(|_| {
                Error::parser(&self.source,
                              lformat!("file {file} contains invalid UTF-8, could not parse it",
                                       file = path.display()))
            })?;
        self.parse(&s)
    }

    /// Parse a string and returns an AST  an Error.
    pub fn parse(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut opts = Options::empty();
        opts.insert(OPTION_ENABLE_TABLES);
        opts.insert(OPTION_ENABLE_FOOTNOTES);
        let mut p = CMParser::new_ext(s, opts);

        let mut res = vec![];
        self.parse_events(&mut p, &mut res, None)?;

        self.parse_footnotes(&mut res)?;

        collapse(&mut res);

        find_standalone(&mut res);
        Ok(res)
    }

    /// Parse an inline string and returns a list of `Token`.
    ///
    /// This function removes the outermost `Paragraph` in most of the
    /// cases, as it is meant to be used for an inline string (e.g. metadata)
    pub fn parse_inline(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut tokens = self.parse(s)?;
        // Unfortunately, parser will put all this in a paragraph, so we might need to remove it.
        if tokens.len() == 1 {
            let res = match tokens[0] {
                Token::Paragraph(ref mut v) => Some(mem::replace(v, vec![])),
                _ => None,
            };
            match res {
                Some(tokens) => Ok(tokens),
                _ => Ok(tokens),
            }
        } else {
            Ok(tokens)
        }
    }

    /// Returns the list of features used by this parser
    pub fn features(&self) -> Features {
        self.features
    }

    /// Replace footnote reference with their definition
    fn parse_footnotes(&mut self, v: &mut Vec<Token>) -> Result<()> {
        for token in v {
            match *token {
                Token::Footnote(ref mut content) => {
                    let reference = if let Token::Str(ref text) = content[0] {
                        text.clone()
                    } else {
                        panic!("Reference is not a vector of a single Token::Str");
                    };
                    if let Some(in_vec) = self.footnotes.get(&reference) {
                        *content = in_vec.clone();
                    } else {
                        return Err(Error::parser(&self.source,
                                                 lformat!("footnote reference {reference} does \
                                                           not have a matching definition",
                                                          reference = &reference)));
                    }
                }
                Token::Paragraph(ref mut vec) |
                Token::Header(_, ref mut vec) |
                Token::Emphasis(ref mut vec) |
                Token::Strong(ref mut vec) |
                Token::Code(ref mut vec) |
                Token::BlockQuote(ref mut vec) |
                Token::CodeBlock(_, ref mut vec) |
                Token::List(ref mut vec) |
                Token::OrderedList(_, ref mut vec) |
                Token::Item(ref mut vec) |
                Token::Table(_, ref mut vec) |
                Token::TableHead(ref mut vec) |
                Token::TableRow(ref mut vec) |
                Token::TableCell(ref mut vec) |
                Token::Link(_, _, ref mut vec) |
                Token::Image(_, _, ref mut vec) => self.parse_footnotes(vec)?,
                _ => (),
            }
        }
        Ok(())
    }

    fn parse_events<'a>(&mut self,
                        p: &mut CMParser<'a>,
                        v: &mut Vec<Token>,
                        current_tag: Option<&Tag>)
                        -> Result<()> {
        while let Some(event) = p.next() {
            match event {
                Event::Html(text) |
                Event::InlineHtml(text) |
                Event::Text(text) => {
                    v.push(Token::Str(text.into_owned()));
                }
                Event::Start(tag) => self.parse_tag(p, v, tag)?,
                Event::End(tag) => {
                    debug_assert!(format!("{:?}", Some(&tag)) == format!("{:?}", current_tag),
                                  format!("Error: opening and closing tags mismatch!\n{:?} ≠ \
                                           {:?}",
                                          tag,
                                          current_tag));
                    break;
                }
                Event::SoftBreak => v.push(Token::SoftBreak),
                Event::HardBreak => v.push(Token::HardBreak),
                Event::FootnoteReference(text) => {
                    v.push(Token::Footnote(vec![Token::Str(text.into_owned())]))
                }
            }
        }
        Ok(())
    }

    fn parse_tag<'a>(&mut self,
                     p: &mut CMParser<'a>,
                     v: &mut Vec<Token>,
                     tag: Tag<'a>)
                     -> Result<()> {
        let mut res = vec![];

        self.parse_events(p, &mut res, Some(&tag))?;

        let token = match tag {
            Tag::Paragraph => Token::Paragraph(res),
            Tag::Emphasis => Token::Emphasis(res),
            Tag::Strong => Token::Strong(res),
            Tag::Code => Token::Code(res),
            Tag::Header(x) => Token::Header(x, res),
            Tag::Link(url, title) => {
                self.features.url = true;
                Token::Link(url.into_owned(), title.into_owned(), res)
            },
            Tag::Image(url, title) => {
                self.features.image = true;
                Token::Image(url.into_owned(), title.into_owned(), res)
            },
            Tag::Rule => Token::Rule,
            Tag::List(opt) => {
                if let Some(n) = opt {
                    self.features.ordered_list = true;
                    Token::OrderedList(n, res)
                } else {
                    Token::List(res)
                }
            }
            Tag::Item => Token::Item(res),
            Tag::BlockQuote => {
                self.features.blockquote = true;
                Token::BlockQuote(res)
            },
            Tag::CodeBlock(language) => {
                self.features.codeblock = true;
                Token::CodeBlock(language.into_owned(), res)
            },
            Tag::Table(v) => {
                self.features.table = true;
                // TODO: actually use v's alignments
                Token::Table(v.len() as i32, res)
            },
            Tag::TableHead => Token::TableHead(res),
            Tag::TableRow => Token::TableRow(res),
            Tag::TableCell => Token::TableCell(res),
            Tag::FootnoteDefinition(reference) => {
                if self.footnotes.contains_key(reference.as_ref()) {
                    Logger::display_warning(lformat!("in {file}, found footnote definition for \
                                                      note '{reference}' but previous \
                                                      definition already exist, overriding it",
                                                     file = self.source,
                                                     reference = reference));
                }
                self.footnotes.insert(reference.into_owned(), res);
                Token::SoftBreak
            }
        };
        v.push(token);
        Ok(())
    }
}

/// Replace consecutives Strs by a Str of both, collapse soft breaks to previous std and so on
fn collapse(ast: &mut Vec<Token>) {
    if ast.len() < 2 {
        return;
    }

    let mut i = 0;
    while i < ast.len() {
        if ast[i].is_str() && i < ast.len() - 1 {
            if ast[i + 1].is_str() {
                // Two consecutives Str, concatenate them
                let token = ast.remove(i + 1);
                if let (&mut Token::Str(ref mut dest), Token::Str(ref source)) = (&mut ast[i],
                                                                                token) {
                    //                        dest.push(' ');
                    dest.push_str(source);
                    continue;
                } else {
                    unreachable!();
                }
            } else if ast[i + 1] == Token::SoftBreak {
                ast.remove(i + 1);
                if let &mut Token::Str(ref mut dest) = &mut ast[i] {
                    dest.push(' ');
                    continue;
                } else {
                    unreachable!();
                }
            }
        }

        // If token is containing others, recurse into them
        if let Some(ref mut inner) = ast[i].inner_mut() {
            collapse(inner);
        }
        i += 1;
    }
}

/// Replace images which are alone in a paragraph by standalone images
fn find_standalone(ast: &mut Vec<Token>) {
    for token in ast {
        let res = if let &mut Token::Paragraph(ref mut inner) = token {
            if inner.len() == 1 {
                if inner[0].is_image() {
                    if let Token::Image(source, title, inner) = mem::replace(&mut inner[0],
                                                                             Token::Rule) {
                        Token::StandaloneImage(source, title, inner)
                    } else {
                        unreachable!();
                    }
                } else {
                    // If paragraph only contains a link only containing an image, ok too
                    // Fixme: messy code and unnecessary clone
                    if let Token::Link(ref url, ref alt, ref mut inner) = inner[0] {
                        if inner[0].is_image() {
                            if let Token::Image(source, title, inner) = mem::replace(&mut inner[0],
                                                                                Token::Rule) {
                                Token::Link(url.clone(), alt.clone(), vec![Token::StandaloneImage(source, title, inner)])
                            } else {
                                unreachable!();
                            }
                        } else {
                            continue;
                        }
                    } else {
                        continue;
                    }
                }
            } else {
                continue;
            }
        } else {
            continue;
        };

        *token = res;
    }
}
<|endoftext|>


<commit_before>extern crate simple_csv;
extern crate png;

use std::env;
use std::fs::File;
use std::io::BufWriter;
use png::HasParameters;

fn main(){
    let mut path = env::current_dir().unwrap();
    path.push("trappist-1.0.png");

    let file = File::create(path).unwrap();
    let ref mut w = BufWriter::new(file);

    let mut encoder = png::Encoder::new(w, 2, 1);
    encoder.set(png::ColorType::RGBA).set(png::BitDepth::Eight);
    let mut writer = encoder.write_header().unwrap();

    let data = [255, 0, 0, 255, 0, 0, 0, 255];
    writer.write_image_data(&data).unwrap();
}
<commit_msg>Create an image from csv<commit_after>extern crate simple_csv;
extern crate png;

use std::env;
use std::fs::File;
use std::io::{BufWriter, BufReader};
use std::str::FromStr;
use png::HasParameters;
use simple_csv::SimpleCsvReader;

fn main(){
    let f = File::open("../long-cadence.csv").unwrap();
    let buf = BufReader::new(f);
    let mut reader = SimpleCsvReader::new(buf);
    let row = reader.next_row().unwrap().unwrap();
    let mut iter = row.iter();
    iter.next(); // dropping time

    let raw: Vec<f64> = iter
        .map(|s| f64::from_str(s).unwrap())
        .collect();
    let maximum = raw
        .iter()
        .fold(std::f64::MIN, |acc, v| acc.max(*v));
    let data: Vec<u8> = raw
        .iter()
        .map(|s| s/maximum)
        .map(|s| 255.0 * s)
        .map(|s| s as u8)
        .collect();

    println!("{}: {:?}", maximum, data);

    let mut path = env::current_dir().unwrap();
    path.push("trappist-1.0.png");

    let file = File::create(path).unwrap();
    let ref mut w = BufWriter::new(file);

    let mut encoder = png::Encoder::new(w, 11, 11);
    encoder.set(png::ColorType::Grayscale).set(png::BitDepth::Eight);
    let mut writer = encoder.write_header().unwrap();

    writer.write_image_data(data.as_slice()).unwrap();
}
<|endoftext|>


<commit_before>use nom::{
    branch::alt,
    bytes::complete::{escaped_transform, is_not, tag, take},
    character::complete::one_of,
    combinator::{complete, eof, map, recognize, value},
    multi::{many0, many1, separated_list0, separated_list1},
    sequence::{delimited, preceded},
    IResult,
};

fn unquoted_token(input: &str) -> IResult<&str, String> {
    let mut parser = map(recognize(is_not("\t ;")), String::from);

    parser(input)
}

fn quoted_token<'a>(input: &'a str) -> IResult<&'a str, String> {
    let parser = escaped_transform(is_not(r#""\"#), '\\', |control_char: &'a str| {
        alt((
            value(r#"""#, tag(r#"""#)),
            value(r#"\"#, tag(r#"\"#)),
            value("\r", tag("r")),
            value("\n", tag("n")),
            value("\t", tag("t")),
            take(1usize), // all other escaped characters are passed through, unmodified
        ))(control_char)
    });

    let double_quote = tag("\"");
    let mut parser = delimited(&double_quote, parser, alt((&double_quote, eof)));

    parser(input)
}

fn token(input: &str) -> IResult<&str, String> {
    let mut parser = alt((quoted_token, unquoted_token));
    parser(input)
}

fn operation_with_args(input: &str) -> IResult<&str, Vec<String>> {
    let mut parser = separated_list1(many1(one_of(" \t")), token);
    parser(input)
}

fn semicolon(input: &str) -> IResult<&str, &str> {
    delimited(many0(one_of(" \t")), tag(";"), many0(one_of(" \t")))(input)
}

fn operation_sequence(input: &str) -> IResult<&str, Vec<Vec<String>>> {
    let parser = separated_list0(many1(semicolon), operation_with_args);
    let parser = delimited(many0(semicolon), parser, many0(semicolon));
    let parser = preceded(many0(one_of(" \t")), parser);

    let mut parser = complete(parser);

    parser(input)
}

/// Split a semicolon-separated list of operations into a vector. Each operation is represented by
/// a non-empty sub-vector, where the first element is the name of the operation, and the rest of
/// the elements are operation's arguments.
///
/// Tokens can be double-quoted. Such tokens can contain spaces and C-like escaped sequences: `\n`
/// for newline, `\r` for carriage return, `\t` for tab, `\"` for double quote, `\\` for backslash.
/// Unsupported sequences are stripped of the escaping, e.g. `\e` turns into `e`.
///
/// This function assumes that the input string:
/// 1. doesn't contain a comment;
/// 2. doesn't contain backticks that need to be processed.
///
/// Returns `None` if the input could not be parsed.
pub fn tokenize_operation_sequence(input: &str) -> Option<Vec<Vec<String>>> {
    match operation_sequence(input) {
        Ok((_leftovers, tokens)) => Some(tokens),
        Err(_error) => None,
    }
}

#[cfg(test)]
mod tests {
    use super::tokenize_operation_sequence;

    #[test]
    fn t_tokenize_operation_sequence_works_for_all_cpp_inputs() {
        assert_eq!(
            tokenize_operation_sequence("").unwrap(),
            Vec::<Vec<String>>::new()
        );
        assert_eq!(
            tokenize_operation_sequence("open").unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence("open-all-unread-in-browser-and-mark-read").unwrap(),
            vec![vec!["open-all-unread-in-browser-and-mark-read"]]
        );
        assert_eq!(
            tokenize_operation_sequence("; ; ; ;").unwrap(),
            Vec::<Vec<String>>::new()
        );
        assert_eq!(
            tokenize_operation_sequence("open ; next").unwrap(),
            vec![vec!["open"], vec!["next"]]
        );
        assert_eq!(
            tokenize_operation_sequence("open ; next ; prev").unwrap(),
            vec![vec!["open"], vec!["next"], vec!["prev"]]
        );
        assert_eq!(
            tokenize_operation_sequence("open ; next ; prev ; quit").unwrap(),
            vec![vec!["open"], vec!["next"], vec!["prev"], vec!["quit"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"set "arg 1""#).unwrap(),
            vec![vec!["set", "arg 1"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"set "arg 1" ; set "arg 2" "arg 3""#).unwrap(),
            vec![vec!["set", "arg 1"], vec!["set", "arg 2", "arg 3"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"set browser "firefox"; open-in-browser"#).unwrap(),
            vec![vec!["set", "browser", "firefox"], vec!["open-in-browser"]]
        );
        assert_eq!(
            tokenize_operation_sequence("set browser firefox; open-in-browser").unwrap(),
            vec![vec!["set", "browser", "firefox"], vec!["open-in-browser"]]
        );
        assert_eq!(
            tokenize_operation_sequence("open-in-browser; quit").unwrap(),
            vec![vec!["open-in-browser"], vec!["quit"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"open; set browser "firefox --private-window"; quit"#)
                .unwrap(),
            vec![
                vec!["open"],
                vec!["set", "browser", "firefox --private-window"],
                vec!["quit"]
            ]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"open ;set browser "firefox --private-window" ;quit"#)
                .unwrap(),
            vec![
                vec!["open"],
                vec!["set", "browser", "firefox --private-window"],
                vec!["quit"]
            ]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"open;set browser "firefox --private-window";quit"#)
                .unwrap(),
            vec![
                vec!["open"],
                vec!["set", "browser", "firefox --private-window"],
                vec!["quit"]
            ]
        );
        assert_eq!(
            tokenize_operation_sequence("; ;; ; open",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(";;; ;; ; open",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(";;; ;; ; open ;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(";;; ;; ; open ;; ;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(";;; ;; ; open ; ;;;;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(";;; open ; ;;;;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence("; open ;; ;; ;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence("open ; ;;; ;;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(
                r#"set browser "sleep 3; do-something ; echo hi"; open-in-browser"#
            )
            .unwrap(),
            vec![
                vec!["set", "browser", "sleep 3; do-something ; echo hi"],
                vec!["open-in-browser"]
            ]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_ignores_escaped_sequences_outside_double_quotes() {
        assert_eq!(
            tokenize_operation_sequence(r#"\t"#).unwrap(),
            vec![vec![r#"\t"#]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"\r"#).unwrap(),
            vec![vec![r#"\r"#]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"\n"#).unwrap(),
            vec![vec![r#"\n"#]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"\v"#).unwrap(),
            vec![vec![r#"\v"#]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"\""#).unwrap(),
            vec![vec![r#"\""#]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"\\"#).unwrap(),
            vec![vec![r#"\\"#]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_expands_escaped_sequences_inside_double_quotes() {
        assert_eq!(
            tokenize_operation_sequence(r#""\t""#).unwrap(),
            vec![vec!["\t"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\r""#).unwrap(),
            vec![vec!["\r"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\n""#).unwrap(),
            vec![vec!["\n"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\"""#).unwrap(),
            vec![vec!["\""]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\\""#).unwrap(),
            vec![vec!["\\"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_passes_through_unsupported_escaped_chars_inside_double_quotes()
    {
        assert_eq!(
            tokenize_operation_sequence(r#""\1""#).unwrap(),
            vec![vec!["1"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\W""#).unwrap(),
            vec![vec!["W"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\b""#).unwrap(),
            vec![vec!["b"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\d""#).unwrap(),
            vec![vec!["d"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\x""#).unwrap(),
            vec![vec!["x"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_implicitly_closes_double_quotes_at_end_of_input() {
        assert_eq!(
            tokenize_operation_sequence(r#"set "arg 1"#).unwrap(),
            vec![vec!["set", "arg 1"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_allows_single_character_unquoted() {
        assert_eq!(
            tokenize_operation_sequence(r#"set a b"#).unwrap(),
            vec![vec!["set", "a", "b"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_ignores_leading_and_trailing_whitespace() {
        assert_eq!(
            tokenize_operation_sequence(" \t set a b \t   ").unwrap(),
            vec![vec!["set", "a", "b"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_allows_tabs_between_arguments() {
        assert_eq!(
            tokenize_operation_sequence("\tset\ta\tb\t;\topen\t").unwrap(),
            vec![vec!["set", "a", "b"], vec!["open"]]
        );
    }
}
<commit_msg>Disallow `--` as unquoted_token in operation sequences<commit_after>use nom::{
    branch::alt,
    bytes::complete::{escaped_transform, is_not, tag, take},
    character::complete::one_of,
    combinator::{complete, eof, map, recognize, value, verify},
    multi::{many0, many1, separated_list0, separated_list1},
    sequence::{delimited, preceded},
    IResult,
};

fn unquoted_token(input: &str) -> IResult<&str, String> {
    let parser = map(recognize(is_not("\t ;")), String::from);
    let mut parser = verify(parser, |t: &str| t != "--");

    parser(input)
}

fn quoted_token<'a>(input: &'a str) -> IResult<&'a str, String> {
    let parser = escaped_transform(is_not(r#""\"#), '\\', |control_char: &'a str| {
        alt((
            value(r#"""#, tag(r#"""#)),
            value(r#"\"#, tag(r#"\"#)),
            value("\r", tag("r")),
            value("\n", tag("n")),
            value("\t", tag("t")),
            take(1usize), // all other escaped characters are passed through, unmodified
        ))(control_char)
    });

    let double_quote = tag("\"");
    let mut parser = delimited(&double_quote, parser, alt((&double_quote, eof)));

    parser(input)
}

fn token(input: &str) -> IResult<&str, String> {
    let mut parser = alt((quoted_token, unquoted_token));
    parser(input)
}

fn operation_with_args(input: &str) -> IResult<&str, Vec<String>> {
    let mut parser = separated_list1(many1(one_of(" \t")), token);
    parser(input)
}

fn semicolon(input: &str) -> IResult<&str, &str> {
    delimited(many0(one_of(" \t")), tag(";"), many0(one_of(" \t")))(input)
}

fn operation_sequence(input: &str) -> IResult<&str, Vec<Vec<String>>> {
    let parser = separated_list0(many1(semicolon), operation_with_args);
    let parser = delimited(many0(semicolon), parser, many0(semicolon));
    let parser = preceded(many0(one_of(" \t")), parser);

    let mut parser = complete(parser);

    parser(input)
}

/// Split a semicolon-separated list of operations into a vector. Each operation is represented by
/// a non-empty sub-vector, where the first element is the name of the operation, and the rest of
/// the elements are operation's arguments.
///
/// Tokens can be double-quoted. Such tokens can contain spaces and C-like escaped sequences: `\n`
/// for newline, `\r` for carriage return, `\t` for tab, `\"` for double quote, `\\` for backslash.
/// Unsupported sequences are stripped of the escaping, e.g. `\e` turns into `e`.
///
/// This function assumes that the input string:
/// 1. doesn't contain a comment;
/// 2. doesn't contain backticks that need to be processed.
///
/// Returns `None` if the input could not be parsed.
pub fn tokenize_operation_sequence(input: &str) -> Option<Vec<Vec<String>>> {
    match operation_sequence(input) {
        Ok((_leftovers, tokens)) => Some(tokens),
        Err(_error) => None,
    }
}

#[cfg(test)]
mod tests {
    use super::tokenize_operation_sequence;

    #[test]
    fn t_tokenize_operation_sequence_works_for_all_cpp_inputs() {
        assert_eq!(
            tokenize_operation_sequence("").unwrap(),
            Vec::<Vec<String>>::new()
        );
        assert_eq!(
            tokenize_operation_sequence("open").unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence("open-all-unread-in-browser-and-mark-read").unwrap(),
            vec![vec!["open-all-unread-in-browser-and-mark-read"]]
        );
        assert_eq!(
            tokenize_operation_sequence("; ; ; ;").unwrap(),
            Vec::<Vec<String>>::new()
        );
        assert_eq!(
            tokenize_operation_sequence("open ; next").unwrap(),
            vec![vec!["open"], vec!["next"]]
        );
        assert_eq!(
            tokenize_operation_sequence("open ; next ; prev").unwrap(),
            vec![vec!["open"], vec!["next"], vec!["prev"]]
        );
        assert_eq!(
            tokenize_operation_sequence("open ; next ; prev ; quit").unwrap(),
            vec![vec!["open"], vec!["next"], vec!["prev"], vec!["quit"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"set "arg 1""#).unwrap(),
            vec![vec!["set", "arg 1"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"set "arg 1" ; set "arg 2" "arg 3""#).unwrap(),
            vec![vec!["set", "arg 1"], vec!["set", "arg 2", "arg 3"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"set browser "firefox"; open-in-browser"#).unwrap(),
            vec![vec!["set", "browser", "firefox"], vec!["open-in-browser"]]
        );
        assert_eq!(
            tokenize_operation_sequence("set browser firefox; open-in-browser").unwrap(),
            vec![vec!["set", "browser", "firefox"], vec!["open-in-browser"]]
        );
        assert_eq!(
            tokenize_operation_sequence("open-in-browser; quit").unwrap(),
            vec![vec!["open-in-browser"], vec!["quit"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"open; set browser "firefox --private-window"; quit"#)
                .unwrap(),
            vec![
                vec!["open"],
                vec!["set", "browser", "firefox --private-window"],
                vec!["quit"]
            ]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"open ;set browser "firefox --private-window" ;quit"#)
                .unwrap(),
            vec![
                vec!["open"],
                vec!["set", "browser", "firefox --private-window"],
                vec!["quit"]
            ]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"open;set browser "firefox --private-window";quit"#)
                .unwrap(),
            vec![
                vec!["open"],
                vec!["set", "browser", "firefox --private-window"],
                vec!["quit"]
            ]
        );
        assert_eq!(
            tokenize_operation_sequence("; ;; ; open",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(";;; ;; ; open",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(";;; ;; ; open ;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(";;; ;; ; open ;; ;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(";;; ;; ; open ; ;;;;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(";;; open ; ;;;;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence("; open ;; ;; ;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence("open ; ;;; ;;",).unwrap(),
            vec![vec!["open"]]
        );
        assert_eq!(
            tokenize_operation_sequence(
                r#"set browser "sleep 3; do-something ; echo hi"; open-in-browser"#
            )
            .unwrap(),
            vec![
                vec!["set", "browser", "sleep 3; do-something ; echo hi"],
                vec!["open-in-browser"]
            ]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_ignores_escaped_sequences_outside_double_quotes() {
        assert_eq!(
            tokenize_operation_sequence(r#"\t"#).unwrap(),
            vec![vec![r#"\t"#]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"\r"#).unwrap(),
            vec![vec![r#"\r"#]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"\n"#).unwrap(),
            vec![vec![r#"\n"#]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"\v"#).unwrap(),
            vec![vec![r#"\v"#]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"\""#).unwrap(),
            vec![vec![r#"\""#]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#"\\"#).unwrap(),
            vec![vec![r#"\\"#]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_expands_escaped_sequences_inside_double_quotes() {
        assert_eq!(
            tokenize_operation_sequence(r#""\t""#).unwrap(),
            vec![vec!["\t"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\r""#).unwrap(),
            vec![vec!["\r"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\n""#).unwrap(),
            vec![vec!["\n"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\"""#).unwrap(),
            vec![vec!["\""]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\\""#).unwrap(),
            vec![vec!["\\"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_passes_through_unsupported_escaped_chars_inside_double_quotes()
    {
        assert_eq!(
            tokenize_operation_sequence(r#""\1""#).unwrap(),
            vec![vec!["1"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\W""#).unwrap(),
            vec![vec!["W"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\b""#).unwrap(),
            vec![vec!["b"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\d""#).unwrap(),
            vec![vec!["d"]]
        );
        assert_eq!(
            tokenize_operation_sequence(r#""\x""#).unwrap(),
            vec![vec!["x"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_implicitly_closes_double_quotes_at_end_of_input() {
        assert_eq!(
            tokenize_operation_sequence(r#"set "arg 1"#).unwrap(),
            vec![vec!["set", "arg 1"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_allows_single_character_unquoted() {
        assert_eq!(
            tokenize_operation_sequence(r#"set a b"#).unwrap(),
            vec![vec!["set", "a", "b"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_ignores_leading_and_trailing_whitespace() {
        assert_eq!(
            tokenize_operation_sequence(" \t set a b \t   ").unwrap(),
            vec![vec!["set", "a", "b"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_allows_tabs_between_arguments() {
        assert_eq!(
            tokenize_operation_sequence("\tset\ta\tb\t;\topen\t").unwrap(),
            vec![vec!["set", "a", "b"], vec!["open"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_does_not_consume_dashdash() {
        assert_eq!(
            tokenize_operation_sequence(r#"set a b -- "name of function""#).unwrap(),
            vec![vec!["set", "a", "b"]]
        );
    }

    #[test]
    fn t_tokenize_operation_sequence_allows_dashdash_in_quoted() {
        assert_eq!(
            tokenize_operation_sequence(r#"set a b "--" "name of function""#).unwrap(),
            vec![vec!["set", "a", "b", "--", "name of function"]]
        );
    }
}
<|endoftext|>


<commit_before>use std::fmt;

#[derive(Clone, Copy, Show, PartialEq)]
pub struct SourcePosition {
    pub line: i32,
    pub column: i32
}
impl SourcePosition {
    fn start() -> SourcePosition {
        SourcePosition { line: 1, column: 1 }
    }

    fn update(&mut self, c: &char) {
        self.column += 1;
        if *c == '\n' {
            self.column = 1;
            self.line += 1;
        }
    }
}

#[derive(Clone, PartialEq, Show)]
pub enum Error {
    Unexpected(char),
    Expected(String),
    Message(String)
}

#[derive(Clone, PartialEq, Show)]
pub enum Consumed {
    Consumed,
    Empty
}

#[derive(Clone, Show, PartialEq)]
pub struct ParseError {
    pub position: SourcePosition,
    pub consumed: Consumed,
    pub errors: Vec<Error>
}

impl ParseError {
    pub fn new(position: SourcePosition, consumed: Consumed, error: Error) -> ParseError {
        ParseError { position: position, consumed: consumed, errors: vec![error] }
    }
    pub fn add_message(&mut self, message: String) {
        self.add_error(Error::Message(message));
    }
    pub fn add_error(&mut self, message: Error) {
        //Don't add duplicate errors
        if self.errors.iter().find(|msg| **msg == message).is_none() {
            self.errors.push(message);
        }
    }
    pub fn merge(mut self, other: ParseError) -> ParseError {
        for message in other.errors.into_iter() {
            self.add_error(message);
        }
        self
    }
}

impl fmt::String for ParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        try!(writeln!(f, "Parse error at {}", self.position));
        for error in self.errors.iter() {
            try!(writeln!(f, "{}", error));
        }
        Ok(())
    }
}
impl fmt::String for SourcePosition {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "line: {}, column: {}", self.line, self.column)
    }
}
impl fmt::String for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Error::Unexpected(c) => write!(f, "Unexpected character '{}'", c),
            Error::Expected(ref s) => write!(f, "Expected {}", s),
            Error::Message(ref msg) => write!(f, "{}", msg),
        }
    }
}

#[derive(Clone, PartialEq, Show)]
pub struct State<I> {
    pub position: SourcePosition,
    pub input: I
}

impl <I: Stream> State<I> {
    fn new(input: I) -> State<I> {
        State { position: SourcePosition::start(), input: input }
    }
    fn uncons<F>(self, f: F) -> ParseResult<<I as Stream>::Item, I>
        where F: FnOnce(&mut SourcePosition, &<I as Stream>::Item) {
        let State { mut position, input } = self;
        match input.uncons() {
            Ok((c, input)) => {
                f(&mut position, &c);
                Ok((c, State { position: position, input: input }))
            }
            Err(()) => Err(ParseError::new(position, Consumed::Empty, Error::Message("End of input".to_string())))
        }
    }
}
impl <I: Stream<Item=char>> State<I> {
    pub fn uncons_char(self) -> ParseResult<<I as Stream>::Item, I> {
        self.uncons(SourcePosition::update)
    }

}

pub type ParseResult<O, I> = Result<(O, State<I>), ParseError>;

pub trait Stream : Clone {
    type Item;
    fn uncons(self) -> Result<(<Self as Stream>::Item, Self), ()>;
}

impl <I: Iterator + Clone> Stream for I {
    type Item = <I as Iterator>::Item;
    fn uncons(mut self) -> Result<(<Self as Stream>::Item, Self), ()> {
        match self.next() {
            Some(x) => Ok((x, self)),
            None => Err(())
        }
    }
}

impl <'a> Stream for &'a str {
    type Item = char;
    fn uncons(self) -> Result<(char, &'a str), ()> {
        match self.slice_shift_char() {
            Some(x) => Ok(x),
            None => Err(())
        }
    }
}

impl <'a, T> Stream for &'a [T] {
    type Item = &'a T;
    fn uncons(self) -> Result<(&'a T, &'a [T]), ()> {
        match self {
            [ref x, rest..] => Ok((x, rest)),
            [] => Err(())
        }
    }
}

pub trait Parser {
    type Input: Stream;
    type Output;

    ///Parses using `input` by calling Stream::uncons one or more times
    ///On success returns `Ok((value, new_state))` on failure it returns `Err(error)`
    fn parse_state(&mut self, input: State<<Self as Parser>::Input>) -> ParseResult<<Self as Parser>::Output, <Self as Parser>::Input>;
    fn parse(&mut self, input: <Self as Parser>::Input) -> ParseResult<<Self as Parser>::Output, <Self as Parser>::Input> {
        self.parse_state(State::new(input))
    }
}
impl <'a, I, O, P> Parser for &'a mut P
    where I: Stream, P: Parser<Input=I, Output=O> {
    type Input = I;
    type Output = O;
    fn parse_state(&mut self, input: State<I>) -> ParseResult<O, I> {
        (*self).parse_state(input)
    }
}
impl <I, O, P> Parser for Box<P>
    where I: Stream, P: Parser<Input=I, Output=O> {
    type Input = I;
    type Output = O;
    fn parse_state(&mut self, input: State<I>) -> ParseResult<O, I> {
        (*self).parse_state(input)
    }
}
<commit_msg>Documented the parse function<commit_after>use std::fmt;

#[derive(Clone, Copy, Show, PartialEq)]
pub struct SourcePosition {
    pub line: i32,
    pub column: i32
}
impl SourcePosition {
    fn start() -> SourcePosition {
        SourcePosition { line: 1, column: 1 }
    }

    fn update(&mut self, c: &char) {
        self.column += 1;
        if *c == '\n' {
            self.column = 1;
            self.line += 1;
        }
    }
}

#[derive(Clone, PartialEq, Show)]
pub enum Error {
    Unexpected(char),
    Expected(String),
    Message(String)
}

#[derive(Clone, PartialEq, Show)]
pub enum Consumed {
    Consumed,
    Empty
}

#[derive(Clone, Show, PartialEq)]
pub struct ParseError {
    pub position: SourcePosition,
    pub consumed: Consumed,
    pub errors: Vec<Error>
}

impl ParseError {
    pub fn new(position: SourcePosition, consumed: Consumed, error: Error) -> ParseError {
        ParseError { position: position, consumed: consumed, errors: vec![error] }
    }
    pub fn add_message(&mut self, message: String) {
        self.add_error(Error::Message(message));
    }
    pub fn add_error(&mut self, message: Error) {
        //Don't add duplicate errors
        if self.errors.iter().find(|msg| **msg == message).is_none() {
            self.errors.push(message);
        }
    }
    pub fn merge(mut self, other: ParseError) -> ParseError {
        for message in other.errors.into_iter() {
            self.add_error(message);
        }
        self
    }
}

impl fmt::String for ParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        try!(writeln!(f, "Parse error at {}", self.position));
        for error in self.errors.iter() {
            try!(writeln!(f, "{}", error));
        }
        Ok(())
    }
}
impl fmt::String for SourcePosition {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "line: {}, column: {}", self.line, self.column)
    }
}
impl fmt::String for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Error::Unexpected(c) => write!(f, "Unexpected character '{}'", c),
            Error::Expected(ref s) => write!(f, "Expected {}", s),
            Error::Message(ref msg) => write!(f, "{}", msg),
        }
    }
}

#[derive(Clone, PartialEq, Show)]
pub struct State<I> {
    pub position: SourcePosition,
    pub input: I
}

impl <I: Stream> State<I> {
    fn new(input: I) -> State<I> {
        State { position: SourcePosition::start(), input: input }
    }
    fn uncons<F>(self, f: F) -> ParseResult<<I as Stream>::Item, I>
        where F: FnOnce(&mut SourcePosition, &<I as Stream>::Item) {
        let State { mut position, input } = self;
        match input.uncons() {
            Ok((c, input)) => {
                f(&mut position, &c);
                Ok((c, State { position: position, input: input }))
            }
            Err(()) => Err(ParseError::new(position, Consumed::Empty, Error::Message("End of input".to_string())))
        }
    }
}
impl <I: Stream<Item=char>> State<I> {
    pub fn uncons_char(self) -> ParseResult<<I as Stream>::Item, I> {
        self.uncons(SourcePosition::update)
    }

}

pub type ParseResult<O, I> = Result<(O, State<I>), ParseError>;

pub trait Stream : Clone {
    type Item;
    fn uncons(self) -> Result<(Self::Item, Self), ()>;
}

impl <I: Iterator + Clone> Stream for I {
    type Item = <I as Iterator>::Item;
    fn uncons(mut self) -> Result<(I::Item, Self), ()> {
        match self.next() {
            Some(x) => Ok((x, self)),
            None => Err(())
        }
    }
}

impl <'a> Stream for &'a str {
    type Item = char;
    fn uncons(self) -> Result<(char, &'a str), ()> {
        match self.slice_shift_char() {
            Some(x) => Ok(x),
            None => Err(())
        }
    }
}

impl <'a, T> Stream for &'a [T] {
    type Item = &'a T;
    fn uncons(self) -> Result<(&'a T, &'a [T]), ()> {
        match self {
            [ref x, rest..] => Ok((x, rest)),
            [] => Err(())
        }
    }
}

pub trait Parser {
    type Input: Stream;
    type Output;

    ///Entrypoint of the parser
    ///Takes some input and tries to parse it returning a `ParseResult`
    fn parse(&mut self, input: Self::Input) -> ParseResult<Self::Output, Self::Input> {
        self.parse_state(State::new(input))
    }
    ///Parses using the state `input` by calling Stream::uncons one or more times
    ///On success returns `Ok((value, new_state))` on failure it returns `Err(error)`
    fn parse_state(&mut self, input: State<Self::Input>) -> ParseResult<Self::Output, Self::Input>;
}
impl <'a, I, O, P> Parser for &'a mut P
    where I: Stream, P: Parser<Input=I, Output=O> {
    type Input = I;
    type Output = O;
    fn parse_state(&mut self, input: State<I>) -> ParseResult<O, I> {
        (*self).parse_state(input)
    }
}
impl <I, O, P> Parser for Box<P>
    where I: Stream, P: Parser<Input=I, Output=O> {
    type Input = I;
    type Output = O;
    fn parse_state(&mut self, input: State<I>) -> ParseResult<O, I> {
        (*self).parse_state(input)
    }
}
<|endoftext|>


<commit_before>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! This module defines the `__rust_probestack` intrinsic which is used in the
//! implementation of "stack probes" on certain platforms.
//!
//! The purpose of a stack probe is to provide a static guarantee that if a
//! thread has a guard page then a stack overflow is guaranteed to hit that
//! guard page. If a function did not have a stack probe then there's a risk of
//! having a stack frame *larger* than the guard page, so a function call could
//! skip over the guard page entirely and then later hit maybe the heap or
//! another thread, possibly leading to security vulnerabilities such as [The
//! Stack Clash], for example.
//!
//! [The Stack Clash]: https://blog.qualys.com/securitylabs/2017/06/19/the-stack-clash
//!
//! The `__rust_probestack` is called in the prologue of functions whose stack
//! size is larger than the guard page, for example larger than 4096 bytes on
//! x86. This function is then responsible for "touching" all pages relevant to
//! the stack to ensure that that if any of them are the guard page we'll hit
//! them guaranteed.
//!
//! The precise ABI for how this function operates is defined by LLVM. There's
//! no real documentation as to what this is, so you'd basically need to read
//! the LLVM source code for reference. Often though the test cases can be
//! illuminating as to the ABI that's generated, or just looking at the output
//! of `llc`.
//!
//! Note that `#[naked]` is typically used here for the stack probe because the
//! ABI corresponds to no actual ABI.
//!
//! Finally it's worth noting that at the time of this writing LLVM only has
//! support for stack probes on x86 and x86_64. There's no support for stack
//! probes on any other architecture like ARM or PowerPC64. LLVM I'm sure would
//! be more than welcome to accept such a change!

#![cfg(not(windows))] // Windows already has builtins to do this

#[naked]
#[no_mangle]
#[cfg(all(target_arch = "x86_64", not(feature = "mangled-names")))]
pub unsafe extern fn __rust_probestack() {
    // Our goal here is to touch each page between %rsp+8 and %rsp+8-%rax,
    // ensuring that if any pages are unmapped we'll make a page fault.
    //
    // The ABI here is that the stack frame size is located in `%eax`. Upon
    // return we're not supposed to modify `%esp` or `%eax`.
    asm!("
        mov    %rax,%r11        // duplicate %rax as we're clobbering %r11

        // Main loop, taken in one page increments. We're decrementing rsp by
        // a page each time until there's less than a page remaining. We're
        // guaranteed that this function isn't called unless there's more than a
        // page needed.
        //
        // Note that we're also testing against `8(%rsp)` to account for the 8
        // bytes pushed on the stack orginally with our return address. Using
        // `8(%rsp)` simulates us testing the stack pointer in the caller's
        // context.
    2:
        sub    $$0x1000,%rsp
        test   %rsp,8(%rsp)
        sub    $$0x1000,%r11
        cmp    $$0x1000,%r11
        ja     2b

        // Finish up the last remaining stack space requested, getting the last
        // bits out of r11
        sub    %r11,%rsp
        test   %rsp,8(%rsp)

        // Restore the stack pointer to what it previously was when entering
        // this function. The caller will readjust the stack pointer after we
        // return.
        add    %rax,%rsp

        ret
    " ::: "memory" : "volatile");
    ::core::intrinsics::unreachable();
}

#[naked]
#[no_mangle]
#[cfg(all(target_arch = "x86", not(feature = "mangled-names")))]
pub unsafe extern fn __rust_probestack() {
    // This is the same as x86_64 above, only translated for 32-bit sizes. Note
    // that on Unix we're expected to restore everything as it was, this
    // function basically can't tamper with anything.
    //
    // The ABI here is the same as x86_64, except everything is 32-bits large.
    asm!("
        push   %ecx
        mov    %eax,%ecx
    2:
        sub    $$0x1000,%esp
        test   %esp,8(%esp)
        sub    $$0x1000,%ecx
        cmp    $$0x1000,%ecx
        ja     2b

        sub    %ecx,%esp
        test   %esp,8(%esp)

        add    %eax,%esp
        pop    %ecx
        ret
    " ::: "memory" : "volatile");
    ::core::intrinsics::unreachable();
}
<commit_msg>Fix segfault on probestack with dynamic alloca.<commit_after>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! This module defines the `__rust_probestack` intrinsic which is used in the
//! implementation of "stack probes" on certain platforms.
//!
//! The purpose of a stack probe is to provide a static guarantee that if a
//! thread has a guard page then a stack overflow is guaranteed to hit that
//! guard page. If a function did not have a stack probe then there's a risk of
//! having a stack frame *larger* than the guard page, so a function call could
//! skip over the guard page entirely and then later hit maybe the heap or
//! another thread, possibly leading to security vulnerabilities such as [The
//! Stack Clash], for example.
//!
//! [The Stack Clash]: https://blog.qualys.com/securitylabs/2017/06/19/the-stack-clash
//!
//! The `__rust_probestack` is called in the prologue of functions whose stack
//! size is larger than the guard page, for example larger than 4096 bytes on
//! x86. This function is then responsible for "touching" all pages relevant to
//! the stack to ensure that that if any of them are the guard page we'll hit
//! them guaranteed.
//!
//! The precise ABI for how this function operates is defined by LLVM. There's
//! no real documentation as to what this is, so you'd basically need to read
//! the LLVM source code for reference. Often though the test cases can be
//! illuminating as to the ABI that's generated, or just looking at the output
//! of `llc`.
//!
//! Note that `#[naked]` is typically used here for the stack probe because the
//! ABI corresponds to no actual ABI.
//!
//! Finally it's worth noting that at the time of this writing LLVM only has
//! support for stack probes on x86 and x86_64. There's no support for stack
//! probes on any other architecture like ARM or PowerPC64. LLVM I'm sure would
//! be more than welcome to accept such a change!

#![cfg(not(windows))] // Windows already has builtins to do this

#[naked]
#[no_mangle]
#[cfg(all(target_arch = "x86_64", not(feature = "mangled-names")))]
pub unsafe extern fn __rust_probestack() {
    // Our goal here is to touch each page between %rsp+8 and %rsp+8-%rax,
    // ensuring that if any pages are unmapped we'll make a page fault.
    //
    // The ABI here is that the stack frame size is located in `%eax`. Upon
    // return we're not supposed to modify `%esp` or `%eax`.
    asm!("
        mov    %rax,%r11        // duplicate %rax as we're clobbering %r11

        // Main loop, taken in one page increments. We're decrementing rsp by
        // a page each time until there's less than a page remaining. We're
        // guaranteed that this function isn't called unless there's more than a
        // page needed.
        //
        // Note that we're also testing against `8(%rsp)` to account for the 8
        // bytes pushed on the stack orginally with our return address. Using
        // `8(%rsp)` simulates us testing the stack pointer in the caller's
        // context.
        cmp    $$0x1000,%r11
        jna    3f
    2:
        sub    $$0x1000,%rsp
        test   %rsp,8(%rsp)
        sub    $$0x1000,%r11
        cmp    $$0x1000,%r11
        ja     2b

    3:
        // Finish up the last remaining stack space requested, getting the last
        // bits out of r11
        sub    %r11,%rsp
        test   %rsp,8(%rsp)

        // Restore the stack pointer to what it previously was when entering
        // this function. The caller will readjust the stack pointer after we
        // return.
        add    %rax,%rsp

        ret
    " ::: "memory" : "volatile");
    ::core::intrinsics::unreachable();
}

#[naked]
#[no_mangle]
#[cfg(all(target_arch = "x86", not(feature = "mangled-names")))]
pub unsafe extern fn __rust_probestack() {
    // This is the same as x86_64 above, only translated for 32-bit sizes. Note
    // that on Unix we're expected to restore everything as it was, this
    // function basically can't tamper with anything.
    //
    // The ABI here is the same as x86_64, except everything is 32-bits large.
    asm!("
        push   %ecx
        mov    %eax,%ecx

        cmp    $$0x1000,%ecx
        jna    3f
    2:
        sub    $$0x1000,%esp
        test   %esp,8(%esp)
        sub    $$0x1000,%ecx
        cmp    $$0x1000,%ecx
        ja     2b

    3:
        sub    %ecx,%esp
        test   %esp,8(%esp)

        add    %eax,%esp
        pop    %ecx
        ret
    " ::: "memory" : "volatile");
    ::core::intrinsics::unreachable();
}
<|endoftext|>


<commit_before>use agents::Controller;
use domains::{Domain, Observation};
use geometry::{ActionSpace, Space};
use policies::Greedy;
use slog::{Logger, Record, Result as LogResult, Serializer, KV};

/// Container for episodic statistics.
#[derive(Debug)]
pub struct Episode {
    /// The number of steps taken to reach the terminal state.
    pub steps: u64,

    /// The total accumulated reward over the episode.
    pub reward: f64,
}

impl KV for Episode {
    fn serialize(&self, _: &Record, serializer: &mut Serializer) -> LogResult {
        serializer.emit_u64("steps", self.steps)?;
        serializer.emit_f64("reward", self.reward)?;

        Ok(())
    }
}

/// Helper function for running experiments.
pub fn run<T>(runner: T, n_episodes: usize, logger: Option<Logger>) -> Vec<Episode>
where T: Iterator<Item = Episode> {
    let exp = runner.take(n_episodes);

    match logger {
        Some(logger) => exp.zip(1..(n_episodes + 1))
            .inspect(|&(ref res, i)| {
                info!(logger, "episode {}", i; res);
            })
            .map(|(res, _)| res)
            .collect(),

        None => exp.collect(),
    }
}

/// Utility for running a single evaluation episode.
pub struct Evaluation<'a, A: 'a, D> {
    agent: &'a mut A,
    domain_factory: Box<Fn() -> D>,

    greedy: Greedy,
}

impl<'a, S: Space, A, D> Evaluation<'a, A, D>
where
    A: Controller<S, ActionSpace>,
    D: Domain<StateSpace = S, ActionSpace = ActionSpace>,
{
    pub fn new(agent: &'a mut A, domain_factory: Box<Fn() -> D>) -> Evaluation<'a, A, D> {
        Evaluation {
            agent: agent,
            domain_factory: domain_factory,

            greedy: Greedy,
        }
    }
}

impl<'a, S: Space, A, D> Iterator for Evaluation<'a, A, D>
where
    A: Controller<S, ActionSpace>,
    D: Domain<StateSpace = S, ActionSpace = ActionSpace>,
{
    type Item = Episode;

    fn next(&mut self) -> Option<Episode> {
        let mut domain = (self.domain_factory)();
        let mut a = self.agent
            .evaluate_policy(&mut self.greedy, &domain.emit().state());

        let mut e = Episode {
            steps: 1,
            reward: 0.0,
        };

        loop {
            let t = domain.step(a);

            e.steps += 1;
            e.reward += t.reward;

            a = match t.to {
                Observation::Terminal(_) => {
                    self.agent.handle_terminal(&t);
                    break;
                },
                _ => self.agent.evaluate_policy(&mut self.greedy, &t.to.state()),
            };
        }

        Some(e)
    }
}

/// Utility for running a sequence of training episodes.
pub struct SerialExperiment<'a, A: 'a, D> {
    agent: &'a mut A,
    domain_factory: Box<Fn() -> D>,

    step_limit: u64,
}

impl<'a, S: Space, A, D> SerialExperiment<'a, A, D>
where
    A: Controller<S, ActionSpace>,
    D: Domain<StateSpace = S, ActionSpace = ActionSpace>,
{
    pub fn new(
        agent: &'a mut A,
        domain_factory: Box<Fn() -> D>,
        step_limit: u64,
    ) -> SerialExperiment<'a, A, D>
    {
        SerialExperiment {
            agent: agent,
            domain_factory: domain_factory,
            step_limit: step_limit,
        }
    }
}

impl<'a, S: Space, A, D> Iterator for SerialExperiment<'a, A, D>
where
    A: Controller<S, ActionSpace>,
    D: Domain<StateSpace = S, ActionSpace = ActionSpace>,
{
    type Item = Episode;

    fn next(&mut self) -> Option<Episode> {
        let mut domain = (self.domain_factory)();
        let mut a = self.agent.mu(domain.emit().state());

        let mut e = Episode {
            steps: 1,
            reward: 0.0,
        };

        for j in 1..(self.step_limit + 1) {
            let t = domain.step(a);

            e.steps = j;
            e.reward += t.reward;

            self.agent.handle_sample(&t);

            // TODO: Clean this mess up!
            if let Observation::Terminal(_) = t.to {
                self.agent.handle_terminal(&t);
                break;
            } else if j >= self.step_limit {
                self.agent.handle_terminal(&t);
                break;
            } else {
                a = self.agent.mu(&t.to.state());
            }
        }

        Some(e)
    }
}
<commit_msg>refactor: decouple experiment types from Space<commit_after>use agents::Controller;
use domains::{Domain, Observation};
use geometry::{ActionSpace, Space};
use policies::Greedy;
use slog::{Logger, Record, Result as LogResult, Serializer, KV};

/// Container for episodic statistics.
#[derive(Debug)]
pub struct Episode {
    /// The number of steps taken to reach the terminal state.
    pub steps: u64,

    /// The total accumulated reward over the episode.
    pub reward: f64,
}

impl KV for Episode {
    fn serialize(&self, _: &Record, serializer: &mut Serializer) -> LogResult {
        serializer.emit_u64("steps", self.steps)?;
        serializer.emit_f64("reward", self.reward)?;

        Ok(())
    }
}

/// Helper function for running experiments.
pub fn run<T>(runner: T, n_episodes: usize, logger: Option<Logger>) -> Vec<Episode>
where T: Iterator<Item = Episode> {
    let exp = runner.take(n_episodes);

    match logger {
        Some(logger) => exp.zip(1..(n_episodes + 1))
            .inspect(|&(ref res, i)| {
                info!(logger, "episode {}", i; res);
            })
            .map(|(res, _)| res)
            .collect(),

        None => exp.collect(),
    }
}

/// Utility for running a single evaluation episode.
pub struct Evaluation<'a, A: 'a, D> {
    agent: &'a mut A,
    domain_factory: Box<Fn() -> D>,

    greedy: Greedy,
}

impl<'a, S: Space, A, D> Evaluation<'a, A, D>
where
    A: Controller<S::Repr, <ActionSpace as Space>::Repr>,
    D: Domain<StateSpace = S, ActionSpace = ActionSpace>,
{
    pub fn new(agent: &'a mut A, domain_factory: Box<Fn() -> D>) -> Evaluation<'a, A, D> {
        Evaluation {
            agent: agent,
            domain_factory: domain_factory,

            greedy: Greedy,
        }
    }
}

impl<'a, S: Space, A, D> Iterator for Evaluation<'a, A, D>
where
    A: Controller<S::Repr, <ActionSpace as Space>::Repr>,
    D: Domain<StateSpace = S, ActionSpace = ActionSpace>,
{
    type Item = Episode;

    fn next(&mut self) -> Option<Episode> {
        let mut domain = (self.domain_factory)();
        let mut a = self.agent
            .evaluate_policy(&mut self.greedy, &domain.emit().state());

        let mut e = Episode {
            steps: 1,
            reward: 0.0,
        };

        loop {
            let t = domain.step(a);

            e.steps += 1;
            e.reward += t.reward;

            a = match t.to {
                Observation::Terminal(_) => {
                    self.agent.handle_terminal(&t);
                    break;
                },
                _ => self.agent.evaluate_policy(&mut self.greedy, &t.to.state()),
            };
        }

        Some(e)
    }
}

/// Utility for running a sequence of training episodes.
pub struct SerialExperiment<'a, A: 'a, D> {
    agent: &'a mut A,
    domain_factory: Box<Fn() -> D>,

    step_limit: u64,
}

impl<'a, S: Space, A, D> SerialExperiment<'a, A, D>
where
    A: Controller<S::Repr, <ActionSpace as Space>::Repr>,
    D: Domain<StateSpace = S, ActionSpace = ActionSpace>,
{
    pub fn new(
        agent: &'a mut A,
        domain_factory: Box<Fn() -> D>,
        step_limit: u64,
    ) -> SerialExperiment<'a, A, D>
    {
        SerialExperiment {
            agent: agent,
            domain_factory: domain_factory,
            step_limit: step_limit,
        }
    }
}

impl<'a, S: Space, A, D> Iterator for SerialExperiment<'a, A, D>
where
    A: Controller<S::Repr, <ActionSpace as Space>::Repr>,
    D: Domain<StateSpace = S, ActionSpace = ActionSpace>,
{
    type Item = Episode;

    fn next(&mut self) -> Option<Episode> {
        let mut domain = (self.domain_factory)();
        let mut a = self.agent.mu(domain.emit().state());

        let mut e = Episode {
            steps: 1,
            reward: 0.0,
        };

        for j in 1..(self.step_limit + 1) {
            let t = domain.step(a);

            e.steps = j;
            e.reward += t.reward;

            self.agent.handle_sample(&t);

            // TODO: Clean this mess up!
            if let Observation::Terminal(_) = t.to {
                self.agent.handle_terminal(&t);
                break;
            } else if j >= self.step_limit {
                self.agent.handle_terminal(&t);
                break;
            } else {
                a = self.agent.mu(&t.to.state());
            }
        }

        Some(e)
    }
}
<|endoftext|>


<commit_before>extern crate chrono;
extern crate clap;
extern crate fern;
#[macro_use]
extern crate log;
extern crate mcts;
extern crate rand;
extern crate thud;
extern crate thud_game;

use ::clap::App;
use ::rand::isaac::IsaacRng;
use ::rand::SeedableRng;

use thud_game::board;

fn main() {
    // Set up arg handling.
    let matches = {
        let app = thud::set_common_args(
            App::new("console_mcts")
                .version("0.1.0")
                .author("Stu Black <trurl@freeshell.org>")
                .about("Plays out Thud MCTS iterations"),
            &[thud::ITERATION_COUNT_FLAG,
              thud::SIMULATION_COUNT_FLAG,
              thud::EXPLORATION_BIAS_FLAG,
              thud::INITIAL_BOARD_FLAG,
              thud::INITIAL_PLAYER_FLAG,
              thud::LOG_LEVEL_FLAG,
              thud::MOVE_SELECTION_CRITERION_FLAG,
              thud::COMPACT_SEARCH_GRAPH_FLAG,]);
        app.get_matches()
    };
    let iteration_count =
        match matches.value_of(thud::ITERATION_COUNT_FLAG).unwrap().parse::<usize>() {
            Ok(x) => x,
            Err(e) => panic!("Bad iteration count: {}", e),
        };
    let simulation_count =
        match matches.value_of(thud::SIMULATION_COUNT_FLAG).unwrap().parse::<usize>() {
            Ok(x) => x,
            Err(e) => panic!("Bad simulation count: {}", e),
        };
    let exploration_bias =
        match matches.value_of(thud::EXPLORATION_BIAS_FLAG).unwrap().parse::<f64>() {
            Ok(x) => x,
            Err(e) => panic!("Bad exploration bias: {}", e),
        };
    let initial_cells =
        match matches.value_of(thud::INITIAL_BOARD_FLAG).map(|x| x.parse::<thud::InitialBoard>()) {
            Some(Ok(x)) => x.cells(),
            Some(Err(e)) => panic!("Bad initial board configuration: {}", e),
            None => board::Cells::default(),
        };
    let toggle_initial_player =
        match matches.value_of(thud::INITIAL_PLAYER_FLAG).map(|x| x.parse::<thud_game::Role>()) {
            None | Some(Ok(thud_game::Role::Dwarf)) => false,
            Some(Ok(thud_game::Role::Troll)) => true,
            Some(Err(x)) => panic!("{}", x),
        };
    let logging_level =
        match matches.value_of(thud::LOG_LEVEL_FLAG).map(|x| x.parse::<log::LogLevelFilter>()) {
            Some(Ok(x)) => x,
            Some(Err(_)) => panic!("Bad logging level '{}'", matches.value_of(thud::LOG_LEVEL_FLAG).unwrap()),
            None => log::LogLevelFilter::Info,
        };
    let move_selection_criterion =
        match matches.value_of(thud::MOVE_SELECTION_CRITERION_FLAG).map(|x| x.parse::<thud::MoveSelectionCriterion>()) {
            Some(Ok(x)) => x,
            Some(Err(e)) => panic!("Bad move selection criterion: {}", e),
            None => thud::MoveSelectionCriterion::VisitCount,
        };
    let rng =
        match matches.value_of(thud::RNG_SEED_FLAG).map(|x| x.parse::<u32>()) {
            Some(Ok(x)) => IsaacRng::from_seed(&[x]),
            Some(Err(e)) => panic!("Bad RNG seed: {}", e),
            None => IsaacRng::new_unseeded(),
        };
    let compact_graph = matches.is_present(thud::COMPACT_SEARCH_GRAPH_FLAG);

    // Set up logging.
    let logger_config = fern::DispatchConfig {
        format: Box::new(|msg: &str, level: &log::LogLevel, _location: &log::LogLocation| {
            format!("[{}][{}] {}",
                    chrono::Local::now().format("%Y-%m-%d %T%.3f%z").to_string(), level, msg)
        }),
        output: vec![fern::OutputConfig::stdout()],
        level: logging_level,
    };
    if let Err(e) = fern::init_global_logger(logger_config, log::LogLevelFilter::Trace) {
        panic!("Failed to initialize global logger: {}", e);
    }

    // Play game.
    let mut state = mcts::State::new(initial_cells);
    if toggle_initial_player {
        state.toggle_active_role();
    }
    let mut graph = mcts::Graph::new();
    thud::initialize_search(state.clone(), &mut graph);

    let mut search_state = mcts::SearchState::new(rng, exploration_bias);
    let mut turn_number = 0;
    loop {
        info!("begin turn {}; board: {}", turn_number, board::format_board(state.board()));
        if graph.get_node(&state).is_none() {
            error!("board not found in playout graph; reinitializing");
            thud::initialize_search(state.clone(), &mut graph);
        }

        // {
        //     let mut available_actions: Vec<game::Action> =
        //         graph.get_node(&state).unwrap().get_child_list().iter()
        //         .map(|c| c.get_data().action).collect();
        //     available_actions.sort_by(thud_game::util::cmp_actions);
        //     let mut state_actions: Vec<game::Action> = state.actions().collect();
        //     state_actions.sort_by(thud_game::util::cmp_actions);
        //     debug!("Checking comparison of state_actions: {:?} vs. available actions: {:?}",
        //            state_actions, available_actions);
        //     assert_eq!(state_actions, available_actions);
        // }

        let mut best_action = None;
        for iteration in 0..iteration_count {
            if iteration % 100 == 0 {
                info!("iteration: {} / {} = {}%", iteration, iteration_count,
                      ((10000.0 * (iteration as f64) / (iteration_count as f64)) as usize as f64) / 100.0);
            }
            match search_state.search(&mut graph, &state,
                                      |_: usize| mcts::SearchSettings {
                                          simulation_count: simulation_count,
                                      }) {
                Ok(stats) => {
                    // TODO: this commented-out block is only valid when we haven't
                    // propagated a multi-visit payoff upwards from conneting to an
                    // extant vertex.
                    // if simulation_count == 1 && toplevel_visits != iteration + 1 {
                    //     console_ui::write_board(state.board());
                    //     for (action, stats) in stats.into_iter() {
                    //         println!("{:?}: [{}, {}] = {:?} / {}", action,
                    //                  (stats.payoff.values[0] as f64) / (stats.visits as f64),
                    //                  (stats.payoff.values[1] as f64) / (stats.visits as f64),
                    //                  stats.payoff, stats.visits);
                    //     }
                    //     panic!("total visits at top level is {}, but iteration count is {}",
                    //            toplevel_visits, iteration)
                    // }
                    if iteration % 1000 == 0 || iteration + 1 == iteration_count {
                        info!("root stats:");
                        let mut best_visits = ::std::u32::MIN;
                        let mut best_ucb = ::std::f64::MIN;
                        for (action, payoff, ucb) in stats.into_iter() {
                            info!("{:?}: {:?}; UCB = {:?}", action, payoff, ucb);
                            best_action = match (best_action, move_selection_criterion) {
                                (None, _) => Some(action),
                                (Some(_), thud::MoveSelectionCriterion::VisitCount) if best_visits < payoff.weight => {
                                    best_visits = payoff.weight;
                                    Some(action)
                                },
                                (Some(_), thud::MoveSelectionCriterion::Ucb) =>
                                    match ucb {
                                        Ok(mcts::UcbProxy::Value(x)) if best_ucb < x => {
                                            best_ucb = x;
                                            Some(action)
                                        },
                                        _ => best_action,
                                    },
                                _ => best_action,
                            }
                        }
                    }
                },
                Err(e) => {
                    error!("Error in seach iteration {}: {}", iteration, e);
                    break
                },
            }
        }

        // Finish turn.
        match best_action {
            Some(action) => {
                info!("turn {}: performing best action {:?}", turn_number, action);
                let mut canonical_state = graph.get_node(&state).unwrap().get_label().clone();
                canonical_state.do_action(&action);
                state.set_from_convolved(&canonical_state);
                if compact_graph {
                    debug!("collecting garbage and compacting search graph");
                    graph.retain_reachable_from(&[&canonical_state]);
                    debug!("done compacting search graph");
                } else {
                    debug!("potential memory leak: not compatcing search graph");
                }
                turn_number += 1;
            },
            None => {
                info!("turn {}: no best action. exiting.", turn_number);
                break
            }
        }
    }
    // console_ui::write_search_graph(&graph, &state);
}
<commit_msg>Make console_mcts exit when the game has finished.<commit_after>extern crate chrono;
extern crate clap;
extern crate fern;
#[macro_use]
extern crate log;
extern crate mcts;
extern crate rand;
extern crate thud;
extern crate thud_game;

use ::clap::App;
use ::rand::isaac::IsaacRng;
use ::rand::SeedableRng;

use thud_game::board;

fn main() {
    // Set up arg handling.
    let matches = {
        let app = thud::set_common_args(
            App::new("console_mcts")
                .version("0.1.0")
                .author("Stu Black <trurl@freeshell.org>")
                .about("Plays out Thud MCTS iterations"),
            &[thud::ITERATION_COUNT_FLAG,
              thud::SIMULATION_COUNT_FLAG,
              thud::EXPLORATION_BIAS_FLAG,
              thud::INITIAL_BOARD_FLAG,
              thud::INITIAL_PLAYER_FLAG,
              thud::LOG_LEVEL_FLAG,
              thud::MOVE_SELECTION_CRITERION_FLAG,
              thud::COMPACT_SEARCH_GRAPH_FLAG,]);
        app.get_matches()
    };
    let iteration_count =
        match matches.value_of(thud::ITERATION_COUNT_FLAG).unwrap().parse::<usize>() {
            Ok(x) => x,
            Err(e) => panic!("Bad iteration count: {}", e),
        };
    let simulation_count =
        match matches.value_of(thud::SIMULATION_COUNT_FLAG).unwrap().parse::<usize>() {
            Ok(x) => x,
            Err(e) => panic!("Bad simulation count: {}", e),
        };
    let exploration_bias =
        match matches.value_of(thud::EXPLORATION_BIAS_FLAG).unwrap().parse::<f64>() {
            Ok(x) => x,
            Err(e) => panic!("Bad exploration bias: {}", e),
        };
    let initial_cells =
        match matches.value_of(thud::INITIAL_BOARD_FLAG).map(|x| x.parse::<thud::InitialBoard>()) {
            Some(Ok(x)) => x.cells(),
            Some(Err(e)) => panic!("Bad initial board configuration: {}", e),
            None => board::Cells::default(),
        };
    let toggle_initial_player =
        match matches.value_of(thud::INITIAL_PLAYER_FLAG).map(|x| x.parse::<thud_game::Role>()) {
            None | Some(Ok(thud_game::Role::Dwarf)) => false,
            Some(Ok(thud_game::Role::Troll)) => true,
            Some(Err(x)) => panic!("{}", x),
        };
    let logging_level =
        match matches.value_of(thud::LOG_LEVEL_FLAG).map(|x| x.parse::<log::LogLevelFilter>()) {
            Some(Ok(x)) => x,
            Some(Err(_)) => panic!("Bad logging level '{}'", matches.value_of(thud::LOG_LEVEL_FLAG).unwrap()),
            None => log::LogLevelFilter::Info,
        };
    let move_selection_criterion =
        match matches.value_of(thud::MOVE_SELECTION_CRITERION_FLAG).map(|x| x.parse::<thud::MoveSelectionCriterion>()) {
            Some(Ok(x)) => x,
            Some(Err(e)) => panic!("Bad move selection criterion: {}", e),
            None => thud::MoveSelectionCriterion::VisitCount,
        };
    let rng =
        match matches.value_of(thud::RNG_SEED_FLAG).map(|x| x.parse::<u32>()) {
            Some(Ok(x)) => IsaacRng::from_seed(&[x]),
            Some(Err(e)) => panic!("Bad RNG seed: {}", e),
            None => IsaacRng::new_unseeded(),
        };
    let compact_graph = matches.is_present(thud::COMPACT_SEARCH_GRAPH_FLAG);

    // Set up logging.
    let logger_config = fern::DispatchConfig {
        format: Box::new(|msg: &str, level: &log::LogLevel, _location: &log::LogLocation| {
            format!("[{}][{}] {}",
                    chrono::Local::now().format("%Y-%m-%d %T%.3f%z").to_string(), level, msg)
        }),
        output: vec![fern::OutputConfig::stdout()],
        level: logging_level,
    };
    if let Err(e) = fern::init_global_logger(logger_config, log::LogLevelFilter::Trace) {
        panic!("Failed to initialize global logger: {}", e);
    }

    // Play game.
    let mut state = mcts::State::new(initial_cells);
    if toggle_initial_player {
        state.toggle_active_role();
    }
    let mut graph = mcts::Graph::new();
    thud::initialize_search(state.clone(), &mut graph);

    let mut search_state = mcts::SearchState::new(rng, exploration_bias);
    let mut turn_number = 0;
    while !state.terminated() {
        info!("begin turn {}; board: {}", turn_number, board::format_board(state.board()));
        if graph.get_node(&state).is_none() {
            error!("board not found in playout graph; reinitializing");
            thud::initialize_search(state.clone(), &mut graph);
        }

        // {
        //     let mut available_actions: Vec<game::Action> =
        //         graph.get_node(&state).unwrap().get_child_list().iter()
        //         .map(|c| c.get_data().action).collect();
        //     available_actions.sort_by(thud_game::util::cmp_actions);
        //     let mut state_actions: Vec<game::Action> = state.actions().collect();
        //     state_actions.sort_by(thud_game::util::cmp_actions);
        //     debug!("Checking comparison of state_actions: {:?} vs. available actions: {:?}",
        //            state_actions, available_actions);
        //     assert_eq!(state_actions, available_actions);
        // }

        let mut best_action = None;
        for iteration in 0..iteration_count {
            if iteration % 100 == 0 {
                info!("iteration: {} / {} = {}%", iteration, iteration_count,
                      ((10000.0 * (iteration as f64) / (iteration_count as f64)) as usize as f64) / 100.0);
            }
            match search_state.search(&mut graph, &state,
                                      |_: usize| mcts::SearchSettings {
                                          simulation_count: simulation_count,
                                      }) {
                Ok(stats) => {
                    // TODO: this commented-out block is only valid when we haven't
                    // propagated a multi-visit payoff upwards from conneting to an
                    // extant vertex.
                    // if simulation_count == 1 && toplevel_visits != iteration + 1 {
                    //     console_ui::write_board(state.board());
                    //     for (action, stats) in stats.into_iter() {
                    //         println!("{:?}: [{}, {}] = {:?} / {}", action,
                    //                  (stats.payoff.values[0] as f64) / (stats.visits as f64),
                    //                  (stats.payoff.values[1] as f64) / (stats.visits as f64),
                    //                  stats.payoff, stats.visits);
                    //     }
                    //     panic!("total visits at top level is {}, but iteration count is {}",
                    //            toplevel_visits, iteration)
                    // }
                    if iteration % 1000 == 0 || iteration + 1 == iteration_count {
                        info!("root stats:");
                        let mut best_visits = ::std::u32::MIN;
                        let mut best_ucb = ::std::f64::MIN;
                        for (action, payoff, ucb) in stats.into_iter() {
                            info!("{:?}: {:?}; UCB = {:?}", action, payoff, ucb);
                            best_action = match (best_action, move_selection_criterion) {
                                (None, _) => Some(action),
                                (Some(_), thud::MoveSelectionCriterion::VisitCount) if best_visits < payoff.weight => {
                                    best_visits = payoff.weight;
                                    Some(action)
                                },
                                (Some(_), thud::MoveSelectionCriterion::Ucb) =>
                                    match ucb {
                                        Ok(mcts::UcbProxy::Value(x)) if best_ucb < x => {
                                            best_ucb = x;
                                            Some(action)
                                        },
                                        _ => best_action,
                                    },
                                _ => best_action,
                            }
                        }
                    }
                },
                Err(e) => {
                    error!("Error in seach iteration {}: {}", iteration, e);
                    break
                },
            }
        }

        // Finish turn.
        match best_action {
            Some(action) => {
                info!("turn {}: performing best action {:?}", turn_number, action);
                let mut canonical_state = graph.get_node(&state).unwrap().get_label().clone();
                canonical_state.do_action(&action);
                state.set_from_convolved(&canonical_state);
                if compact_graph {
                    debug!("collecting garbage and compacting search graph");
                    graph.retain_reachable_from(&[&canonical_state]);
                    debug!("done compacting search graph");
                } else {
                    debug!("potential memory leak: not compatcing search graph");
                }
                turn_number += 1;
            },
            None => {
                info!("turn {}: no best action. exiting.", turn_number);
                break
            }
        }
    }
    info!("game over. final board state: {}", board::format_board(state.cells()));
}
<|endoftext|>