<commit_before>use std::vec::IntoIter;

use clap::ArgMatches;
use regex::Regex;

use module::{CommandEnv, CommandResult, Module, ModuleError};
use module::bm::header::{build_header, get_tags_from_header};
use runtime::Runtime;
use storage::backend::StorageBackendError;
use storage::file::File;
use storage::json::parser::JsonHeaderParser;
use storage::parser::Parser;
use ui::file::{FilePrinter, TablePrinter};

pub fn add_command(module: &Module, env: CommandEnv) -> CommandResult {
    use url::Url;

    let url = env.matches.value_of("url").unwrap();

    if let Err(e) = Url::parse(url) {
        info!("Not an URL: '{}'", url);
        info!("  this will turn into an hard error before 0.1.0");
        debug!("URL parsing error: {:?}", e);
    }

    let tags = get_tags(env.rt, env.matches);
    info!("Adding url '{}' with tags '{:?}'", url, tags);

    let header  = build_header(&String::from(url), &tags);
    let file    = File::new_with_header(module, header);
    let parser  = Parser::new(JsonHeaderParser::new(None));
    let putres  = env.bk.put_file(file, &parser);

    putres.map_err(|sberr| {
        let mut err = ModuleError::new("Storage Backend Error");
        err.caused_by = Some(Box::new(sberr));
        err
    })
}

pub fn list_command(module: &Module, env: CommandEnv) -> CommandResult {
    let printer = TablePrinter::new(env.rt.is_verbose(), env.rt.is_debugging());
    let files   = get_filtered_files_from_backend(module, &env);

    debug!("Printing files now");
    files.map(|f| printer.print_files(f));

    Ok(())
}

pub fn remove_command(module: &Module, env: CommandEnv) -> CommandResult {
    fn remove_by_id(module: &Module, env: CommandEnv, id: String, checked: bool) -> CommandResult {
        let parser = Parser::new(JsonHeaderParser::new(None));
        let file = env.bk
            .get_file_by_id(module, &id.into(), &parser)
            .unwrap_or({
                info!("No files found");
                return Ok(())
            });

        debug!("Remove file: {:?}", file);

        if let Err(e) = env.bk.remove_file(module, file, checked) {
            debug!("Remove failed");
            let mut err = ModuleError::new("Removing file failed");
            err.caused_by = Some(Box::new(e));
            Err(err)
        } else {
            info!("Remove worked");
            Ok(())
        }
    }

    fn remove_by_filtering(module: &Module, env: CommandEnv, checked: bool) -> CommandResult {
        get_filtered_files_from_backend(module, &env).and_then(|files| {
            let nfiles = files.len();
            info!("Removing {} Files", nfiles);

            let errs = files.map(|file| {
                    debug!("Remove file: {:?}", file);
                    env.bk.remove_file(module, file, checked)
                })
                .filter(|e| e.is_err())
                .map(|e| {
                    let err = e.err().unwrap();
                    warn!("Error occured in Filesystem operation: {}", err);
                    err
                })
                .collect::<Vec<StorageBackendError>>();

            if errs.len() != 0 {
                warn!("{} Errors occured while removing {} files", errs.len(), nfiles);
                let moderr = ModuleError::new("File removal failed");

                // TODO : Collect StorageBackendErrors

                Err(moderr)
            } else {
                Ok(())
            }
        })
    }

    let checked : bool = run_removal_checking(&env);
    debug!("Checked mode: {}", checked);

    if let Some(id) = get_id(env.rt, env.matches) {
        debug!("Remove by id: {}", id);
        remove_by_id(module, env, id, checked)
    } else {
        debug!("Remove more than one file");
        remove_by_filtering(module, env, checked)
    }
}

/*
 *
 * Private helpers
 *
 */

fn get_filtered_files_from_backend<'a>(module: &'a Module,
                                       env: &CommandEnv)
    -> Result<IntoIter<File<'a>>, ModuleError>
{
    fn check_tags(tags: &Vec<String>, file: &File) -> bool {
        if tags.len() != 0 {
            debug!("Checking tags of: {:?}", file.id());
            get_tags_from_header(&file.header())
                .iter()
                .any(|t| tags.contains(t))
        } else {
            true
        }
    }

    let parser = Parser::new(JsonHeaderParser::new(None));
    let tags = get_tags(env.rt, env.matches);
    debug!("Tags: {:?}", tags);
    env.bk.iter_files(module, &parser)
        .map(|files| {
            files.filter(|file| {
                debug!("Backend returns file: {:?}", file);
                check_tags(&tags, file)
            }).filter(|file| {
                debug!("Checking matches of: {:?}", file.id());
                get_matcher(env.rt, env.matches)
                    .and_then(|r| Some(file.matches_with(&r)))
                    .unwrap_or(true)
            }).collect::<Vec<File>>().into_iter()
        }).map_err(|e| {
            debug!("Error from Backend: {:?}", e);
            let mut merr = ModuleError::new("Could not filter files");
            merr.caused_by = Some(Box::new(e));
            merr
        })
}

fn get_tags<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Vec<String> {

    fn reject_if_with_spaces(e: &String) -> bool {
        if e.contains(" ") {
            warn!("Tag contains spaces: '{}'", e);
            false
        } else {
            true
        }
    }

    debug!("Fetching tags from commandline");
    sub.value_of("tags").and_then(|tags| {
        Some(tags.split(",")
                 .into_iter()
                 .map(|s| s.to_string())
                 .filter(|e| reject_if_with_spaces(e))
                 .collect()
          )
    }).or(Some(vec![])).unwrap()
}

fn get_matcher<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Option<Regex> {
    debug!("Fetching matcher from commandline");
    if let Some(s) = sub.value_of("match") {
        if let Ok(r) = Regex::new(s) {
            return Some(r)
        } else {
            error!("Regex error, continuing without regex");
        }
    }
    None

}

fn get_id<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Option<String> {
    debug!("Fetching id from commandline");
    sub.value_of("id").and_then(|s| Some(String::from(s)))
}

/*
 * Checks whether the commandline call was set to run the removal "checked",
 * so if another entry from the store refers to this ID, do not remove the file.
 */
fn run_removal_checking(env: &CommandEnv) -> bool {
    env.matches.is_present("check")
}
<commit_msg>Readability fixes<commit_after>use std::vec::IntoIter;

use clap::ArgMatches;
use regex::Regex;

use module::{CommandEnv, CommandResult, Module, ModuleError};
use module::bm::header::{build_header, get_tags_from_header};
use runtime::Runtime;
use storage::backend::StorageBackendError;
use storage::file::File;
use storage::json::parser::JsonHeaderParser;
use storage::parser::Parser;
use ui::file::{FilePrinter, TablePrinter};

pub fn add_command(module: &Module, env: CommandEnv) -> CommandResult {
    use url::Url;

    let url = env.matches.value_of("url").unwrap();

    if let Err(e) = Url::parse(url) {
        info!("Not an URL: '{}'", url);
        info!("  this will turn into an hard error before 0.1.0");
        debug!("URL parsing error: {:?}", e);
    }

    let tags = get_tags(env.rt, env.matches);
    info!("Adding url '{}' with tags '{:?}'", url, tags);

    let header  = build_header(&String::from(url), &tags);
    let file    = File::new_with_header(module, header);
    let parser  = Parser::new(JsonHeaderParser::new(None));
    let putres  = env.bk.put_file(file, &parser);

    putres.map_err(|sberr| {
        let mut err = ModuleError::new("Storage Backend Error");
        err.caused_by = Some(Box::new(sberr));
        err
    })
}

pub fn list_command(module: &Module, env: CommandEnv) -> CommandResult {
    let printer = TablePrinter::new(env.rt.is_verbose(), env.rt.is_debugging());
    let files   = get_filtered_files_from_backend(module, &env);

    debug!("Printing files now");
    files.map(|f| printer.print_files(f));

    Ok(())
}

pub fn remove_command(module: &Module, env: CommandEnv) -> CommandResult {
    fn remove_by_id(module: &Module, env: CommandEnv, id: String, checked: bool) -> CommandResult {
        let parser = Parser::new(JsonHeaderParser::new(None));
        let file = env.bk
            .get_file_by_id(module, &id.into(), &parser)
            .unwrap_or({
                info!("No files found");
                return Ok(())
            });

        debug!("Remove file: {:?}", file);

        if let Err(e) = env.bk.remove_file(module, file, checked) {
            debug!("Remove failed");
            let mut err = ModuleError::new("Removing file failed");
            err.caused_by = Some(Box::new(e));
            Err(err)
        } else {
            info!("Remove worked");
            Ok(())
        }
    }

    fn remove_by_filtering(module: &Module, env: CommandEnv, checked: bool) -> CommandResult {
        get_filtered_files_from_backend(module, &env).and_then(|files| {
            let nfiles = files.len();
            info!("Removing {} Files", nfiles);

            let errs = files.map(|file| {
                    debug!("Remove file: {:?}", file);
                    env.bk.remove_file(module, file, checked)
                })
                .filter(|e| e.is_err())
                .map(|e| {
                    let err = e.err().unwrap();
                    warn!("Error occured in Filesystem operation: {}", err);
                    err
                })
                .collect::<Vec<StorageBackendError>>();

            if errs.len() != 0 {
                warn!("{} Errors occured while removing {} files", errs.len(), nfiles);
                let moderr = ModuleError::new("File removal failed");

                // TODO : Collect StorageBackendErrors

                Err(moderr)
            } else {
                Ok(())
            }
        })
    }

    let checked : bool = run_removal_checking(&env);
    debug!("Checked mode: {}", checked);

    if let Some(id) = get_id(env.rt, env.matches) {
        debug!("Remove by id: {}", id);
        remove_by_id(module, env, id, checked)
    } else {
        debug!("Remove more than one file");
        remove_by_filtering(module, env, checked)
    }
}

/*
 *
 * Private helpers
 *
 */

fn get_filtered_files_from_backend<'a>(module: &'a Module,
                                       env: &CommandEnv)
    -> Result<IntoIter<File<'a>>, ModuleError>
{
    fn check_tags(tags: &Vec<String>, file: &File) -> bool {
        if tags.len() != 0 {
            debug!("Checking tags of: {:?}", file.id());
            get_tags_from_header(&file.header())
                .iter()
                .any(|t| tags.contains(t))
        } else {
            true
        }
    }

    let parser = Parser::new(JsonHeaderParser::new(None));
    let tags = get_tags(env.rt, env.matches);
    debug!("Tags: {:?}", tags);
    env.bk
        .iter_files(module, &parser)
        .map(|files| {
            files.filter(|file| {
                debug!("Backend returns file: {:?}", file);
                check_tags(&tags, file)
            }).filter(|file| {
                debug!("Checking matches of: {:?}", file.id());
                get_matcher(env.rt, env.matches)
                    .map(|r| file.matches_with(&r))
                    .unwrap_or(true)
            })
            .collect::<Vec<File>>()
            .into_iter()
        }).map_err(|e| {
            debug!("Error from Backend: {:?}", e);
            let mut merr = ModuleError::new("Could not filter files");
            merr.caused_by = Some(Box::new(e));
            merr
        })
}

fn get_tags<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Vec<String> {

    fn reject_if_with_spaces(e: &String) -> bool {
        if e.contains(" ") {
            warn!("Tag contains spaces: '{}'", e);
            false
        } else {
            true
        }
    }

    debug!("Fetching tags from commandline");
    sub.value_of("tags").and_then(|tags| {
        Some(tags.split(",")
                 .into_iter()
                 .map(|s| s.to_string())
                 .filter(|e| reject_if_with_spaces(e))
                 .collect()
          )
    }).or(Some(vec![])).unwrap()
}

fn get_matcher<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Option<Regex> {
    debug!("Fetching matcher from commandline");
    if let Some(s) = sub.value_of("match") {
        if let Ok(r) = Regex::new(s) {
            return Some(r)
        } else {
            error!("Regex error, continuing without regex");
        }
    }
    None

}

fn get_id<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Option<String> {
    debug!("Fetching id from commandline");
    sub.value_of("id").and_then(|s| Some(String::from(s)))
}

/*
 * Checks whether the commandline call was set to run the removal "checked",
 * so if another entry from the store refers to this ID, do not remove the file.
 */
fn run_removal_checking(env: &CommandEnv) -> bool {
    env.matches.is_present("check")
}
<|endoftext|>


<commit_before>export parse_sess;
export next_node_id;
export new_parser_from_file;
export new_parser_from_source_str;
export parse_crate_from_file;
export parse_crate_from_crate_file;
export parse_crate_from_source_str;
export parse_expr_from_source_str;
export parse_from_source_str;

import parser::parser;
import ast::node_id;
import util::interner;
import lexer::reader;

type parse_sess = @{
    cm: codemap::codemap,
    mut next_id: node_id,
    span_diagnostic: diagnostic::span_handler,
    // these two must be kept up to date
    mut chpos: uint,
    mut byte_pos: uint
};

fn next_node_id(sess: parse_sess) -> node_id {
    let rv = sess.next_id;
    sess.next_id += 1;
    // ID 0 is reserved for the crate and doesn't actually exist in the AST
    assert rv != 0;
    ret rv;
}

fn new_parser_from_file(sess: parse_sess, cfg: ast::crate_cfg, path: str,
                        ftype: parser::file_type) ->
   parser {
    let src = alt io::read_whole_file_str(path) {
      result::ok(src) {
        // FIXME: This copy is unfortunate
        @src
      }
      result::err(e) {
        sess.span_diagnostic.handler().fatal(e)
      }
    };
    let filemap = codemap::new_filemap(path, src,
                                       sess.chpos, sess.byte_pos);
    sess.cm.files += [filemap];
    let itr = @interner::mk(str::hash, str::eq);
    let rdr = lexer::new_reader(sess.span_diagnostic, filemap, itr);
    ret new_parser(sess, cfg, rdr, ftype);
}

fn new_parser_from_source_str(sess: parse_sess, cfg: ast::crate_cfg,
                              name: str, ss: codemap::file_substr,
                              source: @str) -> parser {
    let ftype = parser::SOURCE_FILE;
    let filemap = codemap::new_filemap_w_substr
        (name, ss, source, sess.chpos, sess.byte_pos);
    sess.cm.files += [filemap];
    let itr = @interner::mk(str::hash, str::eq);
    let rdr = lexer::new_reader(sess.span_diagnostic,
                                filemap, itr);
    ret new_parser(sess, cfg, rdr, ftype);
}

fn new_parser(sess: parse_sess, cfg: ast::crate_cfg, rdr: lexer::reader,
              ftype: parser::file_type) -> parser {
    let tok0 = lexer::next_token(rdr);
    let span0 = ast_util::mk_sp(tok0.chpos, rdr.chpos);
    @{sess: sess,
      cfg: cfg,
      file_type: ftype,
      mut token: tok0.tok,
      mut span: span0,
      mut last_span: span0,
      mut buffer: [],
      mut restriction: parser::UNRESTRICTED,
      reader: rdr,
      binop_precs: prec::binop_prec_table(),
      keywords: token::keyword_table(),
      bad_expr_words: token::bad_expr_word_table()}
}

fn parse_crate_from_crate_file(input: str, cfg: ast::crate_cfg,
                               sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_file(sess, cfg, input, parser::CRATE_FILE);
    let lo = p.span.lo;
    let prefix = path::dirname(p.reader.filemap.name);
    let leading_attrs = attr::parse_inner_attrs_and_next(p);
    let crate_attrs = leading_attrs.inner;
    let first_cdir_attr = leading_attrs.next;
    let cdirs = parser::parse_crate_directives(
        p, token::EOF, first_cdir_attr);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    let cx =
        @{p: p,
          sess: sess,
          cfg: p.cfg};
    let (companionmod, _) = path::splitext(path::basename(input));
    let (m, attrs) = eval::eval_crate_directives_to_mod(
        cx, cdirs, prefix, option::some(companionmod));
    let mut hi = p.span.hi;
    parser::expect(p, token::EOF);
    ret @ast_util::respan(ast_util::mk_sp(lo, hi),
                          {directives: cdirs,
                           module: m,
                           attrs: crate_attrs + attrs,
                           config: p.cfg});
}

fn parse_crate_from_file(input: str, cfg: ast::crate_cfg, sess: parse_sess) ->
   @ast::crate {
    if str::ends_with(input, ".rc") {
        parse_crate_from_crate_file(input, cfg, sess)
    } else if str::ends_with(input, ".rs") {
        parse_crate_from_source_file(input, cfg, sess)
    } else {
        sess.span_diagnostic.handler().fatal("unknown input file type: " +
                                             input)
    }
}

fn parse_crate_from_source_file(input: str, cfg: ast::crate_cfg,
                                sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_file(sess, cfg, input, parser::SOURCE_FILE);
    let r = parser::parse_crate_mod(p, cfg);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_expr_from_source_str(name: str, source: @str, cfg: ast::crate_cfg,
                              sess: parse_sess) -> @ast::expr {
    let p = new_parser_from_source_str(
        sess, cfg, name, codemap::fss_none, source);
    let r = parser::parse_expr(p);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_crate_from_source_str(name: str, source: @str, cfg: ast::crate_cfg,
                               sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_source_str(
        sess, cfg, name, codemap::fss_none, source);
    let r = parser::parse_crate_mod(p, cfg);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_from_source_str<T>(f: fn (p: parser) -> T,
                            name: str, ss: codemap::file_substr,
                            source: @str, cfg: ast::crate_cfg,
                            sess: parse_sess)
    -> T
{
    let p = new_parser_from_source_str(sess, cfg, name, ss, source);
    let r = f(p);
    if !p.reader.is_eof() {
        p.reader.fatal("expected end-of-string");
    }
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}
<commit_msg>syntax: Reorganize mod parse<commit_after>#[doc = "The main parser interface"];

export parse_sess;
export next_node_id;
export new_parser_from_file;
export new_parser_from_source_str;
export parse_crate_from_file;
export parse_crate_from_crate_file;
export parse_crate_from_source_str;
export parse_expr_from_source_str;
export parse_from_source_str;

import parser::parser;
import ast::node_id;
import util::interner;
import lexer::reader;

type parse_sess = @{
    cm: codemap::codemap,
    mut next_id: node_id,
    span_diagnostic: diagnostic::span_handler,
    // these two must be kept up to date
    mut chpos: uint,
    mut byte_pos: uint
};

fn parse_crate_from_file(input: str, cfg: ast::crate_cfg, sess: parse_sess) ->
   @ast::crate {
    if str::ends_with(input, ".rc") {
        parse_crate_from_crate_file(input, cfg, sess)
    } else if str::ends_with(input, ".rs") {
        parse_crate_from_source_file(input, cfg, sess)
    } else {
        sess.span_diagnostic.handler().fatal("unknown input file type: " +
                                             input)
    }
}

fn parse_crate_from_crate_file(input: str, cfg: ast::crate_cfg,
                               sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_file(sess, cfg, input, parser::CRATE_FILE);
    let lo = p.span.lo;
    let prefix = path::dirname(p.reader.filemap.name);
    let leading_attrs = attr::parse_inner_attrs_and_next(p);
    let crate_attrs = leading_attrs.inner;
    let first_cdir_attr = leading_attrs.next;
    let cdirs = parser::parse_crate_directives(
        p, token::EOF, first_cdir_attr);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    let cx =
        @{p: p,
          sess: sess,
          cfg: p.cfg};
    let (companionmod, _) = path::splitext(path::basename(input));
    let (m, attrs) = eval::eval_crate_directives_to_mod(
        cx, cdirs, prefix, option::some(companionmod));
    let mut hi = p.span.hi;
    parser::expect(p, token::EOF);
    ret @ast_util::respan(ast_util::mk_sp(lo, hi),
                          {directives: cdirs,
                           module: m,
                           attrs: crate_attrs + attrs,
                           config: p.cfg});
}

fn parse_crate_from_source_file(input: str, cfg: ast::crate_cfg,
                                sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_file(sess, cfg, input, parser::SOURCE_FILE);
    let r = parser::parse_crate_mod(p, cfg);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_crate_from_source_str(name: str, source: @str, cfg: ast::crate_cfg,
                               sess: parse_sess) -> @ast::crate {
    let p = new_parser_from_source_str(
        sess, cfg, name, codemap::fss_none, source);
    let r = parser::parse_crate_mod(p, cfg);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_expr_from_source_str(name: str, source: @str, cfg: ast::crate_cfg,
                              sess: parse_sess) -> @ast::expr {
    let p = new_parser_from_source_str(
        sess, cfg, name, codemap::fss_none, source);
    let r = parser::parse_expr(p);
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn parse_from_source_str<T>(f: fn (p: parser) -> T,
                            name: str, ss: codemap::file_substr,
                            source: @str, cfg: ast::crate_cfg,
                            sess: parse_sess)
    -> T
{
    let p = new_parser_from_source_str(sess, cfg, name, ss, source);
    let r = f(p);
    if !p.reader.is_eof() {
        p.reader.fatal("expected end-of-string");
    }
    sess.chpos = p.reader.chpos;
    sess.byte_pos = sess.byte_pos + p.reader.pos;
    ret r;
}

fn next_node_id(sess: parse_sess) -> node_id {
    let rv = sess.next_id;
    sess.next_id += 1;
    // ID 0 is reserved for the crate and doesn't actually exist in the AST
    assert rv != 0;
    ret rv;
}

fn new_parser(sess: parse_sess, cfg: ast::crate_cfg, rdr: lexer::reader,
              ftype: parser::file_type) -> parser {
    let tok0 = lexer::next_token(rdr);
    let span0 = ast_util::mk_sp(tok0.chpos, rdr.chpos);
    @{sess: sess,
      cfg: cfg,
      file_type: ftype,
      mut token: tok0.tok,
      mut span: span0,
      mut last_span: span0,
      mut buffer: [],
      mut restriction: parser::UNRESTRICTED,
      reader: rdr,
      binop_precs: prec::binop_prec_table(),
      keywords: token::keyword_table(),
      bad_expr_words: token::bad_expr_word_table()}
}

fn new_parser_from_source_str(sess: parse_sess, cfg: ast::crate_cfg,
                              name: str, ss: codemap::file_substr,
                              source: @str) -> parser {
    let ftype = parser::SOURCE_FILE;
    let filemap = codemap::new_filemap_w_substr
        (name, ss, source, sess.chpos, sess.byte_pos);
    sess.cm.files += [filemap];
    let itr = @interner::mk(str::hash, str::eq);
    let rdr = lexer::new_reader(sess.span_diagnostic,
                                filemap, itr);
    ret new_parser(sess, cfg, rdr, ftype);
}

fn new_parser_from_file(sess: parse_sess, cfg: ast::crate_cfg, path: str,
                        ftype: parser::file_type) ->
   parser {
    let src = alt io::read_whole_file_str(path) {
      result::ok(src) {
        // FIXME: This copy is unfortunate
        @src
      }
      result::err(e) {
        sess.span_diagnostic.handler().fatal(e)
      }
    };
    let filemap = codemap::new_filemap(path, src,
                                       sess.chpos, sess.byte_pos);
    sess.cm.files += [filemap];
    let itr = @interner::mk(str::hash, str::eq);
    let rdr = lexer::new_reader(sess.span_diagnostic, filemap, itr);
    ret new_parser(sess, cfg, rdr, ftype);
}
<|endoftext|>


<commit_before>use std::collections::{VecDeque, BinaryHeap};
use std::cell::RefCell;
use std::sync::Arc;
use std::io;
use std::net::SocketAddr;
use std::ops::Drop;

use time::Timespec;

use super::node::{Node, NodeContext, State};
use super::storage::{Storage, MemoryDB};
use super::messages::{Any, Message, RawMessage, Connect};
use super::events::{Reactor, Event, Timeout};
use super::network::{PeerId, EventSet};
use super::tx_generator::TxGenerator;
use super::crypto::{hash, Hash, PublicKey, SecretKey, gen_keypair};

struct SandboxInner {
    address: SocketAddr,
    time: Timespec,
    sended: VecDeque<(SocketAddr, RawMessage)>,
    timers: BinaryHeap<TimerPair>,
}

#[derive(PartialEq, Eq)]
struct TimerPair(Timespec, Timeout);

impl PartialOrd for TimerPair {
    fn partial_cmp(&self, other: &Self) -> Option<::std::cmp::Ordering> {
        Some((&self.0, &self.1).cmp(&(&other.0, &other.1)).reverse())
    }
}

impl Ord for TimerPair {
    fn cmp(&self, other: &Self) -> ::std::cmp::Ordering {
        (&self.0, &self.1).cmp(&(&other.0, &other.1)).reverse()
    }
}

pub struct Sandbox {
    inner: Arc<RefCell<SandboxInner>>,
    node: RefCell<Node>,
    validators: Vec<(PublicKey, SecretKey)>,
    addresses: Vec<SocketAddr>,
}

pub struct SandboxReactor {
    inner: Arc<RefCell<SandboxInner>>
}

impl Sandbox {
    pub fn new() -> Sandbox {
        let validators = vec![
            gen_keypair(),
            gen_keypair(),
            gen_keypair(),
            gen_keypair(),
        ];

        let addresses = vec![
            "1.1.1.1:1".parse().unwrap(),
            "2.2.2.2:2".parse().unwrap(),
            "3.3.3.3:3".parse().unwrap(),
            "4.4.4.4:4".parse().unwrap(),
        ] : Vec<SocketAddr>;

        let inner = Arc::new(RefCell::new(SandboxInner {
            address: addresses[0].clone(),
            time: Timespec { sec: 0, nsec: 0},
            sended: VecDeque::new(),
            timers: BinaryHeap::new(),
        }));

        let state = State::new(0, validators.iter().map(|&(p, _)| p.clone()).collect());

        let context = NodeContext {
            public_key: validators[0].0.clone(),
            secret_key: validators[0].1.clone(),
            state: state,
            events: Box::new(SandboxReactor {
                inner: inner.clone(),
            }) as Box<Reactor>,
            storage: Storage::new(MemoryDB::new()),
            round_timeout: 1000,
            status_timeout: 5000,
            peer_discovery: Vec::new(),
            tx_generator: TxGenerator::new(),
        };

        let node = Node::with_context(context);

        let sandbox = Sandbox {
            inner: inner,
            node: RefCell::new(node),
            validators: validators,
            addresses: addresses,
        };

        sandbox.initialize();

        return sandbox;
    }
}

impl Reactor for SandboxReactor {
    fn get_time(&self) -> Timespec {
        self.inner.borrow().time
    }

    fn poll(&mut self) -> Event {
        unreachable!();
    }

    fn io(&mut self, _: PeerId, _: EventSet) -> io::Result<()> {
        unreachable!();
    }

    fn bind(&mut self) -> ::std::io::Result<()> {
        Ok(())
    }

    fn send_to(&mut self,
                   address: &SocketAddr,
                   message: RawMessage) -> io::Result<()> {
        self.inner.borrow_mut().sended.push_back((address.clone(), message));
        Ok(())
    }

    fn address(&self) -> SocketAddr {
        self.inner.borrow().address
    }

    fn add_timeout(&mut self, timeout: Timeout, time: Timespec) {
        // assert!(time < self.inner.borrow().time, "Tring to add timeout for the past");
        self.inner.borrow_mut().timers.push(TimerPair(time, timeout));
    }
}

impl Sandbox {
    fn initialize(&self) {
        self.node.borrow_mut().initialize();

        let connect = Connect::new(self.p(0), self.a(0), self.time(), self.s(0));

        self.recv(Connect::new(self.p(1), self.a(1), self.time(), self.s(1)));
        self.send(self.a(1), connect.clone());

        self.recv(Connect::new(self.p(2), self.a(2), self.time(), self.s(2)));
        self.send(self.a(2), connect.clone());

        self.recv(Connect::new(self.p(3), self.a(3), self.time(), self.s(3)));
        self.send(self.a(3), connect.clone());

        self.check_unexpected_message()
    }

    fn check_unexpected_message(&self) {
        let sended = self.inner.borrow_mut().sended.pop_front();
        if let Some((addr, msg)) = sended {
            let any_msg = Any::from_raw(msg.clone())
                              .expect("Send incorrect message");
            panic!("Send unexpected message {:?} to {}", any_msg, addr);
        }
    }

    pub fn p(&self, id: usize) -> &PublicKey {
        &self.validators[id].0
    }

    pub fn s(&self, id: usize) -> &SecretKey {
        &self.validators[id].1
    }

    pub fn a(&self, id: usize) -> SocketAddr {
        self.addresses[id].clone()
    }

    pub fn time(&self) -> Timespec {
        self.inner.borrow().time
    }

    pub fn last_hash(&self) -> Hash {
        // FIXME: temporary hack
        hash(&[])
    }

    pub fn recv<T: Message>(&self, msg: T) {
        self.check_unexpected_message();
        self.node.borrow_mut().handle_message(msg.raw().clone());
    }

    pub fn send<T: Message>(&self, addr: SocketAddr, msg: T) {
        let any_expected_msg = Any::from_raw(msg.raw().clone()).unwrap();
        let sended = self.inner.borrow_mut().sended.pop_front();
        if let Some((real_addr, real_msg)) = sended {
            let any_real_msg = Any::from_raw(real_msg.clone())
                                    .expect("Send incorrect message");
            if real_addr != addr || any_real_msg != any_expected_msg {
                panic!("Expected to send the message {:?} to {} instead sending {:?} to {}",
                       any_expected_msg, addr, any_real_msg, real_addr)
            }
        } else {
            panic!("Expected to send the message {:?} to {} but nothing happened",
                   any_expected_msg, addr);
        }
    }

    pub fn set_time(&self, sec: i64, nsec: i32) {
        self.check_unexpected_message();
        // set time
        let now = Timespec {sec: sec, nsec: nsec};
        self.inner.borrow_mut().time = now;
        // handle timeouts if occurs
        loop {
            let timeout = {
                let ref mut timers = self.inner.borrow_mut().timers;
                if let Some(TimerPair(time, timeout)) = timers.pop() {
                    if time > now {
                        timers.push(TimerPair(time, timeout));
                        break;
                    } else {
                        timeout
                    }
                } else {
                    break;
                }
            };
            self.node.borrow_mut().handle_timeout(timeout);
        }
    }

    pub fn assert_round(&self, round: u32) {
        let actual_round = self.node.borrow().context().state.round();
        assert!(actual_round == round,
                "Incorrect round, actual={}, expected={}", actual_round, round);
    }
}

impl Drop for Sandbox {
    fn drop(&mut self) {
        self.check_unexpected_message();
    }
}

#[test]
fn test_sandbox_init() {
    Sandbox::new();
}

#[test]
fn test_sandbox_recv_and_send() {
    let s = Sandbox::new();
    let (public, secret) = gen_keypair();
    s.recv(Connect::new(&public, s.a(2), s.time(), &secret));
    s.send(s.a(2), Connect::new(s.p(0), s.a(0), s.time(), s.s(0)));
}

#[test]
fn test_sandbox_assert_round() {
    let s = Sandbox::new();
    s.assert_round(1);
    s.set_time(0, 999_999_999);
    s.assert_round(1);
    s.set_time(1, 0);
    s.assert_round(2);
}

#[test]
#[should_panic(expected = "Expected to send the message")]
fn test_sandbox_expected_to_send_but_nothing_happened() {
    let s = Sandbox::new();
    s.send(s.a(1), Connect::new(s.p(0), s.a(0), s.time(), s.s(0)));
}

#[test]
#[should_panic(expected = "Expected to send the message")]
fn test_sandbox_expected_to_send_another_message() {
    let s = Sandbox::new();
    let (public, secret) = gen_keypair();
    s.recv(Connect::new(&public, s.a(2), s.time(), &secret));
    s.send(s.a(1), Connect::new(s.p(0), s.a(0), s.time(), s.s(0)));
}

#[test]
#[should_panic(expected = "Send unexpected message")]
fn test_sandbox_unexpected_message_when_drop() {
    let s = Sandbox::new();
    let (public, secret) = gen_keypair();
    s.recv(Connect::new(&public, s.a(2), s.time(), &secret));
}

#[test]
#[should_panic(expected = "Send unexpected message")]
fn test_sandbox_unexpected_message_when_handle_another_message() {
    let s = Sandbox::new();
    let (public, secret) = gen_keypair();
    s.recv(Connect::new(&public, s.a(2), s.time(), &secret));
    s.recv(Connect::new(&public, s.a(3), s.time(), &secret));
    panic!("Oops! We don't catch unexpected message");
}

#[test]
#[should_panic(expected = "Send unexpected message")]
fn test_sandbox_unexpected_message_when_time_changed() {
    let s = Sandbox::new();
    let (public, secret) = gen_keypair();
    s.recv(Connect::new(&public, s.a(2), s.time(), &secret));
    s.set_time(1, 0);
    panic!("Oops! We don't catch unexpected message");
}
<commit_msg>Sandbox util methods<commit_after>use std::collections::{VecDeque, BinaryHeap, HashSet};
use std::iter::FromIterator;
use std::cell::RefCell;
use std::sync::Arc;
use std::io;
use std::net::SocketAddr;
use std::ops::Drop;

use time::Timespec;

use super::node::{Node, NodeContext, State};
use super::storage::{Storage, MemoryDB};
use super::messages::{Any, Message, RawMessage, Connect};
use super::events::{Reactor, Event, Timeout};
use super::network::{PeerId, EventSet};
use super::tx_generator::TxGenerator;
use super::crypto::{hash, Hash, PublicKey, SecretKey, gen_keypair};

struct SandboxInner {
    address: SocketAddr,
    time: Timespec,
    sended: VecDeque<(SocketAddr, RawMessage)>,
    timers: BinaryHeap<TimerPair>,
}

#[derive(PartialEq, Eq)]
struct TimerPair(Timespec, Timeout);

impl PartialOrd for TimerPair {
    fn partial_cmp(&self, other: &Self) -> Option<::std::cmp::Ordering> {
        Some((&self.0, &self.1).cmp(&(&other.0, &other.1)).reverse())
    }
}

impl Ord for TimerPair {
    fn cmp(&self, other: &Self) -> ::std::cmp::Ordering {
        (&self.0, &self.1).cmp(&(&other.0, &other.1)).reverse()
    }
}

pub struct Sandbox {
    inner: Arc<RefCell<SandboxInner>>,
    node: RefCell<Node>,
    validators: Vec<(PublicKey, SecretKey)>,
    addresses: Vec<SocketAddr>,
}

pub struct SandboxReactor {
    inner: Arc<RefCell<SandboxInner>>
}

impl Sandbox {
    pub fn new() -> Sandbox {
        let validators = vec![
            gen_keypair(),
            gen_keypair(),
            gen_keypair(),
            gen_keypair(),
        ];

        let addresses = vec![
            "1.1.1.1:1".parse().unwrap(),
            "2.2.2.2:2".parse().unwrap(),
            "3.3.3.3:3".parse().unwrap(),
            "4.4.4.4:4".parse().unwrap(),
        ] : Vec<SocketAddr>;

        let inner = Arc::new(RefCell::new(SandboxInner {
            address: addresses[0].clone(),
            time: Timespec { sec: 0, nsec: 0},
            sended: VecDeque::new(),
            timers: BinaryHeap::new(),
        }));

        let state = State::new(0, validators.iter().map(|&(p, _)| p.clone()).collect());

        let context = NodeContext {
            public_key: validators[0].0.clone(),
            secret_key: validators[0].1.clone(),
            state: state,
            events: Box::new(SandboxReactor {
                inner: inner.clone(),
            }) as Box<Reactor>,
            storage: Storage::new(MemoryDB::new()),
            round_timeout: 1000,
            status_timeout: 5000,
            peer_discovery: Vec::new(),
            tx_generator: TxGenerator::new(),
        };

        let node = Node::with_context(context);

        let sandbox = Sandbox {
            inner: inner,
            node: RefCell::new(node),
            validators: validators,
            addresses: addresses,
        };

        sandbox.initialize();

        return sandbox;
    }
}

impl Reactor for SandboxReactor {
    fn get_time(&self) -> Timespec {
        self.inner.borrow().time
    }

    fn poll(&mut self) -> Event {
        unreachable!();
    }

    fn io(&mut self, _: PeerId, _: EventSet) -> io::Result<()> {
        unreachable!();
    }

    fn bind(&mut self) -> ::std::io::Result<()> {
        Ok(())
    }

    fn send_to(&mut self,
                   address: &SocketAddr,
                   message: RawMessage) -> io::Result<()> {
        self.inner.borrow_mut().sended.push_back((address.clone(), message));
        Ok(())
    }

    fn address(&self) -> SocketAddr {
        self.inner.borrow().address
    }

    fn add_timeout(&mut self, timeout: Timeout, time: Timespec) {
        // assert!(time < self.inner.borrow().time, "Tring to add timeout for the past");
        self.inner.borrow_mut().timers.push(TimerPair(time, timeout));
    }
}

impl Sandbox {
    fn initialize(&self) {
        self.node.borrow_mut().initialize();

        let connect = Connect::new(self.p(0), self.a(0), self.time(), self.s(0));

        self.recv(Connect::new(self.p(1), self.a(1), self.time(), self.s(1)));
        self.send(self.a(1), connect.clone());

        self.recv(Connect::new(self.p(2), self.a(2), self.time(), self.s(2)));
        self.send(self.a(2), connect.clone());

        self.recv(Connect::new(self.p(3), self.a(3), self.time(), self.s(3)));
        self.send(self.a(3), connect.clone());

        self.check_unexpected_message()
    }

    fn check_unexpected_message(&self) {
        let sended = self.inner.borrow_mut().sended.pop_front();
        if let Some((addr, msg)) = sended {
            let any_msg = Any::from_raw(msg.clone())
                              .expect("Send incorrect message");
            panic!("Send unexpected message {:?} to {}", any_msg, addr);
        }
    }

    pub fn p(&self, id: usize) -> &PublicKey {
        &self.validators[id].0
    }

    pub fn s(&self, id: usize) -> &SecretKey {
        &self.validators[id].1
    }

    pub fn a(&self, id: usize) -> SocketAddr {
        self.addresses[id].clone()
    }

    pub fn time(&self) -> Timespec {
        self.inner.borrow().time
    }

    pub fn last_hash(&self) -> Hash {
        // FIXME: temporary hack
        hash(&[])
    }

    pub fn recv<T: Message>(&self, msg: T) {
        self.check_unexpected_message();
        self.node.borrow_mut().handle_message(msg.raw().clone());
    }

    pub fn send<T: Message>(&self, addr: SocketAddr, msg: T) {
        let any_expected_msg = Any::from_raw(msg.raw().clone()).unwrap();
        let sended = self.inner.borrow_mut().sended.pop_front();
        if let Some((real_addr, real_msg)) = sended {
            let any_real_msg = Any::from_raw(real_msg.clone())
                                    .expect("Send incorrect message");
            if real_addr != addr || any_real_msg != any_expected_msg {
                panic!("Expected to send the message {:?} to {} instead sending {:?} to {}",
                       any_expected_msg, addr, any_real_msg, real_addr)
            }
        } else {
            panic!("Expected to send the message {:?} to {} but nothing happened",
                   any_expected_msg, addr);
        }
    }

    pub fn broadcast<T: Message>(&self, msg: T) {
        let any_expected_msg = Any::from_raw(msg.raw().clone()).unwrap();
        let mut set = HashSet::from_iter(self.addresses.iter().map(Clone::clone)): HashSet<SocketAddr>;
        for _ in 0..self.validators.len() {
            let sended = self.inner.borrow_mut().sended.pop_front();
            if let Some((real_addr, real_msg)) = sended {
                let any_real_msg = Any::from_raw(real_msg.clone())
                                        .expect("Send incorrect message");
                if any_real_msg != any_expected_msg {
                    panic!("Expected to broadcast the message {:?} instead sending {:?} to {}",
                           any_expected_msg, any_real_msg, real_addr)
                }
                if !set.contains(&real_addr) {
                    panic!("Double send the same message {:?} to {:?} during broadcasting",
                           any_expected_msg, real_addr)
                } else {
                    set.remove(&real_addr);
                }
            } else {
                panic!("Expected to broadcast the message {:?} but someone don't recieve messages: {:?}",
                       any_expected_msg, set);
            }
        }

    }

    pub fn set_time(&self, sec: i64, nsec: i32) {
        self.check_unexpected_message();
        // set time
        let now = Timespec {sec: sec, nsec: nsec};
        self.inner.borrow_mut().time = now;
        // handle timeouts if occurs
        loop {
            let timeout = {
                let ref mut timers = self.inner.borrow_mut().timers;
                if let Some(TimerPair(time, timeout)) = timers.pop() {
                    if time > now {
                        timers.push(TimerPair(time, timeout));
                        break;
                    } else {
                        timeout
                    }
                } else {
                    break;
                }
            };
            self.node.borrow_mut().handle_timeout(timeout);
        }
    }

    pub fn assert_state(&self, height: u64, round: u32) {
        let achual_height = self.node.borrow().context().state.height();
        let actual_round = self.node.borrow().context().state.round();
        assert!(achual_height == height,
                "Incorrect height, actual={}, expected={}", achual_height, height);
        assert!(actual_round == round,
                "Incorrect round, actual={}, expected={}", actual_round, round);
    }

    pub fn assert_lock(&self, round: u32, hash: Option<Hash>) {
        let actual_round = self.node.borrow().context().state.locked_round();
        let actual_hash = self.node.borrow().context().state.locked_propose();
        assert!(actual_round == round,
                "Incorrect height, actual={}, expected={}", actual_round, round);
        assert!(actual_hash == hash,
                "Incorrect round, actual={:?}, expected={:?}", actual_hash, hash);
    }
}

impl Drop for Sandbox {
    fn drop(&mut self) {
        self.check_unexpected_message();
    }
}

#[test]
fn test_sandbox_init() {
    Sandbox::new();
}

#[test]
fn test_sandbox_recv_and_send() {
    let s = Sandbox::new();
    let (public, secret) = gen_keypair();
    s.recv(Connect::new(&public, s.a(2), s.time(), &secret));
    s.send(s.a(2), Connect::new(s.p(0), s.a(0), s.time(), s.s(0)));
}

#[test]
fn test_sandbox_assert_status() {
    // TODO: remove this?
    let s = Sandbox::new();
    s.assert_state(0, 1);
    s.set_time(0, 999_999_999);
    s.assert_state(0, 1);
    s.set_time(1, 0);
    s.assert_state(0, 2);
}

#[test]
#[should_panic(expected = "Expected to send the message")]
fn test_sandbox_expected_to_send_but_nothing_happened() {
    let s = Sandbox::new();
    s.send(s.a(1), Connect::new(s.p(0), s.a(0), s.time(), s.s(0)));
}

#[test]
#[should_panic(expected = "Expected to send the message")]
fn test_sandbox_expected_to_send_another_message() {
    let s = Sandbox::new();
    let (public, secret) = gen_keypair();
    s.recv(Connect::new(&public, s.a(2), s.time(), &secret));
    s.send(s.a(1), Connect::new(s.p(0), s.a(0), s.time(), s.s(0)));
}

#[test]
#[should_panic(expected = "Send unexpected message")]
fn test_sandbox_unexpected_message_when_drop() {
    let s = Sandbox::new();
    let (public, secret) = gen_keypair();
    s.recv(Connect::new(&public, s.a(2), s.time(), &secret));
}

#[test]
#[should_panic(expected = "Send unexpected message")]
fn test_sandbox_unexpected_message_when_handle_another_message() {
    let s = Sandbox::new();
    let (public, secret) = gen_keypair();
    s.recv(Connect::new(&public, s.a(2), s.time(), &secret));
    s.recv(Connect::new(&public, s.a(3), s.time(), &secret));
    panic!("Oops! We don't catch unexpected message");
}

#[test]
#[should_panic(expected = "Send unexpected message")]
fn test_sandbox_unexpected_message_when_time_changed() {
    let s = Sandbox::new();
    let (public, secret) = gen_keypair();
    s.recv(Connect::new(&public, s.a(2), s.time(), &secret));
    s.set_time(1, 0);
    panic!("Oops! We don't catch unexpected message");
}
<|endoftext|>


<commit_before>extern crate html5ever;
extern crate tendril;
extern crate regex;
extern crate hyper;
extern crate string_cache;
extern crate url;

use std::str::FromStr;

use self::tendril::Tendril;

use self::html5ever::rcdom::{Document, Doctype, Text, Comment, Element};
use self::html5ever::rcdom::{RcDom, Handle};
use self::html5ever::{parse, one_input, Attribute};
use std::default::Default;
use std::io::Read;

use self::regex::Regex;
use self::hyper::Client;
use self::hyper::header::Connection;
use self::hyper::header::ConnectionOption;

use Provider;
use Track;

use self::url::percent_encoding::lossy_utf8_percent_decode;

static YOUTUBE_EMBED:    &'static str = r"www.youtube.com/embed/([^\?&].+)";
static YOUTUBE_WATCH:    &'static str = r"www.youtube.com/watch\?v=([^\?&]+)";
static SOUNDCLOUD_TRACK: &'static str = r"api.soundcloud.com/tracks/([^\?&]+)";

pub fn extract_tracks(url: &str) -> Vec<Track> {
    let client = Client::new();
    let mut res = client.get(url)
        .header(Connection(vec![ConnectionOption::Close]))
        .send()
        .unwrap();
    let mut body = String::new();
    res.read_to_string(&mut body).unwrap();
    let input: Tendril<_> = FromStr::from_str(&body).unwrap();
    let dom: RcDom = parse(one_input(input), Default::default());
    let mut tracks  = Vec::new();
    walk(0, dom.document, &mut tracks);
    return tracks
}

// This is not proper HTML serialization, of course.
fn walk(indent: usize, handle: Handle, tracks: &mut Vec<Track>) {
    let node = handle.borrow();
    match node.node {
        Document         => (),
        Doctype(_, _, _) => (),
        Text(_)          => (),
        Comment(_)       => (),
        Element(ref name, ref attrs) => {
            let tag_name = name.local.as_slice();
            match extract_track(tag_name, attrs) {
                Some(track) => {
                    if !(*tracks).contains(&track) {
                        (*tracks).push(track)
                    }
                },
                None => {}
            }
        }
    }
    for child in node.children.iter() {
        walk(indent+4, child.clone(), tracks);
    }
}

fn attr(attr_name: &str, attrs: &Vec<Attribute>) -> Option<String> {
    for attr in attrs.iter() {
        if attr.name.local.as_slice() == attr_name {
            return Some(attr.value.to_string())
        }
    }
    None
}

fn extract_identifier(value: &str, regex_str: &str) -> Option<String> {
    match Regex::new(regex_str) {
        Ok(re) => match re.captures(value) {
            Some(cap) => match cap.at(1) {
                Some(str) => {
                    let strs: Vec<&str> = str.split('?').collect();
                    return Some(strs[0].to_string())
                },
                None => None
            },
            None => None
        },
        Err(_) => None
    }
}

pub fn extract_track(tag_name: &str, attrs: &Vec<Attribute>) -> Option<Track> {
    if tag_name == "iframe" {
        match attr("src", attrs) {
            Some(ref src) => {
                let decoded = lossy_utf8_percent_decode(src.as_bytes());
                match extract_identifier(&decoded, YOUTUBE_EMBED) {
                    Some(identifier) => {
                        return Some(Track {
                                    id: 0,
                              provider: Provider::YouTube,
                                 title: "".to_string(),
                                   url: src.to_string(),
                            identifier: identifier
                        });
                    },
                    None => ()
                }
                match extract_identifier(&decoded, SOUNDCLOUD_TRACK) {
                    Some(identifier) => {
                        return Some(Track {
                                    id: 0,
                              provider: Provider::SoundCloud,
                                 title: "".to_string(),
                                   url: src.to_string(),
                            identifier: identifier
                        });
                    },
                    None => ()
                }
            },
            None => ()
        }
    } else if tag_name == "a" || tag_name == "link" {
        match attr("href", attrs) {
            Some(ref href) => {
                let decoded = lossy_utf8_percent_decode(href.as_bytes());
                match extract_identifier(&decoded, YOUTUBE_WATCH) {
                    Some(identifier) => {
                        return Some(Track {
                                    id: 0,
                              provider: Provider::YouTube,
                                 title: "".to_string(),
                                   url: href.to_string(),
                            identifier: identifier
                        });
                    },
                    None => ()
                }
            },
            None => ()
        }
    }
    return None
}

#[cfg(test)]
mod test {
    use super::extract_identifier;

    #[test]
    fn test_extract_identifier() {
        let soundcloud_src = "https://w.soundcloud.com/player/?url=https%3A//api.soundcloud.com/tracks/195425494&auto_play=false&hide_related=false&show_comments=true&show_user=true&show_reposts=false&visual=true";
        match extract_identifier(soundcloud_src, super::SOUNDCLOUD_TRACK) {
            Some(identifier) => assert_eq!(identifier,
                                           "195425494".to_string()),
            None             => assert!(false)
        }

        let youtube_src = "https://www.youtube.com/embed/X8tOngmlES0?rel=0";
        match extract_identifier(youtube_src, super::YOUTUBE_EMBED) {
            Some(identifier) => assert_eq!(identifier,
                                           "X8tOngmlES0".to_string()),
            None             => assert!(false)
        }

        let youtube_href_src = "https://www.youtube.com/watch?v=oDuif301F-8";
        match extract_identifier(youtube_href_src, super::YOUTUBE_WATCH) {
            Some(identifier) => assert_eq!(identifier,
                                           "oDuif301F-8".to_string()),
            None             => assert!(false)
        }
    }
}
<commit_msg>Ignore /embed/videoseries path for youtube track<commit_after>extern crate html5ever;
extern crate tendril;
extern crate regex;
extern crate hyper;
extern crate string_cache;
extern crate url;

use std::str::FromStr;

use self::tendril::Tendril;

use self::html5ever::rcdom::{Document, Doctype, Text, Comment, Element};
use self::html5ever::rcdom::{RcDom, Handle};
use self::html5ever::{parse, one_input, Attribute};
use std::default::Default;
use std::io::Read;

use self::regex::Regex;
use self::hyper::Client;
use self::hyper::header::Connection;
use self::hyper::header::ConnectionOption;

use Provider;
use Track;

use self::url::percent_encoding::lossy_utf8_percent_decode;

static YOUTUBE_EMBED:    &'static str = r"www.youtube.com/embed/([^\?&{videoseries}].+)";
static YOUTUBE_LIST:     &'static str = r"www.youtube.com/embed/videoseries\?list=([^\?]+)";
static YOUTUBE_WATCH:    &'static str = r"www.youtube.com/watch\?v=([^\?&]+)";
static SOUNDCLOUD_TRACK: &'static str = r"api.soundcloud.com/tracks/([^\?&]+)";

pub fn extract_tracks(url: &str) -> Vec<Track> {
    let client = Client::new();
    let mut res = client.get(url)
        .header(Connection(vec![ConnectionOption::Close]))
        .send()
        .unwrap();
    let mut body = String::new();
    res.read_to_string(&mut body).unwrap();
    let input: Tendril<_> = FromStr::from_str(&body).unwrap();
    let dom: RcDom = parse(one_input(input), Default::default());
    let mut tracks  = Vec::new();
    walk(0, dom.document, &mut tracks);
    return tracks
}

// This is not proper HTML serialization, of course.
fn walk(indent: usize, handle: Handle, tracks: &mut Vec<Track>) {
    let node = handle.borrow();
    match node.node {
        Document         => (),
        Doctype(_, _, _) => (),
        Text(_)          => (),
        Comment(_)       => (),
        Element(ref name, ref attrs) => {
            let tag_name = name.local.as_slice();
            match extract_track(tag_name, attrs) {
                Some(track) => {
                    if !(*tracks).contains(&track) {
                        (*tracks).push(track)
                    }
                },
                None => {}
            }
        }
    }
    for child in node.children.iter() {
        walk(indent+4, child.clone(), tracks);
    }
}

fn attr(attr_name: &str, attrs: &Vec<Attribute>) -> Option<String> {
    for attr in attrs.iter() {
        if attr.name.local.as_slice() == attr_name {
            return Some(attr.value.to_string())
        }
    }
    None
}

fn extract_identifier(value: &str, regex_str: &str) -> Option<String> {
    match Regex::new(regex_str) {
        Ok(re) => match re.captures(value) {
            Some(cap) => match cap.at(1) {
                Some(str) => {
                    let strs: Vec<&str> = str.split('?').collect();
                    return Some(strs[0].to_string())
                },
                None => None
            },
            None => None
        },
        Err(_) => None
    }
}

pub fn extract_track(tag_name: &str, attrs: &Vec<Attribute>) -> Option<Track> {
    if tag_name == "iframe" {
        match attr("src", attrs) {
            Some(ref src) => {
                let decoded = lossy_utf8_percent_decode(src.as_bytes());
                match extract_identifier(&decoded, YOUTUBE_EMBED) {
                    Some(identifier) => {
                        return Some(Track {
                                    id: 0,
                              provider: Provider::YouTube,
                                 title: "".to_string(),
                                   url: src.to_string(),
                            identifier: identifier
                        });
                    },
                    None => ()
                }
                match extract_identifier(&decoded, SOUNDCLOUD_TRACK) {
                    Some(identifier) => {
                        return Some(Track {
                                    id: 0,
                              provider: Provider::SoundCloud,
                                 title: "".to_string(),
                                   url: src.to_string(),
                            identifier: identifier
                        });
                    },
                    None => ()
                }
            },
            None => ()
        }
    } else if tag_name == "a" || tag_name == "link" {
        match attr("href", attrs) {
            Some(ref href) => {
                let decoded = lossy_utf8_percent_decode(href.as_bytes());
                match extract_identifier(&decoded, YOUTUBE_WATCH) {
                    Some(identifier) => {
                        return Some(Track {
                                    id: 0,
                              provider: Provider::YouTube,
                                 title: "".to_string(),
                                   url: href.to_string(),
                            identifier: identifier
                        });
                    },
                    None => ()
                }
            },
            None => ()
        }
    }
    return None
}

#[cfg(test)]
mod test {
    use super::extract_identifier;

    #[test]
    fn test_extract_identifier() {
        let soundcloud_src = "https://w.soundcloud.com/player/?url=https%3A//api.soundcloud.com/tracks/195425494&auto_play=false&hide_related=false&show_comments=true&show_user=true&show_reposts=false&visual=true";
        match extract_identifier(soundcloud_src, super::SOUNDCLOUD_TRACK) {
            Some(identifier) => assert_eq!(identifier, "195425494".to_string()),
            None             => assert!(false)
        }
        let youtube_embed = "https://www.youtube.com/embed/X8tOngmlES0?rel=0";
        match extract_identifier(youtube_embed, super::YOUTUBE_EMBED) {
            Some(identifier) => assert_eq!(identifier, "X8tOngmlES0".to_string()),
            None             => assert!(false)
        }

        let youtube_watch = "https://www.youtube.com/watch?v=oDuif301F-8";
        match extract_identifier(youtube_watch, super::YOUTUBE_WATCH) {
            Some(identifier) => assert_eq!(identifier, "oDuif301F-8".to_string()),
            None             => assert!(false)
        }

        let youtube_list = "https://www.youtube.com/embed/videoseries?list=PLy8LZ8FM-o0ViuGAF68RAaXkQ8V-3dbTX";
        match extract_identifier(youtube_list, super::YOUTUBE_LIST) {
            Some(identifier) => assert_eq!(identifier, "PLy8LZ8FM-o0ViuGAF68RAaXkQ8V-3dbTX".to_string()),
            None             => assert!(false)
        }

        match extract_identifier(youtube_list, super::YOUTUBE_EMBED) {
            Some(identifier) => assert!(false),
            None             => assert!(true)
        }
    }
}
<|endoftext|>


<commit_before>use std::mem;

use self::BufferedParserState::{Beginning, Middle, EndMatch, EndFail};
use self::MatchResult::{Undecided, Committed, Matched, Failed};
use self::ConstantParserState::{AtOffset, AtEnd};

// ----------- Types with lifetimes -------------

// Borrowing encoding of paramaterized types from
// https://github.com/rust-lang/rfcs/blob/master/text/0195-associated-items.md#encoding-higher-kinded-types

pub trait TypeWithLifetime<'a> {
    type Type;
}

pub type At<'a,T> where T: TypeWithLifetime<'a> = T::Type;

pub struct Always<T> (T);

impl<'a,T> TypeWithLifetime<'a> for Always<T> {
    type Type = T;
}

pub type Unit = Always<()>;

// ----------- Types for consumers ------------

pub trait Consumer<T> where T: 'static+for<'a> TypeWithLifetime<'a> {
    fn accept<'a>(&mut self, arg: At<'a,T>);
}

struct DiscardConsumer;

impl Consumer<Unit> for DiscardConsumer {
    fn accept(&mut self, _: ()) {}
}

// ----------- Types for parsers ------------

// State machine transitions are:
//
// init -Undecided->  init
// init -Committed->  committed
// init -Matched(s)-> matched
// init -Failed(b)->  failed(b)
//
// committed -Committed->     committed
// committed -Matched(s)->    matched
// committed -Failed(false)-> failed(false)
//
// matched -Matched(s)-> matched
//
// failed(b) -Failed(b)-> failed(b)
//
// The Failed(b) action carries a boolean indicating if backtracking is allowed.
// Note that there is no transition . -Committed-> . -Failed(true)-> . so
// once a parser has committed, we can clean up space associated with backtracking.

#[derive(Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Debug)]
pub enum MatchResult<T> {
    Undecided,
    Committed,
    Matched(T),
    Failed(bool),
}

pub trait Parser<S,T> where S: 'static+for<'a> TypeWithLifetime<'a>, T: 'static+for<'a> TypeWithLifetime<'a> {
    // If push returns Undecided or Failed(true), it is side-effect-free
    // In the case where T is "list-like" (e.g. &str or &[T])
    // push(nil) is a no-op
    // push(a ++ b) is the same as push(a); push(b)
    fn push<'a>(&mut self, value: At<'a,S>, downstream: &mut Consumer<T>) -> MatchResult<At<'a,S>>;
    // Resets the parser state back to its initial state
    // Returns true if there was a match.
    fn done(&mut self, downstream: &mut Consumer<T>) -> bool;
}

pub trait BufferableMatcher<S,T> where S: 'static+for<'a> TypeWithLifetime<'a>, T: Parser<S,S> {
    fn buffer(self) -> T;
}

// ----------- Always commit ---------------

pub struct CommittedParser<P> {
    parser: P,
}

impl<S,T,P> Parser<S,T> for CommittedParser<P> where P: Parser<S,T>, S: 'static+for<'a> TypeWithLifetime<'a>, T: 'static+for<'a> TypeWithLifetime<'a>  {
    fn push<'a>(&mut self, value: At<'a,S>, downstream: &mut Consumer<T>) -> MatchResult<At<'a,S>> {
        match self.parser.push(value, downstream) {
            Undecided     => Committed,
            Committed     => Committed,
            Matched(rest) => Matched(rest),
            Failed(_)     => Failed(false),
        }
    }
    fn done(&mut self, downstream: &mut Consumer<T>) -> bool {
        self.parser.done(downstream)
    }
}

// ----------- Sequencing ---------------

pub struct AndThenParser<L,R> {
    lhs: L,
    rhs: CommittedParser<R>,
    in_lhs: bool,
}

impl<S,T,L,R> Parser<S,T> for AndThenParser<L,R> where L: Parser<S,T>, R: Parser<S,T>, S: 'static+for<'a> TypeWithLifetime<'a>, T: 'static+for<'a> TypeWithLifetime<'a>  {
    fn push<'a>(&mut self, value: At<'a,S>, downstream: &mut Consumer<T>) -> MatchResult<At<'a,S>> {
        if self.in_lhs {
            match self.lhs.push(value, downstream) {
                Undecided     => Undecided,
                Committed     => Committed,
                Matched(rest) => { self.in_lhs = false; self.rhs.push(rest, downstream) },
                Failed(b)     => Failed(b),
            }
        } else {
            self.rhs.push(value, downstream)
        }
    }
    fn done(&mut self, downstream: &mut Consumer<T>) -> bool {
        self.lhs.done(downstream) && self.rhs.done(downstream)
    }
}

// ----------- Matching strings -------------

pub struct Str;

impl<'a> TypeWithLifetime<'a> for Str {
    type Type = &'a str;
}

// ----------- Constant parsers -------------

#[derive(Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Debug)]
pub enum ConstantParserState {
    AtOffset(usize),
    AtEnd(bool),
}

pub struct ConstantParser {
    constant: String,
    state: ConstantParserState,
}

impl Parser<Str,Unit> for ConstantParser {
    fn push<'a>(&mut self, string: &'a str, downstream: &mut Consumer<Unit>) -> MatchResult<&'a str> {
        match self.state {
            AtOffset(index) if string == &self.constant[index..]           => { downstream.accept(()); self.state = AtEnd(true); Matched("") },
            AtOffset(index) if string.starts_with(&self.constant[index..]) => { downstream.accept(()); self.state = AtEnd(true); Matched(&string[index..]) },
            AtOffset(index) if self.constant[index..].starts_with(string)  => { self.state = AtOffset(index + string.len()); Undecided },
            AtOffset(_)                                                    => { self.state = AtEnd(false); Failed(true) },
            AtEnd(true)                                                    => { Matched(string) },
            AtEnd(false)                                                   => { Failed(true) },
        }
    }
    fn done(&mut self, _: &mut Consumer<Unit>) -> bool {
        let result = self.state == AtEnd(true);
        self.state = AtOffset(0);
        result
    }
}

pub fn constant(string: String) -> ConstantParser {
    ConstantParser{ constant: string, state: AtOffset(0) }
}

// If m is a Parser<Str,Unit> then m.buffer() is a Parser<Str,Str>.
// It does as little buffering as it can, but it does allocate as buffer for the case
// where the boundary marker of the input is misaligned with that of the parser.
// For example, m is matching string literals, and the input is '"abc' followed by 'def"'
// we have to buffer up '"abc'.

#[derive(Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Debug)]
enum BufferedParserState {
    Beginning,
    Middle(String),
    EndMatch,
    EndFail(bool),
}

pub struct BufferedParser<P> {
    parser: P,
    state: BufferedParserState,
}

impl<P> Parser<Str,Str> for BufferedParser<P> where P: Parser<Str,Unit> {
    fn push<'a>(&mut self, string: &'a str, downstream: &mut Consumer<Str>) -> MatchResult<&'a str> {
        match mem::replace(&mut self.state, EndMatch) {
            Beginning => {
                let result = self.parser.push(string, &mut DiscardConsumer);
                match result {
                    Undecided     => self.state = Middle(String::from(string)),
                    Committed     => self.state = Middle(String::from(string)),
                    Failed(b)     => self.state = EndFail(b),
                    Matched(rest) => downstream.accept(&string[..(string.len()-rest.len())]),
                }
                result
            },
            Middle(mut buffer) => {
                let result = self.parser.push(string, &mut DiscardConsumer);
                match result {
                    Undecided     => { buffer.push_str(string); self.state = Middle(buffer); },
                    Committed     => { buffer.push_str(string); self.state = Middle(buffer); },
                    Failed(b)     => { self.state = EndFail(b); },
                    Matched(rest) => { buffer.push_str(&string[..(string.len()-rest.len())]); downstream.accept(&*buffer); },
                }
                result
            }
            EndMatch => Matched(string),
            EndFail(b) => Failed(b),
        }
    }
    fn done(&mut self, downstream: &mut Consumer<Str>) -> bool {
        let result = self.parser.done(&mut DiscardConsumer);
        if result { if let Middle(ref buffer) = self.state { downstream.accept(&*buffer) } }
        self.state = Beginning;
        result
    }
}
<commit_msg>Added a test for constant parsers.<commit_after>use std::mem;

use self::BufferedParserState::{Beginning, Middle, EndMatch, EndFail};
use self::MatchResult::{Undecided, Committed, Matched, Failed};
use self::ConstantParserState::{AtOffset, AtEnd};

// ----------- Types with lifetimes -------------

// Borrowing encoding of paramaterized types from
// https://github.com/rust-lang/rfcs/blob/master/text/0195-associated-items.md#encoding-higher-kinded-types

pub trait TypeWithLifetime<'a> {
    type Type;
}

pub type At<'a,T> where T: TypeWithLifetime<'a> = T::Type;

pub struct Always<T> (T);

impl<'a,T> TypeWithLifetime<'a> for Always<T> {
    type Type = T;
}

pub type Unit = Always<()>;

// ----------- Types for consumers ------------

pub trait Consumer<T> where T: 'static+for<'a> TypeWithLifetime<'a> {
    fn accept<'a>(&mut self, arg: At<'a,T>);
}

struct DiscardConsumer;

impl Consumer<Unit> for DiscardConsumer {
    fn accept(&mut self, _: ()) {}
}

// ----------- Types for parsers ------------

// State machine transitions are:
//
// init -Undecided->  init
// init -Committed->  committed
// init -Matched(s)-> matched
// init -Failed(b)->  failed(b)
//
// committed -Committed->     committed
// committed -Matched(s)->    matched
// committed -Failed(false)-> failed(false)
//
// matched -Matched(s)-> matched
//
// failed(b) -Failed(b)-> failed(b)
//
// The Failed(b) action carries a boolean indicating if backtracking is allowed.
// Note that there is no transition . -Committed-> . -Failed(true)-> . so
// once a parser has committed, we can clean up space associated with backtracking.

#[derive(Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Debug)]
pub enum MatchResult<T> {
    Undecided,
    Committed,
    Matched(T),
    Failed(bool),
}

pub trait Parser<S,T> where S: 'static+for<'a> TypeWithLifetime<'a>, T: 'static+for<'a> TypeWithLifetime<'a> {
    // If push returns Undecided or Failed(true), it is side-effect-free
    // In the case where T is "list-like" (e.g. &str or &[T])
    // push(nil) is a no-op
    // push(a ++ b) is the same as push(a); push(b)
    fn push<'a>(&mut self, value: At<'a,S>, downstream: &mut Consumer<T>) -> MatchResult<At<'a,S>>;
    // Resets the parser state back to its initial state
    // Returns true if there was a match.
    fn done(&mut self, downstream: &mut Consumer<T>) -> bool;
}

pub trait BufferableMatcher<S,T> where S: 'static+for<'a> TypeWithLifetime<'a>, T: Parser<S,S> {
    fn buffer(self) -> T;
}

// ----------- Always commit ---------------

pub struct CommittedParser<P> {
    parser: P,
}

impl<S,T,P> Parser<S,T> for CommittedParser<P> where P: Parser<S,T>, S: 'static+for<'a> TypeWithLifetime<'a>, T: 'static+for<'a> TypeWithLifetime<'a>  {
    fn push<'a>(&mut self, value: At<'a,S>, downstream: &mut Consumer<T>) -> MatchResult<At<'a,S>> {
        match self.parser.push(value, downstream) {
            Undecided     => Committed,
            Committed     => Committed,
            Matched(rest) => Matched(rest),
            Failed(_)     => Failed(false),
        }
    }
    fn done(&mut self, downstream: &mut Consumer<T>) -> bool {
        self.parser.done(downstream)
    }
}

// ----------- Sequencing ---------------

pub struct AndThenParser<L,R> {
    lhs: L,
    rhs: CommittedParser<R>,
    in_lhs: bool,
}

impl<S,T,L,R> Parser<S,T> for AndThenParser<L,R> where L: Parser<S,T>, R: Parser<S,T>, S: 'static+for<'a> TypeWithLifetime<'a>, T: 'static+for<'a> TypeWithLifetime<'a>  {
    fn push<'a>(&mut self, value: At<'a,S>, downstream: &mut Consumer<T>) -> MatchResult<At<'a,S>> {
        if self.in_lhs {
            match self.lhs.push(value, downstream) {
                Undecided     => Undecided,
                Committed     => Committed,
                Matched(rest) => { self.in_lhs = false; self.rhs.push(rest, downstream) },
                Failed(b)     => Failed(b),
            }
        } else {
            self.rhs.push(value, downstream)
        }
    }
    fn done(&mut self, downstream: &mut Consumer<T>) -> bool {
        self.lhs.done(downstream) && self.rhs.done(downstream)
    }
}

// ----------- Matching strings -------------

pub struct Str;

impl<'a> TypeWithLifetime<'a> for Str {
    type Type = &'a str;
}

// ----------- Constant parsers -------------

#[derive(Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Debug)]
pub enum ConstantParserState {
    AtOffset(usize),
    AtEnd(bool),
}

pub struct ConstantParser {
    constant: String,
    state: ConstantParserState,
}

impl Parser<Str,Unit> for ConstantParser {
    fn push<'a>(&mut self, string: &'a str, downstream: &mut Consumer<Unit>) -> MatchResult<&'a str> {
        match self.state {
            AtOffset(index) if string.starts_with(&self.constant[index..]) => { downstream.accept(()); self.state = AtEnd(true); Matched(&string[(self.constant.len() - index)..]) },
            AtOffset(index) if self.constant[index..].starts_with(string)  => { self.state = AtOffset(index + string.len()); Undecided },
            AtOffset(_)                                                    => { self.state = AtEnd(false); Failed(true) },
            AtEnd(true)                                                    => { Matched(string) },
            AtEnd(false)                                                   => { Failed(true) },
        }
    }
    fn done(&mut self, _: &mut Consumer<Unit>) -> bool {
        let result = self.state == AtEnd(true);
        self.state = AtOffset(0);
        result
    }
}

pub fn constant(string: String) -> ConstantParser {
    ConstantParser{ constant: string, state: AtOffset(0) }
}

// If m is a Parser<Str,Unit> then m.buffer() is a Parser<Str,Str>.
// It does as little buffering as it can, but it does allocate as buffer for the case
// where the boundary marker of the input is misaligned with that of the parser.
// For example, m is matching string literals, and the input is '"abc' followed by 'def"'
// we have to buffer up '"abc'.

#[derive(Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Debug)]
enum BufferedParserState {
    Beginning,
    Middle(String),
    EndMatch,
    EndFail(bool),
}

pub struct BufferedParser<P> {
    parser: P,
    state: BufferedParserState,
}

impl<P> Parser<Str,Str> for BufferedParser<P> where P: Parser<Str,Unit> {
    fn push<'a>(&mut self, string: &'a str, downstream: &mut Consumer<Str>) -> MatchResult<&'a str> {
        match mem::replace(&mut self.state, EndMatch) {
            Beginning => {
                let result = self.parser.push(string, &mut DiscardConsumer);
                match result {
                    Undecided     => self.state = Middle(String::from(string)),
                    Committed     => self.state = Middle(String::from(string)),
                    Failed(b)     => self.state = EndFail(b),
                    Matched(rest) => downstream.accept(&string[..(string.len()-rest.len())]),
                }
                result
            },
            Middle(mut buffer) => {
                let result = self.parser.push(string, &mut DiscardConsumer);
                match result {
                    Undecided     => { buffer.push_str(string); self.state = Middle(buffer); },
                    Committed     => { buffer.push_str(string); self.state = Middle(buffer); },
                    Failed(b)     => { self.state = EndFail(b); },
                    Matched(rest) => { buffer.push_str(&string[..(string.len()-rest.len())]); downstream.accept(&*buffer); },
                }
                result
            }
            EndMatch => Matched(string),
            EndFail(b) => Failed(b),
        }
    }
    fn done(&mut self, downstream: &mut Consumer<Str>) -> bool {
        let result = self.parser.done(&mut DiscardConsumer);
        if result { if let Middle(ref buffer) = self.state { downstream.accept(&*buffer) } }
        self.state = Beginning;
        result
    }
}

#[test]
fn test_constant() {
    let mut parser = constant(String::from("abc"));
    assert_eq!(parser.done(&mut DiscardConsumer), false);
    assert_eq!(parser.push("fred", &mut DiscardConsumer), Failed(true));
    assert_eq!(parser.done(&mut DiscardConsumer), false);
    assert_eq!(parser.push("abcdef", &mut DiscardConsumer), Matched("def"));
    assert_eq!(parser.done(&mut DiscardConsumer), true);
    assert_eq!(parser.push("a", &mut DiscardConsumer), Undecided);
    assert_eq!(parser.done(&mut DiscardConsumer), false);
    assert_eq!(parser.push("ab", &mut DiscardConsumer), Undecided);
    assert_eq!(parser.push("cd", &mut DiscardConsumer), Matched("d"));
    assert_eq!(parser.done(&mut DiscardConsumer), true);
}
<|endoftext|>


<commit_before>use container;
use error;

use pe::data_directories;

use scroll::{ctx, LE, Pread, Gread};

/// standard COFF fields
#[repr(C)]
#[derive(Debug, PartialEq, Copy, Clone, Default)]
#[derive(Pread, Pwrite, SizeWith)]
pub struct StandardFields32 {
    pub magic: u16,
    pub major_linker_version: u8,
    pub minor_linker_version: u8,
    pub size_of_code: u32,
    pub size_of_initialized_data: u32,
    pub size_of_uninitialized_data: u32,
    pub address_of_entry_point: u32,
    pub base_of_code: u32,
    /// absent in 64-bit PE32+
    pub base_of_data: u32,
}

pub const SIZEOF_STANDARD_FIELDS_32: usize = 28;

/// standard 64-bit COFF fields
#[repr(C)]
#[derive(Debug, PartialEq, Copy, Clone, Default)]
#[derive(Pread, Pwrite, SizeWith)]
pub struct StandardFields64 {
    pub magic: u16,
    pub major_linker_version: u8,
    pub minor_linker_version: u8,
    pub size_of_code: u32,
    pub size_of_initialized_data: u32,
    pub size_of_uninitialized_data: u32,
    pub address_of_entry_point: u32,
    pub base_of_code: u32,
}

pub const SIZEOF_STANDARD_FIELDS_64: usize = 24;

/// Unified 32/64-bit COFF fields
#[derive(Debug, PartialEq, Copy, Clone, Default)]
pub struct StandardFields {
    pub magic: u16,
    pub major_linker_version: u8,
    pub minor_linker_version: u8,
    pub size_of_code: u64,
    pub size_of_initialized_data: u64,
    pub size_of_uninitialized_data: u64,
    pub address_of_entry_point: u64,
    pub base_of_code: u64,
    /// absent in 64-bit PE32+
    pub base_of_data: u32,
}

impl From<StandardFields32> for StandardFields {
    fn from(fields: StandardFields32) -> Self {
        StandardFields {
            magic: fields.magic,
            major_linker_version: fields.major_linker_version,
            minor_linker_version: fields.minor_linker_version,
            size_of_code: fields.size_of_code as u64,
            size_of_initialized_data: fields.size_of_initialized_data as u64,
            size_of_uninitialized_data: fields.size_of_uninitialized_data as u64,
            address_of_entry_point: fields.address_of_entry_point as u64,
            base_of_code: fields.base_of_code as u64,
            base_of_data: fields.base_of_data,
        }
    }
}

impl From<StandardFields64> for StandardFields {
    fn from(fields: StandardFields64) -> Self {
        StandardFields {
            magic: fields.magic,
            major_linker_version: fields.major_linker_version,
            minor_linker_version: fields.minor_linker_version,
            size_of_code: fields.size_of_code as u64,
            size_of_initialized_data: fields.size_of_initialized_data as u64,
            size_of_uninitialized_data: fields.size_of_uninitialized_data as u64,
            address_of_entry_point: fields.address_of_entry_point as u64,
            base_of_code: fields.base_of_code as u64,
            base_of_data: 0,
        }
    }
}

/// Standard fields magic number for 32-bit binary
pub const MAGIC_32: u16 = 0x10b;
/// Standard fields magic number for 64-bit binary
pub const MAGIC_64: u16 = 0x20b;

/// Windows specific fields
#[repr(C)]
#[derive(Debug, PartialEq, Copy, Clone, Default)]
#[derive(Pread, Pwrite, SizeWith)]
pub struct WindowsFields32 {
    pub image_base: u32,
    pub section_alignment: u32,
    pub file_alignment: u32,
    pub major_operating_system_version: u16,
    pub minor_operating_system_version: u16,
    pub major_image_version: u16,
    pub minor_image_version: u16,
    pub major_subsystem_version: u16,
    pub minor_subsystem_version: u16,
    pub win32_version_value: u32,
    pub size_of_image: u32,
    pub size_of_headers: u32,
    pub check_sum: u32,
    pub subsystem: u16,
    pub dll_characteristics: u16,
    pub size_of_stack_reserve: u32,
    pub size_of_stack_commit: u32,
    pub size_of_heap_reserve: u32,
    pub size_of_heap_commit: u32,
    pub loader_flags: u32,
    pub number_of_rva_and_sizes: u32,
}

pub const SIZEOF_WINDOWS_FIELDS_32: usize = 68;

/// 64-bit Windows specific fields
#[repr(C)]
#[derive(Debug, PartialEq, Copy, Clone, Default)]
#[derive(Pread, Pwrite, SizeWith)]
pub struct WindowsFields64 {
    pub image_base: u64,
    pub section_alignment: u32,
    pub file_alignment: u32,
    pub major_operating_system_version: u16,
    pub minor_operating_system_version: u16,
    pub major_image_version: u16,
    pub minor_image_version: u16,
    pub major_subsystem_version: u16,
    pub minor_subsystem_version: u16,
    pub win32_version_value: u32,
    pub size_of_image: u32,
    pub size_of_headers: u32,
    pub check_sum: u32,
    pub subsystem: u16,
    pub dll_characteristics: u16,
    pub size_of_stack_reserve: u64,
    pub size_of_stack_commit:  u64,
    pub size_of_heap_reserve:  u64,
    pub size_of_heap_commit:   u64,
    pub loader_flags: u32,
    pub number_of_rva_and_sizes: u32,
}

pub const SIZEOF_WINDOWS_FIELDS_64: usize = 88;

// /// Generic 32/64-bit Windows specific fields
// #[derive(Debug, PartialEq, Copy, Clone, Default)]
// pub struct WindowsFields {
//     pub image_base: u64,
//     pub section_alignment: u32,
//     pub file_alignment: u32,
//     pub major_operating_system_version: u16,
//     pub minor_operating_system_version: u16,
//     pub major_image_version: u16,
//     pub minor_image_version: u16,
//     pub major_subsystem_version: u16,
//     pub minor_subsystem_version: u16,
//     pub win32_version_value: u32,
//     pub size_of_image: u32,
//     pub size_of_headers: u32,
//     pub check_sum: u32,
//     pub subsystem: u16,
//     pub dll_characteristics: u16,
//     pub size_of_stack_reserve: u64,
//     pub size_of_stack_commit:  u64,
//     pub size_of_heap_reserve:  u64,
//     pub size_of_heap_commit:   u64,
//     pub loader_flags: u32,
//     pub number_of_rva_and_sizes: u32,
// }

impl From<WindowsFields32> for WindowsFields {
    fn from(windows: WindowsFields32) -> Self {
        WindowsFields {
            image_base: windows.image_base as u64,
            section_alignment: windows.section_alignment,
            file_alignment: windows.file_alignment,
            major_operating_system_version: windows.major_operating_system_version,
            minor_operating_system_version: windows.minor_operating_system_version,
            major_image_version: windows.major_image_version,
            minor_image_version: windows.minor_image_version,
            major_subsystem_version: windows.major_subsystem_version,
            minor_subsystem_version: windows.minor_subsystem_version,
            win32_version_value: windows.win32_version_value,
            size_of_image: windows.size_of_image,
            size_of_headers: windows.size_of_headers,
            check_sum: windows.check_sum,
            subsystem: windows.subsystem,
            dll_characteristics: windows.dll_characteristics,
            size_of_stack_reserve: windows.size_of_stack_reserve as u64,
            size_of_stack_commit: windows.size_of_stack_commit as u64,
            size_of_heap_reserve: windows.size_of_heap_reserve as u64,
            size_of_heap_commit: windows.size_of_heap_commit as u64,
            loader_flags: windows.loader_flags,
            number_of_rva_and_sizes: windows.number_of_rva_and_sizes,
        }
    }
}

// impl From<WindowsFields32> for WindowsFields {
//     fn from(windows: WindowsFields32) -> Self {
//         WindowsFields {
//             image_base: windows.image_base,
//             section_alignment: windows.section_alignment,
//             file_alignment: windows.file_alignment,
//             major_operating_system_version: windows.major_operating_system_version,
//             minor_operating_system_version: windows.minor_operating_system_version,
//             major_image_version: windows.major_image_version,
//             minor_image_version: windows.minor_image_version,
//             major_subsystem_version: windows.major_subsystem_version,
//             minor_subsystem_version: windows.minor_subsystem_version,
//             win32_version_value: windows.win32_version_value,
//             size_of_image: windows.size_of_image,
//             size_of_headers: windows.size_of_headers,
//             check_sum: windows.check_sum,
//             subsystem: windows.subsystem,
//             dll_characteristics: windows.dll_characteristics,
//             size_of_stack_reserve: windows.size_of_stack_reserve,
//             size_of_stack_commit: windows.size_of_stack_commit,
//             size_of_heap_reserve: windows.size_of_heap_reserve,
//             size_of_heap_commit: windows.size_of_heap_commit,
//             loader_flags: windows.loader_flags,
//             number_of_rva_and_sizes: windows.number_of_rva_and_sizes,
//         }
//     }
// }

pub type WindowsFields = WindowsFields64;

#[derive(Debug, PartialEq, Copy, Clone)]
pub struct OptionalHeader {
    pub standard_fields: StandardFields,
    pub windows_fields: WindowsFields,
    pub data_directories: data_directories::DataDirectories
}

impl OptionalHeader {
    pub fn container(&self) -> error::Result<container::Container> {
        match self.standard_fields.magic {
            MAGIC_32 => {
                Ok(container::Container::Little)
            },
            MAGIC_64 => {
                Ok(container::Container::Big)
            },
            magic => {
                Err(error::Error::BadMagic(magic as u64))
            }
        }
    }
}

impl<'a> ctx::TryFromCtx<'a> for OptionalHeader {
    type Error = error::Error;
    fn try_from_ctx(bytes: &'a [u8], (mut offset, _): (usize, ctx::DefaultCtx)) -> error::Result<Self> {
        let magic = bytes.pread_with::<u16>(offset, LE)?;
        let offset = &mut offset;
        let (standard_fields, windows_fields): (StandardFields, WindowsFields) = match magic {
            MAGIC_32 => {
                let standard_fields = bytes.gread_with::<StandardFields32>(offset, LE)?.into();
                let windows_fields = bytes.gread_with::<WindowsFields32>(offset, LE)?.into();
                (standard_fields, windows_fields)
            },
            MAGIC_64 => {
                let standard_fields = bytes.gread_with::<StandardFields64>(offset, LE)?.into();
                let windows_fields = bytes.gread_with::<WindowsFields64>(offset, LE)?;
                (standard_fields, windows_fields)
            },
            _ => return Err(error::Error::BadMagic(magic as u64))
        };
        println!("standard_fields {:#?} windows_fields {:#?}", standard_fields, windows_fields);
        let data_directories = data_directories::DataDirectories::parse(&bytes, windows_fields.number_of_rva_and_sizes as usize, offset)?;
        Ok (OptionalHeader {
            standard_fields: standard_fields,
            windows_fields: windows_fields,
            data_directories: data_directories,
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    #[test]
    fn sizeof_standards32() {
        assert_eq!(::std::mem::size_of::<StandardFields32>(), SIZEOF_STANDARD_FIELDS_32);
    }
    #[test]
    fn sizeof_windows32() {
        assert_eq!(::std::mem::size_of::<WindowsFields32>(), SIZEOF_WINDOWS_FIELDS_32);
    }
    #[test]
    fn sizeof_standards64() {
        assert_eq!(::std::mem::size_of::<StandardFields64>(), SIZEOF_STANDARD_FIELDS_64);
    }
    #[test]
    fn sizeof_windows64() {
        assert_eq!(::std::mem::size_of::<WindowsFields64>(), SIZEOF_WINDOWS_FIELDS_64);
    }
}
<commit_msg>pe.optional_header: remove accidental println<commit_after>use container;
use error;

use pe::data_directories;

use scroll::{ctx, LE, Pread, Gread};

/// standard COFF fields
#[repr(C)]
#[derive(Debug, PartialEq, Copy, Clone, Default)]
#[derive(Pread, Pwrite, SizeWith)]
pub struct StandardFields32 {
    pub magic: u16,
    pub major_linker_version: u8,
    pub minor_linker_version: u8,
    pub size_of_code: u32,
    pub size_of_initialized_data: u32,
    pub size_of_uninitialized_data: u32,
    pub address_of_entry_point: u32,
    pub base_of_code: u32,
    /// absent in 64-bit PE32+
    pub base_of_data: u32,
}

pub const SIZEOF_STANDARD_FIELDS_32: usize = 28;

/// standard 64-bit COFF fields
#[repr(C)]
#[derive(Debug, PartialEq, Copy, Clone, Default)]
#[derive(Pread, Pwrite, SizeWith)]
pub struct StandardFields64 {
    pub magic: u16,
    pub major_linker_version: u8,
    pub minor_linker_version: u8,
    pub size_of_code: u32,
    pub size_of_initialized_data: u32,
    pub size_of_uninitialized_data: u32,
    pub address_of_entry_point: u32,
    pub base_of_code: u32,
}

pub const SIZEOF_STANDARD_FIELDS_64: usize = 24;

/// Unified 32/64-bit COFF fields
#[derive(Debug, PartialEq, Copy, Clone, Default)]
pub struct StandardFields {
    pub magic: u16,
    pub major_linker_version: u8,
    pub minor_linker_version: u8,
    pub size_of_code: u64,
    pub size_of_initialized_data: u64,
    pub size_of_uninitialized_data: u64,
    pub address_of_entry_point: u64,
    pub base_of_code: u64,
    /// absent in 64-bit PE32+
    pub base_of_data: u32,
}

impl From<StandardFields32> for StandardFields {
    fn from(fields: StandardFields32) -> Self {
        StandardFields {
            magic: fields.magic,
            major_linker_version: fields.major_linker_version,
            minor_linker_version: fields.minor_linker_version,
            size_of_code: fields.size_of_code as u64,
            size_of_initialized_data: fields.size_of_initialized_data as u64,
            size_of_uninitialized_data: fields.size_of_uninitialized_data as u64,
            address_of_entry_point: fields.address_of_entry_point as u64,
            base_of_code: fields.base_of_code as u64,
            base_of_data: fields.base_of_data,
        }
    }
}

impl From<StandardFields64> for StandardFields {
    fn from(fields: StandardFields64) -> Self {
        StandardFields {
            magic: fields.magic,
            major_linker_version: fields.major_linker_version,
            minor_linker_version: fields.minor_linker_version,
            size_of_code: fields.size_of_code as u64,
            size_of_initialized_data: fields.size_of_initialized_data as u64,
            size_of_uninitialized_data: fields.size_of_uninitialized_data as u64,
            address_of_entry_point: fields.address_of_entry_point as u64,
            base_of_code: fields.base_of_code as u64,
            base_of_data: 0,
        }
    }
}

/// Standard fields magic number for 32-bit binary
pub const MAGIC_32: u16 = 0x10b;
/// Standard fields magic number for 64-bit binary
pub const MAGIC_64: u16 = 0x20b;

/// Windows specific fields
#[repr(C)]
#[derive(Debug, PartialEq, Copy, Clone, Default)]
#[derive(Pread, Pwrite, SizeWith)]
pub struct WindowsFields32 {
    pub image_base: u32,
    pub section_alignment: u32,
    pub file_alignment: u32,
    pub major_operating_system_version: u16,
    pub minor_operating_system_version: u16,
    pub major_image_version: u16,
    pub minor_image_version: u16,
    pub major_subsystem_version: u16,
    pub minor_subsystem_version: u16,
    pub win32_version_value: u32,
    pub size_of_image: u32,
    pub size_of_headers: u32,
    pub check_sum: u32,
    pub subsystem: u16,
    pub dll_characteristics: u16,
    pub size_of_stack_reserve: u32,
    pub size_of_stack_commit: u32,
    pub size_of_heap_reserve: u32,
    pub size_of_heap_commit: u32,
    pub loader_flags: u32,
    pub number_of_rva_and_sizes: u32,
}

pub const SIZEOF_WINDOWS_FIELDS_32: usize = 68;

/// 64-bit Windows specific fields
#[repr(C)]
#[derive(Debug, PartialEq, Copy, Clone, Default)]
#[derive(Pread, Pwrite, SizeWith)]
pub struct WindowsFields64 {
    pub image_base: u64,
    pub section_alignment: u32,
    pub file_alignment: u32,
    pub major_operating_system_version: u16,
    pub minor_operating_system_version: u16,
    pub major_image_version: u16,
    pub minor_image_version: u16,
    pub major_subsystem_version: u16,
    pub minor_subsystem_version: u16,
    pub win32_version_value: u32,
    pub size_of_image: u32,
    pub size_of_headers: u32,
    pub check_sum: u32,
    pub subsystem: u16,
    pub dll_characteristics: u16,
    pub size_of_stack_reserve: u64,
    pub size_of_stack_commit:  u64,
    pub size_of_heap_reserve:  u64,
    pub size_of_heap_commit:   u64,
    pub loader_flags: u32,
    pub number_of_rva_and_sizes: u32,
}

pub const SIZEOF_WINDOWS_FIELDS_64: usize = 88;

// /// Generic 32/64-bit Windows specific fields
// #[derive(Debug, PartialEq, Copy, Clone, Default)]
// pub struct WindowsFields {
//     pub image_base: u64,
//     pub section_alignment: u32,
//     pub file_alignment: u32,
//     pub major_operating_system_version: u16,
//     pub minor_operating_system_version: u16,
//     pub major_image_version: u16,
//     pub minor_image_version: u16,
//     pub major_subsystem_version: u16,
//     pub minor_subsystem_version: u16,
//     pub win32_version_value: u32,
//     pub size_of_image: u32,
//     pub size_of_headers: u32,
//     pub check_sum: u32,
//     pub subsystem: u16,
//     pub dll_characteristics: u16,
//     pub size_of_stack_reserve: u64,
//     pub size_of_stack_commit:  u64,
//     pub size_of_heap_reserve:  u64,
//     pub size_of_heap_commit:   u64,
//     pub loader_flags: u32,
//     pub number_of_rva_and_sizes: u32,
// }

impl From<WindowsFields32> for WindowsFields {
    fn from(windows: WindowsFields32) -> Self {
        WindowsFields {
            image_base: windows.image_base as u64,
            section_alignment: windows.section_alignment,
            file_alignment: windows.file_alignment,
            major_operating_system_version: windows.major_operating_system_version,
            minor_operating_system_version: windows.minor_operating_system_version,
            major_image_version: windows.major_image_version,
            minor_image_version: windows.minor_image_version,
            major_subsystem_version: windows.major_subsystem_version,
            minor_subsystem_version: windows.minor_subsystem_version,
            win32_version_value: windows.win32_version_value,
            size_of_image: windows.size_of_image,
            size_of_headers: windows.size_of_headers,
            check_sum: windows.check_sum,
            subsystem: windows.subsystem,
            dll_characteristics: windows.dll_characteristics,
            size_of_stack_reserve: windows.size_of_stack_reserve as u64,
            size_of_stack_commit: windows.size_of_stack_commit as u64,
            size_of_heap_reserve: windows.size_of_heap_reserve as u64,
            size_of_heap_commit: windows.size_of_heap_commit as u64,
            loader_flags: windows.loader_flags,
            number_of_rva_and_sizes: windows.number_of_rva_and_sizes,
        }
    }
}

// impl From<WindowsFields32> for WindowsFields {
//     fn from(windows: WindowsFields32) -> Self {
//         WindowsFields {
//             image_base: windows.image_base,
//             section_alignment: windows.section_alignment,
//             file_alignment: windows.file_alignment,
//             major_operating_system_version: windows.major_operating_system_version,
//             minor_operating_system_version: windows.minor_operating_system_version,
//             major_image_version: windows.major_image_version,
//             minor_image_version: windows.minor_image_version,
//             major_subsystem_version: windows.major_subsystem_version,
//             minor_subsystem_version: windows.minor_subsystem_version,
//             win32_version_value: windows.win32_version_value,
//             size_of_image: windows.size_of_image,
//             size_of_headers: windows.size_of_headers,
//             check_sum: windows.check_sum,
//             subsystem: windows.subsystem,
//             dll_characteristics: windows.dll_characteristics,
//             size_of_stack_reserve: windows.size_of_stack_reserve,
//             size_of_stack_commit: windows.size_of_stack_commit,
//             size_of_heap_reserve: windows.size_of_heap_reserve,
//             size_of_heap_commit: windows.size_of_heap_commit,
//             loader_flags: windows.loader_flags,
//             number_of_rva_and_sizes: windows.number_of_rva_and_sizes,
//         }
//     }
// }

pub type WindowsFields = WindowsFields64;

#[derive(Debug, PartialEq, Copy, Clone)]
pub struct OptionalHeader {
    pub standard_fields: StandardFields,
    pub windows_fields: WindowsFields,
    pub data_directories: data_directories::DataDirectories
}

impl OptionalHeader {
    pub fn container(&self) -> error::Result<container::Container> {
        match self.standard_fields.magic {
            MAGIC_32 => {
                Ok(container::Container::Little)
            },
            MAGIC_64 => {
                Ok(container::Container::Big)
            },
            magic => {
                Err(error::Error::BadMagic(magic as u64))
            }
        }
    }
}

impl<'a> ctx::TryFromCtx<'a> for OptionalHeader {
    type Error = error::Error;
    fn try_from_ctx(bytes: &'a [u8], (mut offset, _): (usize, ctx::DefaultCtx)) -> error::Result<Self> {
        let magic = bytes.pread_with::<u16>(offset, LE)?;
        let offset = &mut offset;
        let (standard_fields, windows_fields): (StandardFields, WindowsFields) = match magic {
            MAGIC_32 => {
                let standard_fields = bytes.gread_with::<StandardFields32>(offset, LE)?.into();
                let windows_fields = bytes.gread_with::<WindowsFields32>(offset, LE)?.into();
                (standard_fields, windows_fields)
            },
            MAGIC_64 => {
                let standard_fields = bytes.gread_with::<StandardFields64>(offset, LE)?.into();
                let windows_fields = bytes.gread_with::<WindowsFields64>(offset, LE)?;
                (standard_fields, windows_fields)
            },
            _ => return Err(error::Error::BadMagic(magic as u64))
        };
        let data_directories = data_directories::DataDirectories::parse(&bytes, windows_fields.number_of_rva_and_sizes as usize, offset)?;
        Ok (OptionalHeader {
            standard_fields: standard_fields,
            windows_fields: windows_fields,
            data_directories: data_directories,
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    #[test]
    fn sizeof_standards32() {
        assert_eq!(::std::mem::size_of::<StandardFields32>(), SIZEOF_STANDARD_FIELDS_32);
    }
    #[test]
    fn sizeof_windows32() {
        assert_eq!(::std::mem::size_of::<WindowsFields32>(), SIZEOF_WINDOWS_FIELDS_32);
    }
    #[test]
    fn sizeof_standards64() {
        assert_eq!(::std::mem::size_of::<StandardFields64>(), SIZEOF_STANDARD_FIELDS_64);
    }
    #[test]
    fn sizeof_windows64() {
        assert_eq!(::std::mem::size_of::<WindowsFields64>(), SIZEOF_WINDOWS_FIELDS_64);
    }
}
<|endoftext|>