<commit_before>#![crate_name = "uu_sort"]

/*
 * This file is part of the uutils coreutils package.
 *
 * (c) Michael Yin <mikeyin@mikeyin.org>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

#![allow(dead_code)]

extern crate getopts;
extern crate libc;

#[macro_use]
extern crate uucore;

use std::cmp::Ordering;
use std::fs::File;
use std::io::{BufRead, BufReader, BufWriter, Read, stdin, stdout, Write};
use std::path::Path;
use uucore::fs::is_stdin_interactive;

static NAME: &'static str = "sort";
static VERSION: &'static str = env!("CARGO_PKG_VERSION");

static DECIMAL_PT: char = '.';
static THOUSANDS_SEP: char = ',';

enum SortMode {
    Numeric,
    HumanNumeric,
    Month,
    Default,
}

struct Settings {
    mode: SortMode,
    reverse: bool,
    outfile: Option<String>,
}

impl Default for Settings {
    fn default() -> Settings {
        Settings {
            mode: SortMode::Default,
            reverse: false,
            outfile: None,
        }
    }
}

pub fn uumain(args: Vec<String>) -> i32 {
    let mut settings: Settings = Default::default();
    let mut opts = getopts::Options::new();

    opts.optflag("n", "numeric-sort", "compare according to string numerical value");
    opts.optflag("h", "human-numeric-sort", "compare according to human readable sizes, eg 1M > 100k");
    opts.optflag("M", "month-sort", "compare according to month name abbreviation");
    opts.optflag("r", "reverse", "reverse the output");
    opts.optflag("h", "help", "display this help and exit");
    opts.optflag("", "version", "output version information and exit");
    opts.optopt("o", "output", "write output to FILENAME instead of stdout", "FILENAME");

    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => crash!(1, "Invalid options\n{}", f)
    };
    if matches.opt_present("help") {
        let msg = format!("{0} {1}

Usage:
 {0} [OPTION]... [FILE]...

Write the sorted concatenation of all FILE(s) to standard output.

Mandatory arguments for long options are mandatory for short options too.

With no FILE, or when FILE is -, read standard input.", NAME, VERSION);
        print!("{}", opts.usage(&msg));
        return 0;
    }

    if matches.opt_present("version") {
        println!("{} {}", NAME, VERSION);
        return 0;
    }

    settings.mode = if matches.opt_present("numeric-sort") {
        SortMode::Numeric
    } else if matches.opt_present("human-numeric-sort") {
        SortMode::HumanNumeric
    } else if matches.opt_present("month-sort") {
        SortMode::Month
    } else {
        SortMode::Default
    };

    settings.reverse = matches.opt_present("reverse");
    settings.outfile = matches.opt_str("output");

    let mut files = matches.free;
    if files.is_empty() {
        /* if no file, default to stdin */
        files.push("-".to_owned());
    }

    exec(files, &settings);

    0
}

fn exec(files: Vec<String>, settings: &Settings) {
    for path in &files {
        let (reader, _) = match open(path) {
            Some(x) => x,
            None => continue,
        };

        let buf_reader = BufReader::new(reader);
        let mut lines = Vec::new();

        for line in buf_reader.lines() {
            match line {
                Ok(n) => {
                    lines.push(n);
                },
                _ => break
            }
        }

        match settings.mode {
            SortMode::Numeric => lines.sort_by(numeric_compare),
            SortMode::HumanNumeric => lines.sort_by(human_numeric_size_compare),
            SortMode::Month => lines.sort_by(month_compare),
            SortMode::Default => lines.sort()
        }

        let iter = lines.iter();
        if settings.reverse {
            print_sorted(iter.rev(), &settings.outfile);
        } else {
            print_sorted(iter, &settings.outfile)
        };
    }
}

/// Parse the beginning string into an f64, returning -inf instead of NaN on errors.
fn permissive_f64_parse(a: &str) -> f64{
    //Maybe should be split on non-digit, but then 10e100 won't parse properly.
    //On the flip side, this will give NEG_INFINITY for "1,234", which might be OK
    //because there's no way to handle both CSV and thousands separators without a new flag.
    //GNU sort treats "1,234" as "1" in numeric, so maybe it's fine.
    let sa: &str = a.split_whitespace().next().unwrap();
    match sa.parse::<f64>() {
        Ok(a) => a,
        Err(_) => std::f64::NEG_INFINITY
    }
}

/// Compares two floating point numbers, with errors being assumned to be -inf.
/// Stops coercing at the first whitespace char, so 1e2 will parse as 100 but
/// 1,000 will parse as -inf.
fn numeric_compare(a: &String, b: &String) -> Ordering {
    let fa = permissive_f64_parse(a);
    let fb = permissive_f64_parse(b);
    //f64::cmp isn't implemented because NaN messes with it
    //but we sidestep that with permissive_f64_parse so just fake it
    if fa > fb {
        Ordering::Greater
    }
    else if fa < fb {
        Ordering::Less
    }
    else {
        Ordering::Equal
    }
}

fn human_numeric_convert(a: &String) -> f64 {
    let int_iter = a.chars();
    let suffix_iter = a.chars();
    let int_str: String = int_iter.take_while(|c| c.is_numeric()).collect();
    let suffix = suffix_iter.skip_while(|c| c.is_numeric()).next();
    let int_part = match int_str.parse::<f64>() {
        Ok(i) => i,
        Err(_) => -1f64
    } as f64;
    let suffix: f64 = match suffix.unwrap_or('\0') {
        'K' => 1000f64,
        'M' => 1E6,
        'G' => 1E9,
        'T' => 1E12,
        'P' => 1E15,
        _ => 1f64
    };
    int_part * suffix
}

/// Compare two strings as if they are human readable sizes.
/// AKA 1M > 100k
fn human_numeric_size_compare(a: &String, b: &String) -> Ordering {
    let fa = human_numeric_convert(a);
    let fb = human_numeric_convert(b);
    if fa > fb {
        Ordering::Greater
    }
    else if fa < fb {
        Ordering::Less
    }
    else {
        Ordering::Equal
    }
}

#[derive(Eq, Ord, PartialEq, PartialOrd)]
enum Month {
    Unknown     = 0,
    January     = 1,
    February    = 2,
    March       = 3,
    April       = 4,
    May         = 5,
    June        = 6,
    July        = 7,
    August      = 8,
    September   = 9,
    October     = 10,
    November    = 11,
    December    = 12,
}

/// Parse the beginning string into a Month, returning Month::Unknown on errors.
fn month_parse(line: &String) -> Month {
    match line.split_whitespace().next().unwrap().to_uppercase().as_ref() {
        "JAN" => Month::January,
        "FEB" => Month::February,
        "MAR" => Month::March,
        "APR" => Month::April,
        "MAY" => Month::May,
        "JUN" => Month::June,
        "JUL" => Month::July,
        "AUG" => Month::August,
        "SEP" => Month::September,
        "OCT" => Month::October,
        "NOV" => Month::November,
        "DEC" => Month::December,
        _     => Month::Unknown,
    }
}

fn month_compare(a: &String, b: &String) -> Ordering {
    month_parse(a).cmp(&month_parse(b))
}

fn print_sorted<S, T: Iterator<Item=S>>(iter: T, outfile: &Option<String>) where S: std::fmt::Display {
    let mut file: Box<Write> = match *outfile {
        Some(ref filename) => {
            match File::create(Path::new(&filename)) {
                Ok(f) => Box::new(BufWriter::new(f)) as Box<Write>,
                Err(e) => {
                    show_error!("sort: {0}: {1}", filename, e.to_string());
                    panic!("Could not open output file");
                },
            }
        },
        None => Box::new(stdout()) as Box<Write>,
    };

    for line in iter {
        let str = format!("{}\n", line);
        match file.write_all(str.as_bytes()) {
            Err(e) => {
                show_error!("sort: {0}", e.to_string());
                panic!("Write failed");
            },
            Ok(_) => (),
        }
    }
}

// from cat.rs
fn open(path: &str) -> Option<(Box<Read>, bool)> {
    if path == "-" {
        let stdin = stdin();
        return Some((Box::new(stdin) as Box<Read>, is_stdin_interactive()));
    }

    match File::open(Path::new(path)) {
        Ok(f) => Some((Box::new(f) as Box<Read>, false)),
        Err(e) => {
            show_error!("sort: {0}: {1}", path, e.to_string());
            None
        },
    }
}
<commit_msg>sort: fix whitespace and spelling<commit_after>#![crate_name = "uu_sort"]

/*
 * This file is part of the uutils coreutils package.
 *
 * (c) Michael Yin <mikeyin@mikeyin.org>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

#![allow(dead_code)]

extern crate getopts;
extern crate libc;

#[macro_use]
extern crate uucore;

use std::cmp::Ordering;
use std::fs::File;
use std::io::{BufRead, BufReader, BufWriter, Read, stdin, stdout, Write};
use std::path::Path;
use uucore::fs::is_stdin_interactive;

static NAME: &'static str = "sort";
static VERSION: &'static str = env!("CARGO_PKG_VERSION");

static DECIMAL_PT: char = '.';
static THOUSANDS_SEP: char = ',';

enum SortMode {
    Numeric,
    HumanNumeric,
    Month,
    Default,
}

struct Settings {
    mode: SortMode,
    reverse: bool,
    outfile: Option<String>,
}

impl Default for Settings {
    fn default() -> Settings {
        Settings {
            mode: SortMode::Default,
            reverse: false,
            outfile: None,
        }
    }
}

pub fn uumain(args: Vec<String>) -> i32 {
    let mut settings: Settings = Default::default();
    let mut opts = getopts::Options::new();

    opts.optflag("n", "numeric-sort", "compare according to string numerical value");
    opts.optflag("h", "human-numeric-sort", "compare according to human readable sizes, eg 1M > 100k");
    opts.optflag("M", "month-sort", "compare according to month name abbreviation");
    opts.optflag("r", "reverse", "reverse the output");
    opts.optflag("h", "help", "display this help and exit");
    opts.optflag("", "version", "output version information and exit");
    opts.optopt("o", "output", "write output to FILENAME instead of stdout", "FILENAME");

    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => crash!(1, "Invalid options\n{}", f)
    };
    if matches.opt_present("help") {
        let msg = format!("{0} {1}

Usage:
 {0} [OPTION]... [FILE]...

Write the sorted concatenation of all FILE(s) to standard output.

Mandatory arguments for long options are mandatory for short options too.

With no FILE, or when FILE is -, read standard input.", NAME, VERSION);
        print!("{}", opts.usage(&msg));
        return 0;
    }

    if matches.opt_present("version") {
        println!("{} {}", NAME, VERSION);
        return 0;
    }

    settings.mode = if matches.opt_present("numeric-sort") {
        SortMode::Numeric
    } else if matches.opt_present("human-numeric-sort") {
        SortMode::HumanNumeric
    } else if matches.opt_present("month-sort") {
        SortMode::Month
    } else {
        SortMode::Default
    };

    settings.reverse = matches.opt_present("reverse");
    settings.outfile = matches.opt_str("output");

    let mut files = matches.free;
    if files.is_empty() {
        /* if no file, default to stdin */
        files.push("-".to_owned());
    }

    exec(files, &settings);

    0
}

fn exec(files: Vec<String>, settings: &Settings) {
    for path in &files {
        let (reader, _) = match open(path) {
            Some(x) => x,
            None => continue,
        };

        let buf_reader = BufReader::new(reader);
        let mut lines = Vec::new();

        for line in buf_reader.lines() {
            match line {
                Ok(n) => {
                    lines.push(n);
                },
                _ => break
            }
        }

        match settings.mode {
            SortMode::Numeric => lines.sort_by(numeric_compare),
            SortMode::HumanNumeric => lines.sort_by(human_numeric_size_compare),
            SortMode::Month => lines.sort_by(month_compare),
            SortMode::Default => lines.sort()
        }

        let iter = lines.iter();
        if settings.reverse {
            print_sorted(iter.rev(), &settings.outfile);
        } else {
            print_sorted(iter, &settings.outfile)
        };
    }
}

/// Parse the beginning string into an f64, returning -inf instead of NaN on errors.
fn permissive_f64_parse(a: &str) -> f64 {
    // Maybe should be split on non-digit, but then 10e100 won't parse properly.
    // On the flip side, this will give NEG_INFINITY for "1,234", which might be OK
    // because there's no way to handle both CSV and thousands separators without a new flag.
    // GNU sort treats "1,234" as "1" in numeric, so maybe it's fine.
    let sa: &str = a.split_whitespace().next().unwrap();
    match sa.parse::<f64>() {
        Ok(a) => a,
        Err(_) => std::f64::NEG_INFINITY
    }
}

/// Compares two floating point numbers, with errors being assumed to be -inf.
/// Stops coercing at the first whitespace char, so 1e2 will parse as 100 but
/// 1,000 will parse as -inf.
fn numeric_compare(a: &String, b: &String) -> Ordering {
    let fa = permissive_f64_parse(a);
    let fb = permissive_f64_parse(b);
    // f64::cmp isn't implemented because NaN messes with it
    // but we sidestep that with permissive_f64_parse so just fake it
    if fa > fb {
        Ordering::Greater
    }
    else if fa < fb {
        Ordering::Less
    }
    else {
        Ordering::Equal
    }
}

fn human_numeric_convert(a: &String) -> f64 {
    let int_iter = a.chars();
    let suffix_iter = a.chars();
    let int_str: String = int_iter.take_while(|c| c.is_numeric()).collect();
    let suffix = suffix_iter.skip_while(|c| c.is_numeric()).next();
    let int_part = match int_str.parse::<f64>() {
        Ok(i) => i,
        Err(_) => -1f64
    } as f64;
    let suffix: f64 = match suffix.unwrap_or('\0') {
        'K' => 1000f64,
        'M' => 1E6,
        'G' => 1E9,
        'T' => 1E12,
        'P' => 1E15,
        _ => 1f64
    };
    int_part * suffix
}

/// Compare two strings as if they are human readable sizes.
/// AKA 1M > 100k
fn human_numeric_size_compare(a: &String, b: &String) -> Ordering {
    let fa = human_numeric_convert(a);
    let fb = human_numeric_convert(b);
    if fa > fb {
        Ordering::Greater
    }
    else if fa < fb {
        Ordering::Less
    }
    else {
        Ordering::Equal
    }
}

#[derive(Eq, Ord, PartialEq, PartialOrd)]
enum Month {
    Unknown     = 0,
    January     = 1,
    February    = 2,
    March       = 3,
    April       = 4,
    May         = 5,
    June        = 6,
    July        = 7,
    August      = 8,
    September   = 9,
    October     = 10,
    November    = 11,
    December    = 12,
}

/// Parse the beginning string into a Month, returning Month::Unknown on errors.
fn month_parse(line: &String) -> Month {
    match line.split_whitespace().next().unwrap().to_uppercase().as_ref() {
        "JAN" => Month::January,
        "FEB" => Month::February,
        "MAR" => Month::March,
        "APR" => Month::April,
        "MAY" => Month::May,
        "JUN" => Month::June,
        "JUL" => Month::July,
        "AUG" => Month::August,
        "SEP" => Month::September,
        "OCT" => Month::October,
        "NOV" => Month::November,
        "DEC" => Month::December,
        _     => Month::Unknown,
    }
}

fn month_compare(a: &String, b: &String) -> Ordering {
    month_parse(a).cmp(&month_parse(b))
}

fn print_sorted<S, T: Iterator<Item=S>>(iter: T, outfile: &Option<String>) where S: std::fmt::Display {
    let mut file: Box<Write> = match *outfile {
        Some(ref filename) => {
            match File::create(Path::new(&filename)) {
                Ok(f) => Box::new(BufWriter::new(f)) as Box<Write>,
                Err(e) => {
                    show_error!("sort: {0}: {1}", filename, e.to_string());
                    panic!("Could not open output file");
                },
            }
        },
        None => Box::new(stdout()) as Box<Write>,
    };

    for line in iter {
        let str = format!("{}\n", line);
        match file.write_all(str.as_bytes()) {
            Err(e) => {
                show_error!("sort: {0}", e.to_string());
                panic!("Write failed");
            },
            Ok(_) => (),
        }
    }
}

// from cat.rs
fn open(path: &str) -> Option<(Box<Read>, bool)> {
    if path == "-" {
        let stdin = stdin();
        return Some((Box::new(stdin) as Box<Read>, is_stdin_interactive()));
    }

    match File::open(Path::new(path)) {
        Ok(f) => Some((Box::new(f) as Box<Read>, false)),
        Err(e) => {
            show_error!("sort: {0}: {1}", path, e.to_string());
            None
        },
    }
}
<|endoftext|>


<commit_before>use nom::{self, IResult, Needed};

use std::io::{self, Read};
use std::ptr;
use std::cmp;

use super::StreamProducer;

/// Represent the different kinds of errors.
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum ErrorKind {
  /// Error from I/O.
  IO(io::ErrorKind),
  /// A parser stopped midway and need more bytes to consume.
  Incomplete(usize),
  /// A parser has completes and there is still more bytes to consume.
  Continue,
  /// A parser has completes and there is no more bytes to consume.
  EndOfInput,
  /// A non-specified error.
  Unknown,
  // Parser Error
  /// Failed parsing the "fLaC" header token.
  HeaderParser,
  /// Failed parsing a metadata header.
  MetadataHeaderParser,
  /// Failed parsing the metadata block `StreamInfo`.
  StreamInfoParser,
  /// Failed parsing the metadata block `Padding`.
  PaddingParser,
  /// Failed parsing the metadata block `Application`.
  ApplicationParser,
  /// Failed parsing the metadata block `SeekTable`.
  SeekTableParser,
  /// Failed parsing the metadata block `VorbisComment`.
  VorbisCommentParser,
  /// Failed parsing the metadata block `CueSheet`.
  CueSheetParser,
  /// Failed parsing the metadata block `Picture`.
  PictureParser,
  /// Failed parsing the metadata block `Unknown`.
  UnknownParser,
  /// Failed parsing the blocking strategy inside the frame header.
  BlockingStrategyParser,
  /// Failed parsing the blocking sample inside the frame header.
  BlockingSampleParser,
  /// Failed parsing the channel bits inside the frame header.
  ChannelBitsParser,
  /// Failed parsing the UTF-8 header inside the frame header.
  UTF8HeaderParser,
  /// Failed parsing the UTF-8 body inside the frame header.
  UTF8BodyParser,
  /// Failed parsing the CRC-8 inside the frame header.
  CRC8Parser,
  /// Failed parsing the frame footer, also known as the CRC-16.
  FrameFooterParser,
  SubframeParser,
  LeadingZerosParser,
  ConstantParser,
  VerbatimParser,
  FixedParser,
  LPCParser,
  // Invalid Error
  /// A block type, base on the number, that is outside the range (0-126).
  InvalidBlockType,
  InvalidSyncCode,
  InvalidBlockSample,
  InvalidChannelBits,
  InvalidUTF8,
  InvalidCRC8,
  InvalidCRC16,
  InvalidSubframeHeader,
  // Not Found
  /// Some metadata block was not found with a specific filter.
  NotFound,
}

/// Structure that hold a slice of bytes.
pub struct ByteStream<'a> {
  offset: usize,
  bytes: &'a [u8],
}

impl<'a> ByteStream<'a> {
  /// Construct a `ByteStream` based on the passed in byte slice.
  pub fn new(bytes: &'a [u8]) -> Self {
    ByteStream {
      offset: 0,
      bytes: bytes,
    }
  }

  /// Return the number of bytes that haven't been consumed yet.
  #[inline]
  pub fn len(&self) -> usize {
    self.bytes.len() - self.offset
  }

  /// Return true if the stream contains no more bytes.
  #[inline]
  pub fn is_empty(&self) -> bool {
    self.len() == 0
  }
}

impl<'a> StreamProducer for ByteStream<'a> {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T, ErrorKind> {
    if self.is_empty() {
      return Err(ErrorKind::EndOfInput);
    }

    match f(&self.bytes[self.offset..]) {
      IResult::Done(i, o)    => {
        self.offset += self.len() - i.len();

        Ok(o)
      }
      IResult::Incomplete(n) => {
        let mut needed = self.len();

        if let Needed::Size(size) = n {
          needed = size;
        }

        Err(ErrorKind::Incomplete(needed))
      }
      IResult::Error(e)      => {
        match e {
          nom::Err::Code(k)               |
          nom::Err::Node(k, _)            |
          nom::Err::Position(k, _)        |
          nom::Err::NodePosition(k, _, _) => {
            if let nom::ErrorKind::Custom(kind) = k {
              Err(kind)
            } else {
              Err(ErrorKind::Unknown)
            }
          }
        }
      },
    }
  }
}

// Growable buffer of bytes.
//
// Mainly used to the `ReadStream` structure but can be used seperately for
// manually filling with some `Read` source.
pub struct Buffer {
  data: Vec<u8>,
  filled: usize,
  offset: usize,
}

impl Buffer {
  // Default constructor for `Buffer`
  pub fn new() -> Self {
    Self::with_capacity(1024)
  }

  // Explicitly set the buffer capacity.
  pub fn with_capacity(capacity: usize) -> Self {
    let mut buffer = Vec::with_capacity(capacity);

    unsafe {
      buffer.set_len(capacity);
    }

    Buffer {
      data: buffer,
      filled: 0,
      offset: 0,
    }
  }

  // Return the number of read bytes that haven't been consumed yet.
  #[inline]
  pub fn len(&self) -> usize {
    self.filled - self.offset
  }

  // Return true if buffer contains no more bytes.
  #[inline]
  pub fn is_empty(&self) -> bool {
    self.len() == 0
  }

  // The set length of the unlining buffer.
  #[inline]
  pub fn capacity(&self) -> usize {
    self.data.len()
  }

  // Return a reference to the slice of unread bytes.
  pub fn as_slice(&self) -> &[u8] {
    &self.data[self.offset..self.filled]
  }

  // Fill the buffer with bytes from a `Read` source.
  pub fn fill<R: Read>(&mut self, reader: &mut R) -> io::Result<usize> {
    reader.read(&mut self.data[self.filled..]).map(|consumed| {
      self.filled += consumed;

      consumed
    })
  }

  // Resize the current buffer
  //
  // This will only allocate data when the size requests is larger than the
  // current capacity of the buffer, otherwise it moves the currently filled
  // data to the beginning of the buffer.
  pub fn resize(&mut self, size: usize) {
    if size > self.data.capacity() {
      self.data.reserve(size);

      let capacity = self.data.capacity();

      unsafe {
        self.data.set_len(capacity);
      }
    }

    if self.data.len() - self.filled < size  {
      let length  = self.filled - self.offset;
      let mut_ptr = self.data.as_mut_ptr();

      unsafe {
        let offset_ptr  = self.data.as_ptr().offset(self.offset as isize);

        ptr::copy(offset_ptr, mut_ptr, length);
      }

      self.filled -= self.offset;
      self.offset  = 0;
    }
  }

  // Move the offset by the amount of consumed bytes.
  pub fn consume(&mut self, consumed: usize) {
    self.offset += consumed;
  }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ParserState {
  Incomplete,
  EndOfInput,
}

fn fill<R: Read>(buffer: &mut Buffer, reader: &mut R, needed: usize)
                 -> io::Result<usize> {
  let mut read = 0;

  if buffer.len() < needed {
    buffer.resize(needed);

    while buffer.len() < needed {
      let size_read = try!(buffer.fill(reader));

      if size_read > 0 {
        read += size_read;
      } else {
        break;
      }
    }
  }

  Ok(read)
}

/// Structure that hold a reader for a source of bytes.
pub struct ReadStream<R: Read> {
  reader: R,
  buffer: Buffer,
  needed: usize,
  state: ParserState,
}

impl<R> ReadStream<R> where R: Read {
  /// Constructor for `ReadStream` based on a `Read` source.
  pub fn new(reader: R) -> Self {
    ReadStream {
      reader: reader,
      buffer: Buffer::new(),
      needed: 0,
      state: ParserState::Incomplete,
    }
  }

  // Fill the stream with bytes from a `Read` source.
  fn fill(&mut self) -> io::Result<usize> {
    let needed = cmp::max(1, self.needed);

    fill(&mut self.buffer, &mut self.reader, needed).map(|consumed| {
      if self.buffer.len() < needed {
        self.state = ParserState::EndOfInput;
      }

      consumed
    })
  }
}

fn from_iresult<T>(buffer: &Buffer, result: IResult<&[u8], T, ErrorKind>)
                   -> Result<(usize, T), ErrorKind> {
  match result {
    IResult::Done(i, o)    => Ok((buffer.len() - i.len(), o)),
    IResult::Incomplete(n) => {
      let mut needed = buffer.capacity() + 1024;

      if let Needed::Size(size) = n {
        needed = size;
      }

      Err(ErrorKind::Incomplete(needed))
    }
    IResult::Error(e)      => {
      match e {
        nom::Err::Code(k)               |
        nom::Err::Node(k, _)            |
        nom::Err::Position(k, _)        |
        nom::Err::NodePosition(k, _, _) => {
          if let nom::ErrorKind::Custom(kind) = k {
            Err(kind)
          } else {
            Err(ErrorKind::Unknown)
          }
        }
      }
    },
  }
}

impl<R> StreamProducer for ReadStream<R> where R: Read {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T, ErrorKind> {
    if self.state != ParserState::EndOfInput {
      try!(self.fill().map_err(|e| ErrorKind::IO(e.kind())));
    }

    let mut buffer = &mut self.buffer;

    if buffer.is_empty() {
      self.state = ParserState::EndOfInput;

      return Err(ErrorKind::EndOfInput);
    }

    let result = {
      let iresult = f(buffer.as_slice());

      from_iresult(&buffer, iresult)
    };

    match result {
      Ok((consumed, o)) => {
        buffer.consume(consumed);

        Ok(o)
      }
      Err(kind)         => {
        if let ErrorKind::Incomplete(needed) = kind {
          self.needed = needed;

          Err(ErrorKind::Continue)
        } else {
          Err(kind)
        }
      }
    }
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use utility::StreamProducer;
  use nom::{self, IResult};

  #[inline]
  fn be_u32(input: &[u8]) -> IResult<&[u8], u32, ErrorKind> {
    nom::be_u32(input).map_err(to_custom_error!(Unknown))
  }

  #[test]
  fn test_buffer() {
    let mut buffer = Buffer::new();
    let bytes      = b"Hello World";
    let mut reader = &bytes[..];

    assert!(buffer.is_empty());
    assert_eq!(buffer.capacity(), 1024);

    let bytes_read = buffer.fill(&mut reader).unwrap_or(0);
    let bytes_len  = bytes.len();

    assert_eq!(bytes_read, bytes_len);
    assert_eq!(buffer.len(), bytes_len);
    assert_eq!(buffer.as_slice(), bytes);

    buffer.resize(512);
    assert_eq!(buffer.capacity(), 1024);
  }

  #[test]
  fn test_byte_stream() {
    let bytes      = b"Hello World";
    let mut stream = ByteStream::new(bytes);

    assert_eq!(stream.len(), bytes.len());

    let result = stream.parse(be_u32).unwrap_or(0);

    assert_eq!(result, 1214606444);
    assert_eq!(stream.len(), 7);
  }

  #[test]
  fn test_read_stream() {
    let bytes      = b"Hello World";
    let mut stream = ReadStream::new(&bytes[..]);

    let result = stream.parse(be_u32).unwrap_or(0);

    assert_eq!(result, 1214606444)
  }
}
<commit_msg>Add documentation for frame parser invalidity<commit_after>use nom::{self, IResult, Needed};

use std::io::{self, Read};
use std::ptr;
use std::cmp;

use super::StreamProducer;

/// Represent the different kinds of errors.
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum ErrorKind {
  /// Error from I/O.
  IO(io::ErrorKind),
  /// A parser stopped midway and need more bytes to consume.
  Incomplete(usize),
  /// A parser has completes and there is still more bytes to consume.
  Continue,
  /// A parser has completes and there is no more bytes to consume.
  EndOfInput,
  /// A non-specified error.
  Unknown,
  // Parser Error
  /// Failed parsing the "fLaC" header token.
  HeaderParser,
  /// Failed parsing a metadata header.
  MetadataHeaderParser,
  /// Failed parsing the metadata block `StreamInfo`.
  StreamInfoParser,
  /// Failed parsing the metadata block `Padding`.
  PaddingParser,
  /// Failed parsing the metadata block `Application`.
  ApplicationParser,
  /// Failed parsing the metadata block `SeekTable`.
  SeekTableParser,
  /// Failed parsing the metadata block `VorbisComment`.
  VorbisCommentParser,
  /// Failed parsing the metadata block `CueSheet`.
  CueSheetParser,
  /// Failed parsing the metadata block `Picture`.
  PictureParser,
  /// Failed parsing the metadata block `Unknown`.
  UnknownParser,
  /// Failed parsing the blocking strategy inside the frame header.
  BlockingStrategyParser,
  /// Failed parsing the blocking sample inside the frame header.
  BlockingSampleParser,
  /// Failed parsing the channel bits inside the frame header.
  ChannelBitsParser,
  /// Failed parsing the UTF-8 header inside the frame header.
  UTF8HeaderParser,
  /// Failed parsing the UTF-8 body inside the frame header.
  UTF8BodyParser,
  /// Failed parsing the CRC-8 inside the frame header.
  CRC8Parser,
  /// Failed parsing the frame footer, also known as the CRC-16.
  FrameFooterParser,
  SubframeParser,
  LeadingZerosParser,
  ConstantParser,
  VerbatimParser,
  FixedParser,
  LPCParser,
  // Invalid Error
  /// A block type, base on the number, that is outside the range (0-126).
  InvalidBlockType,
  /// An incorrect sync code with the frame header.
  InvalidSyncCode,
  /// A block sample that could cause sync-fooling.
  InvalidBlockSample,
  /// One or more bits are reserved values.
  InvalidChannelBits,
  /// An error accured in building the UTF-8 value.
  InvalidUTF8,
  /// The stored CRC-8 doesn't match the one generated from the bytes within
  /// the frame header.
  InvalidCRC8,
  /// The stored CRC-16 doesn't match the one generated from the bytes
  /// within the entire frame.
  InvalidCRC16,
  InvalidSubframeHeader,
  // Not Found
  /// Some metadata block was not found with a specific filter.
  NotFound,
}

/// Structure that hold a slice of bytes.
pub struct ByteStream<'a> {
  offset: usize,
  bytes: &'a [u8],
}

impl<'a> ByteStream<'a> {
  /// Construct a `ByteStream` based on the passed in byte slice.
  pub fn new(bytes: &'a [u8]) -> Self {
    ByteStream {
      offset: 0,
      bytes: bytes,
    }
  }

  /// Return the number of bytes that haven't been consumed yet.
  #[inline]
  pub fn len(&self) -> usize {
    self.bytes.len() - self.offset
  }

  /// Return true if the stream contains no more bytes.
  #[inline]
  pub fn is_empty(&self) -> bool {
    self.len() == 0
  }
}

impl<'a> StreamProducer for ByteStream<'a> {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T, ErrorKind> {
    if self.is_empty() {
      return Err(ErrorKind::EndOfInput);
    }

    match f(&self.bytes[self.offset..]) {
      IResult::Done(i, o)    => {
        self.offset += self.len() - i.len();

        Ok(o)
      }
      IResult::Incomplete(n) => {
        let mut needed = self.len();

        if let Needed::Size(size) = n {
          needed = size;
        }

        Err(ErrorKind::Incomplete(needed))
      }
      IResult::Error(e)      => {
        match e {
          nom::Err::Code(k)               |
          nom::Err::Node(k, _)            |
          nom::Err::Position(k, _)        |
          nom::Err::NodePosition(k, _, _) => {
            if let nom::ErrorKind::Custom(kind) = k {
              Err(kind)
            } else {
              Err(ErrorKind::Unknown)
            }
          }
        }
      },
    }
  }
}

// Growable buffer of bytes.
//
// Mainly used to the `ReadStream` structure but can be used seperately for
// manually filling with some `Read` source.
pub struct Buffer {
  data: Vec<u8>,
  filled: usize,
  offset: usize,
}

impl Buffer {
  // Default constructor for `Buffer`
  pub fn new() -> Self {
    Self::with_capacity(1024)
  }

  // Explicitly set the buffer capacity.
  pub fn with_capacity(capacity: usize) -> Self {
    let mut buffer = Vec::with_capacity(capacity);

    unsafe {
      buffer.set_len(capacity);
    }

    Buffer {
      data: buffer,
      filled: 0,
      offset: 0,
    }
  }

  // Return the number of read bytes that haven't been consumed yet.
  #[inline]
  pub fn len(&self) -> usize {
    self.filled - self.offset
  }

  // Return true if buffer contains no more bytes.
  #[inline]
  pub fn is_empty(&self) -> bool {
    self.len() == 0
  }

  // The set length of the unlining buffer.
  #[inline]
  pub fn capacity(&self) -> usize {
    self.data.len()
  }

  // Return a reference to the slice of unread bytes.
  pub fn as_slice(&self) -> &[u8] {
    &self.data[self.offset..self.filled]
  }

  // Fill the buffer with bytes from a `Read` source.
  pub fn fill<R: Read>(&mut self, reader: &mut R) -> io::Result<usize> {
    reader.read(&mut self.data[self.filled..]).map(|consumed| {
      self.filled += consumed;

      consumed
    })
  }

  // Resize the current buffer
  //
  // This will only allocate data when the size requests is larger than the
  // current capacity of the buffer, otherwise it moves the currently filled
  // data to the beginning of the buffer.
  pub fn resize(&mut self, size: usize) {
    if size > self.data.capacity() {
      self.data.reserve(size);

      let capacity = self.data.capacity();

      unsafe {
        self.data.set_len(capacity);
      }
    }

    if self.data.len() - self.filled < size  {
      let length  = self.filled - self.offset;
      let mut_ptr = self.data.as_mut_ptr();

      unsafe {
        let offset_ptr  = self.data.as_ptr().offset(self.offset as isize);

        ptr::copy(offset_ptr, mut_ptr, length);
      }

      self.filled -= self.offset;
      self.offset  = 0;
    }
  }

  // Move the offset by the amount of consumed bytes.
  pub fn consume(&mut self, consumed: usize) {
    self.offset += consumed;
  }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ParserState {
  Incomplete,
  EndOfInput,
}

fn fill<R: Read>(buffer: &mut Buffer, reader: &mut R, needed: usize)
                 -> io::Result<usize> {
  let mut read = 0;

  if buffer.len() < needed {
    buffer.resize(needed);

    while buffer.len() < needed {
      let size_read = try!(buffer.fill(reader));

      if size_read > 0 {
        read += size_read;
      } else {
        break;
      }
    }
  }

  Ok(read)
}

/// Structure that hold a reader for a source of bytes.
pub struct ReadStream<R: Read> {
  reader: R,
  buffer: Buffer,
  needed: usize,
  state: ParserState,
}

impl<R> ReadStream<R> where R: Read {
  /// Constructor for `ReadStream` based on a `Read` source.
  pub fn new(reader: R) -> Self {
    ReadStream {
      reader: reader,
      buffer: Buffer::new(),
      needed: 0,
      state: ParserState::Incomplete,
    }
  }

  // Fill the stream with bytes from a `Read` source.
  fn fill(&mut self) -> io::Result<usize> {
    let needed = cmp::max(1, self.needed);

    fill(&mut self.buffer, &mut self.reader, needed).map(|consumed| {
      if self.buffer.len() < needed {
        self.state = ParserState::EndOfInput;
      }

      consumed
    })
  }
}

fn from_iresult<T>(buffer: &Buffer, result: IResult<&[u8], T, ErrorKind>)
                   -> Result<(usize, T), ErrorKind> {
  match result {
    IResult::Done(i, o)    => Ok((buffer.len() - i.len(), o)),
    IResult::Incomplete(n) => {
      let mut needed = buffer.capacity() + 1024;

      if let Needed::Size(size) = n {
        needed = size;
      }

      Err(ErrorKind::Incomplete(needed))
    }
    IResult::Error(e)      => {
      match e {
        nom::Err::Code(k)               |
        nom::Err::Node(k, _)            |
        nom::Err::Position(k, _)        |
        nom::Err::NodePosition(k, _, _) => {
          if let nom::ErrorKind::Custom(kind) = k {
            Err(kind)
          } else {
            Err(ErrorKind::Unknown)
          }
        }
      }
    },
  }
}

impl<R> StreamProducer for ReadStream<R> where R: Read {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T, ErrorKind> {
    if self.state != ParserState::EndOfInput {
      try!(self.fill().map_err(|e| ErrorKind::IO(e.kind())));
    }

    let mut buffer = &mut self.buffer;

    if buffer.is_empty() {
      self.state = ParserState::EndOfInput;

      return Err(ErrorKind::EndOfInput);
    }

    let result = {
      let iresult = f(buffer.as_slice());

      from_iresult(&buffer, iresult)
    };

    match result {
      Ok((consumed, o)) => {
        buffer.consume(consumed);

        Ok(o)
      }
      Err(kind)         => {
        if let ErrorKind::Incomplete(needed) = kind {
          self.needed = needed;

          Err(ErrorKind::Continue)
        } else {
          Err(kind)
        }
      }
    }
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use utility::StreamProducer;
  use nom::{self, IResult};

  #[inline]
  fn be_u32(input: &[u8]) -> IResult<&[u8], u32, ErrorKind> {
    nom::be_u32(input).map_err(to_custom_error!(Unknown))
  }

  #[test]
  fn test_buffer() {
    let mut buffer = Buffer::new();
    let bytes      = b"Hello World";
    let mut reader = &bytes[..];

    assert!(buffer.is_empty());
    assert_eq!(buffer.capacity(), 1024);

    let bytes_read = buffer.fill(&mut reader).unwrap_or(0);
    let bytes_len  = bytes.len();

    assert_eq!(bytes_read, bytes_len);
    assert_eq!(buffer.len(), bytes_len);
    assert_eq!(buffer.as_slice(), bytes);

    buffer.resize(512);
    assert_eq!(buffer.capacity(), 1024);
  }

  #[test]
  fn test_byte_stream() {
    let bytes      = b"Hello World";
    let mut stream = ByteStream::new(bytes);

    assert_eq!(stream.len(), bytes.len());

    let result = stream.parse(be_u32).unwrap_or(0);

    assert_eq!(result, 1214606444);
    assert_eq!(stream.len(), 7);
  }

  #[test]
  fn test_read_stream() {
    let bytes      = b"Hello World";
    let mut stream = ReadStream::new(&bytes[..]);

    let result = stream.parse(be_u32).unwrap_or(0);

    assert_eq!(result, 1214606444)
  }
}
<|endoftext|>


<commit_before>use std::iter::repeat;

fn max(a: u32, b: u32) -> u32 { if a > b { a } else { b } }

#[derive(Clone)]
struct Edge {
    vertex: u32,
    weight: f32,
}

impl Edge {
    pub fn new(vertex_: u32, weight_: f32) -> Edge {
        Edge{vertex: vertex_, weight: weight_,}
    }
}

struct Graph {
    nodes: Vec<Vec<Edge>>,
}

impl Graph {
    pub fn new() -> Graph { Graph{nodes: Vec::new(),} }

    pub fn add_edge(&mut self, src: u32, dst: u32, weight: f32) {
        // only works with rust nightly:
        // if (max(src,dst)) > self.nodes.len() as u32 {
        //  self.nodes.resize((max(src, dst) + 1) as usize, Vec::new());
        //}

        let len = self.nodes.len();
        if (max(src, dst)) > len as u32 {
            let new_len = (max(src, dst) + 1) as usize;
            self.nodes.extend(repeat(Vec::new()).take(new_len - len))
        }
        self.nodes[src as usize].push(Edge::new(dst, weight));
    }
    pub fn bfs(&self, src: u32) {
        let mut current: u32 = src;
        let mut stack: Vec<u32> = Vec::new();
        let mut visited: Vec<bool> = Vec::new();

        // only works with rust nightly:
        // visited.resize(self.nodes.len(), false);

        let len = self.nodes.len() as usize;
        visited.extend(repeat(false).take(len));
        stack.push(current);

        while stack.len() > 0 {
            current = stack[0 as usize];
            stack.remove(0);
            if !visited[current as usize] {
                println!("current: {}" , current);
                visited[current as usize] = true;
            }

            for n in &self.nodes[current as usize] {
                let neighbor: u32 = n.vertex;
                if !visited[neighbor as usize] { stack.push(neighbor); }
            }
        }
    }
}

fn main() {
    let mut g1 = Graph::new();
    g1.add_edge(0, 1, 1.0);
    g1.add_edge(0, 2, 4.0);
    g1.add_edge(1, 0, 1.0);
    g1.add_edge(1, 2, 2.0);
    g1.add_edge(1, 3, 6.0);
    g1.add_edge(2, 0, 4.0);
    g1.add_edge(2, 1, 2.0);
    g1.add_edge(2, 3, 3.0);
    g1.add_edge(3, 1, 6.0);
    g1.add_edge(3, 2, 3.0);
    g1.bfs(0);
}<commit_msg>added dfs<commit_after>use std::iter::repeat;
use std::collections::VecDeque;

fn max(a: u32, b: u32) -> u32 { if a > b { a } else { b } }

#[derive(Clone)]
struct Edge {
    vertex: u32,
    weight: f32,
}

impl Edge {
    pub fn new(vertex_: u32, weight_: f32) -> Edge {
        Edge{vertex: vertex_, weight: weight_,}
    }
}

struct Graph {
    nodes: Vec<Vec<Edge>>,
}

impl Graph {
    pub fn new() -> Graph { Graph{nodes: Vec::new(),} }

    pub fn add_edge(&mut self, src: u32, dst: u32, weight: f32) {
        // only works with rust nightly:
        // if (max(src,dst)) > self.nodes.len() as u32 {
        //  self.nodes.resize((max(src, dst) + 1) as usize, Vec::new());
        //}

        let len = self.nodes.len();
        if (max(src, dst)) > len as u32 {
            let new_len = (max(src, dst) + 1) as usize;
            self.nodes.extend(repeat(Vec::new()).take(new_len - len))
        }
        self.nodes[src as usize].push(Edge::new(dst, weight));
    }
    pub fn bfs(&self, src: u32) {
        let mut current: u32 = src;
        let mut queue: VecDeque<u32> = VecDeque::new();
        let mut visited: Vec<bool> = Vec::new();

        // only works with rust nightly:
        // visited.resize(self.nodes.len(), false);

        let len = self.nodes.len() as usize;
        visited.extend(repeat(false).take(len));
        queue.push_front(current);

        while queue.len() > 0 {
            current = queue.pop_back();
            if !visited[current as usize] {
                println!("current: {}" , current);
                visited[current as usize] = true;
            }

            for n in &self.nodes[current as usize] {
                let neighbor: u32 = n.vertex;
                if !visited[neighbor as usize] { queue.push_front(neighbor); }
            }
        }
    }
    pub fn dfs(&self, src: u32) {
        let mut current: u32 = src;
        let mut stack: Vec<u32> = Vec::new();
        let mut visited: Vec<bool> = Vec::new();

        // only works with rust nightly:
        // visited.resize(self.nodes.len(), false);

        let len = self.nodes.len() as usize;
        visited.extend(repeat(false).take(len));
        stack.push(current);

        while stack.len() > 0 {
            current = stack[0 as usize];
            stack.remove(0);
            if !visited[current as usize] {
                println!("current: {}" , current);
                visited[current as usize] = true;
            }

            for n in &self.nodes[current as usize] {
                let neighbor: u32 = n.vertex;
                if !visited[neighbor as usize] { stack.push(neighbor); }
            }
        }
    }
}

fn main() {
    let mut g1 = Graph::new();
    g1.add_edge(0, 1, 1.0);
    g1.add_edge(0, 2, 4.0);
    g1.add_edge(1, 0, 1.0);
    g1.add_edge(1, 2, 2.0);
    g1.add_edge(1, 3, 6.0);
    g1.add_edge(2, 0, 4.0);
    g1.add_edge(2, 1, 2.0);
    g1.add_edge(2, 3, 3.0);
    g1.add_edge(3, 1, 6.0);
    g1.add_edge(3, 2, 3.0);
    g1.bfs(0);
}<|endoftext|>


<commit_before>use std::iter::repeat;

fn max(a: u32, b: u32) -> u32 { if a > b { a } else { b } }

#[derive(Clone)]
struct Edge {
    vertex: u32,
    weight: f32,
}

impl Edge {
    pub fn new(vertex_: u32, weight_: f32) -> Edge {
        Edge{vertex: vertex_, weight: weight_,}
    }
}

struct Graph {
    nodes: Vec<Vec<Edge>>,
}

impl Graph {
    pub fn new() -> Graph { Graph{nodes: Vec::new(),} }

    pub fn add_edge(&mut self, src: u32, dst: u32, weight: f32) {
        // only works with rust nightly:
        // if (max(src,dst)) > self.nodes.len() as u32 {
        //  self.nodes.resize((max(src, dst) + 1) as usize, Vec::new());
        //}

        let len = self.nodes.len();
        if (max(src, dst)) > len as u32 {
            use std::iter::repeat;
            let new_len = (max(src, dst) + 1) as usize;
            self.nodes.extend(repeat(Vec::new()).take(new_len - len))
        }
        self.nodes[src as usize].push(Edge::new(dst, weight));
    }
    pub fn bfs(&self, src: u32) {
        let mut current: u32 = src;
        let mut stack: Vec<u32> = Vec::new();
        let mut visited: Vec<bool> = Vec::new();

        // only works with rust nightly:
        // visited.resize(self.nodes.len(), false);

        let len = self.nodes.len() as usize;
        visited.extend(repeat(false).take(len));
        stack.push(current);

        while stack.len() > 0 {
            current = stack[0 as usize];
            stack.remove(0);
            if !visited[current as usize] {
                println!("current: {}" , current);
                visited[current as usize] = true;
            }

            for n in &self.nodes[current as usize] {
                let neighbor: u32 = n.vertex;
                if !visited[neighbor as usize] { stack.push(neighbor); }
            }
        }
    }
}

fn main() {
    let mut g1 = Graph::new();
    g1.add_edge(0, 1, 1.0);
    g1.add_edge(0, 2, 4.0);
    g1.add_edge(1, 0, 1.0);
    g1.add_edge(1, 2, 2.0);
    g1.add_edge(1, 3, 6.0);
    g1.add_edge(2, 0, 4.0);
    g1.add_edge(2, 1, 2.0);
    g1.add_edge(2, 3, 3.0);
    g1.add_edge(3, 1, 6.0);
    g1.add_edge(3, 2, 3.0);
    g1.bfs(0);
}<commit_msg>removed code<commit_after>use std::iter::repeat;

fn max(a: u32, b: u32) -> u32 { if a > b { a } else { b } }

#[derive(Clone)]
struct Edge {
    vertex: u32,
    weight: f32,
}

impl Edge {
    pub fn new(vertex_: u32, weight_: f32) -> Edge {
        Edge{vertex: vertex_, weight: weight_,}
    }
}

struct Graph {
    nodes: Vec<Vec<Edge>>,
}

impl Graph {
    pub fn new() -> Graph { Graph{nodes: Vec::new(),} }

    pub fn add_edge(&mut self, src: u32, dst: u32, weight: f32) {
        // only works with rust nightly:
        // if (max(src,dst)) > self.nodes.len() as u32 {
        //  self.nodes.resize((max(src, dst) + 1) as usize, Vec::new());
        //}

        let len = self.nodes.len();
        if (max(src, dst)) > len as u32 {
            let new_len = (max(src, dst) + 1) as usize;
            self.nodes.extend(repeat(Vec::new()).take(new_len - len))
        }
        self.nodes[src as usize].push(Edge::new(dst, weight));
    }
    pub fn bfs(&self, src: u32) {
        let mut current: u32 = src;
        let mut stack: Vec<u32> = Vec::new();
        let mut visited: Vec<bool> = Vec::new();

        // only works with rust nightly:
        // visited.resize(self.nodes.len(), false);

        let len = self.nodes.len() as usize;
        visited.extend(repeat(false).take(len));
        stack.push(current);

        while stack.len() > 0 {
            current = stack[0 as usize];
            stack.remove(0);
            if !visited[current as usize] {
                println!("current: {}" , current);
                visited[current as usize] = true;
            }

            for n in &self.nodes[current as usize] {
                let neighbor: u32 = n.vertex;
                if !visited[neighbor as usize] { stack.push(neighbor); }
            }
        }
    }
}

fn main() {
    let mut g1 = Graph::new();
    g1.add_edge(0, 1, 1.0);
    g1.add_edge(0, 2, 4.0);
    g1.add_edge(1, 0, 1.0);
    g1.add_edge(1, 2, 2.0);
    g1.add_edge(1, 3, 6.0);
    g1.add_edge(2, 0, 4.0);
    g1.add_edge(2, 1, 2.0);
    g1.add_edge(2, 3, 3.0);
    g1.add_edge(3, 1, 6.0);
    g1.add_edge(3, 2, 3.0);
    g1.bfs(0);
}<|endoftext|>


<commit_before>// Copyright (c) 2015 by Stacy Prowell.  All rights reserved.
//
// Licensed under the BSD 2-Clause license.  See the file LICENSE
// that is part of this distribution.  This file may not be copied,
// modified, or distributed except according to those terms.extern

//! Provide basic definitions of terms.

use num::BigInt;
use std::rc::Rc;

mod root;

/// Export the kinds of terms.
pub use self::TermKind::{RootKind};

/// Represent the kind of a term.
pub enum TermKind {
	/// The special root type.
	RootKind,
}

trait Term {
	fn get_type(&self) -> Rc<Term>;
	fn get_kind(&self) -> TermKind;
}<commit_msg>Commented out the term stuff for now, until we have a better idea how to do this.<commit_after>// Copyright (c) 2015 by Stacy Prowell.  All rights reserved.
//
// Licensed under the BSD 2-Clause license.  See the file LICENSE
// that is part of this distribution.  This file may not be copied,
// modified, or distributed except according to those terms.extern

//! Provide basic definitions of terms.

//use num::BigInt;
use std::rc::Rc;

//mod root;

/// Export the kinds of terms.
pub use self::TermKind::{RootKind};

/// Represent the kind of a term.
#[allow(dead_code)]
pub enum TermKind {
	/// The special root type.
	RootKind,
}

/// Functionality common to all terms.
trait Term {
	/// Get the type of the term.
	fn get_type(&self) -> Rc<Term>;
}<|endoftext|>


<commit_before>extern mod stem;

use std::io;
use std::path;

#[test]
fn lexicon() {
    match io::file_reader(&path::PosixPath("src/test/voc.txt")) {
        Ok(input) => match io::file_reader(&path::PosixPath("src/test/output.txt")) {
            Ok(result) => {
                do input.each_line |word| {
                    match stem::get(word) {
                        Ok(stem) => {
                            let answer = result.read_line();
                            if stem != answer {
                                printfln!("\n[FAILED] '%s' != '%s'", stem, answer);
                            } else {
                                print(".");
                            }
                        },
                        Err(e) => printfln!("\n[FAILED] Cannot get stem for '%s': %s", word, e),
                    }
                    true
                };
            },
            Err(e) => println(e),
        },
        Err(e) => println(e),
    }
    println("");
}
<commit_msg>switched to println!<commit_after>extern mod stem;

use std::io;
use std::path;

#[test]
fn lexicon() {
    match io::file_reader(&path::PosixPath("src/test/voc.txt")) {
        Ok(input) => match io::file_reader(&path::PosixPath("src/test/output.txt")) {
            Ok(result) => {
                do input.each_line |word| {
                    match stem::get(word) {
                        Ok(stem) => {
                            let answer = result.read_line();
                            if stem != answer {
                                println!("\n[FAILED] '{:s}' != '{:s}'", stem, answer);
                            } else {
                                print(".");
                            }
                        },
                        Err(e) => println!("\n[FAILED] Cannot get stem for '{:s}': {:s}", word, e),
                    }
                    true
                };
            },
            Err(e) => println(e),
        },
        Err(e) => println(e),
    }
    println("");
}
<|endoftext|>


<commit_before>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: i64,
        next: Option<Rc<Token>>
    }
}

impl Token {

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ident: ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> i64 {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    return Ok(Token::Identifier {
        ident: query,
        next: None,
    });
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    quickcheck! {
        fn test_array_index(i: i64) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<commit_msg>Impl tokenize_with_seperator to fix test_tokenize_double_token_query<commit_after>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: i64,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    #[cfg(test)]
    pub fn identifier(&self) -> &String {
        match self {
            &Token::Identifier { ident: ref ident, .. } => &ident,
            _ => unreachable!(),
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> i64 {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                let token = String::from(token);

                let mut token = Token::Identifier {
                    ident: token,
                    next: None,
                };

                build_token_tree(split, &mut token);
                last.set_next(token);
            }
        }
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            let mut tok = Token::Identifier { ident: String::from(token), next: None };
            build_token_tree(&mut tokens, &mut tok);
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    quickcheck! {
        fn test_array_index(i: i64) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<|endoftext|>


<commit_before>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    InvalidSymbol,
    ScannerError(String),
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) -> result::Result<(), &str> {
        loop {
            let char = self.scanner.get_char();
            if char.is_none() {
                break;
            }

            let char = char.unwrap();
            if !char.is_whitespace() || char == EOL_CHAR {
                try!(self.scanner.unget_char());
                break;
            }
        }

        Ok(())
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        if let Err(message) = self.ignore_whitespace() {
            return Some(Err(TokenError::ScannerError(message.to_owned())));
        }

        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert!(tokenizer.next().unwrap().unwrap() == Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert!(tokenizer.next().unwrap().unwrap() == Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert!(tokenizer.next().unwrap().unwrap() == ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert!(tokenizer.next().unwrap().unwrap() == ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for("\"");
        assert!(tokenizer.next().unwrap().unwrap() == StringBoundary);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert!(tokenizer.next().unwrap().unwrap() == EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$1111^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::InvalidSymbol);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_invalid_characters() {
        let symbol = "]";
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::UnrecognizedCharacter(']'));
        assert!(tokenizer.next().is_none());

        // Make sure invalid character is caught among valid characters too
        let symbol = "12[3abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::UnrecognizedCharacter('['));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("\"x + y\"\"\"\"");
        let expected = [StringBoundary, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), StringBoundary, StringBoundary, StringBoundary, StringBoundary];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh#*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }
}

<commit_msg>Renamed accidentally duplicated function<commit_after>use std::iter::Iterator;
use std::result;
use std::collections::HashSet;

use token::Token;
use scanner::Scanner;

const EOL_CHAR: char = '\n';

lazy_static! {
    static ref SYMBOL_CHARS: HashSet<char> = vec![
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',
        'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
        'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
        'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
        'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7',
        '8', '9', '-', '_', '$', '^', '&', '*', '!', '@', '%', '+',
        '?', '<', '>', '.', ':', '/', '|', '~', ',', '='
    ].into_iter().collect();
}

#[derive(Debug, Eq, PartialEq)]
pub enum TokenError {
    UnrecognizedCharacter(char),
    InvalidSymbol,
    ScannerError(String),
}

pub type TokenResult = result::Result<Token, TokenError>;

pub struct Tokenizer {
    scanner: Scanner,
    reached_eof: bool,
}

impl Tokenizer {
    pub fn new(scanner: Scanner) -> Tokenizer {
        Tokenizer {
            scanner: scanner,
            reached_eof: false,
        }
    }

    fn ignore_whitespace(&mut self) -> result::Result<(), &str> {
        loop {
            let char = self.scanner.get_char();
            if char.is_none() {
                break;
            }

            let char = char.unwrap();
            if !char.is_whitespace() || char == EOL_CHAR {
                try!(self.scanner.unget_char());
                break;
            }
        }

        Ok(())
    }
}

impl Iterator for Tokenizer {
    type Item = TokenResult;

    fn next(&mut self) -> Option<Self::Item> {
        if self.reached_eof {
            return None;
        }

        if let Err(message) = self.ignore_whitespace() {
            return Some(Err(TokenError::ScannerError(message.to_owned())));
        }

        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::Token::*;
    use scanner::Scanner;

    #[test]
    fn backslash() {
        let mut tokenizer = tokenizer_for("\\");
        assert!(tokenizer.next().unwrap().unwrap() == Backslash);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn equals() {
        let mut tokenizer = tokenizer_for("=");
        assert!(tokenizer.next().unwrap().unwrap() == Equals);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_open() {
        let mut tokenizer = tokenizer_for("(");
        assert!(tokenizer.next().unwrap().unwrap() == ParenOpen);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn paren_close() {
        let mut tokenizer = tokenizer_for(")");
        assert!(tokenizer.next().unwrap().unwrap() == ParenClose);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn string_boundary() {
        let mut tokenizer = tokenizer_for("\"");
        assert!(tokenizer.next().unwrap().unwrap() == StringBoundary);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn eol() {
        let mut tokenizer = tokenizer_for("\n");
        assert!(tokenizer.next().unwrap().unwrap() == EOL);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_multiple_tokens_separated_by_whitespace() {
        let mut tokenizer = tokenizer_for("\\ ) \n ( \\ = = )");
        let expected = [Backslash, ParenClose, EOL, ParenOpen, Backslash, Equals, Equals, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_characters_into_symbols() {
        let symbol = "abqq$$1111^&/|!";
        let mut tokenizer = tokenizer_for(symbol);
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn groups_equals_into_symbols_when_there_is_no_whitespace() {
        let symbol = "=ifsaoi=hfsdMMDS,,,,~~~:=".to_owned();

        // Make sure leading whitespace does not effect the output
        let mut tokenizer = tokenizer_for(&format!("  {}", symbol));

        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_symbols_starting_with_numbers() {
        let symbol = "123abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::InvalidSymbol);
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn disallows_invalid_characters() {
        let symbol = "]";
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::UnrecognizedCharacter(']'));
        assert!(tokenizer.next().is_none());

        // Make sure invalid character is caught among valid characters too
        let symbol = "12[3abc~".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap_err() == TokenError::UnrecognizedCharacter('['));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn accepts_all_valid_symbol_characters_into_a_symbol() {
        let symbol = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_$^&*!@%+?<>.:/|~,=".to_owned();
        let mut tokenizer = tokenizer_for(&symbol);
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol));
        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_parenthesis_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("(x + y)())");
        let expected = [ParenOpen, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), ParenClose, ParenOpen, ParenClose, ParenClose];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn recognizes_string_boundaries_regardless_of_whitespace() {
        let mut tokenizer = tokenizer_for("\"x + y\"\"\"\"");
        let expected = [StringBoundary, Symbol("x".to_owned()), Symbol("+".to_owned()), Symbol("y".to_owned()), StringBoundary, StringBoundary, StringBoundary, StringBoundary];

        for token in expected.into_iter() {
            assert!(tokenizer.next().unwrap().unwrap() == *token);
        }

        assert!(tokenizer.next().is_none());
    }

    #[test]
    fn ignores_leading_whitespace() {
        let symbol = "oh#*$ho".to_owned();
        let mut tokenizer = tokenizer_for(&format!("  \t\t  \t {}", symbol));
        assert!(tokenizer.next().unwrap().unwrap() == Symbol(symbol.to_owned()));
        assert!(tokenizer.next().is_none());
    }

    fn tokenizer_for(string: &str) -> Tokenizer {
        Tokenizer::new(Scanner::from_str(string))
    }
}

<|endoftext|>