<commit_before>//! Implementation and structs for a sequencefile reader

use byteorder::{BigEndian, ByteOrder, ReadBytesExt};
use std::io;
use std::io::prelude::*;
use std::collections::HashMap;
use std::str;
use errors::{Error, Result};
use {ByteString, Header};
use compress;
use compress::CompressionType;

const MAGIC: &'static str = "SEQ";
const SYNC_SIZE: usize = 16;

/// Provides a streaming interface fronted by an Iterator
/// Only buffers when `CompressionType::Block` is used.
#[derive(Debug)]
pub struct Reader<R: io::Read> {
    header: Header,
    reader: R,
    block_buffer: Vec<(ByteString, ByteString)>,
}

impl<R: io::Read> Reader<R> {
    /// Create a new Reader from an io::Read
    ///
    /// # Failures
    /// Returns an `Error` if sequencefile header is malformed, e.g. unsupported version or
    /// invalid compression algorithm
    pub fn new(mut r: R) -> Result<Reader<R>> {
        // TODO: handle this
        let header = try!(read_header(&mut r));

        Ok(Reader {
            header: header,
            reader: r,
            block_buffer: Vec::new(),
        })
    }
}

fn read_header<R: io::Read>(reader: &mut R) -> Result<Header> {
    let mut magic = [0; 3];
    try!(reader.read(&mut magic));
    if magic != MAGIC.as_bytes() {
        return Err(Error::BadMagic(String::from_utf8_lossy(&magic).to_string()));
    }

    let mut version = [0; 1];
    try!(reader.read(&mut version));
    let version = version[0] as u16;

    // Version 4 - block compression
    // Version 5 - custom compression codecs
    // Version 6 - metadata
    if version < 5 || version > 6 {
        return Err(Error::VersionNotSupported(version));
    }

    let key_class = try!(read_string(reader));
    let value_class = try!(read_string(reader));

    let mut flags = [0; 2];
    try!(reader.read(&mut flags));

    let compression_type: CompressionType = {
        // first byte: compression t/f
        // second byte: block t/f
        match (flags[0], flags[1]) {
            (1, 1) => CompressionType::Block,
            (1, 0) => CompressionType::Record,
            (0, 0) => CompressionType::None,
            _ => {
                return Err(Error::CompressionTypeUnknown("undefined compression type".to_string()))
            }
        }
    };

    match compression_type {
        CompressionType::Block => {
            return Err(Error::CompressionTypeUnknown("block compression not yet implemented"
                                                         .to_string()))
        }
        _ => (),
    };

    let compression_codec = if compression_type != CompressionType::None {
        let codec = try!(read_string(reader));

        match compress::codec(&codec) {
            Some(codec) => Some(codec),
            None => return Err(Error::UnsupportedCodec(codec)),
        }
    } else {
        None
    };

    let pairs = try!(reader.read_u32::<BigEndian>());
    for _ in 0..pairs {
        // TODO: do stuff
    }

    let mut sync_marker = [0; SYNC_SIZE];
    try!(reader.read(&mut sync_marker));

    Ok(Header {
        version: version,
        compression_type: compression_type,
        compression_codec: compression_codec,
        key_class: key_class,
        value_class: value_class,
        metadata: HashMap::new(), // TODO
        sync_marker: sync_marker.to_vec(),
    })
}

impl<R: io::Read> Iterator for Reader<R> {
    type Item = (ByteString, ByteString);

    fn next(&mut self) -> Option<(ByteString, ByteString)> {
        let mut last_sync_marker = [0; SYNC_SIZE];
        let mut kv_length = to_opt!(self.reader.read_i32::<BigEndian>()) as isize;
        // ^ todo: Varint reader

        // handle sync marker
        if kv_length == -1 {
            to_opt!(self.reader.read(&mut last_sync_marker));
            if last_sync_marker.to_vec() != self.header.sync_marker {
                panic!(Error::SyncMarkerMismatch);
            }

            kv_length = to_opt!(self.reader.read_i32::<BigEndian>()) as isize;

            if kv_length == -1 {
                return None;
            }
        }

        if self.header.compression_type == CompressionType::Block {
            if self.block_buffer.len() == 0 {
                unimplemented!();
            } else {
                return self.block_buffer.pop();
            }
        }

        let (key, value) = to_opt!(read_kv(kv_length, &self.header, &mut self.reader));

        Some((key, value))
    }
}

fn read_kv<R: io::Read>(kv_length: isize,
                        header: &Header,
                        reader: &mut R)
                        -> Result<(Vec<u8>, Vec<u8>)> {
    let k_length = try!(reader.read_i32::<BigEndian>()) as usize;

    let k_start = 0;
    let k_end = k_start + (k_length - 0);
    let v_start = k_end;
    let v_end = v_start + (kv_length as usize - k_length);

    // TODO: DataInput/DataOutput semantics per type
    // e.g. impl LongWritable...
    // re-implement common writables
    // core interface: Iterator<(u[8], u[8])> or a KV struct
    // mixin interface on KVstruct could extract key/value from
    // bytes
    // debug!("k_start: {:?}, k_end: {:?}, v_start: {:?}, v_end: {:?}",
    //          k_start,
    //          k_end,
    //          v_start,
    //          v_end);
    //
    // debug!("{:?}", header);

    let mut buffer = vec![0; kv_length as usize];
    try!(reader.read(&mut buffer));

    let key = buffer[k_start..k_end].to_vec();

    if header.compression_type == CompressionType::Record {
        if let Some(ref codec) = header.compression_codec {
            let decompressed = try!(compress::decompressor(codec, &buffer[v_start..v_end]));

            Ok((key, decompressed))
        } else {
            panic!("WAT")
        }
    } else {
        let value = buffer[v_start..v_end].to_vec();

        Ok((key, value))
    }
}

fn read_string<R: io::Read>(reader: &mut R) -> Result<String> {
    // read one byte, value len
    let value_length = try!(reader.read_u8()) as usize;
    let mut string = vec![0; value_length];

    try!(reader.read(&mut string));
    str::from_utf8(&string).map(|v| v.to_owned()).map_err(|e| Error::BadEncoding(e))
}

#[cfg(test)]
mod tests {
    use reader;
    use errors::Result;
    use std::path::Path;
    use std::fs::File;

    use byteorder::{BigEndian, ByteOrder};

    #[test]
    fn reads_standard_sequencefile() {
        let kvs = main_read("test_data/abc_long_text_none.seq").unwrap();

        assert_eq!(26, kvs.len());
        assert_eq!((0, "A".to_string()), kvs[0]);
        assert_eq!((25, "Z".to_string()), kvs[25]);
    }

    #[test]
    fn reads_deflate_record() {
        let kvs = main_read("test_data/abc_long_text_deflate_record.seq").unwrap();

        assert_eq!(26, kvs.len());
        assert_eq!((0, "A".to_string()), kvs[0]);
        assert_eq!((25, "Z".to_string()), kvs[25]);
    }

    #[test]
    fn reads_gzip_record() {
        let kvs = main_read("test_data/abc_long_text_gzip_record.seq").unwrap();

        assert_eq!(26, kvs.len());
        assert_eq!((0, "A".to_string()), kvs[0]);
        assert_eq!((25, "Z".to_string()), kvs[25]);
    }

    #[test]
    fn reads_bzip2_record() {
        let kvs = main_read("test_data/abc_long_text_bzip2_record.seq").unwrap();

        assert_eq!(26, kvs.len());
        assert_eq!((0, "A".to_string()), kvs[0]);
        assert_eq!((25, "Z".to_string()), kvs[25]);
    }

    #[test]
    #[should_panic(expected = "unsupported codec")]
    fn reads_snappy_record() {
        match main_read("test_data/abc_long_text_snappy_record.seq") {
            Ok(val) => val,
            Err(err) => panic!("Failed to open sequence file: {}", err),
        };
    }

    #[test]
    #[should_panic(expected = "not yet implemented")]
    fn reads_deflate_block() {
        match main_read("test_data/abc_long_text_deflate_block.seq") {
            Ok(val) => val,
            Err(err) => panic!("Failed to open sequence file: {}", err),
        };
    }

    #[test]
    #[should_panic(expected = "not yet implemented")]
    fn reads_gzip_block() {
        match main_read("test_data/abc_long_text_gzip_block.seq") {
            Ok(val) => val,
            Err(err) => panic!("Failed to open sequence file: {}", err),
        };
    }

    #[test]
    #[should_panic(expected = "not yet implemented")]
    fn reads_bzip2_block() {
        match main_read("test_data/abc_long_text_bzip2_block.seq") {
            Ok(val) => val,
            Err(err) => panic!("Failed to open sequence file: {}", err),
        };
    }

    #[test]
    #[should_panic(expected = "not yet implemented")]
    fn reads_snappy_block() {
        match main_read("test_data/abc_long_text_snappy_block.seq") {
            Ok(val) => val,
            Err(err) => panic!("Failed to open sequence file: {}", err),
        };
    }

    #[test]
    #[should_panic(expected = "bad or missing magic")]
    fn read_checks_magic() {
        match main_read("test_data/bad_magic.seq") {
            Ok(val) => val,
            Err(err) => panic!("Failed to open sequence file: {}", err),
        };
    }

    fn main_read(filename: &str) -> Result<Vec<(i64, String)>> {
        let path = Path::new(filename);

        let file = try!(File::open(&path));
        let seqfile = try!(reader::Reader::new(file));

        let kvs = seqfile.map(|(key, value)| {
            (BigEndian::read_i64(&key),
             String::from_utf8_lossy(&value[2..value.len()]).to_string())
        });

        Ok(kvs.collect())
    }
}
<commit_msg>reader: total WIP implementation of block compression<commit_after>//! Implementation and structs for a sequencefile reader

use byteorder::{BigEndian, ByteOrder, ReadBytesExt};
use std::io;
use std::io::prelude::*;
use std::collections::HashMap;
use std::str;
use errors::{Error, Result};
use {ByteString, Header};
use compress;
use compress::CompressionType;
use util::ZeroCompress;

const MAGIC: &'static str = "SEQ";
const SYNC_SIZE: usize = 16;

/// Provides a streaming interface fronted by an Iterator
/// Only buffers when `CompressionType::Block` is used.
#[derive(Debug)]
pub struct Reader<R: io::Read> {
    header: Header,
    reader: R,
    block_buffer: Vec<(ByteString, ByteString)>,
}

impl<R: io::Read> Reader<R> {
    /// Create a new Reader from an io::Read
    ///
    /// # Failures
    /// Returns an `Error` if sequencefile header is malformed, e.g. unsupported version or
    /// invalid compression algorithm
    pub fn new(mut r: R) -> Result<Reader<R>> {
        // TODO: handle this
        let header = try!(read_header(&mut r));

        Ok(Reader {
            header: header,
            reader: r,
            block_buffer: Vec::new(),
        })
    }
}

fn read_header<R: io::Read>(reader: &mut R) -> Result<Header> {
    let mut magic = [0; 3];
    try!(reader.read(&mut magic));
    if magic != MAGIC.as_bytes() {
        return Err(Error::BadMagic(String::from_utf8_lossy(&magic).to_string()));
    }

    let mut version = [0; 1];
    try!(reader.read(&mut version));
    let version = version[0] as u16;

    // Version 4 - block compression
    // Version 5 - custom compression codecs
    // Version 6 - metadata
    if version < 5 || version > 6 {
        return Err(Error::VersionNotSupported(version));
    }

    let key_class = try!(read_string(reader));
    let value_class = try!(read_string(reader));

    let mut flags = [0; 2];
    try!(reader.read(&mut flags));

    let compression_type: CompressionType = {
        // first byte: compression t/f
        // second byte: block t/f
        match (flags[0], flags[1]) {
            (1, 1) => CompressionType::Block,
            (1, 0) => CompressionType::Record,
            (0, 0) => CompressionType::None,
            _ => {
                return Err(Error::CompressionTypeUnknown("undefined compression type".to_string()))
            }
        }
    };

    let compression_codec = if compression_type != CompressionType::None {
        let codec = try!(read_string(reader));

        match compress::codec(&codec) {
            Some(codec) => Some(codec),
            None => return Err(Error::UnsupportedCodec(codec)),
        }
    } else {
        None
    };

    let pairs = try!(reader.read_u32::<BigEndian>());
    for _ in 0..pairs {
        // TODO: do stuff
    }

    let mut sync_marker = [0; SYNC_SIZE];
    try!(reader.read(&mut sync_marker));

    Ok(Header {
        version: version,
        compression_type: compression_type,
        compression_codec: compression_codec,
        key_class: key_class,
        value_class: value_class,
        metadata: HashMap::new(), // TODO
        sync_marker: sync_marker.to_vec(),
    })
}

impl<R: io::Read> Iterator for Reader<R> {
    type Item = (ByteString, ByteString);

    fn next(&mut self) -> Option<(ByteString, ByteString)> {
        if self.block_buffer.len() == 0 || self.header.compression_type != CompressionType::Block {
            let mut last_sync_marker = [0; SYNC_SIZE];
            let kv_length = to_opt!(self.reader.read_i32::<BigEndian>()) as i64;
            // ^ todo: Varint reader
            // let blocksize = to_opt!(self.reader.decode_vint64());
            println!("kv_length: {:?}", kv_length);
            // let mut kv_length = blocksize;

            // handle sync marker
            if kv_length == -1 {
                to_opt!(self.reader.read(&mut last_sync_marker));
                if last_sync_marker.to_vec() != self.header.sync_marker {
                    panic!("Sync marker mismatch");
                }

                // kv_length = to_opt!(self.reader.decode_vint64());
                //
                // if kv_length == -1 {
                //     return None;
                // }
            }

            if self.header.compression_type != CompressionType::Block {
                let (key, value) = to_opt!(read_kv(kv_length as isize,
                                                   &self.header,
                                                   &mut self.reader));

                return Some((key, value));
            }
        }

        if self.header.compression_type == CompressionType::Block {
            if self.block_buffer.len() == 0 {
                let kv_count = to_opt!(self.reader.decode_vint64()) as usize;
                println!("kv_count: {:?}", kv_count);
                let key_length_buffer_size = to_opt!(self.reader.decode_vint64());
                println!("key_length_buffer_size: {:?}", key_length_buffer_size);
                let mut key_length_buffer = vec![0u8; key_length_buffer_size as usize];
                to_opt!(self.reader.read(&mut key_length_buffer));

                let decompressed_keys_lengths = compress::decompressor(&self.header
                                                                            .compression_codec
                                                                            .unwrap(),
                                                                       key_length_buffer.as_ref());

                let mut c = io::Cursor::new(decompressed_keys_lengths.unwrap());
                // let mut b = Vec::new();
                let mut lens: Vec<usize> = Vec::new();
                for _ in 0..kv_count {
                    let len = c.decode_vint64().unwrap() as usize;
                    println!("len: {:?}", len);
                    lens.push(len);
                }

                let key_length_buffer_size = to_opt!(self.reader.decode_vint64());
                println!("key_length_buffer_size: {:?}", key_length_buffer_size);
                let mut key_length_buffer = vec![0u8; key_length_buffer_size as usize];
                to_opt!(self.reader.read(&mut key_length_buffer));
                let decompressed_keys = compress::decompressor(&self.header
                                                                    .compression_codec
                                                                    .unwrap(),
                                                               key_length_buffer.as_ref());

                let mut c = io::Cursor::new(decompressed_keys.unwrap());
                let mut keys: Vec<ByteString> = Vec::new();
                for i in 0..kv_count {
                    // println!("key: {:?}", c.read_i64::<BigEndian>());
                    let mut k = vec![0; lens[i]];
                    to_opt!(c.read(&mut k)); // todo errors
                    keys.push(k)
                }

                let val_length_buffer_size = to_opt!(self.reader.decode_vint64());
                println!("val_length_buffer_size: {:?}", val_length_buffer_size);
                let mut val_length_buffer = vec![0u8; val_length_buffer_size as usize];
                to_opt!(self.reader.read(&mut val_length_buffer));

                let decompressed_val_lengths = compress::decompressor(&self.header
                                                                           .compression_codec
                                                                           .unwrap(),
                                                                      val_length_buffer.as_ref());

                let mut c = io::Cursor::new(decompressed_val_lengths.unwrap());
                // let mut b = Vec::new();
                let mut lens: Vec<usize> = Vec::new();
                for _ in 0..kv_count {
                    let len = c.decode_vint64().unwrap() as usize;
                    println!("val_len: {:?}", len);
                    lens.push(len);
                }

                println!("{:?}", lens);

                let val_length_buffer_size = to_opt!(self.reader.decode_vint64());
                println!("val_length_buffer_size: {:?}", val_length_buffer_size);
                let mut val_length_buffer = vec![0u8; val_length_buffer_size as usize];
                to_opt!(self.reader.read(&mut val_length_buffer));
                let decompressed_vals = compress::decompressor(&self.header
                                                                    .compression_codec
                                                                    .unwrap(),
                                                               val_length_buffer.as_ref());

                let mut c = io::Cursor::new(decompressed_vals.unwrap());
                for i in 0..kv_count {
                    let mut v = vec![0; lens[i]]; //todo: reuse
                    to_opt!(c.read(&mut v));

                    self.block_buffer.push((keys[i].clone(), v));
                }
            }

            let len = self.block_buffer.len();
            if len > 0 {
                println!("Some");
                return Some(self.block_buffer.remove(0));
            } else {
                println!("None");
                return None;
            }
        } else {
            None
        }
    }
}

fn read_kv<R: io::Read>(kv_length: isize,
                        header: &Header,
                        reader: &mut R)
                        -> Result<(Vec<u8>, Vec<u8>)> {

    if header.compression_type == CompressionType::Block {
        let blocksize = try!(reader.decode_vint64());
        println!("{:?}", blocksize);

        let key = Vec::new();
        let value = Vec::new();
        Ok((key, value))
    } else {
        let k_length = try!(reader.read_i32::<BigEndian>()) as usize;

        let k_start = 0;
        let k_end = k_start + (k_length - 0);
        let v_start = k_end;
        let v_end = v_start + (kv_length as usize - k_length);

        // TODO: DataInput/DataOutput semantics per type
        // e.g. impl LongWritable...
        // re-implement common writables
        // core interface: Iterator<(u[8], u[8])> or a KV struct
        // mixin interface on KVstruct could extract key/value from
        // bytes
        println!("k_start: {:?}, k_end: {:?}, v_start: {:?}, v_end: {:?}",
                 k_start,
                 k_end,
                 v_start,
                 v_end);

        println!("{:?}", header);

        let mut buffer = vec![0; kv_length as usize];
        try!(reader.read(&mut buffer));

        let key = buffer[k_start..k_end].to_vec();

        if header.compression_type == CompressionType::Record {
            if let Some(ref codec) = header.compression_codec {
                let decompressed = try!(compress::decompressor(codec, &buffer[v_start..v_end]));

                Ok((key, decompressed))
            } else {
                panic!("WAT")
            }
        } else {
            let value = buffer[v_start..v_end].to_vec();

            Ok((key, value))
        }
    }
}

fn read_string<R: io::Read>(reader: &mut R) -> Result<String> {
    // read one byte, value len
    let value_length = try!(reader.read_u8()) as usize;
    let mut string = vec![0; value_length];

    try!(reader.read(&mut string));
    str::from_utf8(&string).map(|v| v.to_owned()).map_err(|e| Error::BadEncoding(e))
}

#[cfg(test)]
mod tests {
    use reader;
    use errors::Result;
    use std::path::Path;
    use std::fs::File;

    use byteorder::{BigEndian, ByteOrder};

    #[test]
    fn reads_standard_sequencefile() {
        let kvs = main_read("test_data/abc_long_text_none.seq").unwrap();

        assert_eq!(26, kvs.len());
        assert_eq!((0, "A".to_string()), kvs[0]);
        assert_eq!((25, "Z".to_string()), kvs[25]);
    }

    #[test]
    fn reads_deflate_record() {
        let kvs = main_read("test_data/abc_long_text_deflate_record.seq").unwrap();

        assert_eq!(26, kvs.len());
        assert_eq!((0, "A".to_string()), kvs[0]);
        assert_eq!((25, "Z".to_string()), kvs[25]);
    }

    #[test]
    fn reads_gzip_record() {
        let kvs = main_read("test_data/abc_long_text_gzip_record.seq").unwrap();

        assert_eq!(26, kvs.len());
        assert_eq!((0, "A".to_string()), kvs[0]);
        assert_eq!((25, "Z".to_string()), kvs[25]);
    }

    #[test]
    fn reads_bzip2_record() {
        let kvs = main_read("test_data/abc_long_text_bzip2_record.seq").unwrap();

        assert_eq!(26, kvs.len());
        assert_eq!((0, "A".to_string()), kvs[0]);
        assert_eq!((25, "Z".to_string()), kvs[25]);
    }

    #[test]
    #[should_panic(expected = "unsupported codec")]
    fn reads_snappy_record() {
        match main_read("test_data/abc_long_text_snappy_record.seq") {
            Ok(val) => val,
            Err(err) => panic!("Failed to open sequence file: {}", err),
        };
    }

    #[test]
    fn reads_deflate_block() {
        let kvs = main_read("test_data/abc_long_text_deflate_block.seq").unwrap();

        assert_eq!(26, kvs.len());
        assert_eq!((0, "A".to_string()), kvs[0]);
        assert_eq!((25, "Z".to_string()), kvs[25]);
    }

    #[test]
    fn reads_gzip_block() {
        let kvs = main_read("test_data/abc_long_text_gzip_block.seq").unwrap();

        assert_eq!(26, kvs.len());
        assert_eq!((0, "A".to_string()), kvs[0]);
        assert_eq!((25, "Z".to_string()), kvs[25]);
    }

    #[test]
    fn reads_bzip2_block() {
        let kvs = main_read("test_data/abc_long_text_bzip2_block.seq").unwrap();

        assert_eq!(26, kvs.len());
        assert_eq!((0, "A".to_string()), kvs[0]);
        assert_eq!((25, "Z".to_string()), kvs[25]);
    }

    #[test]
    #[should_panic(expected = "unsupported codec")]
    fn reads_snappy_block() {
        match main_read("test_data/abc_long_text_snappy_block.seq") {
            Ok(val) => val,
            Err(err) => panic!("Failed to open sequence file: {}", err),
        };
    }

    #[test]
    #[should_panic(expected = "bad or missing magic")]
    fn read_checks_magic() {
        match main_read("test_data/bad_magic.seq") {
            Ok(val) => val,
            Err(err) => panic!("Failed to open sequence file: {}", err),
        };
    }

    fn main_read(filename: &str) -> Result<Vec<(i64, String)>> {
        let path = Path::new(filename);

        let file = try!(File::open(&path));
        let seqfile = try!(reader::Reader::new(file));

        let kvs = seqfile.map(|(key, value)| {
            (BigEndian::read_i64(&key),
             String::from_utf8_lossy(&value[2..value.len()]).to_string())
        });

        Ok(kvs.collect())
    }
}
<|endoftext|>


<commit_before>//! # runner
//!
//! Runs the requested tasks.

//! The flow is as follows:
//!
//! * Load env variables
//! * Create an execution plan based on the requested task and its dependencies
//! * Run all tasks defined in the execution plan
//!

use command;
use installer;
use log::Logger;
use std::collections::HashSet;
use std::env;
use std::time::SystemTime;
use types::{Config, ExecutionPlan, Step};

fn run_task(
    logger: &Logger,
    step: &Step,
) {
    logger.info::<()>("Running Task: ", &[&step.name], None);

    installer::install(&logger, &step.config);

    command::run(&logger, &step);
}

fn run_task_flow(
    logger: &Logger,
    execution_plan: &ExecutionPlan,
) {
    for step in &execution_plan.steps {
        run_task(&logger, &step);
    }
}

/// Returns the actual task name to invoke as tasks may have aliases
fn get_task_name(
    logger: &Logger,
    config: &Config,
    name: &str,
) -> String {
    match config.tasks.get(name) {
        Some(task_config) => {
            let alias = if cfg!(windows) {
                match task_config.windows_alias {
                    Some(ref value) => Some(value),
                    _ => None,
                }
            } else if cfg!(target_os = "macos") || cfg!(target_os = "ios") {
                match task_config.mac_alias {
                    Some(ref value) => Some(value),
                    _ => None,
                }
            } else {
                match task_config.linux_alias {
                    Some(ref value) => Some(value),
                    _ => None,
                }
            };

            match alias {
                Some(ref os_alias) => get_task_name(logger, config, os_alias),
                _ => {
                    match task_config.alias {
                        Some(ref alias) => get_task_name(logger, config, alias),
                        _ => name.to_string(),
                    }
                }
            }
        }
        None => {
            // This will actually panic
            logger.error::<()>("Task not found: ", &[&name], None);

            name.to_string()
        }
    }
}

/// Creates an execution plan for the given step based on existing execution plan data
fn create_execution_plan_for_step(
    logger: &Logger,
    config: &Config,
    task: &str,
    steps: &mut Vec<Step>,
    task_names: &mut HashSet<String>,
    root: bool,
) {
    let actual_task = get_task_name(logger, config, task);

    match config.tasks.get(&actual_task) {
        Some(task_config) => {
            match task_config.dependencies {
                Some(ref dependencies) => {
                    for depedency in dependencies {
                        create_execution_plan_for_step(&logger, &config, &depedency, steps, task_names, false);
                    }
                }
                _ => logger.verbose::<()>("No dependencies found for task: ", &[&task], None),
            };

            if !task_names.contains(task) {
                let add = match task_config.disabled {
                    Some(ref disabled) => !disabled,
                    None => true,
                };

                if add {
                    steps.push(Step { name: task.to_string(), config: task_config.clone() });
                    task_names.insert(task.to_string());
                }
            } else if root {
                logger.error::<()>("Circular reference found for task: ", &[&task], None);
            }
        }
        None => logger.error::<()>("Task not found: ", &[&task], None),
    }
}

/// Creates the full execution plan
fn create_execution_plan(
    logger: &Logger,
    config: &Config,
    task: &str,
) -> ExecutionPlan {
    let mut task_names = HashSet::new();
    let mut steps = Vec::new();

    create_execution_plan_for_step(&logger, &config, &task, &mut steps, &mut task_names, true);

    ExecutionPlan { steps }
}

/// Updates the env for the current execution
fn set_env(
    logger: &Logger,
    config: &Config,
) {
    logger.info::<()>("Setting Up Env.", &[], None);

    for (key, value) in &config.env {
        logger.verbose::<()>("Setting env: ", &[&key, "=", &value], None);
        env::set_var(&key, &value);
    }
}

/// Runs the requested tasks.

/// The flow is as follows:
///
/// * Load env variables
/// * Create an execution plan based on the requested task and its dependencies
/// * Run all tasks defined in the execution plan
pub fn run(
    logger: &Logger,
    config: &Config,
    task: &str,
) {
    let start_time = SystemTime::now();

    set_env(logger, config);

    let execution_plan = create_execution_plan(&logger, &config, &task);
    logger.verbose("Created execution plan: ", &[], Some(&execution_plan));

    run_task_flow(logger, &execution_plan);

    let time_string = match start_time.elapsed() {
        Ok(elapsed) => {
            let mut string = " in ".to_string();
            string.push_str(&elapsed.as_secs().to_string());
            string.push_str(" seconds");

            string
        }
        _ => "".to_string(),
    };

    logger.info::<()>("Build done", &[&time_string, "."], None);
}
<commit_msg>platform alias<commit_after>//! # runner
//!
//! Runs the requested tasks.

//! The flow is as follows:
//!
//! * Load env variables
//! * Create an execution plan based on the requested task and its dependencies
//! * Run all tasks defined in the execution plan
//!

use command;
use installer;
use log::Logger;
use std::collections::HashSet;
use std::env;
use std::time::SystemTime;
use types::{Config, ExecutionPlan, Step};

fn run_task(
    logger: &Logger,
    step: &Step,
) {
    logger.info::<()>("Running Task: ", &[&step.name], None);

    installer::install(&logger, &step.config);

    command::run(&logger, &step);
}

fn run_task_flow(
    logger: &Logger,
    execution_plan: &ExecutionPlan,
) {
    for step in &execution_plan.steps {
        run_task(&logger, &step);
    }
}

/// Returns the actual task name to invoke as tasks may have aliases
fn get_task_name(
    logger: &Logger,
    config: &Config,
    name: &str,
) -> String {
    match config.tasks.get(name) {
        Some(task_config) => {
            let alias = if cfg!(windows) {
                match task_config.windows_alias {
                    Some(ref value) => Some(value),
                    _ => None,
                }
            } else if cfg!(target_os = "macos") || cfg!(target_os = "ios") {
                match task_config.mac_alias {
                    Some(ref value) => Some(value),
                    _ => None,
                }
            } else {
                match task_config.linux_alias {
                    Some(ref value) => Some(value),
                    _ => None,
                }
            };

            match alias {
                Some(ref os_alias) => get_task_name(logger, config, os_alias),
                _ => {
                    match task_config.alias {
                        Some(ref alias) => get_task_name(logger, config, alias),
                        _ => name.to_string(),
                    }
                }
            }
        }
        None => {
            // This will actually panic
            logger.error::<()>("Task not found: ", &[&name], None);

            name.to_string()
        }
    }
}

/// Creates an execution plan for the given step based on existing execution plan data
fn create_execution_plan_for_step(
    logger: &Logger,
    config: &Config,
    task: &str,
    steps: &mut Vec<Step>,
    task_names: &mut HashSet<String>,
    root: bool,
) {
    let actual_task = get_task_name(logger, config, task);

    match config.tasks.get(&actual_task) {
        Some(task_config) => {
            match task_config.dependencies {
                Some(ref dependencies) => {
                    for depedency in dependencies {
                        create_execution_plan_for_step(&logger, &config, &depedency, steps, task_names, false);
                    }
                }
                _ => logger.verbose::<()>("No dependencies found for task: ", &[&task], None),
            };

            if !task_names.contains(task) {
                let add = match task_config.disabled {
                    Some(ref disabled) => !disabled,
                    None => true,
                };

                if add {
                    steps.push(Step { name: task.to_string(), config: task_config.clone() });
                    task_names.insert(task.to_string());
                }
            } else if root {
                logger.error::<()>("Circular reference found for task: ", &[&task], None);
            }
        }
        None => logger.error::<()>("Task not found: ", &[&task], None),
    }
}

/// Creates the full execution plan
fn create_execution_plan(
    logger: &Logger,
    config: &Config,
    task: &str,
) -> ExecutionPlan {
    let mut task_names = HashSet::new();
    let mut steps = Vec::new();

    create_execution_plan_for_step(&logger, &config, &task, &mut steps, &mut task_names, true);

    ExecutionPlan { steps }
}

/// Updates the env for the current execution
fn set_env(
    logger: &Logger,
    config: &Config,
) {
    logger.info::<()>("Setting Up Env.", &[], None);

    for (key, value) in &config.env {
        logger.verbose::<()>("Setting env: ", &[&key, "=", &value], None);
        env::set_var(&key, &value);
    }
}

/// Runs the requested tasks.

/// The flow is as follows:
///
/// * Load env variables
/// * Create an execution plan based on the requested task and its dependencies
/// * Run all tasks defined in the execution plan
pub fn run(
    logger: &Logger,
    config: &Config,
    task: &str,
) {
    let start_time = SystemTime::now();

    set_env(logger, config);

    let execution_plan = create_execution_plan(&logger, &config, &task);
    logger.verbose("Created execution plan: ", &[], Some(&execution_plan));

    run_task_flow(logger, &execution_plan);

    let time_string = match start_time.elapsed() {
        Ok(elapsed) => {
            let mut string = " in ".to_string();
            string.push_str(&elapsed.as_secs().to_string());
            string.push_str(" seconds");

            string
        }
        _ => "".to_string(),
    };

    logger.info::<()>("Build done", &[&time_string, "."], None);
}

#[cfg(test)]
mod tests {
    use super::*;
    use log;
    use std::collections::HashMap;
    use types::Task;

    #[test]
    fn set_env_empty() {
        let logger = log::create("error");
        let config = Config { env: HashMap::new(), tasks: HashMap::new() };

        set_env(&logger, &config);
    }

    #[test]
    fn set_env_values() {
        let logger = log::create("error");
        let mut config = Config { env: HashMap::new(), tasks: HashMap::new() };
        config.env.insert("MY_ENV_KEY".to_string(), "MY_ENV_VALUE".to_string());

        assert_eq!(env::var("MY_ENV_KEY").unwrap_or("NONE".to_string()), "NONE".to_string());

        set_env(&logger, &config);

        assert_eq!(env::var("MY_ENV_KEY").unwrap(), "MY_ENV_VALUE");
    }

    #[test]
    #[should_panic]
    fn get_task_name_not_found() {
        let logger = log::create("error");
        let config = Config { env: HashMap::new(), tasks: HashMap::new() };

        get_task_name(&logger, &config, "test");
    }

    #[test]
    fn get_task_name_no_alias() {
        let logger = log::create("error");
        let mut config = Config { env: HashMap::new(), tasks: HashMap::new() };

        config.tasks.insert(
            "test".to_string(),
            Task {
                alias: None,
                linux_alias: None,
                windows_alias: None,
                mac_alias: None,
                disabled: None,
                install_crate: None,
                install_script: None,
                command: None,
                args: None,
                script: None,
                dependencies: None
            }
        );

        let name = get_task_name(&logger, &config, "test");

        assert_eq!(name, "test");
    }

    #[test]
    fn get_task_name_alias() {
        let logger = log::create("error");
        let mut config = Config { env: HashMap::new(), tasks: HashMap::new() };

        config.tasks.insert(
            "test".to_string(),
            Task {
                alias: Some("test2".to_string()),
                linux_alias: None,
                windows_alias: None,
                mac_alias: None,
                disabled: None,
                install_crate: None,
                install_script: None,
                command: None,
                args: None,
                script: None,
                dependencies: None
            }
        );

        config.tasks.insert(
            "test2".to_string(),
            Task {
                alias: None,
                linux_alias: None,
                windows_alias: None,
                mac_alias: None,
                disabled: None,
                install_crate: None,
                install_script: None,
                command: None,
                args: None,
                script: None,
                dependencies: None
            }
        );

        let name = get_task_name(&logger, &config, "test");

        assert_eq!(name, "test2");
    }

    #[test]
    fn get_task_name_platform_alias() {
        let logger = log::create("error");
        let mut config = Config { env: HashMap::new(), tasks: HashMap::new() };

        let mut task = Task {
            alias: None,
            linux_alias: None,
            windows_alias: None,
            mac_alias: None,
            disabled: None,
            install_crate: None,
            install_script: None,
            command: None,
            args: None,
            script: None,
            dependencies: None
        };
        if cfg!(windows) {
            task.windows_alias = Some("test2".to_string());
        } else if cfg!(target_os = "macos") || cfg!(target_os = "ios") {
            task.mac_alias = Some("test2".to_string());
        } else {
            task.linux_alias = Some("test2".to_string());
        };

        config.tasks.insert("test".to_string(), task);

        config.tasks.insert(
            "test2".to_string(),
            Task {
                alias: None,
                linux_alias: None,
                windows_alias: None,
                mac_alias: None,
                disabled: None,
                install_crate: None,
                install_script: None,
                command: None,
                args: None,
                script: None,
                dependencies: None
            }
        );

        let name = get_task_name(&logger, &config, "test");

        assert_eq!(name, "test2");
    }
}
<|endoftext|>


<commit_before>//! TODO Fill in

use std::fmt::{self, Display, Formatter};
use std::default::Default;
use std::rc::Rc;
use rlua::{self, Table, Lua, UserData, ToLua, Value};
use super::object::{Object, Objectable};
use super::class::{self, Class, ClassBuilder};

#[derive(Clone, Debug)]
pub struct TagState {
    // TODO Fill in
    dummy: i32
}

pub struct Tag<'lua>(Table<'lua>);

impl Default for TagState {
    fn default() -> Self {
        TagState {
            dummy: 0
        }
    }
}

impl <'lua> Tag<'lua> {
    fn new(lua: &Lua) -> rlua::Result<Object> {
        let class = class::class_setup(lua, "tag")?;
        Ok(Tag::allocate(lua, class)?.build())
    }
}

impl Display for TagState {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        write!(f, "Tag: {:p}", self)
    }
}

impl <'lua> ToLua<'lua> for Tag<'lua> {
    fn to_lua(self, lua: &'lua Lua) -> rlua::Result<Value<'lua>> {
        self.0.to_lua(lua)
    }
}

impl UserData for TagState {}

pub fn init(lua: &Lua) -> rlua::Result<Class> {
    method_setup(lua, Class::builder(lua, Some(Rc::new(Tag::new)), None, None)?)?
        .save_class("tag")?
        .build()
}

fn method_setup<'lua>(lua: &'lua Lua, builder: ClassBuilder<'lua>) -> rlua::Result<ClassBuilder<'lua>> {
    // TODO Do properly
    use super::dummy;
    builder.method("connect_signal".into(), lua.create_function(dummy))
}

impl_objectable!(Tag, TagState);
<commit_msg>Added __call for Tag<commit_after>//! TODO Fill in

use std::fmt::{self, Display, Formatter};
use std::default::Default;
use std::rc::Rc;
use rlua::{self, Table, Lua, UserData, ToLua, Value};
use super::object::{Object, Objectable};
use super::class::{self, Class, ClassBuilder};

#[derive(Clone, Debug)]
pub struct TagState {
    // TODO Fill in
    dummy: i32
}

pub struct Tag<'lua>(Table<'lua>);

impl Default for TagState {
    fn default() -> Self {
        TagState {
            dummy: 0
        }
    }
}

impl <'lua> Tag<'lua> {
    fn new(lua: &Lua) -> rlua::Result<Object> {
        let class = class::class_setup(lua, "tag")?;
        Ok(Tag::allocate(lua, class)?.build())
    }
}

impl Display for TagState {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        write!(f, "Tag: {:p}", self)
    }
}

impl <'lua> ToLua<'lua> for Tag<'lua> {
    fn to_lua(self, lua: &'lua Lua) -> rlua::Result<Value<'lua>> {
        self.0.to_lua(lua)
    }
}

impl UserData for TagState {}

pub fn init(lua: &Lua) -> rlua::Result<Class> {
    method_setup(lua, Class::builder(lua, Some(Rc::new(Tag::new)), None, None)?)?
        .save_class("tag")?
        .build()
}

fn method_setup<'lua>(lua: &'lua Lua, builder: ClassBuilder<'lua>) -> rlua::Result<ClassBuilder<'lua>> {
    // TODO Do properly
    use super::dummy;
    builder.method("connect_signal".into(), lua.create_function(dummy))?
           .method("__call".into(), lua.create_function(|lua, _: Value| Tag::new(lua)))
}

impl_objectable!(Tag, TagState);
<|endoftext|>


<commit_before>use std::env;
use std::fs::File;
use std::io::Read;
use std::str;
use std::str::FromStr;

fn main() {
    let args: Vec<String> = env::args().collect();
    let filename = &args[1];

    let mut f = File::open(filename).expect("file not found");
    let mut buffer: Vec<u8> = vec!();
    let _ = f.read_to_end(&mut buffer);

    let result: &[u8]= &buffer;

    println!("{:?}", str::from_utf8(&result[2*2880..10*2880]).expect("should be utf8"));
}

<commit_msg>Accept lower and upper bounds for the inspect program<commit_after>use std::env;
use std::fs::File;
use std::io::Read;
use std::str;
use std::str::FromStr;

fn main() {
    let args: Vec<String> = env::args().collect();
    let filename = &args[1];
    let low = usize::from_str(&args[2]).expect("second argument should be a number");
    let high = usize::from_str(&args[3]).expect("thirs argument should be a number");

    let mut f = File::open(filename).expect("file not found");
    let mut buffer: Vec<u8> = vec!();
    let _ = f.read_to_end(&mut buffer);

    let result: &[u8]= &buffer;

    println!("{:?}", str::from_utf8(&result[low..high]).expect("should be utf8"));
}

<|endoftext|>


<commit_before>extern crate rand;
extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

use rand::{Rng, SeedableRng, StdRng};

use timely::progress::timestamp::RootTimestamp;
use timely::dataflow::*;
use timely::dataflow::operators::*;
use timely::dataflow::channels::pact::Exchange;
use timely::drain::DrainExt;

fn main () {

    let node_cnt = std::env::args().skip(1).next().unwrap().parse::<usize>().unwrap();
    let edge_cnt = std::env::args().skip(2).next().unwrap().parse::<usize>().unwrap();

    timely::execute_from_args(std::env::args(), move |root| {

        let index = root.index() as usize;
        let peers = root.peers() as usize;

        let start = time::precise_time_s();

        let mut edges = Vec::new();
        let mut ranks = vec![1.0; (node_cnt / peers) + 1];   // holds ranks
        if (node_cnt % peers) < index { ranks.push(1.0); }
        let mut degrs = vec![0; ranks.len()];

        let mut going = start;

        let mut input = root.scoped(|builder| {

            let (input, graph) = builder.new_input::<(u32, u32)>();
            let (cycle, loopz) = builder.loop_variable::<(u32, f32)>(20, 1);

            graph.binary_notify(&loopz,
                                Exchange::new(|x: &(u32,u32)| x.0 as u64),
                                Exchange::new(|x: &(u32,f32)| x.0 as u64),
                                "pagerank",
                                vec![RootTimestamp::new(0)],
                                move |input1, input2, output, notificator| {

                // receive incoming edges (should only be iter 0)
                while let Some((_index, data)) = input1.next() {
                    for (src,dst) in data.drain_temp() {
                        degrs[src as usize] += 1;
                        edges.push((src / peers,dst));
                    }
                }

                // all inputs received for iter, commence multiplication
                while let Some((iter, _)) = notificator.next() {

                    // record some timings in order to estimate per-iteration times
                    if iter.inner == 0 { println!("src: {}, dst: {}, edges: {}", ranks.len(), node_cnt, edges.len()); }
                    if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                    if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                    // prepare src for transmitting to destinations
                    for s in 0..ranks.len() { ranks[s] = 0.15 + 0.85 * ranks[s] / degrs[s] as f32; }

                    // wander through destinations
                    let mut session = output.session(&iter);
                    for &(src,dst) in &edges {
                        unsafe {
                            session.give((dst, *ranks.get_unchecked(src as usize)));
                        }
                    }

                    for s in &mut ranks { *s = 0.0; }
                }

                // receive data from workers, accumulate in src
                while let Some((iter, data)) = input2.next() {
                    notificator.notify_at(&iter);
                    for &(node, rank) in data.iter() {
                        unsafe { *ranks.get_unchecked_mut(node as usize / peers) += rank; }
                    }
                }
            }).connect_loop(cycle);

            input
        });

        let seed: &[_] = &[1, 2, 3, 4];
        let mut rng: StdRng = SeedableRng::from_seed(seed);

        for _index in 0..(edge_cnt / peers) {
            input.send((rng.gen_range(0, node_cnt as u32), rng.gen_range(0, node_cnt as u32)));
        }
    });
}<commit_msg>reduced size of ranks vector<commit_after>extern crate rand;
extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

use rand::{Rng, SeedableRng, StdRng};

use timely::progress::timestamp::RootTimestamp;
use timely::dataflow::*;
use timely::dataflow::operators::*;
use timely::dataflow::channels::pact::Exchange;
use timely::drain::DrainExt;

fn main () {

    let node_cnt = std::env::args().skip(1).next().unwrap().parse::<usize>().unwrap();
    let edge_cnt = std::env::args().skip(2).next().unwrap().parse::<usize>().unwrap();

    timely::execute_from_args(std::env::args(), move |root| {

        let index = root.index() as usize;
        let peers = root.peers() as usize;

        let start = time::precise_time_s();

        let mut edges = Vec::new();
        let mut ranks = vec![1.0; (node_cnt / peers) + 1];   // holds ranks
        if (node_cnt % peers) < index { ranks.push(1.0); }
        let mut degrs = vec![0; ranks.len()];

        let mut going = start;

        let mut input = root.scoped(|builder| {

            let (input, graph) = builder.new_input::<(u32, u32)>();
            let (cycle, loopz) = builder.loop_variable::<(u32, f32)>(20, 1);

            graph.binary_notify(&loopz,
                                Exchange::new(|x: &(u32,u32)| x.0 as u64),
                                Exchange::new(|x: &(u32,f32)| x.0 as u64),
                                "pagerank",
                                vec![RootTimestamp::new(0)],
                                move |input1, input2, output, notificator| {

                // receive incoming edges (should only be iter 0)
                while let Some((_index, data)) = input1.next() {
                    for (src,dst) in data.drain_temp() {
                        degrs[src as usize] += 1;
                        edges.push((src / (peers as u32),dst));
                    }
                }

                // all inputs received for iter, commence multiplication
                while let Some((iter, _)) = notificator.next() {

                    // record some timings in order to estimate per-iteration times
                    if iter.inner == 0 { println!("src: {}, dst: {}, edges: {}", ranks.len(), node_cnt, edges.len()); }
                    if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                    if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                    // prepare src for transmitting to destinations
                    for s in 0..ranks.len() { ranks[s] = 0.15 + 0.85 * ranks[s] / degrs[s] as f32; }

                    // wander through destinations
                    let mut session = output.session(&iter);
                    for &(src,dst) in &edges {
                        unsafe {
                            session.give((dst, *ranks.get_unchecked(src as usize)));
                        }
                    }

                    for s in &mut ranks { *s = 0.0; }
                }

                // receive data from workers, accumulate in src
                while let Some((iter, data)) = input2.next() {
                    notificator.notify_at(&iter);
                    for &(node, rank) in data.iter() {
                        unsafe { *ranks.get_unchecked_mut(node as usize / peers) += rank; }
                    }
                }
            }).connect_loop(cycle);

            input
        });

        let seed: &[_] = &[1, 2, 3, 4];
        let mut rng: StdRng = SeedableRng::from_seed(seed);

        for _index in 0..(edge_cnt / peers) {
            input.send((rng.gen_range(0, node_cnt as u32), rng.gen_range(0, node_cnt as u32)));
        }
    });
}<|endoftext|>


<commit_before>#[macro_export]
macro_rules! deb {
  ( $e:expr ) => { println!(concat!("\x1b[36m       > ", $e, "\x1b[0m")); };
  ( $e:expr, $($arg:tt)* ) => { println!(concat!("\x1b[36m       > ", $e, "\x1b[0m"), $($arg)*); };
}

#[macro_export]
macro_rules! info {
  ( $e:expr ) => { println!(concat!("\x1b[34m       > ", $e, "\x1b[0m")); };
  ( $e:expr, $($arg:tt)* ) => { println!(concat!("\x1b[34m       > ", $e, "\x1b[0m"), $($arg)*); };
}

#[macro_export]
macro_rules! warn {
  ( $e:expr ) => { println!(concat!("\x1b[33m  /!\\  > ", $e, "\x1b[0m")); };
  ( $e:expr, $($arg:tt)* ) => { println!(concat!("\x1b[33m  /!\\  > ", $e, "\x1b[0m"), $($arg)*); };
}

#[macro_export]
macro_rules! err {
  ( $e:expr ) => { println!(concat!("\x1b[1;31m /!\\  > ", $e, "\x1b[0;0m")); };
  ( $e:expr, $($arg:tt)* ) => { println!(concat!("\x1b[1;31m /!\\  > ", $e, "\x1b[0;0m"), $($arg)*); };
}
<commit_msg>report.rs whitespaces shitstorm fixed.<commit_after>#[macro_export]
macro_rules! deb {
  ( $e:expr ) => { println!(concat!("\x1b[36m    > ", $e, "\x1b[0m")); };
  ( $e:expr, $($arg:tt)* ) => { println!(concat!("\x1b[36m    > ", $e, "\x1b[0m"), $($arg)*); };
}

#[macro_export]
macro_rules! info {
  ( $e:expr ) => { println!(concat!("\x1b[34m    > ", $e, "\x1b[0m")); };
  ( $e:expr, $($arg:tt)* ) => { println!(concat!("\x1b[34m    > ", $e, "\x1b[0m"), $($arg)*); };
}

#[macro_export]
macro_rules! warn {
  ( $e:expr ) => { println!(concat!("\x1b[33m/!\\ > ", $e, "\x1b[0m")); };
  ( $e:expr, $($arg:tt)* ) => { println!(concat!("\x1b[33m/!\\ > ", $e, "\x1b[0m"), $($arg)*); };
}

#[macro_export]
macro_rules! err {
  ( $e:expr ) => { println!(concat!("\x1b[1;31m/!\\ > ", $e, "\x1b[0;0m")); };
  ( $e:expr, $($arg:tt)* ) => { println!(concat!("\x1b[1;31m/!\\ > ", $e, "\x1b[0;0m"), $($arg)*); };
}
<|endoftext|>


<commit_before>use std::collections::HashMap;

use scanner::Scanner;
use regex::Regex;

const COMPARISON           : &'static str = r"^(==|!=|<>|<=?|>=?|contains)";
const SINGLE_STRING_LITERAL: &'static str = r"^'[^']*'";
const DOUBLE_STRING_LITERAL: &'static str = r#"^"[^"]*""#;
const NUMBER_LITERAL:        &'static str = r"^-?\d+(\.\d+)?";
const IDENTIFIER:            &'static str = r"^[a-zA-Z_][\w-]*\??";
const RANGE_OP:              &'static str = r"^\.\.";

#[derive(Clone, Debug, PartialEq)]
pub enum Token {
    Comparison(String),
    Identifier(String),
    Number(String),
    String(String),
    Range(String),
    Pipe,
    Dot,
    Colon,
    Comma,
    OpenSquare,
    CloseSquare,
    OpenRound,
    CloseRound,
    Question,
    Dash
}

macro_rules! token {
    (Range)               => (token!(Range, ".."));
    ($e:ident)            => (Token::$e);
    ($e:ident, $str:expr) => (Token::$e(String::from($str)))
}

pub struct Tokens<'t> {
    scanner: &'t Scanner<'t>,
    specials: HashMap<&'t str, Token>,
    matchers: Vec<Regex>
}

impl<'t> Tokens<'t> {
    fn new<'a>(scanner: &'a Scanner<'a>) -> Tokens<'a> {
        let mut specials = HashMap::new();
        specials.insert("|", Token::Pipe);
        specials.insert(".", Token::Dot);
        specials.insert(":", Token::Colon);
        specials.insert(",", Token::Comma);
        specials.insert("[", Token::OpenSquare);
        specials.insert("]", Token::CloseSquare);
        specials.insert("(", Token::OpenRound);
        specials.insert(")", Token::CloseRound);
        specials.insert("?", Token::Question);
        specials.insert("-", Token::Dash);

        let matchers = vec![
            Regex::new(COMPARISON).unwrap(),
            Regex::new(SINGLE_STRING_LITERAL).unwrap(),
            Regex::new(DOUBLE_STRING_LITERAL).unwrap(),
            Regex::new(NUMBER_LITERAL).unwrap(),
            Regex::new(IDENTIFIER).unwrap(),
            Regex::new(RANGE_OP).unwrap()
        ];

        Tokens { scanner: scanner, specials: specials, matchers: matchers }
    }

    fn token_for(&self, pattern: &Regex, value: &str) -> Token {
        match pattern.as_str() {
            COMPARISON            => token!(Comparison, value),
            SINGLE_STRING_LITERAL => token!(String, value),
            DOUBLE_STRING_LITERAL => token!(String, value),
            NUMBER_LITERAL        => token!(Number, value),
            IDENTIFIER            => token!(Identifier, value),
            RANGE_OP              => token!(Range),
            _                     => unreachable!() // already been checked for existence
        }
    }

    fn next_match(&self) -> Option<Token> {
        match self.matchers.iter().find(|&m| self.scanner.check(m)) {
            Some(regex) => self.matched_token(&regex),
            None => self.matched_special()
        }
    }

    fn matched_token(&self, pattern: &Regex) -> Option<Token> {
        let value = self.scanner.scan(pattern).unwrap();
        Some(self.token_for(pattern, value))
    }

    fn matched_special(&self) -> Option<Token> {
        match self.scanner.get_char() {
            Some(character) => {
                match self.specials.get(character) {
                    Some(token) => Some((*token).clone()),
                    None => unreachable!("Syntax Error")
                }
            },
            None => None
        }
    }
}

impl<'t> Iterator for Tokens<'t> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.next_match()
    }
}

pub struct Lexer<'t> {
    scanner: Scanner<'t>
}

impl<'t> Lexer<'t> {
    pub fn new<'a>(source: &'a str) -> Lexer<'a> {
        Lexer { scanner: Scanner::new(source) }
    }

    pub fn tokens(&self) -> Tokens {
        Tokens::new(&self.scanner)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn new_creates_a_new_instance() {
        let lexer = Lexer::new("doSomthing | filter");
        assert_eq!("doSomthing | filter", lexer.scanner.rest().unwrap());
    }

    #[test]
    fn tokens_when_given_a_blank_string() {
        let lexer              = Lexer::new("");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(0, tokens.len());
    }

    #[test]
    fn tokens_when_given_a_whitespace_only_string() {
        let lexer              = Lexer::new("  \t \n\r ");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(0, tokens.len());
    }

    #[test]
    fn tokens_parses_identifiers() {
        let lexer              = Lexer::new("high five?");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(2, tokens.len());
        assert_eq!(token!(Identifier, "high"), tokens[0]);
        assert_eq!(token!(Identifier, "five?"), tokens[1]);
    }

    #[test]
    fn tokens_knows_that_identifiers_dont_start_with_numbers() {
        let lexer              = Lexer::new("2foo 5.0bar");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(4, tokens.len());
        assert_eq!(token!(Number, "2"), tokens[0]);
        assert_eq!(token!(Identifier, "foo"), tokens[1]);
        assert_eq!(token!(Number, "5.0"), tokens[2]);
        assert_eq!(token!(Identifier, "bar"), tokens[3]);
    }

    #[test]
    fn tokens_parses_string_literals() {
        let lexer              = Lexer::new(r#" 'this is a test""' "wat 'lol'" "#);
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(2, tokens.len());
        assert_eq!(token!(String, r#"'this is a test""'"#), tokens[0]);
        assert_eq!(token!(String, r#""wat 'lol'""#), tokens[1]);
    }

    #[test]
    fn tokens_parses_integers() {
        let lexer              = Lexer::new("hi 50");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(2, tokens.len());
        assert_eq!(token!(Identifier, "hi"), tokens[0]);
        assert_eq!(token!(Number, "50"), tokens[1]);
    }

    #[test]
    fn tokens_parses_floats() {
        let lexer              = Lexer::new("hi 5.0");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(2, tokens.len());
        assert_eq!(token!(Identifier, "hi"), tokens[0]);
        assert_eq!(token!(Number, "5.0"), tokens[1]);
    }

    #[test]
    fn tokens_parses_comparisons() {
        let lexer              = Lexer::new("== <> contains");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(3, tokens.len());
        assert_eq!(token!(Comparison, "=="), tokens[0]);
        assert_eq!(token!(Comparison, "<>"), tokens[1]);
        assert_eq!(token!(Comparison, "contains"), tokens[2]);
    }

    #[test]
    fn tokens_parses_range_operator() {
        let lexer              = Lexer::new("1..10");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(3, tokens.len());
        assert_eq!(token!(Number, "1"), tokens[0]);
        assert_eq!(token!(Range), tokens[1]);
        assert_eq!(token!(Number, "10"), tokens[2]);
    }

    #[test]
    fn tokens_parses_special_characters() {
        let lexer              = Lexer::new("[hi], (| .:) - ?cool");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(12, tokens.len());
        assert_eq!(token!(OpenSquare), tokens[0]);
        assert_eq!(token!(Identifier, "hi"), tokens[1]);
        assert_eq!(token!(CloseSquare), tokens[2]);
        assert_eq!(token!(Comma), tokens[3]);
        assert_eq!(token!(OpenRound), tokens[4]);
        assert_eq!(token!(Pipe), tokens[5]);
        assert_eq!(token!(Dot), tokens[6]);
        assert_eq!(token!(Colon), tokens[7]);
        assert_eq!(token!(CloseRound), tokens[8]);
        assert_eq!(token!(Dash), tokens[9]);
        assert_eq!(token!(Question), tokens[10]);
        assert_eq!(token!(Identifier, "cool"), tokens[11]);
    }

    #[test]
    fn tokens_skips_internal_whitespace() {
        let lexer              = Lexer::new("five|\n\t  ==");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(3, tokens.len());
        assert_eq!(token!(Identifier, "five"), tokens[0]);
        assert_eq!(token!(Pipe), tokens[1]);
        assert_eq!(token!(Comparison, "=="), tokens[2]);
    }

    #[test]
    #[should_panic(expected = "Syntax Error")]
    fn tokens_freaks_out_with_syntax_error() {
        let lexer              = Lexer::new("%");
        let tokens: Vec<Token> = lexer.tokens().collect();
    }
}
<commit_msg>making the tests more readable<commit_after>use std::collections::HashMap;

use scanner::Scanner;
use regex::Regex;

const COMPARISON           : &'static str = r"^(==|!=|<>|<=?|>=?|contains)";
const SINGLE_STRING_LITERAL: &'static str = r"^'[^']*'";
const DOUBLE_STRING_LITERAL: &'static str = r#"^"[^"]*""#;
const NUMBER_LITERAL:        &'static str = r"^-?\d+(\.\d+)?";
const IDENTIFIER:            &'static str = r"^[a-zA-Z_][\w-]*\??";
const RANGE_OP:              &'static str = r"^\.\.";

#[derive(Clone, Debug, PartialEq)]
pub enum Token {
    Comparison(String),
    Identifier(String),
    Number(String),
    String(String),
    Range(String),
    Pipe,
    Dot,
    Colon,
    Comma,
    OpenSquare,
    CloseSquare,
    OpenRound,
    CloseRound,
    Question,
    Dash
}

macro_rules! token {
    (Range)               => (token!(Range, ".."));
    ($e:ident)            => (Token::$e);
    ($e:ident, $str:expr) => (Token::$e(String::from($str)))
}

pub struct Tokens<'t> {
    scanner: &'t Scanner<'t>,
    specials: HashMap<&'t str, Token>,
    matchers: Vec<Regex>
}

impl<'t> Tokens<'t> {
    fn new<'a>(scanner: &'a Scanner<'a>) -> Tokens<'a> {
        let mut specials = HashMap::new();
        specials.insert("|", Token::Pipe);
        specials.insert(".", Token::Dot);
        specials.insert(":", Token::Colon);
        specials.insert(",", Token::Comma);
        specials.insert("[", Token::OpenSquare);
        specials.insert("]", Token::CloseSquare);
        specials.insert("(", Token::OpenRound);
        specials.insert(")", Token::CloseRound);
        specials.insert("?", Token::Question);
        specials.insert("-", Token::Dash);

        let matchers = vec![
            Regex::new(COMPARISON).unwrap(),
            Regex::new(SINGLE_STRING_LITERAL).unwrap(),
            Regex::new(DOUBLE_STRING_LITERAL).unwrap(),
            Regex::new(NUMBER_LITERAL).unwrap(),
            Regex::new(IDENTIFIER).unwrap(),
            Regex::new(RANGE_OP).unwrap()
        ];

        Tokens { scanner: scanner, specials: specials, matchers: matchers }
    }

    fn token_for(&self, pattern: &Regex, value: &str) -> Token {
        match pattern.as_str() {
            COMPARISON            => token!(Comparison, value),
            SINGLE_STRING_LITERAL => token!(String, value),
            DOUBLE_STRING_LITERAL => token!(String, value),
            NUMBER_LITERAL        => token!(Number, value),
            IDENTIFIER            => token!(Identifier, value),
            RANGE_OP              => token!(Range),
            _                     => unreachable!() // already been checked for existence
        }
    }

    fn next_match(&self) -> Option<Token> {
        match self.matchers.iter().find(|&m| self.scanner.check(m)) {
            Some(regex) => self.matched_token(&regex),
            None => self.matched_special()
        }
    }

    fn matched_token(&self, pattern: &Regex) -> Option<Token> {
        let value = self.scanner.scan(pattern).unwrap();
        Some(self.token_for(pattern, value))
    }

    fn matched_special(&self) -> Option<Token> {
        match self.scanner.get_char() {
            Some(character) => {
                match self.specials.get(character) {
                    Some(token) => Some((*token).clone()),
                    None => unreachable!("Syntax Error")
                }
            },
            None => None
        }
    }
}

impl<'t> Iterator for Tokens<'t> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.next_match()
    }
}

pub struct Lexer<'t> {
    scanner: Scanner<'t>
}

impl<'t> Lexer<'t> {
    pub fn new<'a>(source: &'a str) -> Lexer<'a> {
        Lexer { scanner: Scanner::new(source) }
    }

    pub fn tokens(&self) -> Tokens {
        Tokens::new(&self.scanner)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn compare_tokens(lexer: &Lexer, expectedTokens: Vec<Token>) {
        let zipped = lexer.tokens().zip(expectedTokens);

        for token in zipped {
            let (actual, expected) = token;
            assert_eq!(expected, actual);
        }
    }

    #[test]
    fn new_creates_a_new_instance() {
        let lexer = Lexer::new("doSomthing | filter");
        assert_eq!("doSomthing | filter", lexer.scanner.rest().unwrap());
    }

    #[test]
    fn tokens_when_given_a_blank_string() {
        let lexer              = Lexer::new("");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(0, tokens.len());
    }

    #[test]
    fn tokens_when_given_a_whitespace_only_string() {
        let lexer              = Lexer::new("  \t \n\r ");
        let tokens: Vec<Token> = lexer.tokens().collect();

        assert_eq!(0, tokens.len());
    }

    #[test]
    fn tokens_parses_identifiers() {
        let lexer    = Lexer::new("high five?");
        let expected = vec![
            token!(Identifier, "high"),
            token!(Identifier, "five?")
        ];

        compare_tokens(&lexer, expected);
    }

    #[test]
    fn tokens_knows_that_identifiers_dont_start_with_numbers() {
        let lexer    = Lexer::new("2foo 5.0bar");
        let expected = vec![
            token!(Number, "2"),
            token!(Identifier, "foo"),
            token!(Number, "5.0"),
            token!(Identifier, "bar")
        ];

        compare_tokens(&lexer, expected);
    }

    #[test]
    fn tokens_parses_string_literals() {
        let lexer    = Lexer::new(r#" 'this is a test""' "wat 'lol'" "#);
        let expected = vec![
            token!(String, r#"'this is a test""'"#),
            token!(String, r#""wat 'lol'""#)
        ];

        compare_tokens(&lexer, expected);
    }

    #[test]
    fn tokens_parses_integers() {
        let lexer    = Lexer::new("hi 50");
        let expected = vec![
            token!(Identifier, "hi"),
            token!(Number, "50")
        ];

        compare_tokens(&lexer, expected);
    }

    #[test]
    fn tokens_parses_floats() {
        let lexer    = Lexer::new("hi 5.0");
        let expected = vec![
            token!(Identifier, "hi"),
            token!(Number, "5.0")
        ];

        compare_tokens(&lexer, expected);
    }

    #[test]
    fn tokens_parses_comparisons() {
        let lexer    = Lexer::new("== <> contains");
        let expected = vec![
            token!(Comparison, "=="),
            token!(Comparison, "<>"),
            token!(Comparison, "contains")
        ];

        compare_tokens(&lexer, expected);
    }

    #[test]
    fn tokens_parses_range_operator() {
        let lexer    = Lexer::new("1..10");
        let expected = vec![
            token!(Number, "1"),
            token!(Range),
            token!(Number, "10")
        ];

        compare_tokens(&lexer, expected);
    }

    #[test]
    fn tokens_parses_special_characters() {
        let lexer    = Lexer::new("[hi], (| .:) - ?cool");
        let expected = vec![
            token!(OpenSquare),
            token!(Identifier, "hi"),
            token!(CloseSquare),
            token!(Comma),
            token!(OpenRound),
            token!(Pipe),
            token!(Dot),
            token!(Colon),
            token!(CloseRound),
            token!(Dash),
            token!(Question),
            token!(Identifier, "cool")
        ];

        compare_tokens(&lexer, expected);
    }

    #[test]
    fn tokens_skips_internal_whitespace() {
        let lexer    = Lexer::new("five|\n\t ==");
        let expected = vec![
            token!(Identifier, "five"),
            token!(Pipe),
            token!(Comparison, "==")
        ];

        compare_tokens(&lexer, expected);
    }

    #[test]
    #[should_panic(expected = "Syntax Error")]
    fn tokens_freaks_out_with_syntax_error() {
        let lexer              = Lexer::new("%");
        let tokens: Vec<Token> = lexer.tokens().collect();
    }
}
<|endoftext|>


<commit_before>use std::path::{Path, PathBuf};
use std::fs::{self, File};
use std::io::Write;
use std::error::Error;

use {BookConfig, BookItem, theme, parse, utils};
use book::BookItems;
use renderer::{Renderer, HtmlHandlebars};
use utils::{PathExt, create_path};

pub struct MDBook {
    config: BookConfig,
    pub content: Vec<BookItem>,
    renderer: Box<Renderer>,
}

impl MDBook {

    /// Create a new `MDBook` struct with root directory `root`
    ///
    /// - The default source directory is set to `root/src`
    /// - The default output directory is set to `root/book`
    ///
    /// They can both be changed by using [`set_src()`](#method.set_src) and [`set_dest()`](#method.set_dest)

    pub fn new(root: &Path) -> MDBook {

        if !root.exists() || !root.is_dir() {
            output!("{:?} No directory with that name", root);
        }

        MDBook {
            content: vec![],
            config: BookConfig::new(root)
                        .set_src(&root.join("src"))
                        .set_dest(&root.join("book"))
                        .to_owned(),
            renderer: Box::new(HtmlHandlebars::new()),
        }
    }

    /// Returns a flat depth-first iterator over the elements of the book in the form of a tuple:
    /// `(section: String, bookitem: &BookItem)`
    ///
    /// ```no_run
    /// # extern crate mdbook;
    /// # use mdbook::MDBook;
    /// # use std::path::Path;
    /// # fn main() {
    /// # let mut book = MDBook::new(Path::new("mybook"));
    /// for (section, element) in book.iter() {
    ///     println!("{} {}", section, element.name);
    /// }
    ///
    /// // would print something like this:
    /// // 1. Chapter 1
    /// // 1.1 Sub Chapter
    /// // 1.2 Sub Chapter
    /// // 2. Chapter 2
    /// //
    /// // etc.
    /// # }
    /// ```

    pub fn iter(&self) -> BookItems {
        BookItems {
            items: &self.content[..],
            current_index: 0,
            stack: Vec::new(),
        }
    }

    /// `init()` creates some boilerplate files and directories to get you started with your book.
    ///
    /// ```text
    /// book-test/
    /// ├── book
    /// └── src
    ///     ├── chapter_1.md
    ///     └── SUMMARY.md
    /// ```
    ///
    /// It uses the paths given as source and output directories and adds a `SUMMARY.md` and a
    /// `chapter_1.md` to the source directory.

    pub fn init(&mut self) -> Result<(), Box<Error>> {

        debug!("[fn]: init");

        if !self.config.get_root().exists() {
            create_path(self.config.get_root()).unwrap();
            output!("{:?} created", self.config.get_root());
        }

        {
            let dest = self.config.get_dest();
            let src = self.config.get_src();

            if !dest.exists() {
                debug!("[*]: {:?} does not exist, trying to create directory", dest);
                try!(fs::create_dir(&dest));
            }

            if !src.exists() {
                debug!("[*]: {:?} does not exist, trying to create directory", src);
                try!(fs::create_dir(&src));
            }

            let summary = src.join("SUMMARY.md");

            if !summary.exists() {

                // Summary does not exist, create it

                debug!("[*]: {:?} does not exist, trying to create SUMMARY.md", src.join("SUMMARY.md"));
                let mut f = try!(File::create(&src.join("SUMMARY.md")));

                debug!("[*]: Writing to SUMMARY.md");

                try!(writeln!(f, "# Summary"));
                try!(writeln!(f, ""));
                try!(writeln!(f, "- [Chapter 1](./chapter_1.md)"));
            }
        }

        // parse SUMMARY.md, and create the missing item related file
        try!(self.parse_summary());

        debug!("[*]: constructing paths for missing files");
        for item in self.iter() {
            debug!("[*]: item: {:?}", item);
            match item {
                &BookItem::Spacer => continue,
                &BookItem::Chapter(_, ref ch) | &BookItem::Affix(ref ch) => {
                    if ch.path != PathBuf::new() {
                        let path = self.config.get_src().join(&ch.path);

                        if !path.exists() {
                            debug!("[*]: {:?} does not exist, trying to create file", path);
                            try!(::std::fs::create_dir_all(path.parent().unwrap()));
                            let mut f = try!(File::create(path));

                            //debug!("[*]: Writing to {:?}", path);
                            try!(writeln!(f, "# {}", ch.name));
                        }
                    }
                }
            }
        }

        debug!("[*]: init done");
        return Ok(());
    }

    /// The `build()` method is the one where everything happens. First it parses `SUMMARY.md` to
    /// construct the book's structure in the form of a `Vec<BookItem>` and then calls `render()`
    /// method of the current renderer.
    ///
    /// It is the renderer who generates all the output files.

    pub fn build(&mut self) -> Result<(), Box<Error>> {
        debug!("[fn]: build");

        try!(self.init());

        // Clean output directory
        try!(utils::remove_dir_content(&self.config.get_dest()));

        try!(self.renderer.render(&self));

        Ok(())
    }

    pub fn copy_theme(&self) -> Result<(), Box<Error>> {
        debug!("[fn]: copy_theme");

        let theme_dir = self.config.get_src().join("theme");

        if !theme_dir.exists() {
            debug!("[*]: {:?} does not exist, trying to create directory", theme_dir);
            try!(fs::create_dir(&theme_dir));
        }

        // index.hbs
        let mut index = try!(File::create(&theme_dir.join("index.hbs")));
        try!(index.write_all(theme::INDEX));

        // book.css
        let mut css = try!(File::create(&theme_dir.join("book.css")));
        try!(css.write_all(theme::CSS));

        // book.js
        let mut js = try!(File::create(&theme_dir.join("book.js")));
        try!(js.write_all(theme::JS));

        // highlight.css
        let mut highlight_css = try!(File::create(&theme_dir.join("highlight.css")));
        try!(highlight_css.write_all(theme::HIGHLIGHT_CSS));

        // highlight.js
        let mut highlight_js = try!(File::create(&theme_dir.join("highlight.js")));
        try!(highlight_js.write_all(theme::HIGHLIGHT_JS));

        Ok(())
    }

    /// Parses the `book.json` file (if it exists) to extract the configuration parameters.
    /// The `book.json` file should be in the root directory of the book.
    /// The root directory is the one specified when creating a new `MDBook`
    ///
    /// ```no_run
    /// # extern crate mdbook;
    /// # use mdbook::MDBook;
    /// # use std::path::Path;
    /// # fn main() {
    /// let mut book = MDBook::new(Path::new("root_dir"));
    /// # }
    /// ```
    ///
    /// In this example, `root_dir` will be the root directory of our book and is specified in function
    /// of the current working directory by using a relative path instead of an absolute path.

    pub fn read_config(mut self) -> Self {
        let root = self.config.get_root().to_owned();
        self.config.read_config(&root);
        self
    }

    /// You can change the default renderer to another one by using this method. The only requirement
    /// is for your renderer to implement the [Renderer trait](../../renderer/renderer/trait.Renderer.html)
    ///
    /// ```no_run
    /// extern crate mdbook;
    /// use mdbook::MDBook;
    /// use mdbook::renderer::HtmlHandlebars;
    /// # use std::path::Path;
    ///
    /// fn main() {
    ///     let mut book = MDBook::new(Path::new("mybook"))
    ///                         .set_renderer(Box::new(HtmlHandlebars::new()));
    ///
    ///     // In this example we replace the default renderer by the default renderer...
    ///     // Don't forget to put your renderer in a Box
    /// }
    /// ```
    ///
    /// **note:** Don't forget to put your renderer in a `Box` before passing it to `set_renderer()`

    pub fn set_renderer(mut self, renderer: Box<Renderer>) -> Self {
        self.renderer = renderer;
        self
    }

    pub fn set_dest(mut self, dest: &Path) -> Self {

        // Handle absolute and relative paths
        match dest.is_absolute() {
            true => { self.config.set_dest(dest); },
            false => {
                let dest = self.config.get_root().join(dest).to_owned();
                self.config.set_dest(&dest);
            }
        }

        self
    }

    pub fn get_dest(&self) -> &Path {
        self.config.get_dest()
    }

    pub fn set_src(mut self, src: &Path) -> Self {

        // Handle absolute and relative paths
        match src.is_absolute() {
            true => { self.config.set_src(src); },
            false => {
                let src = self.config.get_root().join(src).to_owned();
                self.config.set_src(&src);
            }
        }

        self
    }

    pub fn get_src(&self) -> &Path {
        self.config.get_src()
    }

    pub fn set_title(mut self, title: &str) -> Self {
        self.config.title = title.to_owned();
        self
    }

    pub fn get_title(&self) -> &str {
        &self.config.title
    }

    pub fn set_author(mut self, author: &str) -> Self {
        self.config.author = author.to_owned();
        self
    }

    pub fn get_author(&self) -> &str {
        &self.config.author
    }

    // Construct book
    fn parse_summary(&mut self) -> Result<(), Box<Error>> {
        // When append becomes stable, use self.content.append() ...
        self.content = try!(parse::construct_bookitems(&self.config.get_src().join("SUMMARY.md")));
        Ok(())
    }

}
<commit_msg>fix code in doc<commit_after>use std::path::{Path, PathBuf};
use std::fs::{self, File};
use std::io::Write;
use std::error::Error;

use {BookConfig, BookItem, theme, parse, utils};
use book::BookItems;
use renderer::{Renderer, HtmlHandlebars};
use utils::{PathExt, create_path};

pub struct MDBook {
    config: BookConfig,
    pub content: Vec<BookItem>,
    renderer: Box<Renderer>,
}

impl MDBook {

    /// Create a new `MDBook` struct with root directory `root`
    ///
    /// - The default source directory is set to `root/src`
    /// - The default output directory is set to `root/book`
    ///
    /// They can both be changed by using [`set_src()`](#method.set_src) and [`set_dest()`](#method.set_dest)

    pub fn new(root: &Path) -> MDBook {

        if !root.exists() || !root.is_dir() {
            output!("{:?} No directory with that name", root);
        }

        MDBook {
            content: vec![],
            config: BookConfig::new(root)
                        .set_src(&root.join("src"))
                        .set_dest(&root.join("book"))
                        .to_owned(),
            renderer: Box::new(HtmlHandlebars::new()),
        }
    }

    /// Returns a flat depth-first iterator over the elements of the book, it returns an [BookItem enum](bookitem.html):
    /// `(section: String, bookitem: &BookItem)`
    ///
    /// ```no_run
    /// # extern crate mdbook;
    /// # use mdbook::MDBook;
    /// # use bookitem::BookItem;
    /// # use std::path::Path;
    /// # fn main() {
    /// # let mut book = MDBook::new(Path::new("mybook"));
    /// for item in book.iter() {
    ///     match item {
    ///         BookItem::Chapter(section, chapter) => {},
    ///         BookItem::Affix(chapter) => {},
    ///         BookItem::Spacer => {},
    ///     }
    /// }
    ///
    /// // would print something like this:
    /// // 1. Chapter 1
    /// // 1.1 Sub Chapter
    /// // 1.2 Sub Chapter
    /// // 2. Chapter 2
    /// //
    /// // etc.
    /// # }
    /// ```

    pub fn iter(&self) -> BookItems {
        BookItems {
            items: &self.content[..],
            current_index: 0,
            stack: Vec::new(),
        }
    }

    /// `init()` creates some boilerplate files and directories to get you started with your book.
    ///
    /// ```text
    /// book-test/
    /// ├── book
    /// └── src
    ///     ├── chapter_1.md
    ///     └── SUMMARY.md
    /// ```
    ///
    /// It uses the paths given as source and output directories and adds a `SUMMARY.md` and a
    /// `chapter_1.md` to the source directory.

    pub fn init(&mut self) -> Result<(), Box<Error>> {

        debug!("[fn]: init");

        if !self.config.get_root().exists() {
            create_path(self.config.get_root()).unwrap();
            output!("{:?} created", self.config.get_root());
        }

        {
            let dest = self.config.get_dest();
            let src = self.config.get_src();

            if !dest.exists() {
                debug!("[*]: {:?} does not exist, trying to create directory", dest);
                try!(fs::create_dir(&dest));
            }

            if !src.exists() {
                debug!("[*]: {:?} does not exist, trying to create directory", src);
                try!(fs::create_dir(&src));
            }

            let summary = src.join("SUMMARY.md");

            if !summary.exists() {

                // Summary does not exist, create it

                debug!("[*]: {:?} does not exist, trying to create SUMMARY.md", src.join("SUMMARY.md"));
                let mut f = try!(File::create(&src.join("SUMMARY.md")));

                debug!("[*]: Writing to SUMMARY.md");

                try!(writeln!(f, "# Summary"));
                try!(writeln!(f, ""));
                try!(writeln!(f, "- [Chapter 1](./chapter_1.md)"));
            }
        }

        // parse SUMMARY.md, and create the missing item related file
        try!(self.parse_summary());

        debug!("[*]: constructing paths for missing files");
        for item in self.iter() {
            debug!("[*]: item: {:?}", item);
            match item {
                &BookItem::Spacer => continue,
                &BookItem::Chapter(_, ref ch) | &BookItem::Affix(ref ch) => {
                    if ch.path != PathBuf::new() {
                        let path = self.config.get_src().join(&ch.path);

                        if !path.exists() {
                            debug!("[*]: {:?} does not exist, trying to create file", path);
                            try!(::std::fs::create_dir_all(path.parent().unwrap()));
                            let mut f = try!(File::create(path));

                            //debug!("[*]: Writing to {:?}", path);
                            try!(writeln!(f, "# {}", ch.name));
                        }
                    }
                }
            }
        }

        debug!("[*]: init done");
        return Ok(());
    }

    /// The `build()` method is the one where everything happens. First it parses `SUMMARY.md` to
    /// construct the book's structure in the form of a `Vec<BookItem>` and then calls `render()`
    /// method of the current renderer.
    ///
    /// It is the renderer who generates all the output files.

    pub fn build(&mut self) -> Result<(), Box<Error>> {
        debug!("[fn]: build");

        try!(self.init());

        // Clean output directory
        try!(utils::remove_dir_content(&self.config.get_dest()));

        try!(self.renderer.render(&self));

        Ok(())
    }

    pub fn copy_theme(&self) -> Result<(), Box<Error>> {
        debug!("[fn]: copy_theme");

        let theme_dir = self.config.get_src().join("theme");

        if !theme_dir.exists() {
            debug!("[*]: {:?} does not exist, trying to create directory", theme_dir);
            try!(fs::create_dir(&theme_dir));
        }

        // index.hbs
        let mut index = try!(File::create(&theme_dir.join("index.hbs")));
        try!(index.write_all(theme::INDEX));

        // book.css
        let mut css = try!(File::create(&theme_dir.join("book.css")));
        try!(css.write_all(theme::CSS));

        // book.js
        let mut js = try!(File::create(&theme_dir.join("book.js")));
        try!(js.write_all(theme::JS));

        // highlight.css
        let mut highlight_css = try!(File::create(&theme_dir.join("highlight.css")));
        try!(highlight_css.write_all(theme::HIGHLIGHT_CSS));

        // highlight.js
        let mut highlight_js = try!(File::create(&theme_dir.join("highlight.js")));
        try!(highlight_js.write_all(theme::HIGHLIGHT_JS));

        Ok(())
    }

    /// Parses the `book.json` file (if it exists) to extract the configuration parameters.
    /// The `book.json` file should be in the root directory of the book.
    /// The root directory is the one specified when creating a new `MDBook`
    ///
    /// ```no_run
    /// # extern crate mdbook;
    /// # use mdbook::MDBook;
    /// # use std::path::Path;
    /// # fn main() {
    /// let mut book = MDBook::new(Path::new("root_dir"));
    /// # }
    /// ```
    ///
    /// In this example, `root_dir` will be the root directory of our book and is specified in function
    /// of the current working directory by using a relative path instead of an absolute path.

    pub fn read_config(mut self) -> Self {
        let root = self.config.get_root().to_owned();
        self.config.read_config(&root);
        self
    }

    /// You can change the default renderer to another one by using this method. The only requirement
    /// is for your renderer to implement the [Renderer trait](../../renderer/renderer/trait.Renderer.html)
    ///
    /// ```no_run
    /// extern crate mdbook;
    /// use mdbook::MDBook;
    /// use mdbook::renderer::HtmlHandlebars;
    /// # use std::path::Path;
    ///
    /// fn main() {
    ///     let mut book = MDBook::new(Path::new("mybook"))
    ///                         .set_renderer(Box::new(HtmlHandlebars::new()));
    ///
    ///     // In this example we replace the default renderer by the default renderer...
    ///     // Don't forget to put your renderer in a Box
    /// }
    /// ```
    ///
    /// **note:** Don't forget to put your renderer in a `Box` before passing it to `set_renderer()`

    pub fn set_renderer(mut self, renderer: Box<Renderer>) -> Self {
        self.renderer = renderer;
        self
    }

    pub fn set_dest(mut self, dest: &Path) -> Self {

        // Handle absolute and relative paths
        match dest.is_absolute() {
            true => { self.config.set_dest(dest); },
            false => {
                let dest = self.config.get_root().join(dest).to_owned();
                self.config.set_dest(&dest);
            }
        }

        self
    }

    pub fn get_dest(&self) -> &Path {
        self.config.get_dest()
    }

    pub fn set_src(mut self, src: &Path) -> Self {

        // Handle absolute and relative paths
        match src.is_absolute() {
            true => { self.config.set_src(src); },
            false => {
                let src = self.config.get_root().join(src).to_owned();
                self.config.set_src(&src);
            }
        }

        self
    }

    pub fn get_src(&self) -> &Path {
        self.config.get_src()
    }

    pub fn set_title(mut self, title: &str) -> Self {
        self.config.title = title.to_owned();
        self
    }

    pub fn get_title(&self) -> &str {
        &self.config.title
    }

    pub fn set_author(mut self, author: &str) -> Self {
        self.config.author = author.to_owned();
        self
    }

    pub fn get_author(&self) -> &str {
        &self.config.author
    }

    // Construct book
    fn parse_summary(&mut self) -> Result<(), Box<Error>> {
        // When append becomes stable, use self.content.append() ...
        self.content = try!(parse::construct_bookitems(&self.config.get_src().join("SUMMARY.md")));
        Ok(())
    }

}
<|endoftext|>