<commit_before>use agents::Controller;
use domains::{Domain, Observation};
use geometry::{dimensions::Discrete, Space};
use policies::Greedy;
use slog::{Logger, Record, Result as LogResult, Serializer, KV};

/// Container for episodic statistics.
#[derive(Debug)]
pub struct Episode {
    /// The number of steps taken to reach the terminal state.
    pub steps: u64,

    /// The total accumulated reward over the episode.
    pub reward: f64,
}

impl KV for Episode {
    fn serialize(&self, _: &Record, serializer: &mut Serializer) -> LogResult {
        serializer.emit_u64("steps", self.steps)?;
        serializer.emit_f64("reward", self.reward)?;

        Ok(())
    }
}

/// Helper function for running experiments.
pub fn run<T>(runner: T, n_episodes: usize, logger: Option<Logger>) -> Vec<Episode>
where T: Iterator<Item = Episode> {
    let exp = runner.take(n_episodes);

    match logger {
        Some(logger) => exp.zip(1..(n_episodes + 1))
            .inspect(|&(ref res, i)| {
                info!(logger, "episode {}", i; res);
            })
            .map(|(res, _)| res)
            .collect(),

        None => exp.collect(),
    }
}

/// Utility for running a single evaluation episode.
pub struct Evaluation<'a, A: 'a, D> {
    agent: &'a mut A,
    domain_factory: Box<Fn() -> D>,

    greedy: Greedy,
}

impl<'a, S: Space, A, D> Evaluation<'a, A, D>
where
    A: Controller<S::Value, usize>,
    D: Domain<StateSpace = S, ActionSpace = Discrete>,
{
    pub fn new(agent: &'a mut A, domain_factory: Box<Fn() -> D>) -> Evaluation<'a, A, D> {
        Evaluation {
            agent: agent,
            domain_factory: domain_factory,

            greedy: Greedy,
        }
    }
}

impl<'a, S: Space, A, D> Iterator for Evaluation<'a, A, D>
where
    A: Controller<S::Value, usize>,
    D: Domain<StateSpace = S, ActionSpace = Discrete>,
{
    type Item = Episode;

    fn next(&mut self) -> Option<Episode> {
        let mut domain = (self.domain_factory)();
        let mut a = self.agent
            .evaluate_policy(&mut self.greedy, &domain.emit().state());

        let mut e = Episode {
            steps: 1,
            reward: 0.0,
        };

        loop {
            let t = domain.step(a);

            e.steps += 1;
            e.reward += t.reward;

            a = match t.to {
                Observation::Terminal(_) => {
                    self.agent.handle_terminal(&t);
                    break;
                },
                _ => self.agent.evaluate_policy(&mut self.greedy, &t.to.state()),
            };
        }

        Some(e)
    }
}

/// Utility for running a sequence of training episodes.
pub struct SerialExperiment<'a, A: 'a, D> {
    agent: &'a mut A,
    domain_factory: Box<Fn() -> D>,

    step_limit: u64,
}

impl<'a, S: Space, A, D> SerialExperiment<'a, A, D>
where
    A: Controller<S::Value, usize>,
    D: Domain<StateSpace = S, ActionSpace = Discrete>,
{
    pub fn new(
        agent: &'a mut A,
        domain_factory: Box<Fn() -> D>,
        step_limit: u64,
    ) -> SerialExperiment<'a, A, D>
    {
        SerialExperiment {
            agent: agent,
            domain_factory: domain_factory,
            step_limit: step_limit,
        }
    }
}

impl<'a, S: Space, A, D> Iterator for SerialExperiment<'a, A, D>
where
    A: Controller<S::Value, usize>,
    D: Domain<StateSpace = S, ActionSpace = Discrete>,
{
    type Item = Episode;

    fn next(&mut self) -> Option<Episode> {
        let mut domain = (self.domain_factory)();
        let mut a = self.agent.mu(domain.emit().state());

        let mut e = Episode {
            steps: 1,
            reward: 0.0,
        };

        for j in 1..(self.step_limit + 1) {
            let t = domain.step(a);

            e.steps = j;
            e.reward += t.reward;

            self.agent.handle_sample(&t);

            // TODO: Clean this mess up!
            if let Observation::Terminal(_) = t.to {
                self.agent.handle_terminal(&t);
                break;
            } else if j >= self.step_limit {
                self.agent.handle_terminal(&t);
                break;
            } else {
                a = self.agent.mu(&t.to.state());
            }
        }

        Some(e)
    }
}
<commit_msg>rf: update experiment.rs in light of last 2 commits<commit_after>use agents::Controller;
use core::Handler;
use domains::{Domain, Observation, Transition};
use geometry::{dimensions::Discrete, Space};
use policies::Greedy;
use slog::{Logger, Record, Result as LogResult, Serializer, KV};

/// Container for episodic statistics.
#[derive(Debug)]
pub struct Episode {
    /// The number of steps taken to reach the terminal state.
    pub steps: u64,

    /// The total accumulated reward over the episode.
    pub reward: f64,
}

impl KV for Episode {
    fn serialize(&self, _: &Record, serializer: &mut Serializer) -> LogResult {
        serializer.emit_u64("steps", self.steps)?;
        serializer.emit_f64("reward", self.reward)?;

        Ok(())
    }
}

/// Helper function for running experiments.
pub fn run<T>(runner: T, n_episodes: usize, logger: Option<Logger>) -> Vec<Episode>
where T: Iterator<Item = Episode> {
    let exp = runner.take(n_episodes);

    match logger {
        Some(logger) => exp.zip(1..(n_episodes + 1))
            .inspect(|&(ref res, i)| {
                info!(logger, "episode {}", i; res);
            })
            .map(|(res, _)| res)
            .collect(),

        None => exp.collect(),
    }
}

/// Utility for running a single evaluation episode.
pub struct Evaluation<'a, A: 'a, D> {
    agent: &'a mut A,
    domain_factory: Box<Fn() -> D>,

    greedy: Greedy,
}

impl<'a, S: Space, A, D> Evaluation<'a, A, D>
where
    A: Handler<Transition<S::Value, usize>> + Controller<S::Value, usize>,
    D: Domain<StateSpace = S, ActionSpace = Discrete>,
{
    pub fn new(agent: &'a mut A, domain_factory: Box<Fn() -> D>) -> Evaluation<'a, A, D> {
        Evaluation {
            agent: agent,
            domain_factory: domain_factory,

            greedy: Greedy,
        }
    }
}

impl<'a, S: Space, A, D> Iterator for Evaluation<'a, A, D>
where
    A: Handler<Transition<S::Value, usize>> + Controller<S::Value, usize>,
    D: Domain<StateSpace = S, ActionSpace = Discrete>,
{
    type Item = Episode;

    fn next(&mut self) -> Option<Episode> {
        let mut domain = (self.domain_factory)();
        let mut a = self.agent
            .evaluate_policy(&mut self.greedy, &domain.emit().state());

        let mut e = Episode {
            steps: 1,
            reward: 0.0,
        };

        loop {
            let t = domain.step(a);

            e.steps += 1;
            e.reward += t.reward;

            a = match t.to {
                Observation::Terminal(_) => {
                    self.agent.handle_terminal(&t);
                    break;
                },
                _ => self.agent.evaluate_policy(&mut self.greedy, &t.to.state()),
            };
        }

        Some(e)
    }
}

/// Utility for running a sequence of training episodes.
pub struct SerialExperiment<'a, A: 'a, D> {
    agent: &'a mut A,
    domain_factory: Box<Fn() -> D>,

    step_limit: u64,
}

impl<'a, S: Space, A, D> SerialExperiment<'a, A, D>
where
    A: Controller<S::Value, usize>,
    D: Domain<StateSpace = S, ActionSpace = Discrete>,
{
    pub fn new(
        agent: &'a mut A,
        domain_factory: Box<Fn() -> D>,
        step_limit: u64,
    ) -> SerialExperiment<'a, A, D>
    {
        SerialExperiment {
            agent: agent,
            domain_factory: domain_factory,
            step_limit: step_limit,
        }
    }
}

impl<'a, S: Space, A, D> Iterator for SerialExperiment<'a, A, D>
where
    A: Handler<Transition<S::Value, usize>> + Controller<S::Value, usize>,
    D: Domain<StateSpace = S, ActionSpace = Discrete>,
{
    type Item = Episode;

    fn next(&mut self) -> Option<Episode> {
        let mut domain = (self.domain_factory)();
        let mut a = self.agent.mu(domain.emit().state());

        let mut e = Episode {
            steps: 1,
            reward: 0.0,
        };

        for j in 1..(self.step_limit + 1) {
            let t = domain.step(a);

            e.steps = j;
            e.reward += t.reward;

            self.agent.handle_sample(&t);

            // TODO: Clean this mess up!
            if let Observation::Terminal(_) = t.to {
                self.agent.handle_terminal(&t);
                break;
            } else if j >= self.step_limit {
                self.agent.handle_terminal(&t);
                break;
            } else {
                a = self.agent.mu(&t.to.state());
            }
        }

        Some(e)
    }
}
<|endoftext|>


<commit_before><commit_msg>Build fix<commit_after><|endoftext|>


<commit_before>use std::path::{Path, PathBuf};
use std::env;
use std::fs::DirBuilder;
use std::error::Error;
use std::fmt;
use std::io;

#[derive(Debug)]
pub enum ValidationError {
    UserAbort,
    TargetIsADir,
    EnvError(io::Error),
    BuildDirErr(io::Error),
}

impl Error for ValidationError {
    fn description(&self) -> &str {
        match *self {
            ValidationError::UserAbort => "Abort by user.",
            ValidationError::TargetIsADir => "The selected target is a directory, not a file.",
            ValidationError::EnvError(ref err) => err.description().clone(),
            ValidationError::BuildDirErr(ref err) => err.description().clone(),
        }
    }
}

impl fmt::Display for ValidationError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "[ERROR] {}", self.description())
    }
}

fn ask_user(question: &str) -> bool {
    let mut answer = String::new();
    loop {
        print!("{} (y/n) ", question);
        io::stdin().read_line(&mut answer).unwrap();
        match answer.trim() {
            "y" | "yes" => return true,
            "n" | "no" => return false,
            _ => continue,
        };
    }
}

pub fn validate_target_file(target_path: &str) -> Result<&Path, ValidationError> {
    let mut path = Path::new(target_path);

    // make sure the target directory is absolute
    if path.is_relative() {
        // get the current working directory and adoin the provided relative path onto it
        // potential environment errors are reported back as specialized error type
        let cwd = match env::current_dir() {
            Ok(dir) => dir,
            Err(e) => return Err(ValidationError::EnvError(e)),
        };

        // also, canonicalize the path to generate a clean path w/o relative path descriptions
        // like "../ " and
        path = match cwd.join(path).canonicalize() {
            Ok(dir) => dir.as_path(),
            Err(e) => return Err(ValidationError::EnvError(e)),
        };
    }

    if path.exists() {
        if path.is_file() {
            // The path exists and is a file.
            Ok(path)
        } else {
            Err(ValidationError::TargetIsADir)
        }
    } else {
        // check whether the parent directory exists. If so, create the target directory
        // if not, ask the user if he's sure that he wants to create the directory recursively
        let parent_dir = path.parent().unwrap();
        if parent_dir.exists() {
            Ok(path)
        } else {
            if ask_user("The path to the target directory does not exist. Create it?") {
                let _ = match DirBuilder::new().recursive(true).create(parent_dir) {
                    Ok(b) => b,
                    Err(err) => return Err(ValidationError::BuildDirErr(err)),
                };
                Ok(path)
            } else {
                Err(ValidationError::UserAbort)
            }
        }
    }
}
<commit_msg>fixed lifetime with Absolute value not reference<commit_after>use std::path::PathBuf;
use std::env;
use std::fs::DirBuilder;
use std::error::Error;
use std::fmt;
use std::io;

#[derive(Debug)]
pub enum ValidationError {
    UserAbort,
    TargetIsADir,
    EnvError(io::Error),
    BuildDirErr(io::Error),
}

impl Error for ValidationError {
    fn description(&self) -> &str {
        match *self {
            ValidationError::UserAbort => "Abort by user.",
            ValidationError::TargetIsADir => "The selected target is a directory, not a file.",
            ValidationError::EnvError(ref err) => err.description().clone(),
            ValidationError::BuildDirErr(ref err) => err.description().clone(),
        }
    }
}

impl fmt::Display for ValidationError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "[ERROR] {}", self.description())
    }
}

fn ask_user(question: &str) -> bool {
    let mut answer = String::new();
    loop {
        println!("{} (y/n) ", question);
        io::stdin().read_line(&mut answer).unwrap();
        match answer.trim() {
            "y" | "yes" => return true,
            "n" | "no" => return false,
            _ => continue,
        };
    }
}

pub fn validate_target_file(target_path: &str) -> Result<PathBuf, ValidationError> {
    let mut path = PathBuf::from(target_path);

    // make sure the target directory is absolute
    if path.is_relative() {
        // get the current working directory and adoin the provided relative path onto it
        // potential environment errors are reported back as specialized error type
        let cwd = match env::current_dir() {
            Ok(dir) => dir,
            Err(e) => return Err(ValidationError::EnvError(e)),
        };

        // also, canonicalize the path to generate a clean path w/o relative path descriptions
        // like "../ " and
        path = match cwd.join(path).canonicalize() {
            Ok(dir) => dir,
            Err(e) => return Err(ValidationError::EnvError(e)),
        };
    }

    if path.exists() {
        if path.is_file() {
            // The path exists and is a file.
            Ok(path.to_owned())
        } else {
            Err(ValidationError::TargetIsADir)
        }
    } else {
        // check whether the parent directory exists. If so, create the target directory
        // if not, ask the user if he's sure that he wants to create the directory recursively
        let parent_dir = path.parent().unwrap();
        if parent_dir.exists() {
            Ok(path.to_owned())
        } else {
            if ask_user("The path to the target directory does not exist. Create it?") {
                let _ = match DirBuilder::new().recursive(true).create(parent_dir) {
                    Ok(b) => b,
                    Err(err) => return Err(ValidationError::BuildDirErr(err)),
                };
                Ok(path.to_owned())
            } else {
                Err(ValidationError::UserAbort)
            }
        }
    }
}
<|endoftext|>


<commit_before><commit_msg>refactor: Pull over DepTable from cargo<commit_after><|endoftext|>


<commit_before><commit_msg>The branch operator should contribute zeros to the gradient of the masked input; implement this simply by checking `accumulate` capability.<commit_after><|endoftext|>


<commit_before><commit_msg>Fix transition iterator for nodes with 256 transitions<commit_after><|endoftext|>


<commit_before>use hashes::HashFunction;

static MD2_S_Table: &'static [u8] = &[
    0x29, 0x2E, 0x43, 0xC9, 0xA2, 0xD8, 0x7C, 0x01, 0x3D, 0x36, 0x54, 0xA1, 0xEC, 0xF0, 0x06, 0x13,
    0x62, 0xA7, 0x05, 0xF3, 0xC0, 0xC7, 0x73, 0x8C, 0x98, 0x93, 0x2B, 0xD9, 0xBC, 0x4C, 0x82, 0xCA,
    0x1E, 0x9B, 0x57, 0x3C, 0xFD, 0xD4, 0xE0, 0x16, 0x67, 0x42, 0x6F, 0x18, 0x8A, 0x17, 0xE5, 0x12,
    0xBE, 0x4E, 0xC4, 0xD6, 0xDA, 0x9E, 0xDE, 0x49, 0xA0, 0xFB, 0xF5, 0x8E, 0xBB, 0x2F, 0xEE, 0x7A,
    0xA9, 0x68, 0x79, 0x91, 0x15, 0xB2, 0x07, 0x3F, 0x94, 0xC2, 0x10, 0x89, 0x0B, 0x22, 0x5F, 0x21,
    0x80, 0x7F, 0x5D, 0x9A, 0x5A, 0x90, 0x32, 0x27, 0x35, 0x3E, 0xCC, 0xE7, 0xBF, 0xF7, 0x97, 0x03,
    0xFF, 0x19, 0x30, 0xB3, 0x48, 0xA5, 0xB5, 0xD1, 0xD7, 0x5E, 0x92, 0x2A, 0xAC, 0x56, 0xAA, 0xC6,
    0x4F, 0xB8, 0x38, 0xD2, 0x96, 0xA4, 0x7D, 0xB6, 0x76, 0xFC, 0x6B, 0xE2, 0x9C, 0x74, 0x04, 0xF1,
    0x45, 0x9D, 0x70, 0x59, 0x64, 0x71, 0x87, 0x20, 0x86, 0x5B, 0xCF, 0x65, 0xE6, 0x2D, 0xA8, 0x02,
    0x1B, 0x60, 0x25, 0xAD, 0xAE, 0xB0, 0xB9, 0xF6, 0x1C, 0x46, 0x61, 0x69, 0x34, 0x40, 0x7E, 0x0F,
    0x55, 0x47, 0xA3, 0x23, 0xDD, 0x51, 0xAF, 0x3A, 0xC3, 0x5C, 0xF9, 0xCE, 0xBA, 0xC5, 0xEA, 0x26,
    0x2C, 0x53, 0x0D, 0x6E, 0x85, 0x28, 0x84, 0x09, 0xD3, 0xDF, 0xCD, 0xF4, 0x41, 0x81, 0x4D, 0x52,
    0x6A, 0xDC, 0x37, 0xC8, 0x6C, 0xC1, 0xAB, 0xFA, 0x24, 0xE1, 0x7B, 0x08, 0x0C, 0xBD, 0xB1, 0x4A,
    0x78, 0x88, 0x95, 0x8B, 0xE3, 0x63, 0xE8, 0x6D, 0xE9, 0xCB, 0xD5, 0xFE, 0x3B, 0x00, 0x1D, 0x39,
    0xF2, 0xEF, 0xB7, 0x0E, 0x66, 0x58, 0xD0, 0xE4, 0xA6, 0x77, 0x72, 0xF8, 0xEB, 0x75, 0x4B, 0x0A,
    0x31, 0x44, 0x50, 0xB4, 0x8F, 0xED, 0x1F, 0x1A, 0xDB, 0x99, 0x8D, 0x33, 0x9F, 0x11, 0x83, 0x14
];

/// Struct storing the necessary state for the Message Digest 2 (MD2) hash function
/// Code is ported and `rustified` from libtomcrypt
#[allow(non_camel_case_types)]
pub struct MD2 {
    check_sum : [u8, ..16],
    x         : [u8, ..48],
    buffer    : [u8, ..16],
    cur_len   : uint
}

fn md2_compress(state: &mut MD2) {
   /* copy block to state.x */
   for i in range(0, 16) {
       state.x[16 + i] = state.buffer[i];
       state.x[32 + i] = state.x[i] ^ state.x[16 + i];
   }

   let mut t = 0u8;
   /* perform 18 rounds */
   for round in range(0, 18) {
       for i in range(0, 48) {
           state.x[i] ^= MD2_S_Table[(t & 255) as uint];
           t = state.x[i];
       }
       t = t + round & 255;
   }
}

#[allow(non_snake_case)]
fn md2_update_checksum(state: &mut MD2) {
    let mut L = state.check_sum[15];
    for i in range(0, 16) {
        /* caution, the RFC says its "C[j] = S[M[i*16+j] xor L]" but the reference
         * source code [and test vectors] say otherwise. */
        state.check_sum[i] ^= MD2_S_Table[(state.buffer[i] ^ L) as uint] & 255;
        L = state.check_sum[i];
    }
}

impl MD2 {
    pub fn new() -> MD2 {
        MD2 {
            check_sum : [0, ..16],
            x         : [0, ..48],
            buffer    : [0, ..16],
            cur_len   : 0
        }
    }

    fn reset(&mut self) {
        self.check_sum = [0, ..16];
        self.x         = [0, ..48];
        self.buffer    = [0, ..16];
        self.cur_len   = 0;
    }
}

impl HashFunction for MD2 {
    fn set_input(&mut self, input: &[u8]) {
        use std::cmp::{min};

        self.reset();

        // When is this the case?
        if self.cur_len > self.buffer.len() {
           fail!()
        }

        let mut index = 0u;
        let mut in_len = input.len();

        loop {
            if in_len <= 0 {
                break;
            }

            let n = min(in_len, (16 - self.cur_len));
            for i in range(0, n) {
                self.buffer[self.cur_len + i] = input[index + i];
            }
            self.cur_len += n;
            index         += n;
            in_len       -= n;

            /* if 16 bytes are filled compress and update checksum */
            if self.cur_len == 16 {
                md2_compress(self);
                md2_update_checksum(self);
                self.cur_len = 0;
            }
        }
    }

    fn hash(&mut self) {
        // Again when is this the case?
        if self.cur_len >= self.buffer.len() {
           fail!()
        }

        /* pad the message */
        let k: u8 = 16u8 - self.cur_len as u8;
        for i in range(self.cur_len, 16) {
            self.buffer[i] = k;
        }

        /* hash and update */
        md2_compress(self);
        md2_update_checksum(self);

        /* hash checksum */
        for i in range(0, 16) {
            self.buffer[i] = self.check_sum[i];
        }
        md2_compress(self);
    }

    fn get_output(&mut self, output: &mut [u8]) {
        assert!(output.len() >= self.get_output_length())

        for i in range(0, 16) {
            output[i] = self.x[i];
        }
    }

    fn get_blocksize(&self) -> uint { 16 }

    fn get_output_length_in_bits(&self) -> uint { 128 }
}

#[cfg(test)]
mod tests {
    use hashes::md2::MD2;
    use hashes::test::{HashTestCase, perform_hash_test};

    #[test]
    fn test_md2() {
        let tests = vec![
            HashTestCase {
                input: "",
                output: vec![
                    0x83,0x50,0xe5,0xa3,0xe2,0x4c,0x15,0x3d,
                    0xf2,0x27,0x5c,0x9f,0x80,0x69,0x27,0x73
                ],
                output_str: "8350e5a3e24c153df2275c9f80692773"
            },
            HashTestCase {
                input: "a",
                output: vec![
                    0x32,0xec,0x01,0xec,0x4a,0x6d,0xac,0x72,
                    0xc0,0xab,0x96,0xfb,0x34,0xc0,0xb5,0xd1
                ],
                output_str: "32ec01ec4a6dac72c0ab96fb34c0b5d1"
            },
            HashTestCase {
                input: "message digest",
                output: vec![
                    0xab,0x4f,0x49,0x6b,0xfb,0x2a,0x53,0x0b,
                    0x21,0x9f,0xf3,0x30,0x31,0xfe,0x06,0xb0
                ],
                output_str: "ab4f496bfb2a530b219ff33031fe06b0"
            },
            HashTestCase {
                input: "abcdefghijklmnopqrstuvwxyz",
                output: vec![
                    0x4e,0x8d,0xdf,0xf3,0x65,0x02,0x92,0xab,
                    0x5a,0x41,0x08,0xc3,0xaa,0x47,0x94,0x0b
                ],
                output_str: "4e8ddff3650292ab5a4108c3aa47940b"
            },
            HashTestCase {
                input: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
                output: vec![
                    0xda,0x33,0xde,0xf2,0xa4,0x2d,0xf1,0x39,
                    0x75,0x35,0x28,0x46,0xc3,0x03,0x38,0xcd
                ],
                output_str: "da33def2a42df13975352846c30338cd"
            },
            HashTestCase {
                input: "12345678901234567890123456789012345678901234567890123456789012345678901234567890",
                output: vec![
                    0xd5,0x97,0x6f,0x79,0xd8,0x3d,0x3a,0x0d,
                    0xc9,0x80,0x6c,0x3c,0x66,0xf3,0xef,0xd8
                ],
                output_str: "d5976f79d83d3a0dc9806c3c66f3efd8"
            }
        ];

        let mut md2 = MD2::new();
        for t in tests.iter() {
            perform_hash_test(&mut md2, t);
        }
    }
}
<commit_msg>Added doc comment on MD2::new() and removed some dead code<commit_after>static MD2_S_Table: &'static [u8] = &[
    0x29, 0x2E, 0x43, 0xC9, 0xA2, 0xD8, 0x7C, 0x01, 0x3D, 0x36, 0x54, 0xA1, 0xEC, 0xF0, 0x06, 0x13,
    0x62, 0xA7, 0x05, 0xF3, 0xC0, 0xC7, 0x73, 0x8C, 0x98, 0x93, 0x2B, 0xD9, 0xBC, 0x4C, 0x82, 0xCA,
    0x1E, 0x9B, 0x57, 0x3C, 0xFD, 0xD4, 0xE0, 0x16, 0x67, 0x42, 0x6F, 0x18, 0x8A, 0x17, 0xE5, 0x12,
    0xBE, 0x4E, 0xC4, 0xD6, 0xDA, 0x9E, 0xDE, 0x49, 0xA0, 0xFB, 0xF5, 0x8E, 0xBB, 0x2F, 0xEE, 0x7A,
    0xA9, 0x68, 0x79, 0x91, 0x15, 0xB2, 0x07, 0x3F, 0x94, 0xC2, 0x10, 0x89, 0x0B, 0x22, 0x5F, 0x21,
    0x80, 0x7F, 0x5D, 0x9A, 0x5A, 0x90, 0x32, 0x27, 0x35, 0x3E, 0xCC, 0xE7, 0xBF, 0xF7, 0x97, 0x03,
    0xFF, 0x19, 0x30, 0xB3, 0x48, 0xA5, 0xB5, 0xD1, 0xD7, 0x5E, 0x92, 0x2A, 0xAC, 0x56, 0xAA, 0xC6,
    0x4F, 0xB8, 0x38, 0xD2, 0x96, 0xA4, 0x7D, 0xB6, 0x76, 0xFC, 0x6B, 0xE2, 0x9C, 0x74, 0x04, 0xF1,
    0x45, 0x9D, 0x70, 0x59, 0x64, 0x71, 0x87, 0x20, 0x86, 0x5B, 0xCF, 0x65, 0xE6, 0x2D, 0xA8, 0x02,
    0x1B, 0x60, 0x25, 0xAD, 0xAE, 0xB0, 0xB9, 0xF6, 0x1C, 0x46, 0x61, 0x69, 0x34, 0x40, 0x7E, 0x0F,
    0x55, 0x47, 0xA3, 0x23, 0xDD, 0x51, 0xAF, 0x3A, 0xC3, 0x5C, 0xF9, 0xCE, 0xBA, 0xC5, 0xEA, 0x26,
    0x2C, 0x53, 0x0D, 0x6E, 0x85, 0x28, 0x84, 0x09, 0xD3, 0xDF, 0xCD, 0xF4, 0x41, 0x81, 0x4D, 0x52,
    0x6A, 0xDC, 0x37, 0xC8, 0x6C, 0xC1, 0xAB, 0xFA, 0x24, 0xE1, 0x7B, 0x08, 0x0C, 0xBD, 0xB1, 0x4A,
    0x78, 0x88, 0x95, 0x8B, 0xE3, 0x63, 0xE8, 0x6D, 0xE9, 0xCB, 0xD5, 0xFE, 0x3B, 0x00, 0x1D, 0x39,
    0xF2, 0xEF, 0xB7, 0x0E, 0x66, 0x58, 0xD0, 0xE4, 0xA6, 0x77, 0x72, 0xF8, 0xEB, 0x75, 0x4B, 0x0A,
    0x31, 0x44, 0x50, 0xB4, 0x8F, 0xED, 0x1F, 0x1A, 0xDB, 0x99, 0x8D, 0x33, 0x9F, 0x11, 0x83, 0x14
];

/// Struct storing the necessary state for the Message Digest 2 (MD2) hash function
/// Code is ported and `rustified` from libtomcrypt
#[allow(non_camel_case_types)]
pub struct MD2 {
    check_sum : [u8, ..16],
    x         : [u8, ..48],
    buffer    : [u8, ..16],
    cur_len   : uint
}

fn md2_compress(state: &mut MD2) {
   /* copy block to state.x */
   for i in range(0, 16) {
       state.x[16 + i] = state.buffer[i];
       state.x[32 + i] = state.x[i] ^ state.x[16 + i];
   }

   let mut t = 0u8;
   /* perform 18 rounds */
   for round in range(0, 18) {
       for i in range(0, 48) {
           state.x[i] ^= MD2_S_Table[(t & 255) as uint];
           t = state.x[i];
       }
       t = t + round & 255;
   }
}

#[allow(non_snake_case)]
fn md2_update_checksum(state: &mut MD2) {
    let mut L = state.check_sum[15];
    for i in range(0, 16) {
        /* caution, the RFC says its "C[j] = S[M[i*16+j] xor L]" but the reference
         * source code [and test vectors] say otherwise. */
        state.check_sum[i] ^= MD2_S_Table[(state.buffer[i] ^ L) as uint] & 255;
        L = state.check_sum[i];
    }
}

impl MD2 {
    /// Creates a new MD2 instance
    ///
    /// # Returns
    /// * The created instance
    pub fn new() -> MD2 {
        MD2 {
            check_sum : [0, ..16],
            x         : [0, ..48],
            buffer    : [0, ..16],
            cur_len   : 0
        }
    }

    fn reset(&mut self) {
        self.check_sum = [0, ..16];
        self.x         = [0, ..48];
        self.buffer    = [0, ..16];
        self.cur_len   = 0;
    }
}

impl ::hashes::HashFunction for MD2 {
    fn set_input(&mut self, input: &[u8]) {
        use std::cmp::{min};

        // First reset the hash state
        self.reset();

        let mut index = 0u;
        let mut in_len = input.len();

        loop {
            if in_len <= 0 {
                break;
            }

            let n = min(in_len, (16 - self.cur_len));
            for i in range(0, n) {
                self.buffer[self.cur_len + i] = input[index + i];
            }
            self.cur_len += n;
            index         += n;
            in_len       -= n;

            /* if 16 bytes are filled compress and update checksum */
            if self.cur_len == 16 {
                md2_compress(self);
                md2_update_checksum(self);
                self.cur_len = 0;
            }
        }
    }

    fn hash(&mut self) {
        // When is this the case?
        if self.cur_len >= self.buffer.len() {
           fail!()
        }

        /* pad the message */
        let k: u8 = 16u8 - self.cur_len as u8;
        for i in range(self.cur_len, 16) {
            self.buffer[i] = k;
        }

        /* hash and update */
        md2_compress(self);
        md2_update_checksum(self);

        /* hash checksum */
        for i in range(0, 16) {
            self.buffer[i] = self.check_sum[i];
        }
        md2_compress(self);
    }

    fn get_output(&mut self, output: &mut [u8]) {
        assert!(output.len() >= self.get_output_length())

        for i in range(0, 16) {
            output[i] = self.x[i];
        }
    }

    fn get_blocksize(&self) -> uint { 16 }

    fn get_output_length_in_bits(&self) -> uint { 128 }
}

#[cfg(test)]
mod tests {
    use hashes::md2::MD2;
    use hashes::test::{HashTestCase, perform_hash_test};

    #[test]
    fn test_md2() {
        let tests = vec![
            HashTestCase {
                input: "",
                output: vec![
                    0x83,0x50,0xe5,0xa3,0xe2,0x4c,0x15,0x3d,
                    0xf2,0x27,0x5c,0x9f,0x80,0x69,0x27,0x73
                ],
                output_str: "8350e5a3e24c153df2275c9f80692773"
            },
            HashTestCase {
                input: "a",
                output: vec![
                    0x32,0xec,0x01,0xec,0x4a,0x6d,0xac,0x72,
                    0xc0,0xab,0x96,0xfb,0x34,0xc0,0xb5,0xd1
                ],
                output_str: "32ec01ec4a6dac72c0ab96fb34c0b5d1"
            },
            HashTestCase {
                input: "message digest",
                output: vec![
                    0xab,0x4f,0x49,0x6b,0xfb,0x2a,0x53,0x0b,
                    0x21,0x9f,0xf3,0x30,0x31,0xfe,0x06,0xb0
                ],
                output_str: "ab4f496bfb2a530b219ff33031fe06b0"
            },
            HashTestCase {
                input: "abcdefghijklmnopqrstuvwxyz",
                output: vec![
                    0x4e,0x8d,0xdf,0xf3,0x65,0x02,0x92,0xab,
                    0x5a,0x41,0x08,0xc3,0xaa,0x47,0x94,0x0b
                ],
                output_str: "4e8ddff3650292ab5a4108c3aa47940b"
            },
            HashTestCase {
                input: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
                output: vec![
                    0xda,0x33,0xde,0xf2,0xa4,0x2d,0xf1,0x39,
                    0x75,0x35,0x28,0x46,0xc3,0x03,0x38,0xcd
                ],
                output_str: "da33def2a42df13975352846c30338cd"
            },
            HashTestCase {
                input: "12345678901234567890123456789012345678901234567890123456789012345678901234567890",
                output: vec![
                    0xd5,0x97,0x6f,0x79,0xd8,0x3d,0x3a,0x0d,
                    0xc9,0x80,0x6c,0x3c,0x66,0xf3,0xef,0xd8
                ],
                output_str: "d5976f79d83d3a0dc9806c3c66f3efd8"
            }
        ];

        let mut md2 = MD2::new();
        for t in tests.iter() {
            perform_hash_test(&mut md2, t);
        }
    }
}
<|endoftext|>


<commit_before>use serde::Serialize;
use serde_json::value::{to_value, Value as Json};

pub(crate) static DEFAULT_VALUE: Json = Json::Null;

/// A JSON wrapper designed for handlebars internal use case
///
/// * Constant: the JSON value hardcoded into template
/// * Context:  the JSON value referenced in your provided data context
/// * Derived:  the owned JSON value computed during rendering process
///
#[derive(Debug)]
pub enum ScopedJson<'reg: 'rc, 'rc> {
    Constant(&'reg Json),
    Derived(Json),
    // represents a json reference to context value, its full path
    Context(&'rc Json, Vec<String>),
    Missing,
}

impl<'reg: 'rc, 'rc> ScopedJson<'reg, 'rc> {
    /// get the JSON reference
    pub fn as_json(&self) -> &Json {
        match self {
            ScopedJson::Constant(j) => j,
            ScopedJson::Derived(ref j) => j,
            ScopedJson::Context(j, _) => j,
            _ => &DEFAULT_VALUE,
        }
    }

    pub fn render(&self) -> String {
        self.as_json().render()
    }

    pub fn is_missing(&self) -> bool {
        match self {
            ScopedJson::Missing => true,
            _ => false,
        }
    }

    pub fn into_derived(self) -> ScopedJson<'reg, 'rc> {
        let v = self.as_json();
        ScopedJson::Derived(v.clone())
    }

    pub fn context_path(&self) -> Option<&Vec<String>> {
        match self {
            ScopedJson::Context(_, ref p) => Some(p),
            _ => None,
        }
    }
}

impl<'reg: 'rc, 'rc> From<Json> for ScopedJson<'reg, 'rc> {
    fn from(v: Json) -> ScopedJson<'reg, 'rc> {
        ScopedJson::Derived(v)
    }
}

/// Json wrapper that holds the Json value and reference path information
///
#[derive(Debug)]
pub struct PathAndJson<'reg, 'rc> {
    relative_path: Option<String>,
    value: ScopedJson<'reg, 'rc>,
}

impl<'reg: 'rc, 'rc> PathAndJson<'reg, 'rc> {
    pub fn new(
        relative_path: Option<String>,
        value: ScopedJson<'reg, 'rc>,
    ) -> PathAndJson<'reg, 'rc> {
        PathAndJson {
            relative_path,
            value,
        }
    }

    /// Returns relative path when the value is referenced
    /// If the value is from a literal, the path is `None`
    pub fn relative_path(&self) -> Option<&String> {
        self.relative_path.as_ref()
    }

    /// Returns full path to this value if any
    pub fn context_path(&self) -> Option<&Vec<String>> {
        self.value.context_path()
    }

    /// Returns the value
    pub fn value(&self) -> &Json {
        self.value.as_json()
    }

    /// Test if value is missing
    pub fn is_value_missing(&self) -> bool {
        self.value.is_missing()
    }

    pub fn render(&self) -> String {
        self.value.render()
    }
}

/// Render Json data with default format
pub trait JsonRender {
    fn render(&self) -> String;
}

pub trait JsonTruthy {
    fn is_truthy(&self, include_zero: bool) -> bool;
}

impl JsonRender for Json {
    fn render(&self) -> String {
        match *self {
            Json::String(ref s) => s.to_string(),
            Json::Bool(i) => i.to_string(),
            Json::Number(ref n) => n.to_string(),
            Json::Null => "".to_owned(),
            Json::Array(ref a) => {
                let mut buf = String::new();
                buf.push('[');
                for i in a.iter() {
                    buf.push_str(i.render().as_ref());
                    buf.push_str(", ");
                }
                buf.push(']');
                buf
            }
            Json::Object(_) => "[object]".to_owned(),
        }
    }
}

/// Convert any serializable data into Serde Json type
pub fn to_json<T>(src: T) -> Json
where
    T: Serialize,
{
    to_value(src).unwrap_or_default()
}

pub fn as_string(src: &Json) -> Option<&str> {
    src.as_str()
}

impl JsonTruthy for Json {
    fn is_truthy(&self, include_zero: bool) -> bool {
        match *self {
            Json::Bool(ref i) => *i,
            Json::Number(ref n) => {
                if include_zero {
                    n.as_f64().map(|f| !f.is_nan()).unwrap_or(false)
                } else {
                    // there is no inifity in json/serde_json
                    n.as_f64().map(|f| f.is_normal()).unwrap_or(false)
                }
            }
            Json::Null => false,
            Json::String(ref i) => !i.is_empty(),
            Json::Array(ref i) => !i.is_empty(),
            Json::Object(ref i) => !i.is_empty(),
        }
    }
}

#[test]
fn test_json_render() {
    let raw = "
Hello world
\n<p thing=\"hello\"
";
    let thing = Json::String(raw.to_string());

    assert_eq!(raw, thing.render());
}

#[test]
fn test_json_number_truthy() {
    use std::f64;
    assert!(json!(16i16).is_truthy(false));
    assert!(json!(16i16).is_truthy(true));

    assert!(json!(0i16).is_truthy(true));
    assert!(!json!(0i16).is_truthy(false));

    assert!(json!(1.0f64).is_truthy(false));
    assert!(json!(1.0f64).is_truthy(true));

    assert!(json!(Some(16i16)).is_truthy(false));
    assert!(json!(Some(16i16)).is_truthy(true));

    assert!(!json!(None as Option<i16>).is_truthy(false));
    assert!(!json!(None as Option<i16>).is_truthy(true));

    assert!(!json!(f64::NAN).is_truthy(false));
    assert!(!json!(f64::NAN).is_truthy(true));

    // there is no infinity in json/serde_json
    // assert!(json!(f64::INFINITY).is_truthy(false));
    // assert!(json!(f64::INFINITY).is_truthy(true));

    // assert!(json!(f64::NEG_INFINITY).is_truthy(false));
    // assert!(json!(f64::NEG_INFINITY).is_truthy(true));
}
<commit_msg>(fix) lint error: using matches macro<commit_after>use serde::Serialize;
use serde_json::value::{to_value, Value as Json};

pub(crate) static DEFAULT_VALUE: Json = Json::Null;

/// A JSON wrapper designed for handlebars internal use case
///
/// * Constant: the JSON value hardcoded into template
/// * Context:  the JSON value referenced in your provided data context
/// * Derived:  the owned JSON value computed during rendering process
///
#[derive(Debug)]
pub enum ScopedJson<'reg: 'rc, 'rc> {
    Constant(&'reg Json),
    Derived(Json),
    // represents a json reference to context value, its full path
    Context(&'rc Json, Vec<String>),
    Missing,
}

impl<'reg: 'rc, 'rc> ScopedJson<'reg, 'rc> {
    /// get the JSON reference
    pub fn as_json(&self) -> &Json {
        match self {
            ScopedJson::Constant(j) => j,
            ScopedJson::Derived(ref j) => j,
            ScopedJson::Context(j, _) => j,
            _ => &DEFAULT_VALUE,
        }
    }

    pub fn render(&self) -> String {
        self.as_json().render()
    }

    pub fn is_missing(&self) -> bool {
        matches!(self, ScopedJson::Missing)
    }

    pub fn into_derived(self) -> ScopedJson<'reg, 'rc> {
        let v = self.as_json();
        ScopedJson::Derived(v.clone())
    }

    pub fn context_path(&self) -> Option<&Vec<String>> {
        match self {
            ScopedJson::Context(_, ref p) => Some(p),
            _ => None,
        }
    }
}

impl<'reg: 'rc, 'rc> From<Json> for ScopedJson<'reg, 'rc> {
    fn from(v: Json) -> ScopedJson<'reg, 'rc> {
        ScopedJson::Derived(v)
    }
}

/// Json wrapper that holds the Json value and reference path information
///
#[derive(Debug)]
pub struct PathAndJson<'reg, 'rc> {
    relative_path: Option<String>,
    value: ScopedJson<'reg, 'rc>,
}

impl<'reg: 'rc, 'rc> PathAndJson<'reg, 'rc> {
    pub fn new(
        relative_path: Option<String>,
        value: ScopedJson<'reg, 'rc>,
    ) -> PathAndJson<'reg, 'rc> {
        PathAndJson {
            relative_path,
            value,
        }
    }

    /// Returns relative path when the value is referenced
    /// If the value is from a literal, the path is `None`
    pub fn relative_path(&self) -> Option<&String> {
        self.relative_path.as_ref()
    }

    /// Returns full path to this value if any
    pub fn context_path(&self) -> Option<&Vec<String>> {
        self.value.context_path()
    }

    /// Returns the value
    pub fn value(&self) -> &Json {
        self.value.as_json()
    }

    /// Test if value is missing
    pub fn is_value_missing(&self) -> bool {
        self.value.is_missing()
    }

    pub fn render(&self) -> String {
        self.value.render()
    }
}

/// Render Json data with default format
pub trait JsonRender {
    fn render(&self) -> String;
}

pub trait JsonTruthy {
    fn is_truthy(&self, include_zero: bool) -> bool;
}

impl JsonRender for Json {
    fn render(&self) -> String {
        match *self {
            Json::String(ref s) => s.to_string(),
            Json::Bool(i) => i.to_string(),
            Json::Number(ref n) => n.to_string(),
            Json::Null => "".to_owned(),
            Json::Array(ref a) => {
                let mut buf = String::new();
                buf.push('[');
                for i in a.iter() {
                    buf.push_str(i.render().as_ref());
                    buf.push_str(", ");
                }
                buf.push(']');
                buf
            }
            Json::Object(_) => "[object]".to_owned(),
        }
    }
}

/// Convert any serializable data into Serde Json type
pub fn to_json<T>(src: T) -> Json
where
    T: Serialize,
{
    to_value(src).unwrap_or_default()
}

pub fn as_string(src: &Json) -> Option<&str> {
    src.as_str()
}

impl JsonTruthy for Json {
    fn is_truthy(&self, include_zero: bool) -> bool {
        match *self {
            Json::Bool(ref i) => *i,
            Json::Number(ref n) => {
                if include_zero {
                    n.as_f64().map(|f| !f.is_nan()).unwrap_or(false)
                } else {
                    // there is no inifity in json/serde_json
                    n.as_f64().map(|f| f.is_normal()).unwrap_or(false)
                }
            }
            Json::Null => false,
            Json::String(ref i) => !i.is_empty(),
            Json::Array(ref i) => !i.is_empty(),
            Json::Object(ref i) => !i.is_empty(),
        }
    }
}

#[test]
fn test_json_render() {
    let raw = "
Hello world
\n<p thing=\"hello\"
";
    let thing = Json::String(raw.to_string());

    assert_eq!(raw, thing.render());
}

#[test]
fn test_json_number_truthy() {
    use std::f64;
    assert!(json!(16i16).is_truthy(false));
    assert!(json!(16i16).is_truthy(true));

    assert!(json!(0i16).is_truthy(true));
    assert!(!json!(0i16).is_truthy(false));

    assert!(json!(1.0f64).is_truthy(false));
    assert!(json!(1.0f64).is_truthy(true));

    assert!(json!(Some(16i16)).is_truthy(false));
    assert!(json!(Some(16i16)).is_truthy(true));

    assert!(!json!(None as Option<i16>).is_truthy(false));
    assert!(!json!(None as Option<i16>).is_truthy(true));

    assert!(!json!(f64::NAN).is_truthy(false));
    assert!(!json!(f64::NAN).is_truthy(true));

    // there is no infinity in json/serde_json
    // assert!(json!(f64::INFINITY).is_truthy(false));
    // assert!(json!(f64::INFINITY).is_truthy(true));

    // assert!(json!(f64::NEG_INFINITY).is_truthy(false));
    // assert!(json!(f64::NEG_INFINITY).is_truthy(true));
}
<|endoftext|>


<commit_before>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use token::Token;
use token::Alignment;
use error::{Result, Error, Source};
use logger::Logger;

use std::mem;
use std::fs::File;
use std::path::Path;
use std::convert::AsRef;
use std::io::Read;
use std::collections::HashMap;

use cmark::{Parser as CMParser, Event, Tag, Options, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES};

/// A parser that reads markdown and convert it to AST (a vector of `Token`s)
///
/// This AST can then be used by various renderes.
///
/// As this Parser uses Pulldown-cmark's one, it should be able to parse most
/// *valid* CommonMark variant of Markdown.
///
/// Compared to other Markdown parser, it might fail more often on invalid code, e.g.
/// footnotes references that are not defined anywhere.
///
/// # Examples
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some *valid* Markdown[^1]\n\n[^1]: with a valid footnote");
/// assert!(result.is_ok());
/// ```
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some footnote pointing to nothing[^1] ");
/// assert!(result.is_err());
/// ```
pub struct Parser {
    footnotes: HashMap<String, Vec<Token>>,
    source: Source,
}

impl Parser {
    /// Creates a parser
    pub fn new() -> Parser {
        Parser {
            footnotes: HashMap::new(),
            source: Source::empty(),
        }
    }

    /// Sets a parser's source file
    pub fn set_source_file(&mut self, s: &str) {
        self.source = Source::new(s);
    }

    /// Parse a file and returns an AST or an error
    pub fn parse_file<P: AsRef<Path>>(&mut self, filename: P) -> Result<Vec<Token>> {
        let path: &Path = filename.as_ref();
        let mut f = File::open(path)
            .map_err(|_| {
                Error::file_not_found(&self.source,
                                      lformat!("markdown file"),
                                      format!("{}", path.display()))
            })?;
        let mut s = String::new();

        f.read_to_string(&mut s)
            .map_err(|_| {
                Error::parser(&self.source,
                              lformat!("file {file} contains invalid UTF-8, could not parse it",
                                       file = path.display()))
            })?;
        self.parse(&s)
    }

    /// Parse a string and returns an AST, or an Error.
    pub fn parse(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut opts = Options::empty();
        opts.insert(OPTION_ENABLE_TABLES);
        opts.insert(OPTION_ENABLE_FOOTNOTES);
        let mut p = CMParser::new_ext(s, opts);

        let mut res = vec![];
        self.parse_events(&mut p, &mut res, None)?;

        self.parse_footnotes(&mut res)?;

        collapse(&mut res);

        find_standalone(&mut res);
        Ok(res)
    }

    /// Parse an inline string and returns a list of `Token`.
    ///
    /// This function removes the outermost `Paragraph` in most of the
    /// cases, as it is meant to be used for an inline string (e.g. metadata)
    pub fn parse_inline(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut tokens = self.parse(s)?;
        // Unfortunately, parser will put all this in a paragraph, so we might need to remove it.
        if tokens.len() == 1 {
            let res = match tokens[0] {
                Token::Paragraph(ref mut v) => Some(mem::replace(v, vec![])),
                _ => None,
            };
            match res {
                Some(tokens) => Ok(tokens),
                _ => Ok(tokens),
            }
        } else {
            Ok(tokens)
        }
    }

    /// Replace footnote reference with their definition
    fn parse_footnotes(&mut self, v: &mut Vec<Token>) -> Result<()> {
        for token in v {
            match *token {
                Token::Footnote(ref mut content) => {
                    let reference = if let Token::Str(ref text) = content[0] {
                        text.clone()
                    } else {
                        panic!("Reference is not a vector of a single Token::Str");
                    };
                    if let Some(in_vec) = self.footnotes.get(&reference) {
                        *content = in_vec.clone();
                    } else {
                        return Err(Error::parser(&self.source,
                                                 lformat!("footnote reference {reference} does \
                                                           not have a matching definition",
                                                          reference = &reference)));
                    }
                }
                Token::Paragraph(ref mut vec) |
                Token::Header(_, ref mut vec) |
                Token::Emphasis(ref mut vec) |
                Token::Strong(ref mut vec) |
                Token::Code(ref mut vec) |
                Token::BlockQuote(ref mut vec) |
                Token::CodeBlock(_, ref mut vec) |
                Token::List(ref mut vec) |
                Token::OrderedList(_, ref mut vec) |
                Token::Item(ref mut vec) |
                Token::Table(_, ref mut vec) |
                Token::TableHead(ref mut vec) |
                Token::TableRow(ref mut vec) |
                Token::TableCell(ref mut vec) |
                Token::Link(_, _, ref mut vec) |
                Token::Image(_, _, ref mut vec) => self.parse_footnotes(vec)?,
                _ => (),
            }
        }
        Ok(())
    }

    fn parse_events<'a>(&mut self,
                        p: &mut CMParser<'a>,
                        v: &mut Vec<Token>,
                        current_tag: Option<&Tag>)
                        -> Result<()> {
        while let Some(event) = p.next() {
            match event {
                Event::Html(text) |
                Event::InlineHtml(text) |
                Event::Text(text) => {
                    v.push(Token::Str(text.into_owned()));
                }
                Event::Start(tag) => self.parse_tag(p, v, tag)?,
                Event::End(tag) => {
                    debug_assert!(format!("{:?}", Some(&tag)) == format!("{:?}", current_tag),
                                  format!("Error: opening and closing tags mismatch!\n{:?} ≠ \
                                           {:?}",
                                          tag,
                                          current_tag));
                    break;
                }
                Event::SoftBreak => v.push(Token::SoftBreak),
                Event::HardBreak => v.push(Token::HardBreak),
                Event::FootnoteReference(text) => {
                    v.push(Token::Footnote(vec![Token::Str(text.into_owned())]))
                }
            }
        }
        Ok(())
    }

    fn parse_tag<'a>(&mut self,
                     p: &mut CMParser<'a>,
                     v: &mut Vec<Token>,
                     tag: Tag<'a>)
                     -> Result<()> {
        let mut res = vec![];

        self.parse_events(p, &mut res, Some(&tag))?;

        let token = match tag {
            Tag::Paragraph => Token::Paragraph(res),
            Tag::Emphasis => Token::Emphasis(res),
            Tag::Strong => Token::Strong(res),
            Tag::Code => Token::Code(res),
            Tag::Header(x) => Token::Header(x, res),
            Tag::Link(url, title) => Token::Link(url.into_owned(), title.into_owned(), res),
            Tag::Image(url, title) => Token::Image(url.into_owned(), title.into_owned(), res),
            Tag::Rule => Token::Rule,
            Tag::List(opt) => {
                if let Some(n) = opt {
                    Token::OrderedList(n, res)
                } else {
                    Token::List(res)
                }
            }
            Tag::Item => Token::Item(res),
            Tag::BlockQuote => Token::BlockQuote(res),
            Tag::CodeBlock(language) => Token::CodeBlock(language.into_owned(), res),
            Tag::Table(v) => {
                // TODO: acutally use v's alignments
                let aligns = vec!{Alignment::Center; v.len()};
                Token::Table(aligns, res)
            },
            Tag::TableHead => Token::TableHead(res),
            Tag::TableRow => Token::TableRow(res),
            Tag::TableCell => Token::TableCell(res),
            Tag::FootnoteDefinition(reference) => {
                if self.footnotes.contains_key(reference.as_ref()) {
                    Logger::display_warning(lformat!("in {file}, found footnote definition for \
                                                      note '{reference}' but previous \
                                                      definition already exist, overriding it",
                                                     file = self.source,
                                                     reference = reference));
                }
                self.footnotes.insert(reference.into_owned(), res);
                Token::SoftBreak
            }
        };
        v.push(token);
        Ok(())
    }
}

/// Replace consecutives Strs by a Str of both, collapse soft breaks to previous std and so on
fn collapse(ast: &mut Vec<Token>) {
    if ast.len() < 2 {
        return;
    }

    let mut i = 0;
    while i < ast.len() {
        if ast[i].is_str() && i < ast.len() - 1 {
            if ast[i + 1].is_str() {
                // Two consecutives Str, concatenate them
                let token = ast.remove(i + 1);
                if let (&mut Token::Str(ref mut dest), Token::Str(ref source)) = (&mut ast[i],
                                                                                token) {
                    //                        dest.push(' ');
                    dest.push_str(source);
                    continue;
                } else {
                    unreachable!();
                }
            } else if ast[i + 1] == Token::SoftBreak {
                ast.remove(i + 1);
                if let &mut Token::Str(ref mut dest) = &mut ast[i] {
                    dest.push(' ');
                    continue;
                } else {
                    unreachable!();
                }
            }
        }

        // If token is containing others, recurse into them
        if let Some(ref mut inner) = ast[i].inner_mut() {
            collapse(inner);
        }
        i += 1;
    }
}

/// Replace images which are alone in a paragraph by standalone images
fn find_standalone(ast: &mut Vec<Token>) {
    for token in ast {
        let res = if let &mut Token::Paragraph(ref mut inner) = token {
            if inner.len() == 1 {
                if inner[0].is_image() {
                    if let Token::Image(source, title, inner) = mem::replace(&mut inner[0],
                                                                             Token::Rule) {
                        Token::StandaloneImage(source, title, inner)
                    } else {
                        unreachable!();
                    }
                } else {
                    continue;
                }
            } else {
                continue;
            }
        } else {
            continue;
        };

        *token = res;
    }
}
<commit_msg>Make default alignment to None instead of Center (not used anyway though)<commit_after>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use token::Token;
use token::Alignment;
use error::{Result, Error, Source};
use logger::Logger;

use std::mem;
use std::fs::File;
use std::path::Path;
use std::convert::AsRef;
use std::io::Read;
use std::collections::HashMap;

use cmark::{Parser as CMParser, Event, Tag, Options, OPTION_ENABLE_FOOTNOTES, OPTION_ENABLE_TABLES};

/// A parser that reads markdown and convert it to AST (a vector of `Token`s)
///
/// This AST can then be used by various renderes.
///
/// As this Parser uses Pulldown-cmark's one, it should be able to parse most
/// *valid* CommonMark variant of Markdown.
///
/// Compared to other Markdown parser, it might fail more often on invalid code, e.g.
/// footnotes references that are not defined anywhere.
///
/// # Examples
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some *valid* Markdown[^1]\n\n[^1]: with a valid footnote");
/// assert!(result.is_ok());
/// ```
///
/// ```
/// use crowbook::Parser;
/// let mut parser = Parser::new();
/// let result = parser.parse("Some footnote pointing to nothing[^1] ");
/// assert!(result.is_err());
/// ```
pub struct Parser {
    footnotes: HashMap<String, Vec<Token>>,
    source: Source,
}

impl Parser {
    /// Creates a parser
    pub fn new() -> Parser {
        Parser {
            footnotes: HashMap::new(),
            source: Source::empty(),
        }
    }

    /// Sets a parser's source file
    pub fn set_source_file(&mut self, s: &str) {
        self.source = Source::new(s);
    }

    /// Parse a file and returns an AST or an error
    pub fn parse_file<P: AsRef<Path>>(&mut self, filename: P) -> Result<Vec<Token>> {
        let path: &Path = filename.as_ref();
        let mut f = File::open(path)
            .map_err(|_| {
                Error::file_not_found(&self.source,
                                      lformat!("markdown file"),
                                      format!("{}", path.display()))
            })?;
        let mut s = String::new();

        f.read_to_string(&mut s)
            .map_err(|_| {
                Error::parser(&self.source,
                              lformat!("file {file} contains invalid UTF-8, could not parse it",
                                       file = path.display()))
            })?;
        self.parse(&s)
    }

    /// Parse a string and returns an AST, or an Error.
    pub fn parse(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut opts = Options::empty();
        opts.insert(OPTION_ENABLE_TABLES);
        opts.insert(OPTION_ENABLE_FOOTNOTES);
        let mut p = CMParser::new_ext(s, opts);

        let mut res = vec![];
        self.parse_events(&mut p, &mut res, None)?;

        self.parse_footnotes(&mut res)?;

        collapse(&mut res);

        find_standalone(&mut res);
        Ok(res)
    }

    /// Parse an inline string and returns a list of `Token`.
    ///
    /// This function removes the outermost `Paragraph` in most of the
    /// cases, as it is meant to be used for an inline string (e.g. metadata)
    pub fn parse_inline(&mut self, s: &str) -> Result<Vec<Token>> {
        let mut tokens = self.parse(s)?;
        // Unfortunately, parser will put all this in a paragraph, so we might need to remove it.
        if tokens.len() == 1 {
            let res = match tokens[0] {
                Token::Paragraph(ref mut v) => Some(mem::replace(v, vec![])),
                _ => None,
            };
            match res {
                Some(tokens) => Ok(tokens),
                _ => Ok(tokens),
            }
        } else {
            Ok(tokens)
        }
    }

    /// Replace footnote reference with their definition
    fn parse_footnotes(&mut self, v: &mut Vec<Token>) -> Result<()> {
        for token in v {
            match *token {
                Token::Footnote(ref mut content) => {
                    let reference = if let Token::Str(ref text) = content[0] {
                        text.clone()
                    } else {
                        panic!("Reference is not a vector of a single Token::Str");
                    };
                    if let Some(in_vec) = self.footnotes.get(&reference) {
                        *content = in_vec.clone();
                    } else {
                        return Err(Error::parser(&self.source,
                                                 lformat!("footnote reference {reference} does \
                                                           not have a matching definition",
                                                          reference = &reference)));
                    }
                }
                Token::Paragraph(ref mut vec) |
                Token::Header(_, ref mut vec) |
                Token::Emphasis(ref mut vec) |
                Token::Strong(ref mut vec) |
                Token::Code(ref mut vec) |
                Token::BlockQuote(ref mut vec) |
                Token::CodeBlock(_, ref mut vec) |
                Token::List(ref mut vec) |
                Token::OrderedList(_, ref mut vec) |
                Token::Item(ref mut vec) |
                Token::Table(_, ref mut vec) |
                Token::TableHead(ref mut vec) |
                Token::TableRow(ref mut vec) |
                Token::TableCell(ref mut vec) |
                Token::Link(_, _, ref mut vec) |
                Token::Image(_, _, ref mut vec) => self.parse_footnotes(vec)?,
                _ => (),
            }
        }
        Ok(())
    }

    fn parse_events<'a>(&mut self,
                        p: &mut CMParser<'a>,
                        v: &mut Vec<Token>,
                        current_tag: Option<&Tag>)
                        -> Result<()> {
        while let Some(event) = p.next() {
            match event {
                Event::Html(text) |
                Event::InlineHtml(text) |
                Event::Text(text) => {
                    v.push(Token::Str(text.into_owned()));
                }
                Event::Start(tag) => self.parse_tag(p, v, tag)?,
                Event::End(tag) => {
                    debug_assert!(format!("{:?}", Some(&tag)) == format!("{:?}", current_tag),
                                  format!("Error: opening and closing tags mismatch!\n{:?} ≠ \
                                           {:?}",
                                          tag,
                                          current_tag));
                    break;
                }
                Event::SoftBreak => v.push(Token::SoftBreak),
                Event::HardBreak => v.push(Token::HardBreak),
                Event::FootnoteReference(text) => {
                    v.push(Token::Footnote(vec![Token::Str(text.into_owned())]))
                }
            }
        }
        Ok(())
    }

    fn parse_tag<'a>(&mut self,
                     p: &mut CMParser<'a>,
                     v: &mut Vec<Token>,
                     tag: Tag<'a>)
                     -> Result<()> {
        let mut res = vec![];

        self.parse_events(p, &mut res, Some(&tag))?;

        let token = match tag {
            Tag::Paragraph => Token::Paragraph(res),
            Tag::Emphasis => Token::Emphasis(res),
            Tag::Strong => Token::Strong(res),
            Tag::Code => Token::Code(res),
            Tag::Header(x) => Token::Header(x, res),
            Tag::Link(url, title) => Token::Link(url.into_owned(), title.into_owned(), res),
            Tag::Image(url, title) => Token::Image(url.into_owned(), title.into_owned(), res),
            Tag::Rule => Token::Rule,
            Tag::List(opt) => {
                if let Some(n) = opt {
                    Token::OrderedList(n, res)
                } else {
                    Token::List(res)
                }
            }
            Tag::Item => Token::Item(res),
            Tag::BlockQuote => Token::BlockQuote(res),
            Tag::CodeBlock(language) => Token::CodeBlock(language.into_owned(), res),
            Tag::Table(v) => {
                // TODO: acutally use v's alignments
                let aligns = vec!{Alignment::None; v.len()};
                Token::Table(aligns, res)
            },
            Tag::TableHead => Token::TableHead(res),
            Tag::TableRow => Token::TableRow(res),
            Tag::TableCell => Token::TableCell(res),
            Tag::FootnoteDefinition(reference) => {
                if self.footnotes.contains_key(reference.as_ref()) {
                    Logger::display_warning(lformat!("in {file}, found footnote definition for \
                                                      note '{reference}' but previous \
                                                      definition already exist, overriding it",
                                                     file = self.source,
                                                     reference = reference));
                }
                self.footnotes.insert(reference.into_owned(), res);
                Token::SoftBreak
            }
        };
        v.push(token);
        Ok(())
    }
}

/// Replace consecutives Strs by a Str of both, collapse soft breaks to previous std and so on
fn collapse(ast: &mut Vec<Token>) {
    if ast.len() < 2 {
        return;
    }

    let mut i = 0;
    while i < ast.len() {
        if ast[i].is_str() && i < ast.len() - 1 {
            if ast[i + 1].is_str() {
                // Two consecutives Str, concatenate them
                let token = ast.remove(i + 1);
                if let (&mut Token::Str(ref mut dest), Token::Str(ref source)) = (&mut ast[i],
                                                                                token) {
                    //                        dest.push(' ');
                    dest.push_str(source);
                    continue;
                } else {
                    unreachable!();
                }
            } else if ast[i + 1] == Token::SoftBreak {
                ast.remove(i + 1);
                if let &mut Token::Str(ref mut dest) = &mut ast[i] {
                    dest.push(' ');
                    continue;
                } else {
                    unreachable!();
                }
            }
        }

        // If token is containing others, recurse into them
        if let Some(ref mut inner) = ast[i].inner_mut() {
            collapse(inner);
        }
        i += 1;
    }
}

/// Replace images which are alone in a paragraph by standalone images
fn find_standalone(ast: &mut Vec<Token>) {
    for token in ast {
        let res = if let &mut Token::Paragraph(ref mut inner) = token {
            if inner.len() == 1 {
                if inner[0].is_image() {
                    if let Token::Image(source, title, inner) = mem::replace(&mut inner[0],
                                                                             Token::Rule) {
                        Token::StandaloneImage(source, title, inner)
                    } else {
                        unreachable!();
                    }
                } else {
                    continue;
                }
            } else {
                continue;
            }
        } else {
            continue;
        };

        *token = res;
    }
}
<|endoftext|>


<commit_before>use bindings::*;
use context::Context;
use function::Function;
use util::NativeRef;
use std::str::raw::from_c_str;
/// An ELF binary reader
native_ref!(ReadElf, _reader, jit_readelf_t)
impl ReadElf {
	/// Open a new ELF binary
	pub fn new(filename:&str) -> ReadElf {
		unsafe {
			let mut this = RawPtr::null();
			let code = filename.with_c_str(|c_name|
				jit_readelf_open(&mut this, c_name, 0)
			);
			if this.is_null() {
				fail!("'{}' couldn't be opened due to {}", filename, code);
			} else {
				NativeRef::from_ptr(this)
			}
		}
	}
	#[inline]
	/// Get the name of this ELF binary
	pub fn get_name(&self) -> String {
		unsafe {
			from_c_str(jit_readelf_get_name(self.as_ptr()))
		}
	}
	#[inline]
	pub fn add_to_context(&self, ctx:&Context) {
		unsafe {
			jit_readelf_add_to_context(self.as_ptr(), ctx.as_ptr())
		}
	}
	#[inline]
	/// Get a simple in the ELF binary
	pub unsafe fn get_symbol<T>(&self, symbol:&str) -> *T {
		symbol.with_c_str(|c_symbol|
			jit_readelf_get_symbol(self.as_ptr(), c_symbol) as *T
		)
	}
}
impl Drop for ReadElf {
	#[inline]
	fn drop(&mut self) {
		unsafe {
			jit_readelf_close(self.as_ptr())
		}
	}
}

/// An ELF binary reader
native_ref!(WriteElf, _writer, jit_writeelf_t)
impl WriteElf {
	#[inline]
	/// Create a new ELF binary reader
	pub fn new(lib_name:&str) -> WriteElf {
		lib_name.with_c_str(|c_lib_name| unsafe {
			NativeRef::from_ptr(jit_writeelf_create(c_lib_name))
		})
	}
	#[inline]
	/// Write to the filename given (not implemented by LibJIT yet)
	pub fn write(&self, filename:&str) -> bool {
		filename.with_c_str(|c_filename| unsafe {
			jit_writeelf_write(self.as_ptr(), c_filename) != 0
		})
	}
	#[inline]
	/// Add a function to the ELF
	pub fn add_function(&self, func:&Function, name:&str) -> bool {
		name.with_c_str(|c_name| unsafe {
			jit_writeelf_add_function(self.as_ptr(), func.as_ptr(), c_name) != 0
		})
	}
	#[inline]
	/// Add a dependency to the ELF
	pub fn add_needed(&self, lib_name:&str) -> bool {
		lib_name.with_c_str(|c_lib_name| unsafe {
			jit_writeelf_add_needed(self.as_ptr(), c_lib_name) != 0
		})
	}
}
impl Drop for WriteElf {
	#[inline]
	fn drop(&mut self) {
		unsafe {
			jit_writeelf_destroy(self.as_ptr())
		}
	}
}
#[test]
fn test_elf() {
	let elf = ReadElf::new("/usr/lib/libjit.so.0");
	assert_eq!(elf.get_name().as_slice(), "libjit.so.0");
}<commit_msg>Use generics to allow ELF reader and writer to accept any ToCStr implementors<commit_after>use bindings::*;
use context::Context;
use function::Function;
use util::NativeRef;
use std::fmt::Show;
use std::str::raw::from_c_str;
use std::c_str::ToCStr;
/// An ELF binary reader
native_ref!(ReadElf, _reader, jit_readelf_t)
impl ReadElf {
	/// Open a new ELF binary
	pub fn new<S:ToCStr+Show>(filename:S) -> ReadElf {
		unsafe {
			let mut this = RawPtr::null();
			let code = filename.with_c_str(|c_name|
				jit_readelf_open(&mut this, c_name, 0)
			);
			if this.is_null() {
				fail!("'{}' couldn't be opened due to {}", filename, code);
			} else {
				NativeRef::from_ptr(this)
			}
		}
	}
	#[inline]
	/// Get the name of this ELF binary
	pub fn get_name(&self) -> String {
		unsafe {
			from_c_str(jit_readelf_get_name(self.as_ptr()))
		}
	}
	#[inline]
	pub fn add_to_context(&self, ctx:&Context) {
		unsafe {
			jit_readelf_add_to_context(self.as_ptr(), ctx.as_ptr())
		}
	}
	#[inline]
	/// Get a simple in the ELF binary
	pub unsafe fn get_symbol<T, S:ToCStr>(&self, symbol:S) -> *T {
		symbol.with_c_str(|c_symbol|
			jit_readelf_get_symbol(self.as_ptr(), c_symbol) as *T
		)
	}
}
impl Drop for ReadElf {
	#[inline]
	fn drop(&mut self) {
		unsafe {
			jit_readelf_close(self.as_ptr())
		}
	}
}

/// An ELF binary reader
native_ref!(WriteElf, _writer, jit_writeelf_t)
impl WriteElf {
	#[inline]
	/// Create a new ELF binary reader
	pub fn new<S:ToCStr>(lib_name:S) -> WriteElf {
		lib_name.with_c_str(|c_lib_name| unsafe {
			NativeRef::from_ptr(jit_writeelf_create(c_lib_name))
		})
	}
	#[inline]
	/// Write to the filename given (not implemented by LibJIT yet)
	pub fn write<S:ToCStr>(&self, filename:S) -> bool {
		filename.with_c_str(|c_filename| unsafe {
			jit_writeelf_write(self.as_ptr(), c_filename) != 0
		})
	}
	#[inline]
	/// Add a function to the ELF
	pub fn add_function<S:ToCStr>(&self, func:&Function, name:S) -> bool {
		name.with_c_str(|c_name| unsafe {
			jit_writeelf_add_function(self.as_ptr(), func.as_ptr(), c_name) != 0
		})
	}
	#[inline]
	/// Add a dependency to the ELF
	pub fn add_needed<S:ToCStr>(&self, lib_name:S) -> bool {
		lib_name.with_c_str(|c_lib_name| unsafe {
			jit_writeelf_add_needed(self.as_ptr(), c_lib_name) != 0
		})
	}
}
impl Drop for WriteElf {
	#[inline]
	fn drop(&mut self) {
		unsafe {
			jit_writeelf_destroy(self.as_ptr())
		}
	}
}
#[test]
fn test_elf() {
	let elf = ReadElf::new("/usr/lib/libjit.so.0");
	assert_eq!(elf.get_name().as_slice(), "libjit.so.0");
}<|endoftext|>


<commit_before>// Copyright 2016 `multipart` Crate Developers
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.
use mock::{ClientRequest, HttpBuffer};

use server::{MultipartField, MultipartData, ReadEntry};

use mime::{self, Mime};

use rand::{self, Rng};

use std::collections::{HashMap, HashSet};
use std::fmt;
use std::io::prelude::*;
use std::io::Cursor;
use std::iter::{self, FromIterator};

const MIN_FIELDS: usize = 1;
const MAX_FIELDS: usize = 3;

const MIN_LEN: usize = 2;
const MAX_LEN: usize = 5;
const MAX_DASHES: usize = 2;

fn collect_rand<C: FromIterator<T>, T, F: FnMut() -> T>(mut gen: F) -> C {
    (0 .. rand::thread_rng().gen_range(MIN_FIELDS, MAX_FIELDS))
        .map(|_| gen()).collect()
}

macro_rules! expect_fmt (
    ($val:expr, $($args:tt)*) => (
        match $val {
            Some(val) => val,
            None => panic!($($args)*),
        }
    );
);

#[derive(Debug)]
struct TestFields {
    fields: HashMap<String, HashSet<FieldEntry>>,
}

impl TestFields {
    fn gen() -> Self {
        TestFields {
            fields: collect_rand(|| (gen_string(), FieldEntry::gen_many())),
        }
    }

    fn check_field<M: ReadEntry>(&mut self, field: MultipartField<M>) {
        let field_entries = expect_fmt!(self.fields.remove(&*field.headers.name),
                                        "Got field that wasn't in original dataset: {:?}",
                                        field.headers);

        let test_entry = FieldEntry::from_field(field);

        assert!(field_entries.contains(&test_entry),
            "Got field entry that wasn't in original dataset: {:?}\nEntries: {:?}",
            test_entry, field_entries
        );
    }

    fn assert_is_empty(&self) {
        assert!(self.fields.is_empty(), "Fields were not exhausted! {:?}", self.fields);
    }
}

#[derive(Debug, Hash, PartialEq, Eq)]
struct FieldEntry {
    content_type: Mime,
    filename: Option<String>,
    data: PrintHex,
}

impl FieldEntry {
    fn from_field<M: ReadEntry>(mut field: MultipartField<M>) -> FieldEntry {
        let mut data = Vec::new();
        field.data.read_to_end(&mut data).expect("failed reading field");

        FieldEntry {
            content_type: field.headers.content_type.unwrap_or(mime::APPLICATION_OCTET_STREAM),
            filename: field.headers.filename,
            data: PrintHex(data),
        }
    }

    fn gen_many() -> HashSet<Self> {
        collect_rand(Self::gen)
    }

    fn gen() -> Self {
        let filename = match gen_bool() {
            true => Some(gen_string()),
            false => None,
        };

        let data = PrintHex(match gen_bool() {
            true => gen_string().into_bytes(),
            false => gen_bytes(),
        });

        FieldEntry {
            content_type: rand_mime(),
            filename,
            data,
        }
    }

    fn filename(&self) -> Option<&str> {
        self.filename.as_ref().map(|s| &**s)
    }
}

#[derive(Hash, PartialEq, Eq)]
struct PrintHex(Vec<u8>);

impl fmt::Debug for PrintHex {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "[")?;

        let mut written = false;

        for byte in &self.0 {
            write!(f, "{:X}", byte)?;

            if written {
                write!(f, ", ")?;
            }

            written = true;
        }

        write!(f, "]")
    }
}

macro_rules! do_test (
    ($client_test:ident, $server_test:ident) => (
        let _ = ::env_logger::init();

        info!("Client Test: {:?} Server Test: {:?}", stringify!($client_test),
              stringify!($server_test));

        let mut test_fields = TestFields::gen();

        trace!("Fields for test: {:?}", test_fields);

        let buf = $client_test(&test_fields);

        trace!(
            "\n==Test Buffer Begin==\n{}\n==Test Buffer End==",
            String::from_utf8_lossy(&buf.buf)
        );

        $server_test(buf, &mut test_fields);

        test_fields.assert_is_empty();
    );
);

#[test]
fn reg_client_reg_server() {
    do_test!(test_client, test_server);
}

#[test]
fn reg_client_entry_server() {
    do_test!(test_client, test_server_entry_api);
}

#[test]
fn lazy_client_reg_server() {
    do_test!(test_client_lazy, test_server);
}

#[test]
fn lazy_client_entry_server() {
    do_test!(test_client_lazy, test_server_entry_api);
}

mod extended {
    use super::{test_client, test_server, test_server_entry_api, test_client_lazy, TestFields};

    use std::time::Instant;

    const TIME_LIMIT_SECS: u64 = 300;

    #[test]
    #[ignore]
    fn reg_client_reg_server() {
        let started = Instant::now();

        while started.elapsed().as_secs() < TIME_LIMIT_SECS {
            do_test!(test_client, test_server);
        }
    }

    #[test]
    #[ignore]
    fn reg_client_entry_server() {
        let started = Instant::now();

        while started.elapsed().as_secs() < TIME_LIMIT_SECS {
            do_test!(test_client, test_server_entry_api);
        }
    }

    #[test]
    #[ignore]
    fn lazy_client_reg_server() {
        let started = Instant::now();

        while started.elapsed().as_secs() < TIME_LIMIT_SECS {
            do_test!(test_client_lazy, test_server);
        }
    }

    #[test]
    #[ignore]
    fn lazy_client_entry_server() {
        let started = Instant::now();

        while started.elapsed().as_secs() < TIME_LIMIT_SECS {
            do_test!(test_client_lazy, test_server_entry_api);
        }
    }
}

fn gen_bool() -> bool {
    rand::thread_rng().gen()
}

fn gen_string() -> String {
    let mut rng_1 = rand::thread_rng();
    let mut rng_2 = rand::thread_rng();

    let str_len_1 = rng_1.gen_range(MIN_LEN, MAX_LEN + 1);
    let str_len_2 = rng_2.gen_range(MIN_LEN, MAX_LEN + 1);
    let num_dashes = rng_1.gen_range(0, MAX_DASHES + 1);

    rng_1.gen_ascii_chars().take(str_len_1)
        .chain(iter::repeat('-').take(num_dashes))
        .chain(rng_2.gen_ascii_chars().take(str_len_2))
        .collect()
}

fn gen_bytes() -> Vec<u8> {
    gen_string().into_bytes()
}

fn test_client(test_fields: &TestFields) -> HttpBuffer {
    use client::Multipart;

    let request = ClientRequest::default();

    let mut test_files = test_fields.fields.iter();

    let mut multipart = Multipart::from_request(request).unwrap();

    // Intersperse file fields amongst text fields
    for (name, text) in &test_fields.texts {
        if let Some((file_name, files)) = test_files.next() {
            for file in files {
                multipart.write_stream(file_name, &mut &*file.data.0, file.filename(),
                                       Some(file.content_type.clone())).unwrap();
            }
        }

        multipart.write_text(name, text).unwrap();
    }

    // Write remaining files
    for (file_name, files) in test_files {
        for file in files {
            multipart.write_stream(file_name, &mut &*file.data.0, file.filename(),
                                   Some(file.content_type.clone())).unwrap();
        }
    }

    multipart.send().unwrap()
}

fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer {
    use client::lazy::Multipart;

    let mut multipart = Multipart::new();

    let mut test_files = test_fields.fields.iter();

    for (name, text) in &test_fields.texts {
        if let Some((file_name, files)) = test_files.next() {
            for file in files {
                multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(),
                                     Some(file.content_type.clone()));
            }
        }

        multipart.add_text(&**name, &**text);
    }

    for (file_name, files) in test_files {
        for file in files {
            multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(),
                                 Some(file.content_type.clone()));
        }
    }

    let mut prepared = multipart.prepare().unwrap();

    let mut buf = Vec::new();

    let boundary = prepared.boundary().to_owned();
    let content_len = prepared.content_len();

    prepared.read_to_end(&mut buf).unwrap();

    HttpBuffer::with_buf(buf, boundary, content_len)
}

fn test_server(buf: HttpBuffer, fields: &mut TestFields) {
    use server::Multipart;

    let server_buf = buf.for_server();

    if let Some(content_len) = server_buf.content_len {
        assert!(content_len == server_buf.data.len() as u64, "Supplied content_len different from actual");
    }

    let mut multipart = Multipart::from_request(server_buf)
        .unwrap_or_else(|_| panic!("Buffer should be multipart!"));

    while let Some(mut field) = multipart.read_entry_mut().unwrap_opt() {
        fields.check_field(field);
    }
}

fn test_server_entry_api(buf: HttpBuffer, fields: &mut TestFields) {
    use server::Multipart;

    let server_buf = buf.for_server();

    if let Some(content_len) = server_buf.content_len {
        assert!(content_len == server_buf.data.len() as u64, "Supplied content_len different from actual");
    }

    let multipart = Multipart::from_request(server_buf)
        .unwrap_or_else(|_| panic!("Buffer should be multipart!"));

    let mut entry = multipart.into_entry().expect_alt("Expected entry, got none", "Error reading entry");
    fields.check_field(&mut entry);

    while let Some(entry_) = entry.next_entry().unwrap_opt() {
        entry = entry_;
        fields.check_field(&mut entry);
    }
}

fn rand_mime() -> Mime {
    rand::thread_rng().choose(&[
        // TODO: fill this out, preferably with variants that may be hard to parse
        // i.e. containing hyphens, mainly
        mime::APPLICATION_OCTET_STREAM,
        mime::TEXT_PLAIN,
        mime::IMAGE_PNG,
    ]).unwrap().clone()
}
<commit_msg>fix local_test<commit_after>// Copyright 2016 `multipart` Crate Developers
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.
use mock::{ClientRequest, HttpBuffer};

use server::{MultipartField, MultipartData, ReadEntry, FieldHeaders};

use mime::{self, Mime};

use rand::{self, Rng};

use std::collections::{HashMap, HashSet};
use std::collections::hash_map::{Entry, OccupiedEntry};
use std::fmt;
use std::io::prelude::*;
use std::io::Cursor;
use std::iter::{self, FromIterator};

const MIN_FIELDS: usize = 1;
const MAX_FIELDS: usize = 3;

const MIN_LEN: usize = 2;
const MAX_LEN: usize = 5;
const MAX_DASHES: usize = 2;

fn collect_rand<C: FromIterator<T>, T, F: FnMut() -> T>(mut gen: F) -> C {
    (0 .. rand::thread_rng().gen_range(MIN_FIELDS, MAX_FIELDS))
        .map(|_| gen()).collect()
}

macro_rules! expect_fmt (
    ($val:expr, $($args:tt)*) => (
        match $val {
            Some(val) => val,
            None => panic!($($args)*),
        }
    );
);

/// The error is provided as the `err` format argument
macro_rules! expect_ok_fmt (
    ($val:expr, $($args:tt)*) => (
        match $val {
            Ok(val) => val,
            Err(e) => panic!($($args)*, err=e),
        }
    );
);

fn get_field<'m, V>(field: &FieldHeaders, fields: &'m mut HashMap<String, V>) -> Option<OccupiedEntry<'m, String, V>> {
    match fields.entry(field.name.to_string()) {
        Entry::Occupied(occupied) => Some(occupied),
        Entry::Vacant(_) => None,
    }
}

#[derive(Debug)]
struct TestFields {
    texts: HashMap<String, HashSet<String>>,
    files: HashMap<String, HashSet<FileEntry>>,
}

impl TestFields {
    fn gen() -> Self {
        TestFields {
            texts: collect_rand(|| (gen_string(), collect_rand(gen_string))),
            files: collect_rand(|| (gen_string(), FileEntry::gen_many())),
        }
    }

    fn check_field<M: ReadEntry>(&mut self, mut field: MultipartField<M>) -> M {
        if field.is_text() {
            let mut text_entries = expect_fmt!(get_field(&field.headers, &mut self.texts),
                                        "Got text field that wasn't in original dataset: {:?}",
                                        field.headers);

            let mut text = String::new();
            expect_ok_fmt!(
                field.data.read_to_string(&mut text),
                "error failed to read text data to string: {:?}\n{err}", field.headers
            );

            assert!(
                text_entries.get_mut().remove(&text),
                "Got field text data that wasn't in original data set: {:?}\n{:?}\n{:?}",
                field.headers,
                text,
                text_entries.get(),
            );

            if text_entries.get().is_empty() {
                text_entries.remove_entry();
            }

            return field.data.into_inner();
        }

        let mut file_entries = expect_fmt!(get_field(&field.headers, &mut self.files),
                                        "Got file field that wasn't in original dataset: {:?}",
                                        field.headers);

        let field_name = field.headers.name.clone();
        let (test_entry, inner) = FileEntry::from_field(field);

        assert!(
            file_entries.get_mut().remove(&test_entry),
            "Got field entry that wasn't in original dataset: name: {:?}\n{:?}\nEntries: {:?}",
            field_name,
            test_entry,
            file_entries.get()
        );

        if file_entries.get().is_empty() {
            file_entries.remove_entry();
        }

        return inner;
    }

    fn assert_is_empty(&self) {
        assert!(self.texts.is_empty(), "Text Fields were not exhausted! {:?}", self.texts);
        assert!(self.files.is_empty(), "File Fields were not exhausted! {:?}", self.files);
    }
}

#[derive(Debug, Hash, PartialEq, Eq)]
struct FileEntry {
    content_type: Mime,
    filename: Option<String>,
    data: PrintHex,
}

impl FileEntry {
    fn from_field<M: ReadEntry>(mut field: MultipartField<M>) -> (FileEntry, M) {
        let mut data = Vec::new();
        expect_ok_fmt!(
            field.data.read_to_end(&mut data),
            "Error reading file field: {:?}\n{err}", field.headers
        );

        (
            FileEntry {
                content_type: field.headers.content_type.unwrap_or(mime::APPLICATION_OCTET_STREAM),
                filename: field.headers.filename,
                data: PrintHex(data),
            },
            field.data.into_inner()
        )
    }

    fn gen_many() -> HashSet<FileEntry> {
        collect_rand(Self::gen)
    }

    fn gen() -> Self {
        let filename = match gen_bool() {
            true => Some(gen_string()),
            false => None,
        };

        let data = PrintHex(match gen_bool() {
            true => gen_string().into_bytes(),
            false => gen_bytes(),
        });

        FileEntry {
            content_type: rand_mime(),
            filename,
            data,
        }
    }

    fn filename(&self) -> Option<&str> {
        self.filename.as_ref().map(|s| &**s)
    }
}

#[derive(PartialEq, Eq, Hash)]
struct PrintHex(Vec<u8>);

impl fmt::Debug for PrintHex {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "[")?;

        let mut written = false;

        for byte in &self.0 {
            write!(f, "{:X}", byte)?;

            if written {
                write!(f, ", ")?;
            }

            written = true;
        }

        write!(f, "]")
    }
}

macro_rules! do_test (
    ($client_test:ident, $server_test:ident) => (
        let _ = ::env_logger::init();

        info!("Client Test: {:?} Server Test: {:?}", stringify!($client_test),
              stringify!($server_test));

        let mut test_fields = TestFields::gen();

        trace!("Fields for test: {:?}", test_fields);

        let buf = $client_test(&test_fields);

        trace!(
            "\n==Test Buffer Begin==\n{}\n==Test Buffer End==",
            String::from_utf8_lossy(&buf.buf)
        );

        $server_test(buf, &mut test_fields);

        test_fields.assert_is_empty();
    );
);

#[test]
fn reg_client_reg_server() {
    do_test!(test_client, test_server);
}

#[test]
fn reg_client_entry_server() {
    do_test!(test_client, test_server_entry_api);
}

#[test]
fn lazy_client_reg_server() {
    do_test!(test_client_lazy, test_server);
}

#[test]
fn lazy_client_entry_server() {
    do_test!(test_client_lazy, test_server_entry_api);
}

mod extended {
    use super::{test_client, test_server, test_server_entry_api, test_client_lazy, TestFields};

    use std::time::Instant;

    const TIME_LIMIT_SECS: u64 = 300;

    #[test]
    #[ignore]
    fn reg_client_reg_server() {
        let started = Instant::now();

        while started.elapsed().as_secs() < TIME_LIMIT_SECS {
            do_test!(test_client, test_server);
        }
    }

    #[test]
    #[ignore]
    fn reg_client_entry_server() {
        let started = Instant::now();

        while started.elapsed().as_secs() < TIME_LIMIT_SECS {
            do_test!(test_client, test_server_entry_api);
        }
    }

    #[test]
    #[ignore]
    fn lazy_client_reg_server() {
        let started = Instant::now();

        while started.elapsed().as_secs() < TIME_LIMIT_SECS {
            do_test!(test_client_lazy, test_server);
        }
    }

    #[test]
    #[ignore]
    fn lazy_client_entry_server() {
        let started = Instant::now();

        while started.elapsed().as_secs() < TIME_LIMIT_SECS {
            do_test!(test_client_lazy, test_server_entry_api);
        }
    }
}

fn gen_bool() -> bool {
    rand::thread_rng().gen()
}

fn gen_string() -> String {
    let mut rng_1 = rand::thread_rng();
    let mut rng_2 = rand::thread_rng();

    let str_len_1 = rng_1.gen_range(MIN_LEN, MAX_LEN + 1);
    let str_len_2 = rng_2.gen_range(MIN_LEN, MAX_LEN + 1);
    let num_dashes = rng_1.gen_range(0, MAX_DASHES + 1);

    rng_1.gen_ascii_chars().take(str_len_1)
        .chain(iter::repeat('-').take(num_dashes))
        .chain(rng_2.gen_ascii_chars().take(str_len_2))
        .collect()
}

fn gen_bytes() -> Vec<u8> {
    gen_string().into_bytes()
}

fn test_client(test_fields: &TestFields) -> HttpBuffer {
    use client::Multipart;

    let request = ClientRequest::default();

    let mut test_files = test_fields.files.iter().flat_map(
        |(name, files)| files.iter().map(move |file| (name, file))
    );

    let mut test_texts = test_fields.texts.iter().flat_map(
        |(name, texts)| texts.iter().map(move |text| (name, text))
    );

    let mut multipart = Multipart::from_request(request).unwrap();

    // Intersperse file fields amongst text fields
    for (name, text) in test_texts {
        if let Some((file_name, file)) = test_files.next() {
            multipart.write_stream(file_name, &mut &*file.data.0, file.filename(),
                                   Some(file.content_type.clone())).unwrap();
        }

        multipart.write_text(name, text).unwrap();
    }

    // Write remaining files
    for (file_name, file) in test_files {
        multipart.write_stream(file_name, &mut &*file.data.0, file.filename(),
                               Some(file.content_type.clone())).unwrap();
    }

    multipart.send().unwrap()
}

fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer {
    use client::lazy::Multipart;

    let mut multipart = Multipart::new();

    let mut test_files = test_fields.files.iter().flat_map(
        |(name, files)| files.iter().map(move |file| (name, file))
    );

    let mut test_texts = test_fields.texts.iter().flat_map(
        |(name, texts)| texts.iter().map(move |text| (name, text))
    );

    for (name, text) in test_texts {
        if let Some((file_name, file)) = test_files.next() {
                multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(),
                                     Some(file.content_type.clone()));
        }

        multipart.add_text(&**name, &**text);
    }

    for (file_name, file) in test_files {
        multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(),
                             Some(file.content_type.clone()));
    }

    let mut prepared = multipart.prepare().unwrap();

    let mut buf = Vec::new();

    let boundary = prepared.boundary().to_owned();
    let content_len = prepared.content_len();

    prepared.read_to_end(&mut buf).unwrap();

    HttpBuffer::with_buf(buf, boundary, content_len)
}

fn test_server(buf: HttpBuffer, fields: &mut TestFields) {
    use server::Multipart;

    let server_buf = buf.for_server();

    if let Some(content_len) = server_buf.content_len {
        assert!(content_len == server_buf.data.len() as u64, "Supplied content_len different from actual");
    }

    let mut multipart = Multipart::from_request(server_buf)
        .unwrap_or_else(|_| panic!("Buffer should be multipart!"));

    while let Some(mut field) = multipart.read_entry_mut().unwrap_opt() {
        fields.check_field(field);
    }
}

fn test_server_entry_api(buf: HttpBuffer, fields: &mut TestFields) {
    use server::Multipart;

    let server_buf = buf.for_server();

    if let Some(content_len) = server_buf.content_len {
        assert!(content_len == server_buf.data.len() as u64, "Supplied content_len different from actual");
    }

    let mut multipart = Multipart::from_request(server_buf)
        .unwrap_or_else(|_| panic!("Buffer should be multipart!"));

    let entry = multipart.into_entry().expect_alt("Expected entry, got none", "Error reading entry");
    multipart = fields.check_field(entry);

    while let Some(entry) = multipart.into_entry().unwrap_opt() {
        multipart = fields.check_field(entry);
    }
}

fn rand_mime() -> Mime {
    rand::thread_rng().choose(&[
        // TODO: fill this out, preferably with variants that may be hard to parse
        // i.e. containing hyphens, mainly
        mime::APPLICATION_OCTET_STREAM,
        mime::TEXT_PLAIN,
        mime::IMAGE_PNG,
    ]).unwrap().clone()
}
<|endoftext|>