<commit_before>/// ## purpose
/// to match multiple byte patterns against a byte slice in parallel.
/// we should get all valid matches at the end.
/// does not have to support scanning across the byte slice, only anchored at the start.
/// need support for single character wild cards (`.`).
/// all patterns are the same length
/// (if this needs to change, maybe pad shorter patterns with wildcards).
///
/// ## design:
/// we'll build an NFA with symbols for:
///   - all valid byte values (0-255), and
///   - a wildcard
///
/// a transition table will have 257 columns, one for each of the above symbols, including wildcard.
/// the transition table has a row for each state.
/// entries in each column indicate valid transitions.
/// an entry of `0` indicates "invalid".
/// if a row contains only invalid entries, then its a "terminal state".
/// there's a list associated with each row of "alive" patterns;
/// for a terminal state, these are the patterns that matched.
///
///
/// ### example (no wildcards)
///
/// input patterns:
///
///   p0: A B C D
///   p1: A D C B
///
/// transition table:
///
/// ```ignore
///       A B C D
///   0 | 1        alive: p0, p1
///   1 |   2   4  alive: p0, p1
///   2 |     3    alive: p0
///   3 |          terminal, alive: p0
///   4 |     5    alive: p1
///   5 |          terminal, alive: p1
/// ```
use std::collections::VecDeque;

use nom::IResult;
use nom::multi::many1;
use nom::bytes::complete::tag;
use nom::bytes::complete::take_while_m_n;
use nom::branch::alt;
use nom::combinator::map;
use nom::combinator::map_res;
use bitvec::prelude::*;
use log::Level::Warn;

// u16 because we need 257 possible values, all unsigned.
#[derive(Copy, Clone)]
pub struct Symbol(u16);

// impl note: value 256 is WILDCARD.
pub const WILDCARD: Symbol = Symbol(0x100);

// byte values map directly into their Symbol indices.
impl std::convert::From<u8> for Symbol {
    fn from(v: u8) -> Self {Symbol(v as u16)}
}

// convert to usize so we can index into `State.transitions`.
impl std::convert::Into<usize> for Symbol {
    fn into(self) -> usize {self.0 as usize}
}

impl std::cmp::PartialEq for Symbol {
    fn eq(&self, other: &Symbol) -> bool {
        self.0 == other.0
    }
}

impl std::fmt::Display for Symbol {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        if self.0 == WILDCARD.0 {
            write!(f, "..")
        } else {
            write!(f, "{:02x}", self.0)
        }
    }
}

impl std::fmt::Debug for Symbol {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "{}", self)
    }
}

// a pattern is just a sequence of symbols.
pub struct Pattern(Vec<Symbol>);

impl std::fmt::Display for Pattern {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        let parts: Vec<String> = self.0.iter().map(|s| format!("{}", s)).collect();
        write!(f, "{}", parts.join(""))
    }
}

fn is_hex_digit(c: char) -> bool {
  c.is_digit(16)
}

fn from_hex(input: &str) -> Result<u8, std::num::ParseIntError> {
  u8::from_str_radix(input, 16)
}

/// parse a single hex byte, like `AB`
fn hex(input: &str) -> IResult<&str, u8> {
    map_res(
        take_while_m_n(2, 2, is_hex_digit),
        from_hex
    )(input)
}

/// parse a single byte signature element, which is either a hex byte or a wildcard.
fn sig_element(input: &str) -> IResult<&str, Symbol> {
    alt((
        map(hex, |v| Symbol::from(v)),
        map(tag(".."), |_| WILDCARD),
    ))(input)
}

/// parse byte signature elements, hex or wildcard.
fn byte_signature(input: &str) -> IResult<&str, Pattern> {
    let (input, elems) = many1(sig_element)(input)?;
    Ok((input, Pattern(elems)))
}

/// parse a pattern from a string like `AABB..DD`.
impl std::convert::From<&str> for Pattern {
    fn from(v: &str) -> Self {
        byte_signature(v).expect("failed to parse pattern").1
    }
}

// u32 because we need a fairly large range: these transition tables can have many rows.
// positive values are indexes to a transition table row.
// the zero value indicates an invalid transition, and rows are initially filled with this.
#[derive(Copy, Clone)]
struct Transition(u32);

impl Default for Transition {
    fn default() -> Self {Transition(0)}
}

impl std::fmt::Display for Transition {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "{:x}", self.0)
    }
}

impl std::fmt::Debug for Transition {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "{}", self)
    }
}

// convert to usize so we can index into `StateTable.states`.
impl std::convert::Into<usize> for Transition {
    fn into(self) -> usize {self.0 as usize}
}

impl Transition {
    fn is_valid(&self) -> bool {
        self.0 != 0
    }
}

struct State {
    // 257 to cover the max range of a symbol.
    transitions: [Transition; 257],
    // indices with the bit set indicates the corresponding index in `NFA.patterns` is alive.
    alive: BitVec,
}

impl State {
    fn new(capacity: u32) -> State {
        State {
            transitions: [Default::default(); 257],
            alive: bitvec![0; capacity as usize],
        }
    }

    // a state is terminal if it has no valid transitions from it.
    fn is_terminal(&self) -> bool {
        self.transitions
            .iter()
            .find(|&transition| transition.is_valid())
            .is_none()
    }
}

struct StateTable {
    // total number of patterns in NFA.
    //
    // we have to know this before we start constructing the table,
    // that's why we use the Builder pattern.
    capacity: u32,
    states: Vec<State>,
}

impl StateTable {
    /// add a new state to the end of the existing table.
    /// return the index of the state as a `Transition`.
    fn add_state(&mut self) -> Transition {
        let index = self.states.len() as u32;  // TODO: danger
        self.states.push(State::new(self.capacity));
        Transition(index)
    }
}

pub struct NFA {
    table: StateTable,
    patterns: Vec<Pattern>,
}

impl NFA {
    pub fn r#match(&self, buf: &[u8]) -> Vec<&Pattern> {
        // TODO
        vec![]
    }

    pub fn new() -> NFABuilder {
        NFABuilder {
            patterns: vec![],
        }
    }
}

// output looks something like:
//
//
//     patterns:
//       - pat0: aabbccdd
//       - pat1: aabbcccc
//
//     transition table:
//
//       aa  bb  cc  dd  ..
//       0:  1                   alive: pat0 pat1
//       1:      2               alive: pat0 pat1
//       2:          3           alive: pat0 pat1
//       3:          5   4       alive: pat0 pat1
//       4:                      matches: pat0
//       5:                      matches: pat1
// ````
impl std::fmt::Debug for NFA {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        writeln!(f, "patterns:").unwrap();
        for (i, pattern) in self.patterns.iter().enumerate() {
            writeln!(f, "  - pat{}: {}", i, pattern).unwrap();
        }
        writeln!(f, "").unwrap();

        // TODO: compute these dynamically
        let symbols = [Symbol(0xAA), Symbol(0xBB), Symbol(0xCC), Symbol(0xDD), WILDCARD];
        writeln!(f, "transition table:").unwrap();
        write!(f, "     ");
        for symbol in symbols.iter() {
            write!(f, " {} ", symbol).unwrap();
        }
        writeln!(f, "").unwrap();

        for (i, state) in self.table.states.iter().enumerate() {
            write!(f, "  {:>2}:", i).unwrap();

            for &symbol in symbols.iter() {
                let index: usize = symbol.into();
                let transition = state.transitions[index];
                if transition.is_valid() {
                    write!(f, " {:>2} ", transition.0).unwrap();
                } else {
                    write!(f, "    ").unwrap();
                }
            }

            if state.alive.iter().find(|&b| b).is_some() {
                if state.is_terminal() {
                    write!(f, "  matches:").unwrap();
                } else {
                    write!(f, "  alive:").unwrap();
                }
                for (i, is_alive) in state.alive.iter().enumerate() {
                    if is_alive {
                        write!(f, " pat{}", i).unwrap();
                    }
                }
            }
            writeln!(f, "").unwrap();
        }

        write!(f, "")  // OK
    }
}

pub struct NFABuilder {
    patterns: Vec<Pattern>
}

impl NFABuilder {
    pub fn add_pattern(&mut self, pattern: Pattern) {
        self.patterns.push(pattern)
    }

    pub fn build(self) -> NFA {
        let mut nfa = NFA {
            table: StateTable {
                capacity: self.patterns.len() as u32,  // TODO: danger.
                states: vec![],
            },
            patterns: self.patterns,
        };

        let start_state = nfa.table.add_state();

        #[derive(Debug)]
        struct Step<'a> {
            /// the index to the state that we need to add the next symbol.
            state_pointer: Transition,
            /// the index of the pattern we're currently processing.
            pattern_index: usize,
            /// the remaining symbols that need to be added.
            symbols: &'a[Symbol],
        };

        let mut q: VecDeque<Step> = VecDeque::new();
        for (pattern_index, pattern) in nfa.patterns.iter().enumerate() {
            q.push_back(Step {
                state_pointer: start_state,
                pattern_index: pattern_index,
                symbols: &pattern.0[..]
            })
        }

        while let Some(step) = q.pop_front() {
            println!("state:\n{:?}", nfa);
            println!("step: {:?}", step);

            // I'm not quite sure how to cast this correctly.
            // maybe we need the Index type???
            let index: usize = step.state_pointer.into();
            let mut state = &mut nfa.table.states[index];

            // mark the current pattern as "alive" at the given state.
            state.alive.set(step.pattern_index, true);

            if step.symbols.len() == 0 {
                // this is terminal.
                continue
            }

            // fetch the symbol to insert, like AA
            let symbol = step.symbols[0];

            // fetch the cell from the state table,
            // this will either:
            //
            //   1. be invalid, which means we have to set it, or
            //   2. be already set to an existing transition
            //
            // if the value is unset, then we need to allocate a new row,
            // and point this cell towards it.
            let index: usize = symbol.into();
            let transition: Transition = state.transitions[index];

            if step.symbols[0] == WILDCARD {
                // wildcard

            } else {
                // byte value
                let next_state = if transition.is_valid() {
                    // there is already a pointer to a state
                    transition
                } else {
                    // need to alloc a new state
                    let next_state = nfa.table.add_state();

                    let index: usize = step.state_pointer.into();
                    let mut state = &mut nfa.table.states[index];

                    let index: usize = step.symbols[0].into();
                    state.transitions[index] = next_state;

                    next_state
                };

                q.push_back(Step {
                    state_pointer: next_state,
                    pattern_index: step.pattern_index,
                    symbols: &step.symbols[1..]
                })
            }
        }

        println!("final state:\n{:?}", nfa);

        nfa
    }

}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_empty_build() {
        let nfa = NFA::new().build();
    }

    // patterns:
    //   - pat0: aabbccdd
    //
    // transition table:
    //
    //  aa  bb  cc  dd  ..
    //  0:  1                   alive: pat0
    //  1:      2               alive: pat0
    //  2:          3           alive: pat0
    //  3:              4       alive: pat0
    //  4:                      matches: pat0
    #[test]
    fn test_add_one_pattern() {
        let mut b = NFA::new();
        b.add_pattern(Pattern::from("AABBCCDD"));

        let nfa = b.build();
    }

    // patterns:
    //   - pat0: aabbccdd
    //   - pat1: aabbcccc
    //
    // transition table:
    //
    //  aa  bb  cc  dd  ..
    //  0:  1                   alive: pat0 pat1
    //  1:      2               alive: pat0 pat1
    //  2:          3           alive: pat0 pat1
    //  3:          5   4       alive: pat0 pat1
    //  4:                      matches: pat0
    //  5:                      matches: pat1
    #[test]
    fn test_add_two_patterns() {
        let mut b = NFA::new();
        b.add_pattern(Pattern::from("AABBCCDD"));
        b.add_pattern(Pattern::from("AABBCCCC"));

        let nfa = b.build();
        assert!(false);
    }
}
<commit_msg>nfa: add matching of basic patterns<commit_after>/// ## purpose
/// to match multiple byte patterns against a byte slice in parallel.
/// we should get all valid matches at the end.
/// does not have to support scanning across the byte slice, only anchored at the start.
/// need support for single character wild cards (`.`).
/// all patterns are the same length
/// (if this needs to change, maybe pad shorter patterns with wildcards).
///
/// ## design:
/// we'll build an NFA with symbols for:
///   - all valid byte values (0-255), and
///   - a wildcard
///
/// a transition table will have 257 columns, one for each of the above symbols, including wildcard.
/// the transition table has a row for each state.
/// entries in each column indicate valid transitions.
/// an entry of `0` indicates "invalid".
/// if a row contains only invalid entries, then its a "terminal state".
/// there's a list associated with each row of "alive" patterns;
/// for a terminal state, these are the patterns that matched.
///
///
/// ### example (no wildcards)
///
/// input patterns:
///
///   p0: A B C D
///   p1: A D C B
///
/// transition table:
///
/// ```ignore
///       A B C D
///   0 | 1        alive: p0, p1
///   1 |   2   4  alive: p0, p1
///   2 |     3    alive: p0
///   3 |          terminal, alive: p0
///   4 |     5    alive: p1
///   5 |          terminal, alive: p1
/// ```
use std::collections::VecDeque;

use nom::IResult;
use nom::multi::many1;
use nom::bytes::complete::tag;
use nom::bytes::complete::take_while_m_n;
use nom::branch::alt;
use nom::combinator::map;
use nom::combinator::map_res;
use bitvec::prelude::*;
use log::{trace};
use crate::SigElement::Wildcard;

// u16 because we need 257 possible values, all unsigned.
#[derive(Copy, Clone)]
pub struct Symbol(u16);

// impl note: value 256 is WILDCARD.
pub const WILDCARD: Symbol = Symbol(0x100);

// byte values map directly into their Symbol indices.
impl std::convert::From<u8> for Symbol {
    fn from(v: u8) -> Self {Symbol(v as u16)}
}

// convert to usize so we can index into `State.transitions`.
impl std::convert::Into<usize> for Symbol {
    fn into(self) -> usize {self.0 as usize}
}

impl std::cmp::PartialEq for Symbol {
    fn eq(&self, other: &Symbol) -> bool {
        self.0 == other.0
    }
}

impl std::fmt::Display for Symbol {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        if self.0 == WILDCARD.0 {
            write!(f, "..")
        } else {
            write!(f, "{:02x}", self.0)
        }
    }
}

impl std::fmt::Debug for Symbol {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "{}", self)
    }
}

// a pattern is just a sequence of symbols.
pub struct Pattern(Vec<Symbol>);

impl std::fmt::Display for Pattern {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        let parts: Vec<String> = self.0.iter().map(|s| format!("{}", s)).collect();
        write!(f, "{}", parts.join(""))
    }
}

fn is_hex_digit(c: char) -> bool {
  c.is_digit(16)
}

fn from_hex(input: &str) -> Result<u8, std::num::ParseIntError> {
  u8::from_str_radix(input, 16)
}

/// parse a single hex byte, like `AB`
fn hex(input: &str) -> IResult<&str, u8> {
    map_res(
        take_while_m_n(2, 2, is_hex_digit),
        from_hex
    )(input)
}

/// parse a single byte signature element, which is either a hex byte or a wildcard.
fn sig_element(input: &str) -> IResult<&str, Symbol> {
    alt((
        map(hex, |v| Symbol::from(v)),
        map(tag(".."), |_| WILDCARD),
    ))(input)
}

/// parse byte signature elements, hex or wildcard.
fn byte_signature(input: &str) -> IResult<&str, Pattern> {
    let (input, elems) = many1(sig_element)(input)?;
    Ok((input, Pattern(elems)))
}

/// parse a pattern from a string like `AABB..DD`.
impl std::convert::From<&str> for Pattern {
    fn from(v: &str) -> Self {
        byte_signature(v).expect("failed to parse pattern").1
    }
}

// u32 because we need a fairly large range: these transition tables can have many rows.
// positive values are indexes to a transition table row.
// the zero value indicates an invalid transition, and rows are initially filled with this.
#[derive(Copy, Clone)]
struct Transition(u32);

impl Default for Transition {
    fn default() -> Self {Transition(0)}
}

impl std::fmt::Display for Transition {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "{:x}", self.0)
    }
}

impl std::fmt::Debug for Transition {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "{}", self)
    }
}

// convert to usize so we can index into `StateTable.states`.
impl std::convert::Into<usize> for Transition {
    fn into(self) -> usize {self.0 as usize}
}

impl Transition {
    fn is_valid(&self) -> bool {
        self.0 != 0
    }
}

struct State {
    // 257 to cover the max range of a symbol.
    transitions: [Transition; 257],
    // indices with the bit set indicates the corresponding index in `NFA.patterns` is alive.
    alive: BitVec,
}

impl State {
    fn new(capacity: u32) -> State {
        State {
            transitions: [Default::default(); 257],
            alive: bitvec![0; capacity as usize],
        }
    }

    // a state is terminal if it has no valid transitions from it.
    fn is_terminal(&self) -> bool {
        self.transitions
            .iter()
            .find(|&transition| transition.is_valid())
            .is_none()
    }
}

struct StateTable {
    // total number of patterns in NFA.
    //
    // we have to know this before we start constructing the table,
    // that's why we use the Builder pattern.
    capacity: u32,
    states: Vec<State>,
}

impl StateTable {
    /// add a new state to the end of the existing table.
    /// return the index of the state as a `Transition`.
    fn add_state(&mut self) -> Transition {
        let index = self.states.len() as u32;  // TODO: danger
        self.states.push(State::new(self.capacity));
        Transition(index)
    }

    fn initial_state(&self) -> Transition {
        Transition(0)
    }
}

pub struct NFA {
    table: StateTable,
    patterns: Vec<Pattern>,
}

impl NFA {
    pub fn r#match(&self, buf: &[u8]) -> Vec<&Pattern> {
        #[derive(Debug)]
        struct Step<'a> {
            /// the index to the state that we need to match next.
            state_pointer: Transition,
            /// the remaining bytes that need to be matched.
            buf: &'a[u8],
        };

        let mut q: VecDeque<Step> = VecDeque::new();
        q.push_back(Step {
            state_pointer: self.table.initial_state(),
            buf: buf,
        });

        let mut matches = vec![];
        while let Some(step) = q.pop_front() {
            trace!("match step: {:?}", step);

            let index: usize = step.state_pointer.into();
            let state: &State = &self.table.states[index];

            if state.is_terminal() {
                trace!("match: found terminal state: {}", step.state_pointer);

                let match_indices = state.alive
                    .iter()
                    .enumerate()
                    .filter(|(_, is_alive)| *is_alive)
                    .map(|(i, _)| i);

                for i in match_indices {
                    matches.push(&self.patterns[i])
                }

                continue
            }

            let literal_transition = state.transitions[step.buf[0] as usize];
            trace!("match: found transition to {} for input {:02x}", literal_transition, step.buf[0]);
            if literal_transition.is_valid() {
                q.push_back(Step {
                    state_pointer: literal_transition,
                    buf: &step.buf[1..],
                })
            }

            let index: usize = WILDCARD.into();
            let wildcard_transition = state.transitions[index];
            if wildcard_transition.is_valid() {
                q.push_back(Step {
                    state_pointer: wildcard_transition,
                    buf: &step.buf[1..],
                })
            }
        }

        matches
    }

    pub fn new() -> NFABuilder {
        NFABuilder {
            patterns: vec![],
        }
    }
}

// output looks something like:
//
//
//     patterns:
//       - pat0: aabbccdd
//       - pat1: aabbcccc
//
//     transition table:
//
//       aa  bb  cc  dd  ..
//       0:  1                   alive: pat0 pat1
//       1:      2               alive: pat0 pat1
//       2:          3           alive: pat0 pat1
//       3:          5   4       alive: pat0 pat1
//       4:                      matches: pat0
//       5:                      matches: pat1
// ````
impl std::fmt::Debug for NFA {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        writeln!(f, "patterns:").unwrap();
        for (i, pattern) in self.patterns.iter().enumerate() {
            writeln!(f, "  - pat{}: {}", i, pattern).unwrap();
        }
        writeln!(f, "").unwrap();

        // TODO: compute these dynamically
        let symbols = [Symbol(0xAA), Symbol(0xBB), Symbol(0xCC), Symbol(0xDD), WILDCARD];
        writeln!(f, "transition table:").unwrap();
        write!(f, "     ").unwrap();
        for symbol in symbols.iter() {
            write!(f, " {} ", symbol).unwrap();
        }
        writeln!(f, "").unwrap();

        for (i, state) in self.table.states.iter().enumerate() {
            write!(f, "  {:>2}:", i).unwrap();

            for &symbol in symbols.iter() {
                let index: usize = symbol.into();
                let transition = state.transitions[index];
                if transition.is_valid() {
                    write!(f, " {:>2} ", transition.0).unwrap();
                } else {
                    write!(f, "    ").unwrap();
                }
            }

            if state.alive.iter().find(|&b| b).is_some() {
                if state.is_terminal() {
                    write!(f, "  matches:").unwrap();
                } else {
                    write!(f, "  alive:").unwrap();
                }
                for (i, is_alive) in state.alive.iter().enumerate() {
                    if is_alive {
                        write!(f, " pat{}", i).unwrap();
                    }
                }
            }
            writeln!(f, "").unwrap();
        }

        write!(f, "")  // OK
    }
}

pub struct NFABuilder {
    patterns: Vec<Pattern>
}

impl NFABuilder {
    pub fn add_pattern(&mut self, pattern: Pattern) {
        self.patterns.push(pattern)
    }

    pub fn build(self) -> NFA {
        let mut nfa = NFA {
            table: StateTable {
                capacity: self.patterns.len() as u32,  // TODO: danger.
                states: vec![],
            },
            patterns: self.patterns,
        };

        let start_state = nfa.table.add_state();

        #[derive(Debug)]
        struct Step<'a> {
            /// the index to the state that we need to add the next symbol.
            state_pointer: Transition,
            /// the index of the pattern we're currently processing.
            pattern_index: usize,
            /// the remaining symbols that need to be added.
            symbols: &'a[Symbol],
        };

        let mut q: VecDeque<Step> = VecDeque::new();
        for (pattern_index, pattern) in nfa.patterns.iter().enumerate() {
            q.push_back(Step {
                state_pointer: start_state,
                pattern_index: pattern_index,
                symbols: &pattern.0[..]
            })
        }

        while let Some(step) = q.pop_front() {
            println!("state:\n{:?}", nfa);
            println!("step: {:?}", step);

            // I'm not quite sure how to cast this correctly.
            // maybe we need the Index type???
            let index: usize = step.state_pointer.into();
            let state = &mut nfa.table.states[index];

            // mark the current pattern as "alive" at the given state.
            state.alive.set(step.pattern_index, true);

            if step.symbols.len() == 0 {
                // this is terminal.
                continue
            }

            // fetch the symbol to insert, like AA
            let symbol = step.symbols[0];

            // fetch the cell from the state table,
            // this will either:
            //
            //   1. be invalid, which means we have to set it, or
            //   2. be already set to an existing transition
            //
            // if the value is unset, then we need to allocate a new row,
            // and point this cell towards it.
            let index: usize = symbol.into();
            let transition: Transition = state.transitions[index];

            if step.symbols[0] == WILDCARD {
                // wildcard

            } else {
                // byte value
                let next_state = if transition.is_valid() {
                    // there is already a pointer to a state
                    transition
                } else {
                    // need to alloc a new state
                    let next_state = nfa.table.add_state();

                    let index: usize = step.state_pointer.into();
                    let mut state = &mut nfa.table.states[index];

                    let index: usize = step.symbols[0].into();
                    state.transitions[index] = next_state;

                    next_state
                };

                q.push_back(Step {
                    state_pointer: next_state,
                    pattern_index: step.pattern_index,
                    symbols: &step.symbols[1..]
                })
            }
        }

        println!("final state:\n{:?}", nfa);

        nfa
    }

}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_empty_build() {
        NFA::new().build();
    }

    // patterns:
    //   - pat0: aabbccdd
    //
    // transition table:
    //
    //  aa  bb  cc  dd  ..
    //  0:  1                   alive: pat0
    //  1:      2               alive: pat0
    //  2:          3           alive: pat0
    //  3:              4       alive: pat0
    //  4:                      matches: pat0
    #[test]
    fn test_add_one_pattern() {
        let mut b = NFA::new();
        b.add_pattern(Pattern::from("AABBCCDD"));

        let nfa = b.build();
        println!("{:?}", nfa);
    }

    // patterns:
    //   - pat0: aabbccdd
    //   - pat1: aabbcccc
    //
    // transition table:
    //
    //  aa  bb  cc  dd  ..
    //  0:  1                   alive: pat0 pat1
    //  1:      2               alive: pat0 pat1
    //  2:          3           alive: pat0 pat1
    //  3:          5   4       alive: pat0 pat1
    //  4:                      matches: pat0
    //  5:                      matches: pat1
    #[test]
    fn test_add_two_patterns() {
        let mut b = NFA::new();
        b.add_pattern(Pattern::from("AABBCCDD"));
        b.add_pattern(Pattern::from("AABBCCCC"));

        let nfa = b.build();
        println!("{:?}", nfa);
    }

    // we don't match when we don't have any patterns.
    #[test]
    fn test_match_empty() {
        let nfa = NFA::new().build();
        assert_eq!(nfa.r#match(b"\xAA\xBB\xCC\xDD").len(), 0);
    }

    // we match things we want to, and don't match other data.
    #[test]
    fn test_match_one() {
        let mut b = NFA::new();
        b.add_pattern(Pattern::from("AABBCCDD"));
        let nfa = b.build();

        // true positive
        assert_eq!(nfa.r#match(b"\xAA\xBB\xCC\xDD").len(), 1);
        // true negative
        assert_eq!(nfa.r#match(b"\xAA\xBB\xCC\xEE").len(), 0);
    }

    // we match from the beginning of the buffer onwards,
    // ignoring trailing bytes beyond the length of the pattern.
    #[test]
    fn test_match_long() {
        let mut b = NFA::new();
        b.add_pattern(Pattern::from("AABBCCDD"));
        let nfa = b.build();

        assert_eq!(nfa.r#match(b"\xAA\xBB\xCC\xDD\x00").len(), 1);
        assert_eq!(nfa.r#match(b"\xAA\xBB\xCC\xDD\x11").len(), 1);
    }

}
<|endoftext|>


<commit_before>#![cfg(test)]
use super::*;
use crate::{test_utils::*, *};

fn new_state<T: EthSpec>(validator_count: usize, slot: Slot) -> BeaconState<T> {
    let spec = &T::spec();

    let mut builder =
        TestingBeaconStateBuilder::from_single_keypair(validator_count, &Keypair::random(), spec);

    builder.teleport_to_slot(slot, spec);

    let (state, _keypairs) = builder.build();

    state
}

#[test]
fn fails_without_validators() {
    let state = new_state::<FewValidatorsEthSpec>(0, Slot::new(0));
    let spec = &FewValidatorsEthSpec::spec();

    assert_eq!(
        EpochCache::initialized(&state, state.current_epoch(), &spec),
        Err(BeaconStateError::InsufficientValidators)
    );
}

#[test]
fn initializes_with_the_right_epoch() {
    let state = new_state::<FewValidatorsEthSpec>(16, Slot::new(0));
    let spec = &FewValidatorsEthSpec::spec();

    let cache = EpochCache::default();
    assert_eq!(cache.initialized_epoch, None);

    let cache = EpochCache::initialized(&state, state.current_epoch(), &spec).unwrap();
    assert_eq!(cache.initialized_epoch, Some(state.current_epoch()));

    let cache = EpochCache::initialized(&state, state.previous_epoch(), &spec).unwrap();
    assert_eq!(cache.initialized_epoch, Some(state.previous_epoch()));

    let cache = EpochCache::initialized(&state, state.next_epoch(), &spec).unwrap();
    assert_eq!(cache.initialized_epoch, Some(state.next_epoch()));
}

#[test]
fn shuffles_for_the_right_epoch() {
    let num_validators = FewValidatorsEthSpec::minimum_validator_count() * 2;
    let epoch = Epoch::new(100_000_000);
    let slot = epoch.start_slot(FewValidatorsEthSpec::slots_per_epoch());

    let mut state = new_state::<FewValidatorsEthSpec>(num_validators, slot);
    let spec = &FewValidatorsEthSpec::spec();

    let distinct_hashes: Vec<Hash256> = (0..FewValidatorsEthSpec::latest_randao_mixes_length())
        .into_iter()
        .map(|i| Hash256::from(i as u64))
        .collect();

    state.latest_randao_mixes = FixedLenVec::from(distinct_hashes);

    let previous_seed = state.generate_seed(state.previous_epoch(), spec).unwrap();
    let current_seed = state.generate_seed(state.current_epoch(), spec).unwrap();
    let next_seed = state.generate_seed(state.next_epoch(), spec).unwrap();

    assert!((previous_seed != current_seed) && (current_seed != next_seed));

    let shuffling_with_seed = |seed: Hash256| {
        shuffle_list(
            (0..num_validators).collect(),
            spec.shuffle_round_count,
            &seed[..],
            false,
        )
        .unwrap()
    };

    let cache = EpochCache::initialized(&state, state.current_epoch(), spec).unwrap();
    assert_eq!(cache.shuffling, shuffling_with_seed(current_seed));

    let cache = EpochCache::initialized(&state, state.previous_epoch(), spec).unwrap();
    assert_eq!(cache.shuffling, shuffling_with_seed(previous_seed));

    let cache = EpochCache::initialized(&state, state.next_epoch(), spec).unwrap();
    assert_eq!(cache.shuffling, shuffling_with_seed(next_seed));
}
<commit_msg>Add more epoch_cache tests<commit_after>#![cfg(test)]
use super::*;
use crate::{test_utils::*, *};
use fixed_len_vec::typenum::*;
use serde_derive::{Deserialize, Serialize};

fn new_state<T: EthSpec>(validator_count: usize, slot: Slot) -> BeaconState<T> {
    let spec = &T::spec();

    let mut builder =
        TestingBeaconStateBuilder::from_single_keypair(validator_count, &Keypair::random(), spec);

    builder.teleport_to_slot(slot, spec);

    let (state, _keypairs) = builder.build();

    state
}

#[test]
fn fails_without_validators() {
    let state = new_state::<FewValidatorsEthSpec>(0, Slot::new(0));
    let spec = &FewValidatorsEthSpec::spec();

    assert_eq!(
        EpochCache::initialized(&state, state.current_epoch(), &spec),
        Err(BeaconStateError::InsufficientValidators)
    );
}

#[test]
fn initializes_with_the_right_epoch() {
    let state = new_state::<FewValidatorsEthSpec>(16, Slot::new(0));
    let spec = &FewValidatorsEthSpec::spec();

    let cache = EpochCache::default();
    assert_eq!(cache.initialized_epoch, None);

    let cache = EpochCache::initialized(&state, state.current_epoch(), &spec).unwrap();
    assert_eq!(cache.initialized_epoch, Some(state.current_epoch()));

    let cache = EpochCache::initialized(&state, state.previous_epoch(), &spec).unwrap();
    assert_eq!(cache.initialized_epoch, Some(state.previous_epoch()));

    let cache = EpochCache::initialized(&state, state.next_epoch(), &spec).unwrap();
    assert_eq!(cache.initialized_epoch, Some(state.next_epoch()));
}

#[test]
fn shuffles_for_the_right_epoch() {
    let num_validators = FewValidatorsEthSpec::minimum_validator_count() * 2;
    let epoch = Epoch::new(100_000_000);
    let slot = epoch.start_slot(FewValidatorsEthSpec::slots_per_epoch());

    let mut state = new_state::<FewValidatorsEthSpec>(num_validators, slot);
    let spec = &FewValidatorsEthSpec::spec();

    let distinct_hashes: Vec<Hash256> = (0..FewValidatorsEthSpec::latest_randao_mixes_length())
        .into_iter()
        .map(|i| Hash256::from(i as u64))
        .collect();

    state.latest_randao_mixes = FixedLenVec::from(distinct_hashes);

    let previous_seed = state.generate_seed(state.previous_epoch(), spec).unwrap();
    let current_seed = state.generate_seed(state.current_epoch(), spec).unwrap();
    let next_seed = state.generate_seed(state.next_epoch(), spec).unwrap();

    assert!((previous_seed != current_seed) && (current_seed != next_seed));

    let shuffling_with_seed = |seed: Hash256| {
        shuffle_list(
            (0..num_validators).collect(),
            spec.shuffle_round_count,
            &seed[..],
            false,
        )
        .unwrap()
    };

    let cache = EpochCache::initialized(&state, state.current_epoch(), spec).unwrap();
    assert_eq!(cache.shuffling, shuffling_with_seed(current_seed));

    let cache = EpochCache::initialized(&state, state.previous_epoch(), spec).unwrap();
    assert_eq!(cache.shuffling, shuffling_with_seed(previous_seed));

    let cache = EpochCache::initialized(&state, state.next_epoch(), spec).unwrap();
    assert_eq!(cache.shuffling, shuffling_with_seed(next_seed));
}

#[test]
fn can_start_on_any_shard() {
    let num_validators = FewValidatorsEthSpec::minimum_validator_count() * 2;
    let epoch = Epoch::new(100_000_000);
    let slot = epoch.start_slot(FewValidatorsEthSpec::slots_per_epoch());

    let mut state = new_state::<FewValidatorsEthSpec>(num_validators, slot);
    let spec = &FewValidatorsEthSpec::spec();

    for i in 0..FewValidatorsEthSpec::shard_count() as u64 {
        state.latest_start_shard = i;

        let cache = EpochCache::initialized(&state, state.current_epoch(), spec).unwrap();
        assert_eq!(cache.shuffling_start_shard, i);

        let cache = EpochCache::initialized(&state, state.previous_epoch(), spec).unwrap();
        assert_eq!(cache.shuffling_start_shard, i);

        let cache = EpochCache::initialized(&state, state.next_epoch(), spec).unwrap();
        assert_eq!(cache.shuffling_start_shard, i);
    }
}

#[derive(Clone, PartialEq, Debug, Default, Serialize, Deserialize)]
pub struct ExcessShardsEthSpec;

impl EthSpec for ExcessShardsEthSpec {
    type ShardCount = U128;
    type SlotsPerHistoricalRoot = U8192;
    type LatestRandaoMixesLength = U8192;
    type LatestActiveIndexRootsLength = U8192;
    type LatestSlashedExitLength = U8192;

    fn spec() -> ChainSpec {
        ChainSpec::few_validators()
    }
}

#[test]
fn starts_on_the_correct_shard() {
    let spec = &ExcessShardsEthSpec::spec();

    let num_validators = ExcessShardsEthSpec::shard_count();

    let epoch = Epoch::new(100_000_000);
    let slot = epoch.start_slot(ExcessShardsEthSpec::slots_per_epoch());

    let mut state = new_state::<ExcessShardsEthSpec>(num_validators, slot);

    let validator_count = state.validator_registry.len();

    let previous_epoch = state.previous_epoch();
    let current_epoch = state.current_epoch();
    let next_epoch = state.next_epoch();

    for (i, mut v) in state.validator_registry.iter_mut().enumerate() {
        let epoch = if i < validator_count / 4 {
            previous_epoch
        } else if i < validator_count / 2 {
            current_epoch
        } else {
            next_epoch
        };

        v.activation_epoch = epoch;
    }

    assert_eq!(
        get_active_validator_count(&state.validator_registry, previous_epoch),
        validator_count / 4
    );
    assert_eq!(
        get_active_validator_count(&state.validator_registry, current_epoch),
        validator_count / 2
    );
    assert_eq!(
        get_active_validator_count(&state.validator_registry, next_epoch),
        validator_count
    );

    let previous_shards = ExcessShardsEthSpec::get_epoch_committee_count(
        get_active_validator_count(&state.validator_registry, previous_epoch),
    );
    let current_shards = ExcessShardsEthSpec::get_epoch_committee_count(
        get_active_validator_count(&state.validator_registry, current_epoch),
    );
    let next_shards = ExcessShardsEthSpec::get_epoch_committee_count(get_active_validator_count(
        &state.validator_registry,
        next_epoch,
    ));

    assert_eq!(
        previous_shards as usize,
        ExcessShardsEthSpec::shard_count() / 4
    );
    assert_eq!(
        current_shards as usize,
        ExcessShardsEthSpec::shard_count() / 2
    );
    assert_eq!(next_shards as usize, ExcessShardsEthSpec::shard_count());

    let shard_count = ExcessShardsEthSpec::shard_count();
    for i in 0..ExcessShardsEthSpec::shard_count() {
        state.latest_start_shard = i as u64;

        let cache = EpochCache::initialized(&state, state.current_epoch(), spec).unwrap();
        assert_eq!(cache.shuffling_start_shard as usize, i);

        let cache = EpochCache::initialized(&state, state.previous_epoch(), spec).unwrap();
        assert_eq!(
            cache.shuffling_start_shard as usize,
            (i + shard_count - previous_shards) % shard_count
        );

        let cache = EpochCache::initialized(&state, state.next_epoch(), spec).unwrap();
        assert_eq!(
            cache.shuffling_start_shard as usize,
            (i + current_shards) % shard_count
        );
    }
}
<|endoftext|>


<commit_before>extern crate indexmap;
extern crate timely;
extern crate differential_dataflow;

use std::io::{BufRead, BufReader};
use std::fs::File;

use indexmap::IndexMap;

use timely::Allocate;

use timely::dataflow::Scope;
use timely::dataflow::scopes::{ScopeParent, Root, Child};
use timely::progress::timestamp::RootTimestamp;
use timely::progress::nested::product::Product;
use timely::progress::Timestamp;

use differential_dataflow::Collection;
use differential_dataflow::lattice::Lattice;
use differential_dataflow::input::{Input, InputSession};
use differential_dataflow::operators::arrange::{ArrangeByKey, ArrangeBySelf};
use differential_dataflow::operators::iterate::Variable;
use differential_dataflow::operators::{Threshold, JoinCore, Consolidate};

type Node = usize;
type Edge = (Node, Node);

/// A direction we might traverse an edge.
#[derive(Debug)]
pub enum Relation {
    /// The forward direction of the relation, A(x,y).
    Forward(String),
    /// The reverse direction of the relation, A(y,x).
    Reverse(String),
}

/// A rule for producing edges based on paths.
///
/// The intent is that we should add to the left hand relation any pair `(x,y)` for which
/// there exists a path x -> z0 -> z1 -> .. -> y through the specified relations.
#[derive(Debug)]
pub struct Production {
    /// The name of the edge set to populate.
    pub left_hand: String,
    /// A sequence of edges to follow.
    pub relations: Vec<Relation>,
}

impl<'a> From<&'a str> for Production {
    fn from(text: &'a str) -> Production {

        let mut names = text.split_whitespace();

        // Extract the target.
        let left_hand = names.next().expect("All rules must have a target.").to_string();

        // Collect all remaining names.
        let relations = names.map(|name| {
            if name.starts_with('~') { Relation::Reverse(name.split_at(1).1.to_string()) }
            else                     { Relation::Forward(name.to_string()) }
        }).collect();

        Production { left_hand, relations }
    }
}

#[derive(Debug)]
pub struct Query {
    /// A sequence of productions.
    ///
    /// Empty productions are a great way to define input relations, but are otherwise ignored.
    pub productions: Vec<Production>,
}

use differential_dataflow::trace::implementations::ord::{OrdValSpine, OrdKeySpine};
use differential_dataflow::operators::arrange::{Arranged, TraceAgent};

type TraceKeyHandle<K,T,R> = TraceAgent<K, (), T, R, OrdKeySpine<K, T, R>>;
type TraceValHandle<K,V,T,R> = TraceAgent<K, V, T, R, OrdValSpine<K, V, T, R>>;
type Arrange<G, K, V, R> = Arranged<G, K, V, R, TraceValHandle<K, V, <G as ScopeParent>::Timestamp, R>>;

/// An evolving set of edges.
///
/// An edge variable represents the distinct set of edges found in any of a set of productions
/// which the variable. A newly created edge variable supports the addition of new productions
/// up until the variable is "completed" at which point the definition is locked in.
///
/// An edge variable provides arranged representations of its contents, even before they are
/// completely defined, in support of recursively defined productions.
pub struct EdgeVariable<'a, G: Scope> where G::Timestamp : Lattice {
    variable: Variable<'a, G, Edge, isize>,
    current: Collection<Child<'a, G, u64>, Edge, isize>,
    forward: Option<Arrange<Child<'a, G, u64>, Node, Node, isize>>,
    reverse: Option<Arrange<Child<'a, G, u64>, Node, Node, isize>>,
}

impl<'a, G: Scope> EdgeVariable<'a, G> where G::Timestamp : Lattice {
    /// Creates a new variable initialized with `source`.
    pub fn from(source: &Collection<Child<'a, G, u64>, Edge>) -> Self {
        let variable = Variable::from(source.filter(|_| false));
        EdgeVariable {
            variable: variable,
            current: source.clone(),
            forward: None,
            reverse: None,
        }
    }
    /// Concatenates `production` into the definition of the variable.
    pub fn add_production(&mut self, production: &Collection<Child<'a, G, u64>, Edge, isize>) {
        self.current = self.current.concat(production);
    }
    /// Finalizes the variable, connecting its recursive definition.
    ///
    /// Failure to call `complete` on a variable results in a non-recursively defined
    /// collection, whose contents are just its initial `source` data.
    pub fn complete(self) {
        let distinct = self.current.distinct();
        // distinct.map(|_| ()).consolidate().inspect(|x| println!("{:?}", x));
        self.variable.set(&distinct);
    }
    /// The collection arranged in the forward direction.
    pub fn forward(&mut self) -> &Arrange<Child<'a, G, u64>, Node, Node, isize> {
        if self.forward.is_none() {
            self.forward = Some(self.variable.arrange_by_key());
        }
        self.forward.as_ref().unwrap()
    }
    /// The collection arranged in the reverse direction.
    pub fn reverse(&mut self) -> &Arrange<Child<'a, G, u64>, Node, Node, isize> {
        if self.reverse.is_none() {
            self.reverse = Some(self.variable.map(|(x,y)| (y,x)).arrange_by_key());
        }
        self.reverse.as_ref().unwrap()
    }
}

/// Handles to inputs and outputs of a computation.
pub struct RelationHandles<T: Timestamp+Lattice> {
    /// An input handle supporting arbitrary changes.
    pub input: InputSession<T, Edge, isize>,
    /// An output trace handle which can be used in other computations.
    pub trace: TraceKeyHandle<Edge, Product<RootTimestamp, T>, isize>,
}

impl Query {

    /// Creates a new query from a sequence of productions.
    ///
    /// Each production is a `String` of the form
    ///
    ///    Target Rel1 Rel2 ..
    ///
    /// Where each Rel may begin with a '~' character, indicating the reverse direction.
    /// Target must not begin with a '~'; rewrite the rule in the other order, silly.
    pub fn build_from<'a>(iterator: impl IntoIterator<Item=&'a str>) -> Self {
        Query { productions: iterator.into_iter().map(|text| Production::from(text)).collect() }
    }

    /// Creates a dataflow implementing the query, and returns input and trace handles.
    pub fn render_in<'a, A, T>(&self, scope: &mut Child<'a, Root<A>, T>) -> IndexMap<String, RelationHandles<T>>
    where A: Allocate, T: Timestamp+Lattice {

        // Create new input (handle, stream) pairs
        let mut input_map = IndexMap::new();
        for production in self.productions.iter() {
            input_map.entry(production.left_hand.clone()).or_insert_with(|| scope.new_collection());
        }

        // We need a subscope to allow iterative development of variables.
        scope.scoped(|subscope| {

            // create map from relation name to input handle and collection.
            let mut result_map = IndexMap::new();
            let mut variable_map = IndexMap::new();

            // create variables and result handles for each named relation.
            for (name, (input, collection)) in input_map.drain(..) {
                let edge_variable = EdgeVariable::from(&collection.enter(subscope));
                let trace = edge_variable.variable.leave().arrange_by_self().trace;
                result_map.insert(name.clone(), RelationHandles { input, trace });
                variable_map.insert(name.clone(), edge_variable);
            }

            // For each rule, add to the productions for the relation.
            for production in self.productions.iter() {

                let name = &production.left_hand;
                let rule = &production.relations;

                // We need to start somewhere; ignore empty rules.
                if rule.len() > 0 {

                    // We'll track the path transposed, so that it is indexed by *destination* rather than source.
                    let mut transposed = match &rule[0] {
                        Relation::Forward(name) => variable_map[name].reverse().clone(),
                        Relation::Reverse(name) => variable_map[name].forward().clone(),
                    };

                    for relation in rule[1..].iter() {
                        let to_join = match relation {
                            Relation::Forward(name) => variable_map[name].forward(),
                            Relation::Reverse(name) => variable_map[name].reverse(),
                        };

                        transposed =
                        transposed
                            .join_core(to_join, |_k,&x,&y| Some((y,x)))
                            .arrange_by_key();
                    }

                    // Reverse the direction before adding it as a production.
                    variable_map[name].add_production(&transposed.as_collection(|&dst,&src| (src,dst)));
                }
            }

            for (_name, variable) in variable_map.drain(..) {
                variable.complete();
            }
            result_map
        })
    }
}

fn main() {

    timely::execute_from_args(std::env::args(), move |worker| {

        let timer = ::std::time::Instant::now();

        let peers = worker.peers();
        let index = worker.index();

        let query_filename = std::env::args().nth(1).expect("Argument 1 (query filename) missing.");
        let query_text = std::fs::read_to_string(query_filename).expect("Failed to read query file");
        let query = Query::build_from(query_text.lines());

        let mut relation_map = worker.dataflow::<(),_,_>(|scope| query.render_in(scope));

        if index == 0 { println!("{:?}:\tDataflow assembled for {:?}", timer.elapsed(), query); }

        // Build a dataflow to report final sizes.
        worker.dataflow(|scope| {
            for (name, data) in relation_map.iter_mut() {
                let name = name.to_string();
                data.trace
                    .import(scope)
                    .as_collection(|&_kv,&()| ())
                    .consolidate()
                    .inspect(move |x| println!("{:?}\tfinal size of relation '{}': {:?}", timer.elapsed(), name, x.2));
            }
        });

        // snag a filename to use for the input graph.
        let data_filename = std::env::args().nth(2).expect("Argument 2 (data filename) missing.");
        let file = BufReader::new(File::open(data_filename).expect("Failed to read data file"));

        for readline in file.lines() {
            let line = readline.ok().expect("read error");
            if !line.starts_with('#') && line.len() > 0 {
                let mut elts = line[..].split_whitespace();
                let src = elts.next().expect("data line with no src (1st) element").parse().expect("malformed src");
                if (src as usize) % peers == index {
                    let dst = elts.next().expect("data line with no dst (2nd) element").parse().expect("malformed dst");
                    let val: &str = elts.next().expect("data line with no val (3rd) element");
                    if let Some(handle) = relation_map.get_mut(val) {
                        handle.input.insert((src, dst));
                    }
                    else {
                        panic!("couldn't find the named relation: {:?}", val);
                    }
                }
            }
        }

        if index == 0 { println!("{:?}:\tData loaded", timer.elapsed()); }

    }).expect("Timely computation did not complete cleanly");
}
<commit_msg>fixed comm use<commit_after>extern crate indexmap;
extern crate timely;
extern crate differential_dataflow;

use std::io::{BufRead, BufReader};
use std::fs::File;

use indexmap::IndexMap;

use timely::communication::Allocate;

use timely::dataflow::Scope;
use timely::dataflow::scopes::{ScopeParent, Root, Child};
use timely::progress::timestamp::RootTimestamp;
use timely::progress::nested::product::Product;
use timely::progress::Timestamp;

use differential_dataflow::Collection;
use differential_dataflow::lattice::Lattice;
use differential_dataflow::input::{Input, InputSession};
use differential_dataflow::operators::arrange::{ArrangeByKey, ArrangeBySelf};
use differential_dataflow::operators::iterate::Variable;
use differential_dataflow::operators::{Threshold, JoinCore, Consolidate};

type Node = usize;
type Edge = (Node, Node);

/// A direction we might traverse an edge.
#[derive(Debug)]
pub enum Relation {
    /// The forward direction of the relation, A(x,y).
    Forward(String),
    /// The reverse direction of the relation, A(y,x).
    Reverse(String),
}

/// A rule for producing edges based on paths.
///
/// The intent is that we should add to the left hand relation any pair `(x,y)` for which
/// there exists a path x -> z0 -> z1 -> .. -> y through the specified relations.
#[derive(Debug)]
pub struct Production {
    /// The name of the edge set to populate.
    pub left_hand: String,
    /// A sequence of edges to follow.
    pub relations: Vec<Relation>,
}

impl<'a> From<&'a str> for Production {
    fn from(text: &'a str) -> Production {

        let mut names = text.split_whitespace();

        // Extract the target.
        let left_hand = names.next().expect("All rules must have a target.").to_string();

        // Collect all remaining names.
        let relations = names.map(|name| {
            if name.starts_with('~') { Relation::Reverse(name.split_at(1).1.to_string()) }
            else                     { Relation::Forward(name.to_string()) }
        }).collect();

        Production { left_hand, relations }
    }
}

#[derive(Debug)]
pub struct Query {
    /// A sequence of productions.
    ///
    /// Empty productions are a great way to define input relations, but are otherwise ignored.
    pub productions: Vec<Production>,
}

use differential_dataflow::trace::implementations::ord::{OrdValSpine, OrdKeySpine};
use differential_dataflow::operators::arrange::{Arranged, TraceAgent};

type TraceKeyHandle<K,T,R> = TraceAgent<K, (), T, R, OrdKeySpine<K, T, R>>;
type TraceValHandle<K,V,T,R> = TraceAgent<K, V, T, R, OrdValSpine<K, V, T, R>>;
type Arrange<G, K, V, R> = Arranged<G, K, V, R, TraceValHandle<K, V, <G as ScopeParent>::Timestamp, R>>;

/// An evolving set of edges.
///
/// An edge variable represents the distinct set of edges found in any of a set of productions
/// which the variable. A newly created edge variable supports the addition of new productions
/// up until the variable is "completed" at which point the definition is locked in.
///
/// An edge variable provides arranged representations of its contents, even before they are
/// completely defined, in support of recursively defined productions.
pub struct EdgeVariable<'a, G: Scope> where G::Timestamp : Lattice {
    variable: Variable<'a, G, Edge, isize>,
    current: Collection<Child<'a, G, u64>, Edge, isize>,
    forward: Option<Arrange<Child<'a, G, u64>, Node, Node, isize>>,
    reverse: Option<Arrange<Child<'a, G, u64>, Node, Node, isize>>,
}

impl<'a, G: Scope> EdgeVariable<'a, G> where G::Timestamp : Lattice {
    /// Creates a new variable initialized with `source`.
    pub fn from(source: &Collection<Child<'a, G, u64>, Edge>) -> Self {
        let variable = Variable::from(source.filter(|_| false));
        EdgeVariable {
            variable: variable,
            current: source.clone(),
            forward: None,
            reverse: None,
        }
    }
    /// Concatenates `production` into the definition of the variable.
    pub fn add_production(&mut self, production: &Collection<Child<'a, G, u64>, Edge, isize>) {
        self.current = self.current.concat(production);
    }
    /// Finalizes the variable, connecting its recursive definition.
    ///
    /// Failure to call `complete` on a variable results in a non-recursively defined
    /// collection, whose contents are just its initial `source` data.
    pub fn complete(self) {
        let distinct = self.current.distinct();
        // distinct.map(|_| ()).consolidate().inspect(|x| println!("{:?}", x));
        self.variable.set(&distinct);
    }
    /// The collection arranged in the forward direction.
    pub fn forward(&mut self) -> &Arrange<Child<'a, G, u64>, Node, Node, isize> {
        if self.forward.is_none() {
            self.forward = Some(self.variable.arrange_by_key());
        }
        self.forward.as_ref().unwrap()
    }
    /// The collection arranged in the reverse direction.
    pub fn reverse(&mut self) -> &Arrange<Child<'a, G, u64>, Node, Node, isize> {
        if self.reverse.is_none() {
            self.reverse = Some(self.variable.map(|(x,y)| (y,x)).arrange_by_key());
        }
        self.reverse.as_ref().unwrap()
    }
}

/// Handles to inputs and outputs of a computation.
pub struct RelationHandles<T: Timestamp+Lattice> {
    /// An input handle supporting arbitrary changes.
    pub input: InputSession<T, Edge, isize>,
    /// An output trace handle which can be used in other computations.
    pub trace: TraceKeyHandle<Edge, Product<RootTimestamp, T>, isize>,
}

impl Query {

    /// Creates a new query from a sequence of productions.
    ///
    /// Each production is a `String` of the form
    ///
    ///    Target Rel1 Rel2 ..
    ///
    /// Where each Rel may begin with a '~' character, indicating the reverse direction.
    /// Target must not begin with a '~'; rewrite the rule in the other order, silly.
    pub fn build_from<'a>(iterator: impl IntoIterator<Item=&'a str>) -> Self {
        Query { productions: iterator.into_iter().map(|text| Production::from(text)).collect() }
    }

    /// Creates a dataflow implementing the query, and returns input and trace handles.
    pub fn render_in<'a, A, T>(&self, scope: &mut Child<'a, Root<A>, T>) -> IndexMap<String, RelationHandles<T>>
    where A: Allocate, T: Timestamp+Lattice {

        // Create new input (handle, stream) pairs
        let mut input_map = IndexMap::new();
        for production in self.productions.iter() {
            input_map.entry(production.left_hand.clone()).or_insert_with(|| scope.new_collection());
        }

        // We need a subscope to allow iterative development of variables.
        scope.scoped(|subscope| {

            // create map from relation name to input handle and collection.
            let mut result_map = IndexMap::new();
            let mut variable_map = IndexMap::new();

            // create variables and result handles for each named relation.
            for (name, (input, collection)) in input_map.drain(..) {
                let edge_variable = EdgeVariable::from(&collection.enter(subscope));
                let trace = edge_variable.variable.leave().arrange_by_self().trace;
                result_map.insert(name.clone(), RelationHandles { input, trace });
                variable_map.insert(name.clone(), edge_variable);
            }

            // For each rule, add to the productions for the relation.
            for production in self.productions.iter() {

                let name = &production.left_hand;
                let rule = &production.relations;

                // We need to start somewhere; ignore empty rules.
                if rule.len() > 0 {

                    // We'll track the path transposed, so that it is indexed by *destination* rather than source.
                    let mut transposed = match &rule[0] {
                        Relation::Forward(name) => variable_map[name].reverse().clone(),
                        Relation::Reverse(name) => variable_map[name].forward().clone(),
                    };

                    for relation in rule[1..].iter() {
                        let to_join = match relation {
                            Relation::Forward(name) => variable_map[name].forward(),
                            Relation::Reverse(name) => variable_map[name].reverse(),
                        };

                        transposed =
                        transposed
                            .join_core(to_join, |_k,&x,&y| Some((y,x)))
                            .arrange_by_key();
                    }

                    // Reverse the direction before adding it as a production.
                    variable_map[name].add_production(&transposed.as_collection(|&dst,&src| (src,dst)));
                }
            }

            for (_name, variable) in variable_map.drain(..) {
                variable.complete();
            }
            result_map
        })
    }
}

fn main() {

    timely::execute_from_args(std::env::args(), move |worker| {

        let timer = ::std::time::Instant::now();

        let peers = worker.peers();
        let index = worker.index();

        let query_filename = std::env::args().nth(1).expect("Argument 1 (query filename) missing.");
        let query_text = std::fs::read_to_string(query_filename).expect("Failed to read query file");
        let query = Query::build_from(query_text.lines());

        let mut relation_map = worker.dataflow::<(),_,_>(|scope| query.render_in(scope));

        if index == 0 { println!("{:?}:\tDataflow assembled for {:?}", timer.elapsed(), query); }

        // Build a dataflow to report final sizes.
        worker.dataflow(|scope| {
            for (name, data) in relation_map.iter_mut() {
                let name = name.to_string();
                data.trace
                    .import(scope)
                    .as_collection(|&_kv,&()| ())
                    .consolidate()
                    .inspect(move |x| println!("{:?}\tfinal size of relation '{}': {:?}", timer.elapsed(), name, x.2));
            }
        });

        // snag a filename to use for the input graph.
        let data_filename = std::env::args().nth(2).expect("Argument 2 (data filename) missing.");
        let file = BufReader::new(File::open(data_filename).expect("Failed to read data file"));

        for readline in file.lines() {
            let line = readline.ok().expect("read error");
            if !line.starts_with('#') && line.len() > 0 {
                let mut elts = line[..].split_whitespace();
                let src = elts.next().expect("data line with no src (1st) element").parse().expect("malformed src");
                if (src as usize) % peers == index {
                    let dst = elts.next().expect("data line with no dst (2nd) element").parse().expect("malformed dst");
                    let val: &str = elts.next().expect("data line with no val (3rd) element");
                    if let Some(handle) = relation_map.get_mut(val) {
                        handle.input.insert((src, dst));
                    }
                    else {
                        panic!("couldn't find the named relation: {:?}", val);
                    }
                }
            }
        }

        if index == 0 { println!("{:?}:\tData loaded", timer.elapsed()); }

    }).expect("Timely computation did not complete cleanly");
}
<|endoftext|>


<commit_before>extern crate num;

// The `AdditiveIterator` trait adds the `sum` method to iterators
use std::iter::{AdditiveIterator,count};
// The `Integer` trait adds the `is_odd` method to integer primitives
use num::Integer;

fn main() {
    println!("Find the sum of all the squared odd numbers under 1000");
    let upper: int = 1000;

    // Imperative approach
    // Declare accumulator variable
    let mut acc = 0;
    // Iterate: 0, 1, 2, ... to infinity
    for n in count(0, 1) {
        // Square the number
        let n_squared = n * n;

        if n_squared >= upper {
            // Break loop if exceeded the upper limit
            break;
        } else if n_squared.is_odd() {
            // Accumulate value, if it's odd
            acc += n_squared;
        }
    }
    println!("imperative style: {}", acc);

    // Functional approach
    let sum_of_squared_odd_numbers =
        // All natural numbers
        count(0, 1).
        // Squared
        map(|n| n * n).
        // Below upper limit
        take_while(|&n| n < upper).
        // That are odd
        filter(|n| n.is_odd()).
        // Sum them
        sum();
    println!("functional style: {}", sum_of_squared_odd_numbers);
}
<commit_msg>hof: import `std::iter`<commit_after>extern crate num;

// The `AdditiveIterator` trait adds the `sum` method to iterators
use std::iter::AdditiveIterator;
use std::iter;
// The `Integer` trait adds the `is_odd` method to integer primitives
use num::Integer;

fn main() {
    println!("Find the sum of all the squared odd numbers under 1000");
    let upper: int = 1000;

    // Imperative approach
    // Declare accumulator variable
    let mut acc = 0;
    // Iterate: 0, 1, 2, ... to infinity
    for n in iter::count(0, 1) {
        // Square the number
        let n_squared = n * n;

        if n_squared >= upper {
            // Break loop if exceeded the upper limit
            break;
        } else if n_squared.is_odd() {
            // Accumulate value, if it's odd
            acc += n_squared;
        }
    }
    println!("imperative style: {}", acc);

    // Functional approach
    let sum_of_squared_odd_numbers =
        // All natural numbers
        iter::count(0, 1).
        // Squared
        map(|n| n * n).
        // Below upper limit
        take_while(|&n| n < upper).
        // That are odd
        filter(|n| n.is_odd()).
        // Sum them
        sum();
    println!("functional style: {}", sum_of_squared_odd_numbers);
}
<|endoftext|>


<commit_before>use plexus::integration::nalgebra;

use lazy_static::lazy_static;
use nalgebra::{Isometry3, Matrix4, Orthographic3, Perspective3, Point3, Vector3};
use wgpu::SwapChainDescriptor;

lazy_static! {
    #[rustfmt::skip]
    static ref OPENGL_TO_WGPU_TRANSFORM: Matrix4<f32> = Matrix4::new(
        1.0, 0.0, 0.0, 0.0,
        0.0, 1.0, 0.0, 0.0,
        0.0, 0.0, 0.5, 0.0,
        0.0, 0.0, 0.5, 1.0,
    );
}

pub enum Projection {
    Perspective(Perspective3<f32>),
    Orthographic(Orthographic3<f32>),
}

impl Projection {
    pub fn perspective(aspect: f32, fov: f32, near: f32, far: f32) -> Self {
        Projection::Perspective(Perspective3::new(aspect, fov, near, far))
    }

    pub fn orthographic(left: f32, right: f32, bottom: f32, top: f32, near: f32, far: f32) -> Self {
        Projection::Orthographic(Orthographic3::new(left, right, bottom, top, near, far))
    }
}

impl AsRef<Matrix4<f32>> for Projection {
    fn as_ref(&self) -> &Matrix4<f32> {
        match *self {
            Projection::Perspective(ref perspective) => perspective.as_matrix(),
            Projection::Orthographic(ref orthographic) => orthographic.as_matrix(),
        }
    }
}

pub struct Camera {
    pub projection: Projection,
    view: Isometry3<f32>,
}

impl Camera {
    pub fn look_at(&mut self, from: &Point3<f32>, to: &Point3<f32>) {
        self.view = Isometry3::look_at_rh(from, to, &Vector3::y());
    }

    pub fn reproject(&mut self, descriptor: &SwapChainDescriptor) {
        if let Projection::Perspective(ref mut perspective) = self.projection {
            perspective.set_aspect(descriptor.width as f32 / descriptor.height as f32);
        }
    }

    pub fn transform(&self) -> Matrix4<f32> {
        *OPENGL_TO_WGPU_TRANSFORM * self.projection.as_ref() * self.view.to_homogeneous()
    }
}

impl From<Projection> for Camera {
    fn from(projection: Projection) -> Self {
        Camera {
            projection,
            view: Isometry3::look_at_rh(
                &Point3::new(0.0, 0.0, 1.0),
                &Point3::origin(),
                &Vector3::y(),
            ),
        }
    }
}
<commit_msg>Adjust orthographic projections when the view is resized.<commit_after>use plexus::integration::nalgebra;

use lazy_static::lazy_static;
use nalgebra::{Isometry3, Matrix4, Orthographic3, Perspective3, Point3, Vector3};
use wgpu::SwapChainDescriptor;

lazy_static! {
    #[rustfmt::skip]
    static ref OPENGL_TO_WGPU_TRANSFORM: Matrix4<f32> = Matrix4::new(
        1.0, 0.0, 0.0, 0.0,
        0.0, 1.0, 0.0, 0.0,
        0.0, 0.0, 0.5, 0.0,
        0.0, 0.0, 0.5, 1.0,
    );
}

pub enum Projection {
    Perspective(Perspective3<f32>),
    Orthographic(Orthographic3<f32>),
}

impl Projection {
    pub fn perspective(aspect: f32, fov: f32, near: f32, far: f32) -> Self {
        Projection::Perspective(Perspective3::new(aspect, fov, near, far))
    }

    pub fn orthographic(left: f32, right: f32, bottom: f32, top: f32, near: f32, far: f32) -> Self {
        Projection::Orthographic(Orthographic3::new(left, right, bottom, top, near, far))
    }
}

impl AsRef<Matrix4<f32>> for Projection {
    fn as_ref(&self) -> &Matrix4<f32> {
        match *self {
            Projection::Perspective(ref perspective) => perspective.as_matrix(),
            Projection::Orthographic(ref orthographic) => orthographic.as_matrix(),
        }
    }
}

pub struct Camera {
    pub projection: Projection,
    view: Isometry3<f32>,
}

impl Camera {
    pub fn look_at(&mut self, from: &Point3<f32>, to: &Point3<f32>) {
        self.view = Isometry3::look_at_rh(from, to, &Vector3::y());
    }

    pub fn reproject(&mut self, descriptor: &SwapChainDescriptor) {
        match self.projection {
            Projection::Perspective(ref mut perspective) => {
                perspective.set_aspect(descriptor.width as f32 / descriptor.height as f32);
            }
            Projection::Orthographic(ref mut orthographic) => {
                let inverse = descriptor.height as f32 / descriptor.width as f32;
                let radius = (orthographic.right() - orthographic.left()) * inverse * 0.5;
                orthographic.set_bottom_and_top(-radius, radius);
            }
        }
    }

    pub fn transform(&self) -> Matrix4<f32> {
        *OPENGL_TO_WGPU_TRANSFORM * self.projection.as_ref() * self.view.to_homogeneous()
    }
}

impl From<Projection> for Camera {
    fn from(projection: Projection) -> Self {
        Camera {
            projection,
            view: Isometry3::look_at_rh(
                &Point3::new(0.0, 0.0, 1.0),
                &Point3::origin(),
                &Vector3::y(),
            ),
        }
    }
}
<|endoftext|>


<commit_before>fn function() {
    println!("called `function()`");
}

// A module named `my`
mod my {
    // A module can contain items like functions
    #[allow(dead_code)]
    fn function() {
        println!("called `my::function()`");
    }

    // Modules can be nested
    mod nested {
        #[allow(dead_code)]
        fn function() {
            println!("called `my::nested::function()`");
        }
    }
}

fn main() {
    function();

    // Items inside a module can be called using their full path
    // The `println` function lives in the `stdio` module
    // The `stdio` module lives in the `io` module
    // And the `io` module lives in the `std` crate
    std::io::stdio::println("Hello World!");

    // Error! `my::function` is private
    my::function();
    // TODO ^ Comment out this line
}
<commit_msg>Fix missing io to old_io for mod<commit_after>fn function() {
    println!("called `function()`");
}

// A module named `my`
mod my {
    // A module can contain items like functions
    #[allow(dead_code)]
    fn function() {
        println!("called `my::function()`");
    }

    // Modules can be nested
    mod nested {
        #[allow(dead_code)]
        fn function() {
            println!("called `my::nested::function()`");
        }
    }
}

fn main() {
    function();

    // Items inside a module can be called using their full path
    // The `println` function lives in the `stdio` module
    // The `stdio` module lives in the `io` module
    // And the `io` module lives in the `std` crate
    std::old_io::stdio::println("Hello World!");

    // Error! `my::function` is private
    my::function();
    // TODO ^ Comment out this line
}
<|endoftext|>


<commit_before>use std::{cmp, io::{self, Write, Read}, iter::repeat, ptr};

#[derive(Debug,PartialEq,Clone)]
pub struct Buffer {
  memory:   Vec<u8>,
  capacity: usize,
  position: usize,
  end:      usize
}

impl Buffer {
  pub fn with_capacity(capacity: usize) -> Buffer {
    let mut v = Vec::with_capacity(capacity);
    v.extend(repeat(0).take(capacity));
    Buffer {
      memory:   v,
      capacity: capacity,
      position: 0,
      end:      0
    }
  }

  pub fn from_slice(sl: &[u8]) -> Buffer {
    Buffer {
      memory:   Vec::from(sl),
      capacity: sl.len(),
      position: 0,
      end:      sl.len()
    }
  }

  pub fn grow(&mut self, new_size: usize) -> bool {
    if self.capacity >= new_size {
      return false;
    }

    self.memory.resize(new_size, 0);
    self.capacity = new_size;
    true
  }

  pub fn available_data(&self) -> usize {
    self.end - self.position
  }

  pub fn available_space(&self) -> usize {
    self.capacity - self.end
  }

  pub fn capacity(&self) -> usize {
    self.capacity
  }

  pub fn empty(&self) -> bool {
    self.position == self.end
  }

  pub fn consume(&mut self, count: usize) -> usize {
    let cnt        = cmp::min(count, self.available_data());
    self.position += cnt;
    if self.position > self.capacity / 2 {
      //trace!("consume shift: pos {}, end {}", self.position, self.end);
      self.shift();
    }
    cnt
  }

  pub fn fill(&mut self, count: usize) -> usize {
    let cnt   = cmp::min(count, self.available_space());
    self.end += cnt;
    if self.available_space() < self.available_data() + cnt {
      //trace!("fill shift: pos {}, end {}", self.position, self.end);
      self.shift();
    }

    cnt
  }

  pub fn reset(&mut self) {
    self.position = 0;
    self.end      = 0;
  }

  pub fn data(&self) -> &[u8] {
    &self.memory[self.position..self.end]
  }

  pub fn space(&mut self) -> &mut[u8] {
    &mut self.memory[self.end..self.capacity]
  }

  pub fn shift(&mut self) {
    if self.position > 0 {
      unsafe {
        let length = self.end - self.position;
        ptr::copy( (&self.memory[self.position..self.end]).as_ptr(), (&mut self.memory[..length]).as_mut_ptr(), length);
        self.position = 0;
        self.end      = length;
      }
    }
  }

  pub fn delete_slice(&mut self, start: usize, length: usize) -> Option<usize> {
    if start + length >= self.available_data() {
      return None
    }

    unsafe {
      let begin    = self.position + start;
      let next_end = self.end - length;
      ptr::copy(
        (&self.memory[begin+length..self.end]).as_ptr(),
        (&mut self.memory[begin..next_end]).as_mut_ptr(),
        self.end - (begin+length)
      );
      self.end = next_end;
    }
    Some(self.available_data())
  }

  pub fn replace_slice(&mut self, data: &[u8], start: usize, length: usize) -> Option<usize> {
    let data_len = data.len();
    if start + length > self.available_data() ||
      self.position + start + data_len > self.capacity {
      return None
    }

    unsafe {
      let begin     = self.position + start;
      let slice_end = begin + data_len;
      // we reduced the data size
      if data_len < length {
        ptr::copy(data.as_ptr(), (&mut self.memory[begin..slice_end]).as_mut_ptr(), data_len);

        ptr::copy((&self.memory[start+length..self.end]).as_ptr(), (&mut self.memory[slice_end..]).as_mut_ptr(), self.end - (start + length));
        self.end = self.end - (length - data_len);

      // we put more data in the buffer
      } else {
        ptr::copy((&self.memory[start+length..self.end]).as_ptr(), (&mut self.memory[start+data_len..]).as_mut_ptr(), self.end - (start + length));
        ptr::copy(data.as_ptr(), (&mut self.memory[begin..slice_end]).as_mut_ptr(), data_len);
        self.end = self.end + data_len - length;
      }
    }
    Some(self.available_data())
  }

  pub fn insert_slice(&mut self, data: &[u8], start: usize) -> Option<usize> {
    let data_len = data.len();
    if start > self.available_data() ||
      self.position + self.end + data_len > self.capacity {
      return None
    }

    unsafe {
      let begin     = self.position + start;
      let slice_end = begin + data_len;
      ptr::copy((&self.memory[start..self.end]).as_ptr(), (&mut self.memory[start+data_len..]).as_mut_ptr(), self.end - start);
      ptr::copy(data.as_ptr(), (&mut self.memory[begin..slice_end]).as_mut_ptr(), data_len);
      self.end = self.end + data_len;
    }
    Some(self.available_data())
  }
}

impl Write for Buffer {
  fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
    match self.space().write(buf) {
      Ok(size) => { self.fill(size); Ok(size) },
      err      => err
    }
  }

  fn flush(&mut self) -> io::Result<()> {
    Ok(())
  }
}

impl Read for Buffer {
  fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
    let len = cmp::min(self.available_data(), buf.len());
    unsafe {
      ptr::copy((&self.memory[self.position..len]).as_ptr(), buf.as_mut_ptr(), len);
      self.position += len;
    }
    Ok(len)
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use std::io::Write;

  #[test]
  fn fill_and_consume() {
    let mut b = Buffer::with_capacity(10);
    assert_eq!(b.available_data(), 0);
    assert_eq!(b.available_space(), 10);
    let res = b.write(&b"abcd"[..]);
    assert_eq!(res.ok(), Some(4));
    assert_eq!(b.available_data(), 4);
    assert_eq!(b.available_space(), 6);

    assert_eq!(b.data(), &b"abcd"[..]);

    b.consume(2);
    assert_eq!(b.available_data(), 2);
    assert_eq!(b.available_space(), 6);
    assert_eq!(b.data(), &b"cd"[..]);

    b.shift();
    assert_eq!(b.available_data(), 2);
    assert_eq!(b.available_space(), 8);
    assert_eq!(b.data(), &b"cd"[..]);

    assert_eq!(b.write(&b"efghijklmnop"[..]).ok(), Some(8));
    assert_eq!(b.available_data(), 10);
    assert_eq!(b.available_space(), 0);
    assert_eq!(b.data(), &b"cdefghijkl"[..]);
    b.shift();
    assert_eq!(b.available_data(), 10);
    assert_eq!(b.available_space(), 0);
    assert_eq!(b.data(), &b"cdefghijkl"[..]);
  }

  #[test]
  fn delete() {
    let mut b = Buffer::with_capacity(10);
    b.write(&b"abcdefgh"[..]).expect("Failed to write to buffer");
    assert_eq!(b.available_data(), 8);
    assert_eq!(b.available_space(), 2);

    assert_eq!(b.delete_slice(2, 3), Some(5));
    assert_eq!(b.available_data(), 5);
    assert_eq!(b.available_space(), 5);
    assert_eq!(b.data(), &b"abfgh"[..]);

    assert_eq!(b.delete_slice(5, 2), None);
    assert_eq!(b.delete_slice(4, 2), None);
  }

  #[test]
  fn replace() {
    let mut b = Buffer::with_capacity(10);
    b.write(&b"abcdefgh"[..]).expect("Failed to write to buffer");
    assert_eq!(b.available_data(), 8);
    assert_eq!(b.available_space(), 2);

    assert_eq!(b.replace_slice(&b"ABC"[..], 2, 3), Some(8));
    assert_eq!(b.available_data(), 8);
    assert_eq!(b.available_space(), 2);
    assert_eq!(b.data(), &b"abABCfgh"[..]);

    assert_eq!(b.replace_slice(&b"XYZ"[..], 8, 3), None);
    assert_eq!(b.replace_slice(&b"XYZ"[..], 6, 3), None);

    assert_eq!(b.replace_slice(&b"XYZ"[..], 2, 4), Some(7));
    assert_eq!(b.available_data(), 7);
    assert_eq!(b.available_space(), 3);
    assert_eq!(b.data(), &b"abXYZgh"[..]);

    assert_eq!(b.replace_slice(&b"123"[..], 2, 2), Some(8));
    assert_eq!(b.available_data(), 8);
    assert_eq!(b.available_space(), 2);
    assert_eq!(b.data(), &b"ab123Zgh"[..]);
  }
}
<commit_msg>buffer: drop dead code<commit_after>use std::{cmp, io::{self, Write, Read}, iter::repeat, ptr};

#[derive(Debug,PartialEq,Clone)]
pub struct Buffer {
  memory:   Vec<u8>,
  capacity: usize,
  position: usize,
  end:      usize
}

impl Buffer {
  pub fn with_capacity(capacity: usize) -> Buffer {
    let mut v = Vec::with_capacity(capacity);
    v.extend(repeat(0).take(capacity));
    Buffer {
      memory:   v,
      capacity: capacity,
      position: 0,
      end:      0
    }
  }

  pub fn grow(&mut self, new_size: usize) -> bool {
    if self.capacity >= new_size {
      return false;
    }

    self.memory.resize(new_size, 0);
    self.capacity = new_size;
    true
  }

  pub fn available_data(&self) -> usize {
    self.end - self.position
  }

  pub fn available_space(&self) -> usize {
    self.capacity - self.end
  }

  pub fn consume(&mut self, count: usize) -> usize {
    let cnt        = cmp::min(count, self.available_data());
    self.position += cnt;
    if self.position > self.capacity / 2 {
      //trace!("consume shift: pos {}, end {}", self.position, self.end);
      self.shift();
    }
    cnt
  }

  pub fn fill(&mut self, count: usize) -> usize {
    let cnt   = cmp::min(count, self.available_space());
    self.end += cnt;
    if self.available_space() < self.available_data() + cnt {
      //trace!("fill shift: pos {}, end {}", self.position, self.end);
      self.shift();
    }

    cnt
  }

  pub fn data(&self) -> &[u8] {
    &self.memory[self.position..self.end]
  }

  pub fn space(&mut self) -> &mut[u8] {
    &mut self.memory[self.end..self.capacity]
  }

  fn shift(&mut self) {
    if self.position > 0 {
      unsafe {
        let length = self.end - self.position;
        ptr::copy( (&self.memory[self.position..self.end]).as_ptr(), (&mut self.memory[..length]).as_mut_ptr(), length);
        self.position = 0;
        self.end      = length;
      }
    }
  }
}

impl Write for Buffer {
  fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
    match self.space().write(buf) {
      Ok(size) => { self.fill(size); Ok(size) },
      err      => err
    }
  }

  fn flush(&mut self) -> io::Result<()> {
    Ok(())
  }
}

impl Read for Buffer {
  fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
    let len = cmp::min(self.available_data(), buf.len());
    unsafe {
      ptr::copy((&self.memory[self.position..len]).as_ptr(), buf.as_mut_ptr(), len);
      self.position += len;
    }
    Ok(len)
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use std::io::Write;

  #[test]
  fn fill_and_consume() {
    let mut b = Buffer::with_capacity(10);
    assert_eq!(b.available_data(), 0);
    assert_eq!(b.available_space(), 10);
    let res = b.write(&b"abcd"[..]);
    assert_eq!(res.ok(), Some(4));
    assert_eq!(b.available_data(), 4);
    assert_eq!(b.available_space(), 6);

    assert_eq!(b.data(), &b"abcd"[..]);

    b.consume(2);
    assert_eq!(b.available_data(), 2);
    assert_eq!(b.available_space(), 6);
    assert_eq!(b.data(), &b"cd"[..]);

    b.shift();
    assert_eq!(b.available_data(), 2);
    assert_eq!(b.available_space(), 8);
    assert_eq!(b.data(), &b"cd"[..]);

    assert_eq!(b.write(&b"efghijklmnop"[..]).ok(), Some(8));
    assert_eq!(b.available_data(), 10);
    assert_eq!(b.available_space(), 0);
    assert_eq!(b.data(), &b"cdefghijkl"[..]);
    b.shift();
    assert_eq!(b.available_data(), 10);
    assert_eq!(b.available_space(), 0);
    assert_eq!(b.data(), &b"cdefghijkl"[..]);
  }
}
<|endoftext|>


<commit_before>extern crate websocket;
extern crate futures;
extern crate tokio_core;
extern crate serde;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;

use std::fmt::Debug;

use websocket::message::OwnedMessage;
use websocket::server::InvalidConnection;
use websocket::client::async::Framed;
use websocket::async::{Server, MessageCodec};

use tokio_core::net::TcpStream;
use tokio_core::reactor::{Handle, Core};

use futures::{Future, Sink, Stream};
use futures::future::{self, Loop};

fn main() {
    let mut core = Core::new().unwrap();
    let handle = core.handle();
    let server = Server::bind("localhost:8081", &handle).unwrap();

    let f = server.incoming()
        // we don't wanna save the stream if it drops
        .map_err(|InvalidConnection { error, .. }| error)
        .for_each(|(upgrade, addr)| {
            println!("Got a connection from: {}", addr);
            if !upgrade.protocols().iter().any(|s| s == "rust-websocket") {
                spawn_future(upgrade.reject(), "Upgrade Rejection", &handle);
                return Ok(());
            }

            let client = upgrade
                .use_protocol("rust-websocket")
                .accept();
            let f = client
                .and_then(|(stream, _)| {
                 future::loop_fn(stream, |stream| {
                        stream
                            .into_future()
                            .or_else(|(err, stream)| {
                                println!("Could not send message: {:?}", err);
                                stream.send(OwnedMessage::Close(None)).map(|s| (None, s))
                            })
                            .and_then(|(msg, stream)|{
                                let mut state = State::new();
                                handle_incoming(&mut state, &msg);
                                Ok((msg, stream))
                            }).map(|(msg, stream)| {
                                match msg {
                                    Some(_) => Loop::Continue(stream),
                                    None => Loop::Break(()),
                                }
                            })
                            .boxed()
                })
            });
            spawn_future(f, "Client Status", &handle);
            Ok(())
        });

    core.run(f).unwrap();
}

fn spawn_future<F, I, E>(f: F, desc: &'static str, handle: &Handle)
    where F: Future<Item = I, Error = E> + 'static,
          E: Debug
{
    handle.spawn(f.map_err(move |e| println!("{}: '{:?}'", desc, e))
                     .map(move |_| println!("{}: Finished.", desc)));
}

fn handle_incoming(state: &mut State, msg: &Option<OwnedMessage>) {
    match msg {
        &Some(OwnedMessage::Text(ref txt)) => {
            println!("Received message: {}", txt);
            state.msg = txt.clone();
        }
        _ => {}
    }
}

type FramedStream = Framed<TcpStream, MessageCodec<OwnedMessage>>;

fn send(state: &State, stream: FramedStream) -> futures::sink::Send<FramedStream> {
    let msg = serde_json::to_string(&state).unwrap();
    println!("Sending message: {}", msg);
    stream.send(OwnedMessage::Text(msg))
}

#[derive(Serialize)]
struct State {
    msg: String,
}

impl State {
    fn new() -> State {
        State { msg: "git gud".to_string() }
    }
}
<commit_msg>Cargo: Fix indentation<commit_after>extern crate websocket;
extern crate futures;
extern crate tokio_core;
extern crate serde;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;

use std::fmt::Debug;

use websocket::message::OwnedMessage;
use websocket::server::InvalidConnection;
use websocket::client::async::Framed;
use websocket::async::{Server, MessageCodec};

use tokio_core::net::TcpStream;
use tokio_core::reactor::{Handle, Core};

use futures::{Future, Sink, Stream};
use futures::future::{self, Loop};

fn main() {
    let mut core = Core::new().unwrap();
    let handle = core.handle();
    let server = Server::bind("localhost:8081", &handle).unwrap();

    let f = server.incoming()
        // we don't wanna save the stream if it drops
        .map_err(|InvalidConnection { error, .. }| error)
        .for_each(|(upgrade, addr)| {
            println!("Got a connection from: {}", addr);
            if !upgrade.protocols().iter().any(|s| s == "rust-websocket") {
                spawn_future(upgrade.reject(), "Upgrade Rejection", &handle);
                return Ok(());
            }

            let client = upgrade
                .use_protocol("rust-websocket")
                .accept();
            let f = client
                .and_then(|(stream, _)| {
                    future::loop_fn(stream, |stream| {
                        stream
                            .into_future()
                            .or_else(|(err, stream)| {
                                println!("Could not send message: {:?}", err);
                                stream.send(OwnedMessage::Close(None)).map(|s| (None, s))
                            })
                            .and_then(|(msg, stream)|{
                                let mut state = State::new();
                                handle_incoming(&mut state, &msg);
                                Ok((msg, stream))
                            }).map(|(msg, stream)| {
                                match msg {
                                    Some(_) => Loop::Continue(stream),
                                    None => Loop::Break(()),
                                }
                            })
                            .boxed()
                    })
                });
            spawn_future(f, "Client Status", &handle);
            Ok(())
        });

    core.run(f).unwrap();
}

fn spawn_future<F, I, E>(f: F, desc: &'static str, handle: &Handle)
    where F: Future<Item = I, Error = E> + 'static,
          E: Debug
{
    handle.spawn(f.map_err(move |e| println!("{}: '{:?}'", desc, e))
                     .map(move |_| println!("{}: Finished.", desc)));
}

fn handle_incoming(state: &mut State, msg: &Option<OwnedMessage>) {
    match msg {
        &Some(OwnedMessage::Text(ref txt)) => {
            println!("Received message: {}", txt);
            state.msg = txt.clone();
        }
        _ => {}
    }
}

type FramedStream = Framed<TcpStream, MessageCodec<OwnedMessage>>;

fn send(state: &State, stream: FramedStream) -> futures::sink::Send<FramedStream> {
    let msg = serde_json::to_string(&state).unwrap();
    println!("Sending message: {}", msg);
    stream.send(OwnedMessage::Text(msg))
}

#[derive(Serialize)]
struct State {
    msg: String,
}

impl State {
    fn new() -> State {
        State { msg: "git gud".to_string() }
    }
}
<|endoftext|>