<commit_before>use std::collections::{BTreeSet};
use automata::{M, StateSet, State};

pub type RelationMatrixRow = Vec<bool>;
pub type RelationMatrix = Vec<RelationMatrixRow>;

pub fn get_relation_matrix(m: &M) -> RelationMatrix {
    let mut matrix: RelationMatrix = Vec::new();

    for qi in m.k.iter() {
        let mut row: RelationMatrixRow = Vec::new();
        for qj in m.k.iter() {
            let mut i_relation_j = false;

            for a in m.alphabet.iter() {
                if m.get_next_states(qi, &a).contains(qj) {
                    i_relation_j = true
                }
            }

            row.push(i_relation_j);
        }
        matrix.push(row);
    }

    matrix
}

pub fn warshall(matrix: &RelationMatrix) -> RelationMatrix {
    let n = matrix.len();
    let mut r = matrix.clone();

    for k in 0..n {
        for i in 0..n {
            for j in 0..n {
                r[i][j] = r[i][j] || ( r[i][k] && r[k][j] )
            }
        }
    }

    r
}

pub fn get_reachable_states(m: &M, r: &RelationMatrix) -> StateSet {
    let states: Vec<State> = m.k.iter().cloned().collect();
    let q0_index = m.k.iter().position(|s| *s == m.q0).unwrap();
    let ref reachable_state_row = r[q0_index];
    let reachable_states = {
        let mut res: StateSet = BTreeSet::new();
        for (i, &reachable) in reachable_state_row.iter().enumerate() {
            if reachable {
                res.insert(states[i].clone());
            }
        }
        res
    };

    reachable_states
}

pub fn remove_unreachable_states(mut m: M, reachable_states: StateSet) -> M {
    for u in m.k.difference(&reachable_states) {
        let _ = m.delta.remove(u);
    }

    m.k = reachable_states;

    m
}

pub type EquivalenceClass = StateSet;
pub type Quotient = BTreeSet<EquivalenceClass>;

pub fn get_equivalence_class(states: &StateSet, quotient: &Quotient) -> EquivalenceClass {
    for eq_class in quotient.iter() {
        if states.is_subset(&eq_class) {
            return eq_class.clone();
        }
    }

    stateset!()
}

pub fn get_quotient(m: &M) -> Quotient {
    let k_f: EquivalenceClass = m.k.difference(&m.f).cloned().collect();
    let f: EquivalenceClass = m.f.clone();

    let mut quotient: Quotient = {
        let mut q = BTreeSet::new();
        q.insert(k_f);
        q.insert(f);
        q
    };

    let mut fin = false;

    while !fin {
        let mut next_quotient: Quotient = BTreeSet::new();
        for x in &quotient {
            let mut x_marked: EquivalenceClass = BTreeSet::new();

            while x != &x_marked {
                for state in x {
                    if x_marked.contains(state) { continue }
                    println!("STATE {:?}", state);
                    let mut x1: EquivalenceClass = stateset!(state);

                    x_marked.insert(state.clone());

                    for other_state in x {
                        if x_marked.contains(other_state) { continue }
                        println!("OTHER_STATE {:?}", other_state);
                        let mut flag = true;
                        for a in &m.alphabet {
                            let state_eq_class = get_equivalence_class(&m.get_next_states(&state, &a), &quotient);
                            let other_state_eq_class = get_equivalence_class(&m.get_next_states(&other_state, &a), &quotient);
                            if  state_eq_class == other_state_eq_class  {
                                flag = flag && true
                            } else {
                                flag = flag && false
                            }
                        }
                        if flag {
                            println!("SAME CLASS {:?}, {:?}", state, other_state);
                            x1.insert(other_state.clone());
                            x_marked.insert(other_state.clone());
                        }
                    }

                    next_quotient.insert(x1);
                }
            }
        }

        println!("     QUOTIENT {:?}", quotient);
        println!("Next QUOTIENT {:?}", next_quotient);
        if next_quotient != quotient {
            quotient = next_quotient;
            fin = false;
        } else {
            fin = true;
        }
    }

    quotient
}

#[cfg(test)]
mod tests {
    #[test]
    fn get_relation_matrix_test() {
        use super::get_relation_matrix;
        use automata::M;

        let k = stateset!("q0", "q1", "q2", "q3");
        let alphabet = alphabet!('a', 'b');
        let q0 = "q0".to_string();
        let f = stateset!("q1");
        let delta = delta!(
            ("q0", 'a', "q0"),
            ("q0", 'b', "q1"),
            ("q1", 'a', "q1"),
            ("q1", 'b', "q2"),
            ("q2", 'a', "q0"),
            ("q2", 'b', "q2"),
            ("q3", 'a', "q3")
        );

        let m = M::new(k, alphabet, q0, f, delta);

        let rm = get_relation_matrix(&m);
        //for i in rm {
            //println!("{:?}", i);
        //}

        let rm_expected = vec![
            vec![true, true, false, false],
            vec![false, true, true, false],
            vec![true, false, true, false],
            vec![false, false, false, true]
        ];

        assert!(rm == rm_expected);
    }

    #[test]
    fn warshall_test() {
        use super::warshall;

        let rm = vec![
            vec![true, true, false, false],
            vec![false, true, true, false],
            vec![true, false, true, false],
            vec![false, false, false, true]
        ];

        let r = warshall(&rm);
        let r_expected = vec![
            vec![true, true, true, false],
            vec![true, true, true, false],
            vec![true, true, true, false],
            vec![false, false, false, true]
        ];

        //for i in &r {
            //println!("{:?}", i);
        //}

        assert!(r == r_expected);
    }

    #[test]
    fn get_reachable_states_test() {
        use super::get_reachable_states;
        use automata::M;

        let k = stateset!("q0", "q1", "q2", "q3");
        let alphabet = alphabet!('a', 'b');
        let q0 = "q0".to_string();
        let f = stateset!("q1");
        let delta = delta!(
            ("q0", 'a', "q0"),
            ("q0", 'b', "q1"),
            ("q1", 'a', "q1"),
            ("q1", 'b', "q2"),
            ("q2", 'a', "q0"),
            ("q2", 'b', "q2"),
            ("q3", 'a', "q3")
        );

        let m = M::new(k, alphabet, q0, f, delta);
        let r = vec![
            vec![true, true, true, false],
            vec![true, true, true, false],
            vec![true, true, true, false],
            vec![false, false, false, true]
        ];

        let states = get_reachable_states(&m, &r);
        let states_expected = stateset!("q0", "q1", "q2");

        //for i in &states {
            //println!("{:?}", i);
        //}

        assert!(states == states_expected);
    }

    #[test]
    fn remove_unreachable_states_test() {
        use super::remove_unreachable_states;
        use automata::{M, to_delta_inner};

        let k = stateset!("q0", "q1", "q2", "q3");
        let alphabet = alphabet!('a', 'b');
        let q0 = "q0".to_string();
        let f = stateset!("q1");
        let delta = delta!(
            ("q0", 'a', "q0"),
            ("q0", 'b', "q1"),
            ("q1", 'a', "q1"),
            ("q1", 'b', "q2"),
            ("q2", 'a', "q0"),
            ("q2", 'b', "q2"),
            ("q3", 'a', "q3")
        );

        let m = M::new(k, alphabet, q0, f, delta);
        let reachable_states = stateset!("q0", "q1", "q2");

        let m_new = remove_unreachable_states(m, reachable_states.clone());

        let delta_expected = delta!(
            ("q0", 'a', "q0"),
            ("q0", 'b', "q1"),
            ("q1", 'a', "q1"),
            ("q1", 'b', "q2"),
            ("q2", 'a', "q0"),
            ("q2", 'b', "q2")
        );

        let delta_expected = to_delta_inner(delta_expected);

        assert!(m_new.k == reachable_states);
        assert!(m_new.delta == delta_expected);
    }

    #[test]
    fn get_quotient_test() {
        use std::collections::{BTreeSet};
        use super::{Quotient, get_quotient};
        use automata::{M};

        let k = stateset!("q0", "q1", "q2", "q3");
        let alphabet = alphabet!('a', 'b');
        let q0 = "q0".to_string();
        let f = stateset!("q3");
        let delta = delta!(
            ("q0", 'a', "q1"),
            ("q0", 'b', "q2"),
            ("q1", 'a', "q3"),
            ("q2", 'a', "q3")
        );

        let m = M::new(k, alphabet, q0, f, delta);

        let quotient: Quotient = get_quotient(&m);
        let quotient_expected: Quotient = {
            let mut q = BTreeSet::new();
            q.insert(stateset!("q0"));
            q.insert(stateset!("q1", "q2"));
            q.insert(stateset!("q3"));
            q
        };

        assert_eq!(quotient, quotient_expected);
    }
}
<commit_msg>fn apply_quotient is working<commit_after>use std::collections::{BTreeSet};
use automata::{M, StateSet, State};
use automata_operators::stateset_name;

pub type RelationMatrixRow = Vec<bool>;
pub type RelationMatrix = Vec<RelationMatrixRow>;

pub fn get_relation_matrix(m: &M) -> RelationMatrix {
    let mut matrix: RelationMatrix = Vec::new();

    for qi in m.k.iter() {
        let mut row: RelationMatrixRow = Vec::new();
        for qj in m.k.iter() {
            let mut i_relation_j = false;

            for a in m.alphabet.iter() {
                if m.get_next_states(qi, &a).contains(qj) {
                    i_relation_j = true
                }
            }

            row.push(i_relation_j);
        }
        matrix.push(row);
    }

    matrix
}

pub fn warshall(matrix: &RelationMatrix) -> RelationMatrix {
    let n = matrix.len();
    let mut r = matrix.clone();

    for k in 0..n {
        for i in 0..n {
            for j in 0..n {
                r[i][j] = r[i][j] || ( r[i][k] && r[k][j] )
            }
        }
    }

    r
}

pub fn get_reachable_states(m: &M, r: &RelationMatrix) -> StateSet {
    let states: Vec<State> = m.k.iter().cloned().collect();
    let q0_index = m.k.iter().position(|s| *s == m.q0).unwrap();
    let ref reachable_state_row = r[q0_index];
    let reachable_states = {
        let mut res: StateSet = BTreeSet::new();
        for (i, &reachable) in reachable_state_row.iter().enumerate() {
            if reachable {
                res.insert(states[i].clone());
            }
        }
        res
    };

    reachable_states
}

pub fn remove_unreachable_states(mut m: M, reachable_states: StateSet) -> M {
    for u in m.k.difference(&reachable_states) {
        let _ = m.delta.remove(u);
    }

    m.k = reachable_states;

    m
}

pub type EquivalenceClass = StateSet;
pub type Quotient = BTreeSet<EquivalenceClass>;

pub fn get_equivalence_class(states: &StateSet, quotient: &Quotient) -> EquivalenceClass {
    for eq_class in quotient.iter() {
        if states.is_subset(&eq_class) {
            return eq_class.clone();
        }
    }

    stateset!()
}

pub fn get_quotient(m: &M) -> Quotient {
    let k_f: EquivalenceClass = m.k.difference(&m.f).cloned().collect();
    let f: EquivalenceClass = m.f.clone();

    let mut quotient: Quotient = {
        let mut q = BTreeSet::new();
        q.insert(k_f);
        q.insert(f);
        q
    };

    let mut fin = false;

    while !fin {
        let mut next_quotient: Quotient = BTreeSet::new();
        for x in &quotient {
            let mut x_marked: EquivalenceClass = BTreeSet::new();

            while x != &x_marked {
                for state in x {
                    if x_marked.contains(state) { continue }
                    println!("STATE {:?}", state);
                    let mut x1: EquivalenceClass = stateset!(state);

                    x_marked.insert(state.clone());

                    for other_state in x {
                        if x_marked.contains(other_state) { continue }
                        println!("OTHER_STATE {:?}", other_state);
                        let mut flag = true;
                        for a in &m.alphabet {
                            let state_eq_class = get_equivalence_class(&m.get_next_states(&state, &a), &quotient);
                            let other_state_eq_class = get_equivalence_class(&m.get_next_states(&other_state, &a), &quotient);
                            if  state_eq_class == other_state_eq_class  {
                                flag = flag && true
                            } else {
                                flag = flag && false
                            }
                        }
                        if flag {
                            println!("SAME CLASS {:?}, {:?}", state, other_state);
                            x1.insert(other_state.clone());
                            x_marked.insert(other_state.clone());
                        }
                    }

                    next_quotient.insert(x1);
                }
            }
        }

        println!("     QUOTIENT {:?}", quotient);
        println!("Next QUOTIENT {:?}", next_quotient);
        if next_quotient != quotient {
            quotient = next_quotient;
            fin = false;
        } else {
            fin = true;
        }
    }

    quotient
}

pub fn apply_quotient(m: &M, quotient: &Quotient) -> M {
    let states: StateSet = quotient.iter().map(|eq_class| stateset_name(eq_class)).collect();

    let q0 = stateset_name(&get_equivalence_class(&stateset!(m.q0), quotient));
    let mut f = stateset!();
    for state in &m.f {
        let final_state = stateset_name(&get_equivalence_class(&stateset!(state), quotient));
        f.insert(final_state);
    }

    let mut delta = delta!();
    for (state, delta_value) in &m.delta {
        for (a, next_states) in delta_value {
            for next_state in next_states {
                let s = stateset_name(&get_equivalence_class(&stateset!(state), quotient));
                let ns = stateset_name(&get_equivalence_class(&stateset!(next_state), quotient));
                delta.insert( (s, a.clone(), ns) );
            }
        }
    }

    M::new(
        states,
        m.alphabet.clone(),
        q0,
        f,
        delta
    )
}

#[cfg(test)]
mod tests {
    #[test]
    fn get_relation_matrix_test() {
        use super::get_relation_matrix;
        use automata::M;

        let k = stateset!("q0", "q1", "q2", "q3");
        let alphabet = alphabet!('a', 'b');
        let q0 = "q0".to_string();
        let f = stateset!("q1");
        let delta = delta!(
            ("q0", 'a', "q0"),
            ("q0", 'b', "q1"),
            ("q1", 'a', "q1"),
            ("q1", 'b', "q2"),
            ("q2", 'a', "q0"),
            ("q2", 'b', "q2"),
            ("q3", 'a', "q3")
        );

        let m = M::new(k, alphabet, q0, f, delta);

        let rm = get_relation_matrix(&m);
        //for i in rm {
            //println!("{:?}", i);
        //}

        let rm_expected = vec![
            vec![true, true, false, false],
            vec![false, true, true, false],
            vec![true, false, true, false],
            vec![false, false, false, true]
        ];

        assert!(rm == rm_expected);
    }

    #[test]
    fn warshall_test() {
        use super::warshall;

        let rm = vec![
            vec![true, true, false, false],
            vec![false, true, true, false],
            vec![true, false, true, false],
            vec![false, false, false, true]
        ];

        let r = warshall(&rm);
        let r_expected = vec![
            vec![true, true, true, false],
            vec![true, true, true, false],
            vec![true, true, true, false],
            vec![false, false, false, true]
        ];

        //for i in &r {
            //println!("{:?}", i);
        //}

        assert!(r == r_expected);
    }

    #[test]
    fn get_reachable_states_test() {
        use super::get_reachable_states;
        use automata::M;

        let k = stateset!("q0", "q1", "q2", "q3");
        let alphabet = alphabet!('a', 'b');
        let q0 = "q0".to_string();
        let f = stateset!("q1");
        let delta = delta!(
            ("q0", 'a', "q0"),
            ("q0", 'b', "q1"),
            ("q1", 'a', "q1"),
            ("q1", 'b', "q2"),
            ("q2", 'a', "q0"),
            ("q2", 'b', "q2"),
            ("q3", 'a', "q3")
        );

        let m = M::new(k, alphabet, q0, f, delta);
        let r = vec![
            vec![true, true, true, false],
            vec![true, true, true, false],
            vec![true, true, true, false],
            vec![false, false, false, true]
        ];

        let states = get_reachable_states(&m, &r);
        let states_expected = stateset!("q0", "q1", "q2");

        //for i in &states {
            //println!("{:?}", i);
        //}

        assert!(states == states_expected);
    }

    #[test]
    fn remove_unreachable_states_test() {
        use super::remove_unreachable_states;
        use automata::{M, to_delta_inner};

        let k = stateset!("q0", "q1", "q2", "q3");
        let alphabet = alphabet!('a', 'b');
        let q0 = "q0".to_string();
        let f = stateset!("q1");
        let delta = delta!(
            ("q0", 'a', "q0"),
            ("q0", 'b', "q1"),
            ("q1", 'a', "q1"),
            ("q1", 'b', "q2"),
            ("q2", 'a', "q0"),
            ("q2", 'b', "q2"),
            ("q3", 'a', "q3")
        );

        let m = M::new(k, alphabet, q0, f, delta);
        let reachable_states = stateset!("q0", "q1", "q2");

        let m_new = remove_unreachable_states(m, reachable_states.clone());

        let delta_expected = delta!(
            ("q0", 'a', "q0"),
            ("q0", 'b', "q1"),
            ("q1", 'a', "q1"),
            ("q1", 'b', "q2"),
            ("q2", 'a', "q0"),
            ("q2", 'b', "q2")
        );

        let delta_expected = to_delta_inner(delta_expected);

        assert!(m_new.k == reachable_states);
        assert!(m_new.delta == delta_expected);
    }

    #[test]
    fn get_quotient_test() {
        use std::collections::{BTreeSet};
        use super::{Quotient, get_quotient};
        use automata::{M};

        let k = stateset!("q0", "q1", "q2", "q3");
        let alphabet = alphabet!('a', 'b');
        let q0 = "q0".to_string();
        let f = stateset!("q3");
        let delta = delta!(
            ("q0", 'a', "q1"),
            ("q0", 'b', "q2"),
            ("q1", 'a', "q3"),
            ("q2", 'a', "q3")
        );

        let m = M::new(k, alphabet, q0, f, delta);

        let quotient: Quotient = get_quotient(&m);
        let quotient_expected: Quotient = {
            let mut q = BTreeSet::new();
            q.insert(stateset!("q0"));
            q.insert(stateset!("q1", "q2"));
            q.insert(stateset!("q3"));
            q
        };

        assert_eq!(quotient, quotient_expected);
    }

    #[test]
    fn apply_quotient_test() {
        use std::collections::{BTreeSet};
        use super::{Quotient, apply_quotient};
        use automata::{M, to_delta_inner};

        let k = stateset!("q0", "q1", "q2", "q3");
        let alphabet = alphabet!('a', 'b');
        let q0 = "q0".to_string();
        let f = stateset!("q3");
        let delta = delta!(
            ("q0", 'a', "q1"),
            ("q0", 'b', "q2"),
            ("q1", 'a', "q3"),
            ("q2", 'a', "q3")
        );

        let m = M::new(k, alphabet, q0, f, delta);

        let quotient: Quotient = {
            let mut q = BTreeSet::new();
            q.insert(stateset!("q0"));
            q.insert(stateset!("q1", "q2"));
            q.insert(stateset!("q3"));
            q
        };

        let min_m = apply_quotient(&m, &quotient);

        let delta_expected = delta!(
            ("q0", 'a', "q1q2"),
            ("q0", 'b', "q1q2"),
            ("q1q2", 'a', "q3")
        );

        assert_eq!(min_m.alphabet, m.alphabet);
        assert_eq!(min_m.k, stateset!("q0", "q1q2", "q3"));
        assert_eq!(min_m.q0, "q0");
        assert_eq!(min_m.f, stateset!("q3"));
        assert_eq!(min_m.delta, to_delta_inner(delta_expected));
    }
}
<|endoftext|>


<commit_before>// Copyright 2015-2017 James Bostock. See the LICENSE file at the top-level
// directory of this distribution.

use std::fs;
use std::io::{self, Read, Seek, SeekFrom};
use std::fs::File;

// Utility routines

// Taken from stackoverflow:
// http://stackoverflow.com/questions/27588416/how-to-send-output-to-stderr
macro_rules! errln(
    ($($arg:tt)*) => (
        use std::io::Write;
        match writeln!(&mut ::std::io::stderr(), $($arg)* ) {
            Ok(_) => {},
            Err(x) => panic!("Unable to write to stderr: {}", x),
        }
    )
);

// An input source.
//
// We use an enum so that, when the input is from a file, we can seek on
// it (as needed by the od command).
//
// Taken from stackoverflow:
// http://stackoverflow.com/questions/37223741/how-can-i-take-input-from-either-stdin-or-a-file-if-i-cannot-seek-stdin

/// The `Input` type.
pub enum Input {
    /// Input is from a file
    File(fs::File),
    /// Input is from the standard input
    Stdin(io::Stdin),
}

impl Read for Input {
    /// Pull some bytes from this source into the specified buffer,
    /// returning how many bytes were read.
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        match *self {
            Input::File(ref mut file) => file.read(buf),
            Input::Stdin(ref mut stdin) => stdin.read(buf),
        }
    }
}

impl Seek for Input {
    /// Seek to an offset, in bytes, in a stream.
    ///
    /// # Errors
    ///
    /// Seeking on the standard input is an error.
    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
        match *self {
            Input::File(ref mut file) => file.seek(pos),
            Input::Stdin(_) => {
                Err(io::Error::new(
                    io::ErrorKind::Other,
                    "not supported by stdin-input",
                ))
            },
        }
    }
}

impl Input {
    /// Create a new Input by opening the specified file or, if the
    /// filename is "-", the standard input.
    pub fn open(filename: &str) -> io::Result<Input> {
        if filename == "-" {
            Ok(Input::Stdin(io::stdin()))
        } else {
            Ok(Input::File(fs::File::open(filename)?))
        }
    }
}
<commit_msg>Shorten Stack Overflow URL.<commit_after>// Copyright 2015-2017 James Bostock. See the LICENSE file at the top-level
// directory of this distribution.

use std::fs;
use std::io::{self, Read, Seek, SeekFrom};
use std::fs::File;

// Utility routines

// Taken from stackoverflow:
// http://stackoverflow.com/questions/27588416/how-to-send-output-to-stderr
macro_rules! errln(
    ($($arg:tt)*) => (
        use std::io::Write;
        match writeln!(&mut ::std::io::stderr(), $($arg)* ) {
            Ok(_) => {},
            Err(x) => panic!("Unable to write to stderr: {}", x),
        }
    )
);

// An input source.
//
// We use an enum so that, when the input is from a file, we can seek on
// it (as needed by the od command).
//
// Taken from stackoverflow: http://bit.ly/2kbtASY

/// The `Input` type.
pub enum Input {
    /// Input is from a file
    File(fs::File),
    /// Input is from the standard input
    Stdin(io::Stdin),
}

impl Read for Input {
    /// Pull some bytes from this source into the specified buffer,
    /// returning how many bytes were read.
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        match *self {
            Input::File(ref mut file) => file.read(buf),
            Input::Stdin(ref mut stdin) => stdin.read(buf),
        }
    }
}

impl Seek for Input {
    /// Seek to an offset, in bytes, in a stream.
    ///
    /// # Errors
    ///
    /// Seeking on the standard input is an error.
    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
        match *self {
            Input::File(ref mut file) => file.seek(pos),
            Input::Stdin(_) => {
                Err(io::Error::new(
                    io::ErrorKind::Other,
                    "not supported by stdin-input",
                ))
            },
        }
    }
}

impl Input {
    /// Create a new Input by opening the specified file or, if the
    /// filename is "-", the standard input.
    pub fn open(filename: &str) -> io::Result<Input> {
        if filename == "-" {
            Ok(Input::Stdin(io::stdin()))
        } else {
            Ok(Input::File(fs::File::open(filename)?))
        }
    }
}
<|endoftext|>


<commit_before>use super::BrotliEncoderParams;
use std::io::{self, Error, ErrorKind, Read, Write, Seek, SeekFrom};
use super::{Rebox, HeapAllocator, IoWriterWrapper};
use brotli::{DecompressorWriterCustomIo, HuffmanCode, CustomWrite};
use alloc_no_stdlib::{Allocator, SliceWrapper,SliceWrapperMut};

struct Tee<OutputA:Write, OutputB:Write>(OutputA, OutputB);
impl <OutputA:Write, OutputB:Write> Write for Tee<OutputA, OutputB> {
    fn write(&mut self, data:&[u8]) -> Result<usize, io::Error> {
        match self.0.write(data) {
            Err(err) => return Err(err),
            Ok(size) => match self.1.write_all(&data[..size]) {
                Ok(_) => Ok(size),
                Err(err) => Err(err),
            }
        }
    }
    fn flush(&mut self) -> Result<(), io::Error> {
        match self.0.flush() {
            Err(err) => return Err(err),
            Ok(_) => loop {
                match self.1.flush() {
                    Err(e) => match e.kind() {
                        ErrorKind::Interrupted => continue,
                        _ => return Err(e),
                    }
                    Ok(e) => return Ok(e),
                }
            }
        }
    }
}

struct DecompressAndValidate<'a, OutputType:Write+'a>(
    DecompressorWriterCustomIo<io::Error,
                               IoWriterWrapper<'a, OutputType>,
                               Rebox<u8>, // buffer type
                               HeapAllocator,
                               HeapAllocator,
                               HeapAllocator>
);

impl<'a, OutputType:Write> Write for DecompressAndValidate<'a, OutputType> {
    fn write(&mut self, data:&[u8]) -> Result<usize, io::Error> {
        self.0.write(data)
    }
    fn flush(&mut self) -> Result<(), io::Error> {
        self.0.flush()
    }
}

fn make_sha_writer() -> io::Sink {
    io::sink()
}

struct ShaReader<InputType:Read> {
    reader:InputType,
}

pub fn compress_validate<InputType:Read, OutputType: Write>(r: &mut InputType,
                                                            w: &mut OutputType,
                                                            buffer_size: usize,
                                                            params: &BrotliEncoderParams,
                                                            custom_dictionary:Rebox<u8>,
                                                            num_threads: usize) -> Result<(), io::Error> {
    let mut m8 = HeapAllocator::default();
    let buffer = <HeapAllocator as Allocator<u8>>::alloc_cell(&mut m8, buffer_size);
    // FIXME: could reuse the dictionary to seed the compressor, but that violates the abstraction right now
    // also dictionaries are not very popular since they are mostly an internal concept, given their deprecation in
    // the standard brotli spec
    let mut dict = Vec::<u8>::new();

    dict.extend_from_slice(custom_dictionary.slice());
    let mut sha_writer = make_sha_writer();
    let mut validate_writer = DecompressAndValidate(DecompressorWriterCustomIo::new_with_custom_dictionary(
        IoWriterWrapper(&mut sha_writer),
        buffer,
        m8,
        HeapAllocator::default(),
        HeapAllocator::default(),
        custom_dictionary,
        Error::new(ErrorKind::InvalidData,
                   "Invalid Data")));
    let mut overarching_writer = Tee(validate_writer, w);
    let ret = super::compress(r, &mut overarching_writer, buffer_size, params, &dict[..], num_threads);
    match ret {
        //FIXME: check sig here
        Ok(ret) => Ok(()),
        Err(e) => Err(e),
    }
}
<commit_msg>some fixes for older compilers and checking the shasum<commit_after>use super::BrotliEncoderParams;
use std::io::{self, Error, ErrorKind, Read, Write};
use super::{Rebox, HeapAllocator, IoWriterWrapper};
use brotli::{DecompressorWriterCustomIo, CustomWrite};
use alloc_no_stdlib::{Allocator, SliceWrapper};

struct Tee<OutputA:Write, OutputB:Write>(OutputA, OutputB);
impl <OutputA:Write, OutputB:Write> Write for Tee<OutputA, OutputB> {
    fn write(&mut self, data:&[u8]) -> Result<usize, io::Error> {
        match self.0.write(data) {
            Err(err) => return Err(err),
            Ok(size) => match self.1.write_all(&data[..size]) {
                Ok(_) => Ok(size),
                Err(err) => Err(err),
            }
        }
    }
    fn flush(&mut self) -> Result<(), io::Error> {
        match self.0.flush() {
            Err(err) => return Err(err),
            Ok(_) => {
                loop {
                    match self.1.flush() {
                        Err(e) => match e.kind() {
                            ErrorKind::Interrupted => continue,
                            _ => return Err(e),
                        }
                        Ok(e) => return Ok(e),
                    }
                }
            },
        }
    }
}

struct DecompressAndValidate<'a, OutputType:Write+'a>(
    DecompressorWriterCustomIo<io::Error,
                               IoWriterWrapper<'a, OutputType>,
                               Rebox<u8>, // buffer type
                               HeapAllocator,
                               HeapAllocator,
                               HeapAllocator>
);

impl<'a, OutputType:Write> Write for DecompressAndValidate<'a, OutputType> {
    fn write(&mut self, data:&[u8]) -> Result<usize, io::Error> {
        self.0.write(data)
    }
    fn flush(&mut self) -> Result<(), io::Error> {
        self.0.flush()
    }
}

fn make_sha_writer() -> io::Sink {
    io::sink()
}

fn make_sha_reader<InputType:Read>(r:&mut InputType) -> &mut InputType {
    r
}

fn sha_ok<InputType:Read>(_writer: io::Sink, _reader: &mut InputType) -> bool {
    false
}

struct ShaReader<InputType:Read> {
    reader:InputType,
}

pub fn compress_validate<InputType:Read, OutputType: Write>(r: &mut InputType,
                                                            w: &mut OutputType,
                                                            buffer_size: usize,
                                                            params: &BrotliEncoderParams,
                                                            custom_dictionary:Rebox<u8>,
                                                            num_threads: usize) -> Result<(), io::Error> {
    let mut m8 = HeapAllocator::default();
    let buffer = <HeapAllocator as Allocator<u8>>::alloc_cell(&mut m8, buffer_size);
    // FIXME: could reuse the dictionary to seed the compressor, but that violates the abstraction right now
    // also dictionaries are not very popular since they are mostly an internal concept, given their deprecation in
    // the standard brotli spec
    let mut dict = Vec::<u8>::new();

    dict.extend_from_slice(custom_dictionary.slice());
    let mut sha_writer = make_sha_writer();
    let mut sha_reader = make_sha_reader(r);
    let ret;
    {
        let validate_writer = DecompressAndValidate(DecompressorWriterCustomIo::new_with_custom_dictionary(
            IoWriterWrapper(&mut sha_writer),
            buffer,
            m8,
            HeapAllocator::default(),
            HeapAllocator::default(),
            custom_dictionary,
            Error::new(ErrorKind::InvalidData,
                       "Invalid Data")));
        let mut overarching_writer = Tee(validate_writer, w);
        ret = super::compress(sha_reader, &mut overarching_writer, buffer_size, params, &dict[..], num_threads);
    }
    match ret {
        Ok(_ret) => {
            if sha_ok(sha_writer, sha_reader) {
                return Ok(());
            } else {
                return Err(Error::new(ErrorKind::InvalidData, "Validation failed"));
            }
        },
        Err(e) => Err(e),
    }
}
<|endoftext|>


<commit_before>extern crate difference;
extern crate regex;
extern crate term;

use difference::{Changeset, Difference};

use std::fs;
use std::io::{Cursor, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use term::Terminal;

// Given the default Rust test runner doesn't expose programmatic test cases
// we are using single "check_expansions" test to process all the data files.
//
// This is similar approach to what rustfmt does.

struct TestConfig
{
    crate_path: PathBuf,
    data_path: PathBuf,
}

#[test]
fn check_expansions()
{
    let root_path = find_root().unwrap();

    let crate_path = root_path.join("intercom-attributes");
    let config = TestConfig {
        data_path: crate_path.join("tests/data"),
        crate_path,
    };
    // Running "cargo test" in a clean build directory does not
    // finalize the compilation of all the crates.
    // The final binaries are unavailable in the target directory.
    // Force the building here to ensure they are available
    // for the tests.
    build_crate("intercom");
    build_crate("intercom-fmt");

    let failed =
        test_path(&config, "macro", TestMode::Macro) + test_path(&config, "ui", TestMode::UI);

    // Ensure there were no failures.
    //
    // If we fail here, cargo will display our printlns to the user.
    assert_eq!(failed, 0, "{} tests failed", failed);
}

struct OutputResult
{
    message: String,
    actual: String,
    expected_path: Option<String>,
    changeset: Changeset,
}

impl OutputResult
{
    /// Prints the diff.
    pub fn show_diff(&self, ctx_size: usize, t: &mut dyn Terminal<Output = std::io::Stdout>)
    {
        enum Lines
        {
            Expected(String),
            Actual(String),
            Same(String),
        };

        // Resolve the changed lines so we can print only the context.
        let mut all_lines = vec![];
        for cs in &self.changeset.diffs {
            match cs {
                Difference::Same(ref x) => all_lines.push(
                    x.lines()
                        .map(|l| Lines::Same(format!("  {}", l)))
                        .collect::<Vec<_>>(),
                ),
                Difference::Add(ref x) => all_lines.push(
                    x.lines()
                        .map(|l| Lines::Expected(format!("E {}", l)))
                        .collect::<Vec<_>>(),
                ),
                Difference::Rem(ref x) => all_lines.push(
                    x.lines()
                        .map(|l| Lines::Actual(format!("A {}", l)))
                        .collect::<Vec<_>>(),
                ),
            }
        }
        let all_lines = all_lines.into_iter().flat_map(|i| i).collect::<Vec<_>>();

        let mut ctx_counter = 0;
        let mut ctx_lines = vec![];
        for l in &all_lines {
            match l {
                Lines::Expected(..) | Lines::Actual(..) => ctx_counter = ctx_size * 2 + 1,
                _ if ctx_counter > 0 => ctx_counter -= 1,
                _ => {}
            }
            ctx_lines.push(ctx_counter);
        }
        for _ in 0..ctx_size {
            if ctx_counter > 0 {
                ctx_counter -= 1
            }
            ctx_lines.push(ctx_counter);
        }

        let mut skip = false;
        for i in 0..all_lines.len() {
            if ctx_lines[i + ctx_size] == 0 {
                if !skip {
                    writeln!(t, "...snip").expect("Write failed");
                    skip = true;
                }
                continue;
            }
            skip = false;
            match all_lines[i] {
                Lines::Same(ref x) => {
                    t.reset().unwrap();
                    writeln!(t, "{}", x).expect("Write failed");
                }
                Lines::Expected(ref x) => {
                    t.fg(term::color::GREEN).unwrap();
                    writeln!(t, "{}", x).expect("Write failed");
                }
                Lines::Actual(ref x) => {
                    t.fg(term::color::RED).unwrap();
                    writeln!(t, "{}", x).expect("Write failed");
                }
            }
        }
        t.reset().unwrap();
    }
}

/// Compiles a single file using rustc using similar options than what
/// cargo would have used.
fn build(cwd: &str, path: &str, mode: TestMode) -> (bool, String, String)
{
    #[cfg(debug_assertions)]
    let conf = "debug";

    #[cfg(not(debug_assertions))]
    let conf = "release";

    // Launch rustc.
    #[rustfmt::skip]
    let mut cmd = std::process::Command::new("rustc");
    cmd.current_dir(cwd)
        .env("CARGO_PKG_NAME", "TestLib")
        .args(&[
            "--crate-name",
            "source",
            "--crate-type",
            "lib",
            path,
            "--out-dir",
            "tests/out",
            "-L",
            &format!("dependency=../target/{}/deps", conf),
            "--extern",
            &format!("intercom=../target/{}/libintercom.rlib", conf),
        ]);

    // In expansion mode add the 'pretty=expanded' option.
    if let TestMode::Macro = mode {
        cmd.arg("--pretty=expanded");
        cmd.arg("-Z").arg("unstable-options");
    }

    // Get the output.
    let output = cmd.output().expect("Failed to execute");

    // stdout/err is utf8 byte stream. Parse it into a string.
    (
        output.status.success(),
        String::from_utf8(output.stdout).expect("Bad output"),
        String::from_utf8(output.stderr).expect("Bad stderr"),
    )
}

fn format(code: &str) -> String
{
    let intercom_fmt = find_intercom_fmt().unwrap();
    let mut formatter = Command::new(&intercom_fmt)
        .arg("-p")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .spawn()
        .expect(&format!("Failed to launch formatter {:?}", intercom_fmt));

    // Send the code to the formatter.
    // "stdin" of the formatter will be closed automatically when the scope terminates.
    {
        let mut stdin = formatter.stdin.as_mut().unwrap();
        std::io::copy(&mut Cursor::new(code.as_bytes()), &mut stdin)
            .expect("Failed to send data for formatting.");
    }

    let output = formatter
        .wait_with_output()
        .expect("Failed to read formatter results.");
    let status = output.status;
    assert!(
        status.success() && status.code() == Some(0),
        "Formatting failed with status \"{:?}\".",
        status.code()
    );

    // Convert the UTF-8 output into a string.
    String::from_utf8(output.stdout).expect("Bad output")
}

/// Builds the "intercom" library
fn build_crate(module: &str)
{
    let mut cmd = Command::new("cargo");
    cmd.arg("build");

    #[cfg(not(debug_assertions))]
    {
        cmd.arg("--release");
    }

    let status = cmd
        .current_dir(find_root().unwrap().join(module))
        .status()
        .expect(&format!("Failed to build crate \"{0}\"", module));
    assert!(status.success());
}

fn find_root() -> std::io::Result<PathBuf>
{
    let mut root_path = std::env::current_exe()?;
    loop {
        if root_path.join("Cargo.toml").exists() {
            break;
        }
        assert!(root_path.pop());
    }

    Ok(root_path)
}

fn find_intercom_fmt() -> std::io::Result<PathBuf>
{
    // Avoid the need to determine the current build target by basing the search on the current
    // executable.
    let mut intercom_fmt_dir = std::env::current_exe()?;
    assert!(intercom_fmt_dir.pop()); // The name of the executable.
    let original_path = format!("{:?}", intercom_fmt_dir);
    loop {
        // Stop search when intercom-fmt has been found.
        if let Some(intercom_fmt) = has_intercom_fmt(&intercom_fmt_dir) {
            return Ok(intercom_fmt);
        }

        // Move towards root
        assert!(
            intercom_fmt_dir.pop(),
            format!(
                "Could not locate intercom-fmt. Search started from \"{0}\".",
                original_path
            )
        );
    }
}

/// Determines if the given directory has intercom-fmt
fn has_intercom_fmt(dir: &PathBuf) -> Option<PathBuf>
{
    #[cfg(windows)]
    let intercom_fmt = dir.join("intercom-fmt.exe");

    #[cfg(not(windows))]
    let intercom_fmt = dir.join("intercom-fmt");

    if intercom_fmt.exists() && intercom_fmt.metadata().unwrap().is_file() {
        Some(intercom_fmt)
    } else {
        None
    }
}

#[derive(Clone, Copy)]
enum TestMode
{
    Macro,
    UI,
}

fn test_path(config: &TestConfig, sub_path: &str, mode: TestMode) -> usize
{
    // Get the source test data files.
    let test_data = fs::read_dir(config.data_path.join(sub_path)).unwrap();
    let source_paths = test_data
        .into_iter()
        .map(|e| e.expect("Failed to read entry").path())
        .map(|p| p.to_str().unwrap().to_owned())
        .map(|s| s.replace("\\", "/"))
        .filter(|p| p.ends_with(".rs"));
    let mut failed = 0;

    for source_path in source_paths {
        println!("Testing {}", source_path);

        // Get the source and target code.

        // The source is compiled using rustc
        let (_, result_code, result_stderr) =
            build(config.crate_path.to_str().unwrap(), &source_path, mode);

        // Generate diffs for both sources
        // Ensure the linebreaks are the same for both. This seems to be
        // somewhat of an issue on AppVeyor.
        let result_code = result_code.replace("\r", "");
        let result_stderr = result_stderr.replace("\r", "");

        let mut results = vec![];

        // Use rustfmt to format both pieces of code so that we have a
        // canonical format for them. Without rustfmt we'd need to match the
        // compiler pretty print format in the reference target files - which,
        // despite its name, isn't very pretty.
        match mode {
            TestMode::Macro => {
                results.extend(assert_macro(result_code, result_stderr, &source_path))
            }
            TestMode::UI => {
                results.extend(assert_compile(result_code, result_stderr, &source_path))
            }
        }

        // If these were equal, there's only one "Same" diff segment.
        // If there is more than one, they differed.
        let mut t = term::stdout().unwrap();
        for r in results {
            if std::env::var("UPDATE_TARGETS").is_ok() && r.expected_path.is_some() {
                // The user wants to update the targets.
                let mut target_file = fs::File::create(r.expected_path.as_ref().unwrap()).unwrap();
                target_file.write_all(r.actual.as_bytes()).expect(&format!(
                    "Writing target file {} failed",
                    r.expected_path.as_ref().unwrap()
                ));
                writeln!(
                    t,
                    "-----------------------------------------------------------------"
                )
                .unwrap();
                writeln!(t, "UPDATED: {}", r.expected_path.unwrap()).unwrap();
                writeln!(
                    t,
                    "-----------------------------------------------------------------"
                )
                .unwrap();
            } else {
                writeln!(
                    t,
                    "-----------------------------------------------------------------"
                )
                .unwrap();
                writeln!(t, "FAIL: {}", r.message).unwrap();
                writeln!(t, "").unwrap();
                r.show_diff(5, &mut *t);
                writeln!(t, "").unwrap();
                writeln!(
                    t,
                    "-----------------------------------------------------------------"
                )
                .unwrap();
                failed += 1;
            }
        }
    }
    failed
}

fn assert_macro(result_code: String, result_stderr: String, source_path: &str)
    -> Vec<OutputResult>
{
    // Construct the target file path by replacing the ".source.rs" with a
    // ".target.rs". There's a small discrepancy here as the .source.rs had
    // to be at the end for the file to count as source file, but here
    // we are replacing the .target. everywhere in the file name.
    //
    // This shouldn't matter in practice as these are test files and we can
    // decide on their naming as we write them.
    vec![
        assert_output_with(result_code, source_path, "stdout", |s| format(s.trim())),
        assert_output_with(result_stderr, source_path, "stderr", |s| {
            strip_path(source_path, s)
        }),
    ]
    .into_iter()
    .flat_map(|v| v)
    .collect()
}

fn assert_compile(
    result_code: String,
    result_stderr: String,
    source_path: &str,
) -> Vec<OutputResult>
{
    vec![
        assert_output(result_code, source_path, "stdout"),
        assert_output_with(result_stderr, source_path, "stderr", |s| {
            strip_path(source_path, s)
        }),
    ]
    .into_iter()
    .flat_map(|v| v)
    .collect()
}

fn assert_output(actual: String, actual_path: &str, output_kind: &str) -> Option<OutputResult>
{
    assert_output_with(actual, actual_path, output_kind, |input| input)
}

fn assert_output_with(
    actual: String,
    actual_path: &str,
    output_kind: &str,
    sanitize: impl Fn(String) -> String,
) -> Option<OutputResult>
{
    let name = Path::new(actual_path)
        .file_name()
        .unwrap()
        .to_string_lossy();
    let actual = sanitize(actual.clone());
    let expected_path_string = format!("{}.{}", actual_path, output_kind);
    let expected_path = Path::new(&expected_path_string);
    if expected_path.exists() {
        let expected = std::fs::read_to_string(expected_path)
            .unwrap()
            .replace("\r", "");
        let expected = sanitize(expected);
        if std::env::var("FORCE_UPDATE_TARGETS").is_ok() {
            Some(OutputResult {
                message: format!("Forcing update"),
                changeset: Changeset::new(&actual, &expected, "\n"),
                actual: actual,
                expected_path: Some(expected_path_string),
            })
        } else if expected != actual {
            Some(OutputResult {
                message: format!("{} {} output differs", name, output_kind),
                changeset: Changeset::new(&actual, &expected, "\n"),
                actual: actual,
                expected_path: Some(expected_path_string),
            })
        } else {
            None
        }
    } else if !actual.is_empty() {
        Some(OutputResult {
            message: format!("{} {} output was not expected", name, output_kind),
            changeset: Changeset::new(&actual, "", "\n"),
            actual: actual,
            expected_path: None,
        })
    } else {
        None
    }
}

fn strip_path(path: &str, source: String) -> String
{
    let root = Path::new(path).parent().unwrap().to_string_lossy() + "/";
    eprintln!("{}", root);
    source.replace(root.as_ref(), "")
}
<commit_msg>Define a fallback terminal<commit_after>extern crate difference;
extern crate regex;
extern crate term;

use difference::{Changeset, Difference};

use std::fs;
use std::io::{Cursor, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use term::Terminal;

// Given the default Rust test runner doesn't expose programmatic test cases
// we are using single "check_expansions" test to process all the data files.
//
// This is similar approach to what rustfmt does.

struct TestConfig
{
    crate_path: PathBuf,
    data_path: PathBuf,
}

#[test]
fn check_expansions()
{
    let root_path = find_root().unwrap();

    let crate_path = root_path.join("intercom-attributes");
    let config = TestConfig {
        data_path: crate_path.join("tests/data"),
        crate_path,
    };
    // Running "cargo test" in a clean build directory does not
    // finalize the compilation of all the crates.
    // The final binaries are unavailable in the target directory.
    // Force the building here to ensure they are available
    // for the tests.
    build_crate("intercom");
    build_crate("intercom-fmt");

    let failed =
        test_path(&config, "macro", TestMode::Macro) + test_path(&config, "ui", TestMode::UI);

    // Ensure there were no failures.
    //
    // If we fail here, cargo will display our printlns to the user.
    assert_eq!(failed, 0, "{} tests failed", failed);
}

struct OutputResult
{
    message: String,
    actual: String,
    expected_path: Option<String>,
    changeset: Changeset,
}

impl OutputResult
{
    /// Prints the diff.
    pub fn show_diff(&self, ctx_size: usize, t: &mut dyn Terminal<Output = std::io::Stdout>)
    {
        enum Lines
        {
            Expected(String),
            Actual(String),
            Same(String),
        };

        // Resolve the changed lines so we can print only the context.
        let mut all_lines = vec![];
        for cs in &self.changeset.diffs {
            match cs {
                Difference::Same(ref x) => all_lines.push(
                    x.lines()
                        .map(|l| Lines::Same(format!("  {}", l)))
                        .collect::<Vec<_>>(),
                ),
                Difference::Add(ref x) => all_lines.push(
                    x.lines()
                        .map(|l| Lines::Expected(format!("E {}", l)))
                        .collect::<Vec<_>>(),
                ),
                Difference::Rem(ref x) => all_lines.push(
                    x.lines()
                        .map(|l| Lines::Actual(format!("A {}", l)))
                        .collect::<Vec<_>>(),
                ),
            }
        }
        let all_lines = all_lines.into_iter().flat_map(|i| i).collect::<Vec<_>>();

        let mut ctx_counter = 0;
        let mut ctx_lines = vec![];
        for l in &all_lines {
            match l {
                Lines::Expected(..) | Lines::Actual(..) => ctx_counter = ctx_size * 2 + 1,
                _ if ctx_counter > 0 => ctx_counter -= 1,
                _ => {}
            }
            ctx_lines.push(ctx_counter);
        }
        for _ in 0..ctx_size {
            if ctx_counter > 0 {
                ctx_counter -= 1
            }
            ctx_lines.push(ctx_counter);
        }

        let mut skip = false;
        for i in 0..all_lines.len() {
            if ctx_lines[i + ctx_size] == 0 {
                if !skip {
                    writeln!(t, "...snip").expect("Write failed");
                    skip = true;
                }
                continue;
            }
            skip = false;
            match all_lines[i] {
                Lines::Same(ref x) => {
                    t.reset().unwrap();
                    writeln!(t, "{}", x).expect("Write failed");
                }
                Lines::Expected(ref x) => {
                    t.fg(term::color::GREEN).unwrap();
                    writeln!(t, "{}", x).expect("Write failed");
                }
                Lines::Actual(ref x) => {
                    t.fg(term::color::RED).unwrap();
                    writeln!(t, "{}", x).expect("Write failed");
                }
            }
        }
        t.reset().unwrap();
    }
}

/// Compiles a single file using rustc using similar options than what
/// cargo would have used.
fn build(cwd: &str, path: &str, mode: TestMode) -> (bool, String, String)
{
    #[cfg(debug_assertions)]
    let conf = "debug";

    #[cfg(not(debug_assertions))]
    let conf = "release";

    // Launch rustc.
    #[rustfmt::skip]
    let mut cmd = std::process::Command::new("rustc");
    cmd.current_dir(cwd)
        .env("CARGO_PKG_NAME", "TestLib")
        .args(&[
            "--crate-name",
            "source",
            "--crate-type",
            "lib",
            path,
            "--out-dir",
            "tests/out",
            "-L",
            &format!("dependency=../target/{}/deps", conf),
            "--extern",
            &format!("intercom=../target/{}/libintercom.rlib", conf),
        ]);

    // In expansion mode add the 'pretty=expanded' option.
    if let TestMode::Macro = mode {
        cmd.arg("--pretty=expanded");
        cmd.arg("-Z").arg("unstable-options");
    }

    // Get the output.
    let output = cmd.output().expect("Failed to execute");

    // stdout/err is utf8 byte stream. Parse it into a string.
    (
        output.status.success(),
        String::from_utf8(output.stdout).expect("Bad output"),
        String::from_utf8(output.stderr).expect("Bad stderr"),
    )
}

fn format(code: &str) -> String
{
    let intercom_fmt = find_intercom_fmt().unwrap();
    let mut formatter = Command::new(&intercom_fmt)
        .arg("-p")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .spawn()
        .expect(&format!("Failed to launch formatter {:?}", intercom_fmt));

    // Send the code to the formatter.
    // "stdin" of the formatter will be closed automatically when the scope terminates.
    {
        let mut stdin = formatter.stdin.as_mut().unwrap();
        std::io::copy(&mut Cursor::new(code.as_bytes()), &mut stdin)
            .expect("Failed to send data for formatting.");
    }

    let output = formatter
        .wait_with_output()
        .expect("Failed to read formatter results.");
    let status = output.status;
    assert!(
        status.success() && status.code() == Some(0),
        "Formatting failed with status \"{:?}\".",
        status.code()
    );

    // Convert the UTF-8 output into a string.
    String::from_utf8(output.stdout).expect("Bad output")
}

/// Builds the "intercom" library
fn build_crate(module: &str)
{
    let mut cmd = Command::new("cargo");
    cmd.arg("build");

    #[cfg(not(debug_assertions))]
    {
        cmd.arg("--release");
    }

    let status = cmd
        .current_dir(find_root().unwrap().join(module))
        .status()
        .expect(&format!("Failed to build crate \"{0}\"", module));
    assert!(status.success());
}

fn find_root() -> std::io::Result<PathBuf>
{
    let mut root_path = std::env::current_exe()?;
    loop {
        if root_path.join("Cargo.toml").exists() {
            break;
        }
        assert!(root_path.pop());
    }

    Ok(root_path)
}

fn find_intercom_fmt() -> std::io::Result<PathBuf>
{
    // Avoid the need to determine the current build target by basing the search on the current
    // executable.
    let mut intercom_fmt_dir = std::env::current_exe()?;
    assert!(intercom_fmt_dir.pop()); // The name of the executable.
    let original_path = format!("{:?}", intercom_fmt_dir);
    loop {
        // Stop search when intercom-fmt has been found.
        if let Some(intercom_fmt) = has_intercom_fmt(&intercom_fmt_dir) {
            return Ok(intercom_fmt);
        }

        // Move towards root
        assert!(
            intercom_fmt_dir.pop(),
            format!(
                "Could not locate intercom-fmt. Search started from \"{0}\".",
                original_path
            )
        );
    }
}

/// Determines if the given directory has intercom-fmt
fn has_intercom_fmt(dir: &PathBuf) -> Option<PathBuf>
{
    #[cfg(windows)]
    let intercom_fmt = dir.join("intercom-fmt.exe");

    #[cfg(not(windows))]
    let intercom_fmt = dir.join("intercom-fmt");

    if intercom_fmt.exists() && intercom_fmt.metadata().unwrap().is_file() {
        Some(intercom_fmt)
    } else {
        None
    }
}

#[derive(Clone, Copy)]
enum TestMode
{
    Macro,
    UI,
}

fn test_path(config: &TestConfig, sub_path: &str, mode: TestMode) -> usize
{
    // Get the source test data files.
    let test_data = fs::read_dir(config.data_path.join(sub_path)).unwrap();
    let source_paths = test_data
        .into_iter()
        .map(|e| e.expect("Failed to read entry").path())
        .map(|p| p.to_str().unwrap().to_owned())
        .map(|s| s.replace("\\", "/"))
        .filter(|p| p.ends_with(".rs"));
    let mut failed = 0;

    for source_path in source_paths {
        println!("Testing {}", source_path);

        // Get the source and target code.

        // The source is compiled using rustc
        let (_, result_code, result_stderr) =
            build(config.crate_path.to_str().unwrap(), &source_path, mode);

        // Generate diffs for both sources
        // Ensure the linebreaks are the same for both. This seems to be
        // somewhat of an issue on AppVeyor.
        let result_code = result_code.replace("\r", "");
        let result_stderr = result_stderr.replace("\r", "");

        let mut results = vec![];

        // Use rustfmt to format both pieces of code so that we have a
        // canonical format for them. Without rustfmt we'd need to match the
        // compiler pretty print format in the reference target files - which,
        // despite its name, isn't very pretty.
        match mode {
            TestMode::Macro => {
                results.extend(assert_macro(result_code, result_stderr, &source_path))
            }
            TestMode::UI => {
                results.extend(assert_compile(result_code, result_stderr, &source_path))
            }
        }

        // Acquire a terminal.
        //
        // Especially in CI environments there is no real terminal but since we
        // are still passing the TerminfoTerminal through the functions, we need
        // one. We'll try ANSI terminal as a fallback.
        let mut t = term::stdout().unwrap_or_else(|| {
            Box::new(term::TerminfoTerminal::new_with_terminfo(
                std::io::stdout(),
                term::terminfo::TermInfo::from_name("ansi").expect("ansi terminal not supported"),
            ))
        });
        for r in results {
            if std::env::var("UPDATE_TARGETS").is_ok() && r.expected_path.is_some() {
                // The user wants to update the targets.
                let mut target_file = fs::File::create(r.expected_path.as_ref().unwrap()).unwrap();
                target_file.write_all(r.actual.as_bytes()).expect(&format!(
                    "Writing target file {} failed",
                    r.expected_path.as_ref().unwrap()
                ));
                writeln!(
                    t,
                    "-----------------------------------------------------------------"
                )
                .unwrap();
                writeln!(t, "UPDATED: {}", r.expected_path.unwrap()).unwrap();
                writeln!(
                    t,
                    "-----------------------------------------------------------------"
                )
                .unwrap();
            } else {
                writeln!(
                    t,
                    "-----------------------------------------------------------------"
                )
                .unwrap();
                writeln!(t, "FAIL: {}", r.message).unwrap();
                writeln!(t, "").unwrap();
                r.show_diff(5, &mut *t);
                writeln!(t, "").unwrap();
                writeln!(
                    t,
                    "-----------------------------------------------------------------"
                )
                .unwrap();
                failed += 1;
            }
        }
    }
    failed
}

fn assert_macro(result_code: String, result_stderr: String, source_path: &str)
    -> Vec<OutputResult>
{
    // Construct the target file path by replacing the ".source.rs" with a
    // ".target.rs". There's a small discrepancy here as the .source.rs had
    // to be at the end for the file to count as source file, but here
    // we are replacing the .target. everywhere in the file name.
    //
    // This shouldn't matter in practice as these are test files and we can
    // decide on their naming as we write them.
    vec![
        assert_output_with(result_code, source_path, "stdout", |s| format(s.trim())),
        assert_output_with(result_stderr, source_path, "stderr", |s| {
            strip_path(source_path, s)
        }),
    ]
    .into_iter()
    .flat_map(|v| v)
    .collect()
}

fn assert_compile(
    result_code: String,
    result_stderr: String,
    source_path: &str,
) -> Vec<OutputResult>
{
    vec![
        assert_output(result_code, source_path, "stdout"),
        assert_output_with(result_stderr, source_path, "stderr", |s| {
            strip_path(source_path, s)
        }),
    ]
    .into_iter()
    .flat_map(|v| v)
    .collect()
}

fn assert_output(actual: String, actual_path: &str, output_kind: &str) -> Option<OutputResult>
{
    assert_output_with(actual, actual_path, output_kind, |input| input)
}

fn assert_output_with(
    actual: String,
    actual_path: &str,
    output_kind: &str,
    sanitize: impl Fn(String) -> String,
) -> Option<OutputResult>
{
    let name = Path::new(actual_path)
        .file_name()
        .unwrap()
        .to_string_lossy();
    let actual = sanitize(actual.clone());
    let expected_path_string = format!("{}.{}", actual_path, output_kind);
    let expected_path = Path::new(&expected_path_string);
    if expected_path.exists() {
        let expected = std::fs::read_to_string(expected_path)
            .unwrap()
            .replace("\r", "");
        let expected = sanitize(expected);
        if std::env::var("FORCE_UPDATE_TARGETS").is_ok() {
            Some(OutputResult {
                message: format!("Forcing update"),
                changeset: Changeset::new(&actual, &expected, "\n"),
                actual: actual,
                expected_path: Some(expected_path_string),
            })
        } else if expected != actual {
            Some(OutputResult {
                message: format!("{} {} output differs", name, output_kind),
                changeset: Changeset::new(&actual, &expected, "\n"),
                actual: actual,
                expected_path: Some(expected_path_string),
            })
        } else {
            None
        }
    } else if !actual.is_empty() {
        Some(OutputResult {
            message: format!("{} {} output was not expected", name, output_kind),
            changeset: Changeset::new(&actual, "", "\n"),
            actual: actual,
            expected_path: None,
        })
    } else {
        None
    }
}

fn strip_path(path: &str, source: String) -> String
{
    let root = Path::new(path).parent().unwrap().to_string_lossy() + "/";
    source.replace(root.as_ref(), "")
}
<|endoftext|>


<commit_before>use std;
use std::collections::HashMap;
use std::sync::Arc;
use std::thread;
use std::sync::atomic::{AtomicU64, Ordering};
use parking_lot::{RwLock, RwLockWriteGuard};
use serde;

use bifrost_hasher::{hash_str, hash_bytes};
use membership::client::{ObserverClient as MembershipClient, Member, WatchResult};
use conshash::weights::DEFAULT_SERVICE_ID;
use conshash::weights::client::{SMClient as WeightSMClient};
use raft::client::{RaftClient};
use utils::bincode::{serialize};
use rand;

pub mod weights;

pub static DEFAULT_NODE_LIST_SIZE: f64 = 2048f64;

#[derive(Debug)]
pub enum Action {
    Joined,
    Left,
}

#[derive(Debug)]
pub enum InitTableError {
    GroupNotExisted,
    NoWeightService,
    NoWeightGroup,
    NoWeightInfo,
    Unknown,
}

#[derive(Debug)]
pub enum CHError {
    WatchError(WatchResult),
    InitTableError(InitTableError),
}

#[derive(Clone)]
struct Node {
    start: u64,
    server: u64,
}

struct LookupTables {
    nodes: Vec<Node>,
    addrs: HashMap<u64, String>
}

pub struct ConsistentHashing {
    tables: RwLock<LookupTables>,
    membership: Arc<MembershipClient>,
    weight_sm_client: WeightSMClient,
    group_name: String,
    watchers: RwLock<Vec<Box<Fn(&Member, &Action, &LookupTables, &Vec<Node>) + Send + Sync>>>,
    version: AtomicU64
}

impl ConsistentHashing {
    pub fn new<'a>(group: &'a str, raft_client: &Arc<RaftClient>) -> Result<Arc<ConsistentHashing>, CHError>  {
        let membership = Arc::new(MembershipClient::new(raft_client));
        let ch = Arc::new(ConsistentHashing {
            tables: RwLock::new(LookupTables {
                nodes: Vec::new(),
                addrs: HashMap::new()
            }),
            membership: membership.clone(),
            weight_sm_client: WeightSMClient::new(DEFAULT_SERVICE_ID, &raft_client),
            group_name: group.to_string(),
            watchers: RwLock::new(Vec::new()),
            version: AtomicU64::new(0)
        });
        {
            let ch = ch.clone();
            let res = membership.on_group_member_joined(move |r| {
                if let Ok((member, version)) = r { server_joined(&ch, member, version); }
            }, group);
            if let Ok(Ok(_)) = res {} else {return Err(CHError::WatchError(res));}
        }
        {
            let ch = ch.clone();
            let res = membership.on_group_member_online(move |r| {
                if let Ok((member, version)) = r { server_joined(&ch, member, version); }
            }, group);
            if let Ok(Ok(_)) = res {} else {return Err(CHError::WatchError(res));}
        }
        {
            let ch = ch.clone();
            let res = membership.on_group_member_left(move |r| {
                if let Ok((member, version)) = r { server_left(&ch, member, version); }
            }, group);
            if let Ok(Ok(_)) = res {} else {return Err(CHError::WatchError(res));}
        }
        {
            let ch = ch.clone();
            let res = membership.on_group_member_offline(move |r| {
                if let Ok((member, version)) = r { server_left(&ch, member, version); }
            }, group);
            if let Ok(Ok(_)) = res {} else {return Err(CHError::WatchError(res));}
        }
        Ok(ch)
    }
    pub fn new_client<'a>(group: &'a str, raft_client: &Arc<RaftClient>) -> Result<Arc<ConsistentHashing>, CHError> {
        match ConsistentHashing::new(group, raft_client) {
            Err(e) => Err(e),
            Ok(ch) => {
                match ch.init_table() {
                    Err(e) => Err(CHError::InitTableError(e)),
                    Ok(_) =>  Ok(ch.clone())
                }
            },
        }
    }
    pub fn init_table(&self) -> Result<(), InitTableError> {
        let mut table = &mut self.tables.write();
        self.init_table_(&mut table)
    }
    pub fn to_server_name(&self, server_id: Option<u64>) -> Option<String> {
        let lookup_table = self.tables.read();
        match server_id {
            Some(id) => Some(lookup_table.addrs.get(&id).unwrap().clone()),
            None => None
        }
    }
    pub fn get_by_server_id(&self, hash: u64) -> Option<u64> {
        let lookup_table = self.tables.read();
        let nodes = &lookup_table.nodes;
        let len = nodes.len();
        if len == 0 {return None;}
        let first_node = nodes.first().unwrap();
        let last_node = nodes.last().unwrap();
        if hash < first_node.start {return Some(first_node.server)}
        if hash >= last_node.start {return Some(last_node.server)}
        let mut low = 0;
        let mut high = len - 1;
        while low <= high {
            let mid = low + (high - low) / 2;
            let curr_node = &nodes[mid];
            let next_node = &nodes[mid + 1];
            if hash >= curr_node.start && hash < next_node.start {
                return Some(curr_node.server);
            } else if hash < curr_node.start {
                high = mid - 1;
            } else {
                low = mid + 1;
            }
        }
        None
    }
    pub fn get_server(&self, hash: u64) -> Option<String> {
        self.to_server_name(self.get_by_server_id(hash))
    }
    pub fn get_server_by_string(&self, string: &String) -> Option<String> {
        self.get_server(hash_str(string))
    }
    pub fn get_server_by<T>(&self, obj: &T) -> Option<String> where T: serde::Serialize {
        self.get_server(hash_bytes(serialize(obj).as_slice()))
    }
    pub fn get_server_id_by_string(&self, string: &String) -> Option<u64> {
        self.get_by_server_id(hash_str(string))
    }
    pub fn get_server_id_by<T>(&self, obj: &T) -> Option<u64> where T: serde::Serialize {
        self.get_by_server_id(hash_bytes(serialize(obj).as_slice()))
    }
    pub fn rand_server(&self) -> Option<String> {
        let rand = rand::random::<u64>();
        self.get_server(rand)
    }
    pub fn nodes_count(&self) -> usize {
        let lookup_table = self.tables.read();
        return lookup_table.nodes.len();
    }
    pub fn set_weight(&self, server_name: &String, weight: u64) -> bool {
        let group_id = hash_str(&self.group_name);
        let server_id = hash_str(server_name);
        if let Ok(Ok(_)) = self.weight_sm_client.set_weight(&group_id, &server_id, &weight) { true } else { false }
    }
    pub fn watch_all_actions<F>(&self, f: F)
        where F: Fn(&Member, &Action, &LookupTables, &Vec<Node>) + 'static + Send + Sync {
        let mut watchers = self.watchers.write();
        watchers.push(Box::new(f));
    }
    pub fn watch_server_nodes_range_changed<F>(&self, server: &String, f: F)
        // return ranges [...,...)
        where F: Fn(Vec<(u64, u64)>) + 'static + Send + Sync {
        let server_id = hash_str(server);
        let wrapper = move |
            _: &Member,_: &Action,
            lookup_table: &LookupTables, _: &Vec<Node>| {
            let nodes = &lookup_table.nodes;
            let node_len = nodes.len();
            let mut ranges: Vec<(u64, u64)> = Vec::new();
            for ni in 0..node_len {
                let node = &nodes[ni];
                if node.server == server_id {
                    ranges.push((node.start, if ni == node_len - 1 {
                        std::u64::MAX
                    } else {
                        nodes[ni + 1].start
                    }));
                }
            }
            if !ranges.is_empty() {f(ranges);}
        };
        self.watch_all_actions(wrapper);
    }

    fn init_table_(&self, lookup_table: &mut RwLockWriteGuard<LookupTables>) -> Result<(), InitTableError> {
        if let Ok(Ok((members, version))) = self.membership.group_members(&self.group_name, true) {
            let group_id = hash_str(&self.group_name);
            match self.weight_sm_client.get_weights(&group_id) {
                Ok(Ok(Some(weights))) =>  {
                    if let Some(min_weight) = weights.values().min() {
                        lookup_table.nodes.clear(); // refresh nodes
                        let mut factors: HashMap<u64, f64> = HashMap::new();
                        let min_weight = *min_weight as f64;
                        for member in members.iter() {
                            let k = member.id;
                            let w = match weights.get(&k) {
                                Some(w) => *w as f64,
                                None => min_weight,
                            };
                            factors.insert(k, w / min_weight);
                        }
                        let factor_sum: f64 = factors.values().sum();
                        let nodes_size = if factor_sum > DEFAULT_NODE_LIST_SIZE {factor_sum} else {DEFAULT_NODE_LIST_SIZE};
                        let factor_scale = nodes_size / factor_sum;
                        for member in members.iter() {lookup_table.addrs.insert(member.id, member.address.clone());}
                        for (k, f) in factors.iter() {
                            let weight = (*f * factor_scale) as u64;
                            for i in 0..weight {
                                let node_key = format!("{}_{}", k, i);
                                let hash = hash_str(&node_key);
                                lookup_table.nodes.push(Node{
                                    start: hash, server: *k
                                });
                            }
                        };
                        lookup_table.nodes.sort_by(|n1, n2| n1.start.cmp(&n2.start));
                        self.version.store(version, Ordering::Relaxed);
                        Ok(())
                    } else {
                        Err(InitTableError::NoWeightInfo)
                    }
                },
                Err(_) => Err(InitTableError::NoWeightService),
                Ok(Ok(None)) => Err(InitTableError::NoWeightGroup),
                _ => Err(InitTableError::Unknown)

            }
        } else {
            Err(InitTableError::GroupNotExisted)
        }
    }
}

fn server_joined(ch: &Arc<ConsistentHashing>, member: Member, version: u64) {
    server_changed(ch, member, Action::Joined, version);
}
fn server_left(ch: &Arc<ConsistentHashing>, member: Member, version: u64) {
    server_changed(ch, member, Action::Left, version);
}
fn server_changed(ch: &Arc<ConsistentHashing>, member: Member, action: Action, version: u64) {
    let ch_version = ch.version.load(Ordering::Relaxed);
    if ch_version < version {
        let ch = ch.clone();
        thread::spawn(move || {
            let mut lookup_table = ch.tables.write();
            let watchers = ch.watchers.read();
            let ch_version = ch.version.load(Ordering::Relaxed);
            if ch_version >= version {return;}
            let old_nodes = lookup_table.nodes.clone();
            ch.init_table_(&mut lookup_table);
            for watch in watchers.iter() {
                watch(&member, &action, &*lookup_table, &old_nodes);
            }
        });
    }
}<commit_msg>fix naming<commit_after>use std;
use std::collections::HashMap;
use std::sync::Arc;
use std::thread;
use std::sync::atomic::{AtomicU64, Ordering};
use parking_lot::{RwLock, RwLockWriteGuard};
use serde;

use bifrost_hasher::{hash_str, hash_bytes};
use membership::client::{ObserverClient as MembershipClient, Member, WatchResult};
use conshash::weights::DEFAULT_SERVICE_ID;
use conshash::weights::client::{SMClient as WeightSMClient};
use raft::client::{RaftClient};
use utils::bincode::{serialize};
use rand;

pub mod weights;

pub static DEFAULT_NODE_LIST_SIZE: f64 = 2048f64;

#[derive(Debug)]
pub enum Action {
    Joined,
    Left,
}

#[derive(Debug)]
pub enum InitTableError {
    GroupNotExisted,
    NoWeightService,
    NoWeightGroup,
    NoWeightInfo,
    Unknown,
}

#[derive(Debug)]
pub enum CHError {
    WatchError(WatchResult),
    InitTableError(InitTableError),
}

#[derive(Clone)]
struct Node {
    start: u64,
    server: u64,
}

struct LookupTables {
    nodes: Vec<Node>,
    addrs: HashMap<u64, String>
}

pub struct ConsistentHashing {
    tables: RwLock<LookupTables>,
    membership: Arc<MembershipClient>,
    weight_sm_client: WeightSMClient,
    group_name: String,
    watchers: RwLock<Vec<Box<Fn(&Member, &Action, &LookupTables, &Vec<Node>) + Send + Sync>>>,
    version: AtomicU64
}

impl ConsistentHashing {
    pub fn new<'a>(group: &'a str, raft_client: &Arc<RaftClient>) -> Result<Arc<ConsistentHashing>, CHError>  {
        let membership = Arc::new(MembershipClient::new(raft_client));
        let ch = Arc::new(ConsistentHashing {
            tables: RwLock::new(LookupTables {
                nodes: Vec::new(),
                addrs: HashMap::new()
            }),
            membership: membership.clone(),
            weight_sm_client: WeightSMClient::new(DEFAULT_SERVICE_ID, &raft_client),
            group_name: group.to_string(),
            watchers: RwLock::new(Vec::new()),
            version: AtomicU64::new(0)
        });
        {
            let ch = ch.clone();
            let res = membership.on_group_member_joined(move |r| {
                if let Ok((member, version)) = r { server_joined(&ch, member, version); }
            }, group);
            if let Ok(Ok(_)) = res {} else {return Err(CHError::WatchError(res));}
        }
        {
            let ch = ch.clone();
            let res = membership.on_group_member_online(move |r| {
                if let Ok((member, version)) = r { server_joined(&ch, member, version); }
            }, group);
            if let Ok(Ok(_)) = res {} else {return Err(CHError::WatchError(res));}
        }
        {
            let ch = ch.clone();
            let res = membership.on_group_member_left(move |r| {
                if let Ok((member, version)) = r { server_left(&ch, member, version); }
            }, group);
            if let Ok(Ok(_)) = res {} else {return Err(CHError::WatchError(res));}
        }
        {
            let ch = ch.clone();
            let res = membership.on_group_member_offline(move |r| {
                if let Ok((member, version)) = r { server_left(&ch, member, version); }
            }, group);
            if let Ok(Ok(_)) = res {} else {return Err(CHError::WatchError(res));}
        }
        Ok(ch)
    }
    pub fn new_client<'a>(group: &'a str, raft_client: &Arc<RaftClient>) -> Result<Arc<ConsistentHashing>, CHError> {
        match ConsistentHashing::new(group, raft_client) {
            Err(e) => Err(e),
            Ok(ch) => {
                match ch.init_table() {
                    Err(e) => Err(CHError::InitTableError(e)),
                    Ok(_) =>  Ok(ch.clone())
                }
            },
        }
    }
    pub fn init_table(&self) -> Result<(), InitTableError> {
        let mut table = &mut self.tables.write();
        self.init_table_(&mut table)
    }
    pub fn to_server_name(&self, server_id: Option<u64>) -> Option<String> {
        let lookup_table = self.tables.read();
        match server_id {
            Some(id) => Some(lookup_table.addrs.get(&id).unwrap().clone()),
            None => None
        }
    }
    pub fn get_server_id(&self, hash: u64) -> Option<u64> {
        let lookup_table = self.tables.read();
        let nodes = &lookup_table.nodes;
        let len = nodes.len();
        if len == 0 {return None;}
        let first_node = nodes.first().unwrap();
        let last_node = nodes.last().unwrap();
        if hash < first_node.start {return Some(first_node.server)}
        if hash >= last_node.start {return Some(last_node.server)}
        let mut low = 0;
        let mut high = len - 1;
        while low <= high {
            let mid = low + (high - low) / 2;
            let curr_node = &nodes[mid];
            let next_node = &nodes[mid + 1];
            if hash >= curr_node.start && hash < next_node.start {
                return Some(curr_node.server);
            } else if hash < curr_node.start {
                high = mid - 1;
            } else {
                low = mid + 1;
            }
        }
        None
    }
    pub fn get_server(&self, hash: u64) -> Option<String> {
        self.to_server_name(self.get_server_id(hash))
    }
    pub fn get_server_by_string(&self, string: &String) -> Option<String> {
        self.get_server(hash_str(string))
    }
    pub fn get_server_by<T>(&self, obj: &T) -> Option<String> where T: serde::Serialize {
        self.get_server(hash_bytes(serialize(obj).as_slice()))
    }
    pub fn get_server_id_by_string(&self, string: &String) -> Option<u64> {
        self.get_server_id(hash_str(string))
    }
    pub fn get_server_id_by<T>(&self, obj: &T) -> Option<u64> where T: serde::Serialize {
        self.get_server_id(hash_bytes(serialize(obj).as_slice()))
    }
    pub fn rand_server(&self) -> Option<String> {
        let rand = rand::random::<u64>();
        self.get_server(rand)
    }
    pub fn nodes_count(&self) -> usize {
        let lookup_table = self.tables.read();
        return lookup_table.nodes.len();
    }
    pub fn set_weight(&self, server_name: &String, weight: u64) -> bool {
        let group_id = hash_str(&self.group_name);
        let server_id = hash_str(server_name);
        if let Ok(Ok(_)) = self.weight_sm_client.set_weight(&group_id, &server_id, &weight) { true } else { false }
    }
    pub fn watch_all_actions<F>(&self, f: F)
        where F: Fn(&Member, &Action, &LookupTables, &Vec<Node>) + 'static + Send + Sync {
        let mut watchers = self.watchers.write();
        watchers.push(Box::new(f));
    }
    pub fn watch_server_nodes_range_changed<F>(&self, server: &String, f: F)
        // return ranges [...,...)
        where F: Fn(Vec<(u64, u64)>) + 'static + Send + Sync {
        let server_id = hash_str(server);
        let wrapper = move |
            _: &Member,_: &Action,
            lookup_table: &LookupTables, _: &Vec<Node>| {
            let nodes = &lookup_table.nodes;
            let node_len = nodes.len();
            let mut ranges: Vec<(u64, u64)> = Vec::new();
            for ni in 0..node_len {
                let node = &nodes[ni];
                if node.server == server_id {
                    ranges.push((node.start, if ni == node_len - 1 {
                        std::u64::MAX
                    } else {
                        nodes[ni + 1].start
                    }));
                }
            }
            if !ranges.is_empty() {f(ranges);}
        };
        self.watch_all_actions(wrapper);
    }

    fn init_table_(&self, lookup_table: &mut RwLockWriteGuard<LookupTables>) -> Result<(), InitTableError> {
        if let Ok(Ok((members, version))) = self.membership.group_members(&self.group_name, true) {
            let group_id = hash_str(&self.group_name);
            match self.weight_sm_client.get_weights(&group_id) {
                Ok(Ok(Some(weights))) =>  {
                    if let Some(min_weight) = weights.values().min() {
                        lookup_table.nodes.clear(); // refresh nodes
                        let mut factors: HashMap<u64, f64> = HashMap::new();
                        let min_weight = *min_weight as f64;
                        for member in members.iter() {
                            let k = member.id;
                            let w = match weights.get(&k) {
                                Some(w) => *w as f64,
                                None => min_weight,
                            };
                            factors.insert(k, w / min_weight);
                        }
                        let factor_sum: f64 = factors.values().sum();
                        let nodes_size = if factor_sum > DEFAULT_NODE_LIST_SIZE {factor_sum} else {DEFAULT_NODE_LIST_SIZE};
                        let factor_scale = nodes_size / factor_sum;
                        for member in members.iter() {lookup_table.addrs.insert(member.id, member.address.clone());}
                        for (k, f) in factors.iter() {
                            let weight = (*f * factor_scale) as u64;
                            for i in 0..weight {
                                let node_key = format!("{}_{}", k, i);
                                let hash = hash_str(&node_key);
                                lookup_table.nodes.push(Node{
                                    start: hash, server: *k
                                });
                            }
                        };
                        lookup_table.nodes.sort_by(|n1, n2| n1.start.cmp(&n2.start));
                        self.version.store(version, Ordering::Relaxed);
                        Ok(())
                    } else {
                        Err(InitTableError::NoWeightInfo)
                    }
                },
                Err(_) => Err(InitTableError::NoWeightService),
                Ok(Ok(None)) => Err(InitTableError::NoWeightGroup),
                _ => Err(InitTableError::Unknown)

            }
        } else {
            Err(InitTableError::GroupNotExisted)
        }
    }
}

fn server_joined(ch: &Arc<ConsistentHashing>, member: Member, version: u64) {
    server_changed(ch, member, Action::Joined, version);
}
fn server_left(ch: &Arc<ConsistentHashing>, member: Member, version: u64) {
    server_changed(ch, member, Action::Left, version);
}
fn server_changed(ch: &Arc<ConsistentHashing>, member: Member, action: Action, version: u64) {
    let ch_version = ch.version.load(Ordering::Relaxed);
    if ch_version < version {
        let ch = ch.clone();
        thread::spawn(move || {
            let mut lookup_table = ch.tables.write();
            let watchers = ch.watchers.read();
            let ch_version = ch.version.load(Ordering::Relaxed);
            if ch_version >= version {return;}
            let old_nodes = lookup_table.nodes.clone();
            ch.init_table_(&mut lookup_table);
            for watch in watchers.iter() {
                watch(&member, &action, &*lookup_table, &old_nodes);
            }
        });
    }
}<|endoftext|>


<commit_before>extern crate libc;

pub const TB_DEFAULT   : u16 = 0x00;
pub const TB_BLACK     : u16 = 0x01;
pub const TB_RED       : u16 = 0x02;
pub const TB_GREEN     : u16 = 0x03;
pub const TB_YELLOW    : u16 = 0x04;
pub const TB_BLUE      : u16 = 0x05;
pub const TB_MAGENTA   : u16 = 0x06;
pub const TB_CYAN      : u16 = 0x07;
pub const TB_WHITE     : u16 = 0x08;

pub const TB_BOLD      : u16 = 0x0100;
pub const TB_UNDERLINE : u16 = 0x0200;
pub const TB_REVERSE   : u16 = 0x0400;

#[repr(C)]
pub struct tb_cell {
    pub ch: u32,
    pub fg: u16,
    pub bg: u16,
}

pub const TB_EUNSUPPORTED_TERMINAL : libc::c_int = -1;
pub const TB_EFAILED_TO_OPEN_TTY   : libc::c_int = -2;

pub const TB_HIDE_CURSOR      : libc::c_int = -1;

pub const TB_OUTPUT_CURRENT   : libc::c_int = 0;
pub const TB_OUTPUT_NORMAL    : libc::c_int = 1;
pub const TB_OUTPUT_256       : libc::c_int = 2;
pub const TB_OUTPUT_216       : libc::c_int = 3;
pub const TB_OUTPUT_GRAYSCALE : libc::c_int = 4;

extern {
    pub fn tb_init() -> libc::c_int;
    pub fn tb_resize();
    pub fn tb_shutdown();
    pub fn tb_width() -> libc::c_int;
    pub fn tb_height() -> libc::c_int;
    pub fn tb_clear() -> libc::c_int;
    pub fn tb_set_clear_attributes(fg: u16, bg: u16);
    pub fn tb_present();
    pub fn tb_set_cursor(cx: libc::c_int, cy: libc::c_int);
    pub fn tb_put_cell(x: libc::c_int, y: libc::c_int, cell: tb_cell);
    pub fn tb_change_cell(x: libc::c_int, y: libc::c_int, ch: u32, fg: u16, bg: u16);
    pub fn tb_cell_buffer() -> *mut tb_cell;
    pub fn tb_select_output_mode(mode: libc::c_int);
}
<commit_msg>termbox: Implement higher-level API based on an abstract Termbox struct<commit_after>extern crate libc;

pub const TB_DEFAULT   : u16 = 0x00;
pub const TB_BLACK     : u16 = 0x01;
pub const TB_RED       : u16 = 0x02;
pub const TB_GREEN     : u16 = 0x03;
pub const TB_YELLOW    : u16 = 0x04;
pub const TB_BLUE      : u16 = 0x05;
pub const TB_MAGENTA   : u16 = 0x06;
pub const TB_CYAN      : u16 = 0x07;
pub const TB_WHITE     : u16 = 0x08;

pub const TB_BOLD      : u16 = 0x0100;
pub const TB_UNDERLINE : u16 = 0x0200;
pub const TB_REVERSE   : u16 = 0x0400;

#[repr(C)]
pub struct Cell {
    pub ch: u32,
    pub fg: u16,
    pub bg: u16,
}

const TB_EUNSUPPORTED_TERMINAL : libc::c_int = -1;
const TB_EFAILED_TO_OPEN_TTY   : libc::c_int = -2;

const TB_HIDE_CURSOR      : libc::c_int = -1;

const TB_OUTPUT_CURRENT   : libc::c_int = 0;
const TB_OUTPUT_NORMAL    : libc::c_int = 1;
// These are not used, we just std::mem::transmute the value if it's in range
// const TB_OUTPUT_256       : libc::c_int = 2;
// const TB_OUTPUT_216       : libc::c_int = 3;
const TB_OUTPUT_GRAYSCALE : libc::c_int = 4;

extern {
    fn tb_init() -> libc::c_int;
    fn tb_resize();
    fn tb_shutdown();
    fn tb_width() -> libc::c_int;
    fn tb_height() -> libc::c_int;
    fn tb_clear() -> libc::c_int;
    fn tb_set_clear_attributes(fg: u16, bg: u16);
    fn tb_present();
    fn tb_set_cursor(cx: libc::c_int, cy: libc::c_int);
    fn tb_put_cell(x: libc::c_int, y: libc::c_int, cell: Cell);
    fn tb_change_cell(x: libc::c_int, y: libc::c_int, ch: u32, fg: u16, bg: u16);
    // fn tb_cell_buffer() -> *mut tb_cell;
    fn tb_select_output_mode(mode: libc::c_int) -> libc::c_int;
}

pub struct Termbox {}

#[derive(Debug)]
pub enum InitError { UnsupportedTerminal, FailedToOpenTty }

#[repr(C)]
pub enum OutputMode {
    OutputNormal = 1, Output256, Output216, OutputGrayscale
}

impl Termbox {
    pub fn init() -> Result<Termbox, InitError> {
        let ret = unsafe { tb_init() };
        if ret == TB_EUNSUPPORTED_TERMINAL {
            Err(InitError::UnsupportedTerminal)
        } else if ret == TB_EFAILED_TO_OPEN_TTY {
            Err(InitError::FailedToOpenTty)
        } else {
            Ok(Termbox {})
        }
    }

    pub fn resize(&mut self) {
        unsafe { tb_resize(); }
    }

    pub fn width(&self) -> i32 {
        unsafe { tb_width() as i32 }
    }

    pub fn height(&self) -> i32 {
        unsafe { tb_height() as i32 }
    }

    pub fn clear(&mut self) {
        unsafe { tb_clear(); }
    }

    pub fn set_clear_attributes(&mut self, fg: u16, bg: u16) {
        unsafe { tb_set_clear_attributes(fg, bg) }
    }

    pub fn present(&mut self) {
        unsafe { tb_present() }
    }

    pub fn hide_cursor(&mut self) {
        unsafe { tb_set_cursor(TB_HIDE_CURSOR, TB_HIDE_CURSOR); }
    }

    pub fn set_cursor(&mut self, cx: i32, cy: i32) {
        unsafe { tb_set_cursor(cx as libc::c_int, cy as libc::c_int) }
    }

    pub fn put_cell(&mut self, x: i32, y: i32, cell: Cell) {
        unsafe { tb_put_cell(x as libc::c_int, y as libc::c_int, cell) }
    }

    pub fn change_cell(&mut self, x: i32, y: i32, ch: char, fg: u16, bg: u16) {
        // FIXME: This is assuming that the char is represented as its utf-8 encoding!
        unsafe { tb_change_cell(x as libc::c_int, y as libc::c_int, ch as u32, fg, bg) }
    }

    pub fn get_output_mode(&self) -> OutputMode {
        let ret = unsafe { tb_select_output_mode(TB_OUTPUT_CURRENT) };
        if ret >= TB_OUTPUT_NORMAL && ret <= TB_OUTPUT_GRAYSCALE {
            unsafe { std::mem::transmute(ret) }
        } else {
            panic!("get_output_mode(): Invalid output mode: {}", ret)
        }
    }

    pub fn set_output_mode(&mut self, mode: OutputMode) {
        unsafe { tb_select_output_mode(std::mem::transmute(mode)); }
    }
}

impl Drop for Termbox {
    fn drop(&mut self) {
        unsafe { tb_shutdown(); }
    }
}
<|endoftext|>


<commit_before><commit_msg>do not panic on initial load_state<commit_after><|endoftext|>


<commit_before>use crate::codecs::DataFormat;
use crate::codecs::Encodable;
use std::io::Result;
use std::io::Write;

const ID: u64 = 3;
const MEDIA_TYPE: &str = "text/plain";

///Simple unstructured text format. Any applications which just want to exchange plain text may used
/// (e.g. a chat clients, or an application used to exchang text files between peers).
pub struct PlainTextDataFormat;
impl DataFormat for PlainTextDataFormat {
    ///Returns three, the id of the most simple text encoder
    #[inline]
    fn id() -> u64 {
        ID
    }
    #[inline]
    fn media_type() -> &'static str {
        MEDIA_TYPE
    }
}

impl Encodable<PlainTextDataFormat> for String {
    #[inline]
    fn encode_to(&self, _format: &PlainTextDataFormat, w: &mut impl Write) -> Result<usize> {
        w.write(self.as_bytes())
    }
}

impl<'a> Encodable<PlainTextDataFormat> for &'a str {
    #[inline]
    fn encode_to(&self, _format: &PlainTextDataFormat, w: &mut impl Write) -> Result<usize> {
        w.write(self.as_bytes())
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::io::Cursor;

    #[test]
    fn check_plain_text_encoder() {
        let mut vec = Vec::<u8>::new();
        let mut cursor = Cursor::new(&mut vec);
        let df = PlainTextDataFormat;
        let msg = "They are who we thought they are";
        msg.encode_to(&df, &mut cursor).unwrap();
        assert_eq!(cursor.position() as usize, msg.len());
        msg.to_string().encode_to(&df, &mut cursor).unwrap();
        assert_eq!(cursor.position() as usize, 2 * msg.len());
    }

    #[test]
    fn check_data_format() {
        assert_eq!(PlainTextDataFormat::id(), ID);
        assert_eq!(PlainTextDataFormat::media_type(), MEDIA_TYPE);
    }
}
<commit_msg>Change to a generic blanket implementation for PlainTextDataFormat.<commit_after>use crate::codecs::DataFormat;
use crate::codecs::Encodable;
use std::io::Result;
use std::io::Write;

const ID: u64 = 3;
const MEDIA_TYPE: &str = "text/plain";

///Simple unstructured text format. Any applications which just want to exchange plain text may used
/// (e.g. a chat clients, or an application used to exchang text files between peers).
pub struct PlainTextDataFormat;
impl DataFormat for PlainTextDataFormat {
    ///Returns three, the id of the most simple text encoder
    #[inline]
    fn id() -> u64 {
        ID
    }
    #[inline]
    fn media_type() -> &'static str {
        MEDIA_TYPE
    }
}

impl<T: AsRef<str>> Encodable<PlainTextDataFormat> for T {
    #[inline]
    fn encode_to(&self, _format: &PlainTextDataFormat, w: &mut impl Write) -> Result<usize> {
        w.write(self.as_ref().as_bytes())
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::io::Cursor;

    #[test]
    fn check_plain_text_encoder() {
        let mut vec = Vec::<u8>::new();
        let mut cursor = Cursor::new(&mut vec);
        let df = PlainTextDataFormat;
        let msg = "They are who we thought they are";
        msg.encode_to(&df, &mut cursor).unwrap();
        assert_eq!(cursor.position() as usize, msg.len());
        msg.to_string().encode_to(&df, &mut cursor).unwrap();
        assert_eq!(cursor.position() as usize, 2 * msg.len());
    }

    #[test]
    fn check_data_format() {
        assert_eq!(PlainTextDataFormat::id(), ID);
        assert_eq!(PlainTextDataFormat::media_type(), MEDIA_TYPE);
    }
}
<|endoftext|>


<commit_before>// The size of an ELF header tag, in bytes. This was calculated
// from reading the ELF format specification.
const ELF_SECTION_HEADER_SIZE: u64 = 56;

#[repr(C, packed)]
pub struct ElfSectionTag {
    tag_type: u32,
    size: u32,
    num: u16,
    entsize: u16,
    shndx: u16,
    reserved: u16,
    first_section: ElfSectionHeader
}

impl ElfSectionTag {
    pub fn elf_sections(&'static self) -> ElfSectionIter {
        ElfSectionIter {
            current_section: &self.first_section,
            section_index: 0,
            total_sections: self.num as u32,
            entry_size: self.entsize as u64
        }
    }

    pub fn section_count(&self) -> usize {
        self.num as usize
    }

    pub fn size(&self) -> usize {
        self.size as usize
    }

    pub fn entry_size(&self) -> usize {
        self.entsize as usize
    }
}

#[repr(C)]
pub struct ElfSectionHeader {
    sh_name: u32,
    sh_type: u32,
    sh_flags: u64,
    sh_addr: u64,
    sh_offset: u64,
    sh_size: u64,
    sh_link: u32,
    sh_info: u32,
    sh_addralign: u64,
    sh_entsize: u64
}

impl ElfSectionHeader {
    pub fn name(&self) -> usize {
        self.sh_name as usize
    }

    pub fn section_type(&self) -> usize {
        self.sh_type as usize
    }

    pub fn start_address(&self) -> usize {
        self.sh_addr as usize
    }

    pub fn end_address(&self) -> usize {
        (self.sh_addr + self.sh_size) as usize
    }

    pub fn size(&self) -> usize {
        self.sh_size as usize
    }

    pub fn flags(&self) -> usize {
        self.sh_flags as usize
    }

    pub fn entry_size(&self) -> usize {
        self.sh_entsize as usize
    }
}

pub struct ElfSectionIter {
    current_section: &'static ElfSectionHeader,
    section_index: u32,
    total_sections: u32,
    entry_size: u64
}

impl Iterator for ElfSectionIter {
    type Item = &'static ElfSectionHeader;

    fn next(&mut self) -> Option<Self::Item> {
        if self.section_index >= self.total_sections {
            return None;
        } else {
            let section = self.current_section;
            let next_section_addr =
                (self.current_section as *const ElfSectionHeader as u64) + self.entry_size;
            let next_section = unsafe {
                &*(next_section_addr as *const ElfSectionHeader)
            };

            self.current_section = next_section;
            self.section_index += 1;

            Some(section)
        }
    }
}
<commit_msg>Bug fixes.<commit_after>// The size of an ELF header tag, in bytes. This was calculated
// from reading the ELF format specification.
const ELF_SECTION_HEADER_SIZE: u64 = 56;

#[repr(C, packed)]
pub struct ElfSectionTag {
    tag_type: u32,
    size: u32,
    num: u32,
    entsize: u32,
    shndx: u32,
    //reserved: u16,
    first_section: ElfSectionHeader
}

impl ElfSectionTag {
    pub fn elf_sections(&'static self) -> ElfSectionIter {
        ElfSectionIter {
            current_section: &self.first_section,
            section_index: 0,
            total_sections: self.num as u32,
            entry_size: self.entsize as u64
        }
    }

    pub fn section_count(&self) -> usize {
        self.num as usize
    }

    pub fn size(&self) -> usize {
        self.size as usize
    }

    pub fn entry_size(&self) -> usize {
        self.entsize as usize
    }
}

#[repr(C)]
pub struct ElfSectionHeader {
    sh_name: u32,
    sh_type: u32,
    sh_flags: u64,
    sh_addr: u64,
    sh_offset: u64,
    sh_size: u64,
    sh_link: u32,
    sh_info: u32,
    sh_addralign: u64,
    sh_entsize: u64
}

impl ElfSectionHeader {
    pub fn section_type(&self) -> usize {
        self.sh_type as usize
    }

    pub fn section_start_address(&self) -> usize {
        self.sh_addr as usize
    }

    pub fn section_end_address(&self) -> usize {
        (self.sh_addr + self.sh_size) as usize
    }

    pub fn size(&self) -> usize {
        self.sh_size as usize
    }

    pub fn flags(&self) -> usize {
        self.sh_flags as usize
    }

    pub fn entry_size(&self) -> usize {
        self.sh_entsize as usize
    }
}

pub struct ElfSectionIter {
    current_section: &'static ElfSectionHeader,
    section_index: u32,
    total_sections: u32,
    entry_size: u64
}

impl Iterator for ElfSectionIter {
    type Item = &'static ElfSectionHeader;

    fn next(&mut self) -> Option<Self::Item> {
        if self.section_index >= self.total_sections {
            return None;
        } else {
            let section = self.current_section;
            let next_section_addr =
                (self.current_section as *const ElfSectionHeader as u64) + self.entry_size;
            let next_section = unsafe {
                &*(next_section_addr as *const ElfSectionHeader)
            };

            self.current_section = next_section;
            self.section_index += 1;

            Some(section)
        }
    }
}
<|endoftext|>