<commit_before>use std::path::Path;
use std::fs::File;
use std::f64;

use image::{ DynamicImage, GenericImage, ImageFormat };
use image;

use bitreader::BitReader;

use num_traits::float::Float;

pub struct CoverImage {
    cover_image: DynamicImage
}

impl CoverImage {
    pub fn new(file_path: &str) -> CoverImage {
        CoverImage {
            cover_image: image::open(&Path::new(&file_path)).unwrap()
        }
    }

    pub fn encode_with(&self, message: &str) {
        let bit_vector = get_bit_vec(message);

        encode(&self.cover_image, &bit_vector);

        let ref mut fout = File::create(&Path::new("/home/hugh/Pictures/yurt.png")).unwrap();

        let _ = self.cover_image.save(fout, ImageFormat::PNG).unwrap();
    }
}

// Get a Vector of Bits to Encode the Image with
fn get_bit_vec(message: &str) -> Vec<u8> {
    let mut bit_vector = Vec::new();
    let mut reader = BitReader::new(&message.as_bytes());

    // Multiplied by 8 because it's a Vec of bits not bytes
    for _ in 0..message.len() * 8 {
        bit_vector.push(reader.read_u8(1).unwrap());
    }

    bit_vector
}

// Encode the image with the Vector of Bits
fn encode(c_image: &DynamicImage, bit_vec: &Vec<u8>) {
    let mut tiled_image_vec = tile_image(&c_image);

    let (width, height) = c_image.dimensions();

    let encoded_image = encode_image(&mut tiled_image_vec, bit_vec, width, height);
}

fn tile_image(c_image: &DynamicImage) -> Vec<image::Rgba<u8>> {
    let (width, height) = c_image.dimensions();

    let mut image_blocks = Vec::new();

    for row_index in 0..(height / 8) as u32 {
        for col_index in 0..(width / 8) as u32 {
            for row in 0..8 {
                for column in 0..8 {
                    image_blocks.push(c_image.get_pixel(column + (col_index * 8), row + (row_index * 8)));
                }
            }
        }
    }

    image_blocks
}

fn encode_image(tiled_image: &mut Vec<image::Rgba<u8>>, message: &Vec<u8>, width: u32, height: u32) {
    // 1
    // - + Cv + Cu
    // 4
    let constants = 1 / 128;

    /*let mut cu = 0.0;
    let mut cv = 0.0;*/
    let mut total = 0.0;
    let mut index: usize = 0;
    let num_of_iterations = if message.len() / 3 == 0 {
                                    (message.len() / 3) as u32
                                } else {
                                    ((message.len() / 3) + 1) as u32
                                };

    for _ in 0..num_of_iterations {
        for channel in 0..3 {
            index = 36 + (num_of_iterations * 64) as usize;

            let colour_value = tiled_image[index].data[channel];

            for y in 0..8 {
                for x in 0..8 {
                    /*if u == 0 {
                        cu = 0.0
                    } else {
                        cu = 1.0 / 2.0.sqrt()
                    }

                    if v == 0 {
                        cv = 0.0
                    } else {
                        cv = 1.0 / 2.0.sqrt()
                    }*/

                    total = total + (4.0 * f64::consts::PI * ((2.0 * y as f64) + 1.0)).cos() *
                                    (4.0 * f64::consts::PI * ((2.0 * x as f64) + 1.0)).cos();
                }
            }

            total = total * constants;
        }
    }
}

/**************************************************************************************************
 *                                                                                *
 *                                         Tests                                                  *
 *                                                                                *
 **************************************************************************************************/

#[cfg(test)]
#[test]
fn test_get_bit_vec_len() {
    let bits = get_bit_vec("h");

    assert_eq!(8, bits.len());
}

#[cfg(test)]
#[test]
fn test_get_bit_vec() {
    let test_bits = vec![0, 1, 1, 0, 1, 0, 0, 0];
    let bits = get_bit_vec("h");

    assert_eq!(bits, test_bits);
}

#[cfg(test)]
#[test]
fn test_encode_red_channel_lsb_set() {
    let mut img = image::open(&Path::new("/home/hugh/Pictures/scenery.jpg")).unwrap();
    let bit_vec = vec![0, 1, 1, 0, 1, 0, 0, 0];

    encode(&mut img, &bit_vec);

    let pixel = img.get_pixel(0, 0);

    assert_eq!(pixel.data[0] % 2, bit_vec[0]);
}

#[cfg(test)]
#[test]
fn test_encode_green_channel_lsb_set() {
    let mut img = image::open(&Path::new("/home/hugh/Pictures/scenery.jpg")).unwrap();
    let bit_vec = vec![0, 1, 1, 0, 1, 0, 0, 0];

    encode(&mut img, &bit_vec);

    let pixel = img.get_pixel(0, 0);

    assert_eq!(pixel.data[1] % 2, bit_vec[1]);
}

#[cfg(test)]
#[test]
fn test_encode_blue_channel_lsb_set() {
    let mut img = image::open(&Path::new("/home/hugh/Pictures/scenery.jpg")).unwrap();
    let bit_vec = vec![0, 1, 1, 0, 1, 0, 0, 0];

    encode(&mut img, &bit_vec);

    let pixel = img.get_pixel(0, 0);

    assert_eq!(pixel.data[2] % 2 as u8, bit_vec[2]);
}

#[cfg(test)]
#[test]
fn test_full_byte_is_encoded() {
    let mut img = image::open(&Path::new("/home/hugh/Pictures/scenery.jpg")).unwrap();
    let mut encoded_bit_vec = Vec::new();
    let mut count = 0;

    let bit_vec = vec![0, 1, 1, 0, 1, 0, 0, 0];

    encode(&mut img, &bit_vec);

    'outer: for x_co_ord in 0..3 {
        let pixel = img.get_pixel(x_co_ord, 0);

        for channel in 0..3 {
            if count >= 8 {
                break 'outer;
            }

            encoded_bit_vec.push(pixel.data[channel] % 2);

            count += 1;
        }
    }

    assert_eq!(encoded_bit_vec, bit_vec);
}

#[cfg(test)]
#[test]
fn test_tile_image_length() {
    let img = image::open(&Path::new("/home/hugh/Pictures/colour.jpg")).unwrap();

    let vec = tile_image(&img);

    assert_eq!(vec.len(), 91200);
}

#[cfg(test)]
#[test]
fn test_tile_image_first_pixel() {
    let img = image::open(&Path::new("/home/hugh/Pictures/project.jpg")).unwrap();

    let pixel = img.get_pixel(0, 0);

    let vec = tile_image(&img);

    assert_eq!(vec[0], pixel);
}

#[cfg(test)]
#[test]
fn test_tile_image_random_pixel1() {
    let img = image::open(&Path::new("/home/hugh/Pictures/project.jpg")).unwrap();

    let pixel = img.get_pixel(15, 7);

    let vec = tile_image(&img);

    assert_eq!(vec[127], pixel);
}

#[cfg(test)]
#[test]
fn test_tile_image_random_pixel2() {
    let img = image::open(&Path::new("/home/hugh/Pictures/project.jpg")).unwrap();

    let pixel = img.get_pixel(40, 16);

    let vec = tile_image(&img);

    assert_eq!(vec[6720], pixel);
}

#[cfg(test)]
#[test]
fn test_tile_image_random_pixel3() {
    let img = image::open(&Path::new("/home/hugh/Pictures/project.jpg")).unwrap();

    let pixel = img.get_pixel(15, 7);

    let vec = tile_image(&img);

    assert_eq!(vec[127], pixel);
}

#[cfg(test)]
#[test]
fn test_tile_image_last_pixel() {
    let img = image::open(&Path::new("/home/hugh/Pictures/project.jpg")).unwrap();

    let pixel = img.get_pixel(399, 399);

    let vec = tile_image(&img);

    assert_eq!(vec[159999], pixel);
}
<commit_msg>Start endocing message<commit_after>use std::path::Path;
use std::fs::File;
use std::f64;

use image::{ DynamicImage, GenericImage, ImageFormat };
use image;

use bitreader::BitReader;

use num_traits::float::Float;

pub struct CoverImage {
    cover_image: DynamicImage
}

impl CoverImage {
    pub fn new(file_path: &str) -> CoverImage {
        CoverImage {
            cover_image: image::open(&Path::new(&file_path)).unwrap()
        }
    }

    pub fn encode_with(&self, message: &str) {
        let bit_vector = get_bit_vec(message);

        encode(&self.cover_image, &bit_vector);

        let ref mut fout = File::create(&Path::new("/home/hugh/Pictures/yurt.png")).unwrap();

        let _ = self.cover_image.save(fout, ImageFormat::PNG).unwrap();
    }
}

// Get a Vector of Bits to Encode the Image with
fn get_bit_vec(message: &str) -> Vec<u8> {
    let mut bit_vector = Vec::new();
    let mut reader = BitReader::new(&message.as_bytes());

    // Multiplied by 8 because it's a Vec of bits not bytes
    for _ in 0..message.len() * 8 {
        bit_vector.push(reader.read_u8(1).unwrap());
    }

    bit_vector
}

// Encode the image with the Vector of Bits
fn encode(c_image: &DynamicImage, bit_vec: &Vec<u8>) {
    let mut tiled_image_vec = tile_image(&c_image);

    let (width, height) = c_image.dimensions();

    let encoded_image = encode_image(&mut tiled_image_vec, bit_vec, width, height);
}

fn tile_image(c_image: &DynamicImage) -> Vec<image::Rgba<u8>> {
    let (width, height) = c_image.dimensions();

    let mut image_blocks = Vec::new();

    for row_index in 0..(height / 8) as u32 {
        for col_index in 0..(width / 8) as u32 {
            for row in 0..8 {
                for column in 0..8 {
                    image_blocks.push(c_image.get_pixel(column + (col_index * 8), row + (row_index * 8)));
                }
            }
        }
    }

    image_blocks
}

fn encode_image(tiled_image: &mut Vec<image::Rgba<u8>>, message: &Vec<u8>, width: u32, height: u32) {
    /*let mut cu = 0.0;
    let mut cv = 0.0;*/
    let mut total = 0.0;
    let mut index: usize = 0;
    let mut count = 0;
    let num_of_iterations = if message.len() / 3 == 0 {
                                    (message.len() / 3) as u32
                                } else {
                                    ((message.len() / 3) + 1) as u32
                                };

    for _ in 0..num_of_iterations {
        for channel in 0..3 {
            index = 36 + (num_of_iterations * 64) as usize;

            let colour_value = tiled_image[index].data[channel];

            for y in 0..8 {
                for x in 0..8 {
                    // These are scaling factors, needed in future
                    /*if u == 0 {
                        cu = 0.0
                    } else {
                        cu = 1.0 / 2.0.sqrt()
                    }

                    if v == 0 {
                        cv = 0.0
                    } else {
                        cv = 1.0 / 2.0.sqrt()
                    }*/

                    total = total + (4.0 * f64::consts::PI * ((2.0 * y as f64) + 1.0) / 16.0).cos() *
                                    (4.0 * f64::consts::PI * ((2.0 * x as f64) + 1.0) / 16.0).cos() *
                                    colour_value as f64;
                }
            }

            // 0.25 is another scaling factor
            total = total * 0.25;

            // Encode the message
            if total.trunc() % 2.0 == 0.0 {
                if message[count] == 1 {
                    total = total + 1.0
                }
            }
            else {
                if message[count] == 0 {
                    total = total + 1.0
                }
            }
        }
    }
}

/**************************************************************************************************
 *                                                                                *
 *                                         Tests                                                  *
 *                                                                                *
 **************************************************************************************************/

#[cfg(test)]
#[test]
fn test_get_bit_vec_len() {
    let bits = get_bit_vec("h");

    assert_eq!(8, bits.len());
}

#[cfg(test)]
#[test]
fn test_get_bit_vec() {
    let test_bits = vec![0, 1, 1, 0, 1, 0, 0, 0];
    let bits = get_bit_vec("h");

    assert_eq!(bits, test_bits);
}

#[cfg(test)]
#[test]
fn test_encode_red_channel_lsb_set() {
    let mut img = image::open(&Path::new("/home/hugh/Pictures/scenery.jpg")).unwrap();
    let bit_vec = vec![0, 1, 1, 0, 1, 0, 0, 0];

    encode(&mut img, &bit_vec);

    let pixel = img.get_pixel(0, 0);

    assert_eq!(pixel.data[0] % 2, bit_vec[0]);
}

#[cfg(test)]
#[test]
fn test_encode_green_channel_lsb_set() {
    let mut img = image::open(&Path::new("/home/hugh/Pictures/scenery.jpg")).unwrap();
    let bit_vec = vec![0, 1, 1, 0, 1, 0, 0, 0];

    encode(&mut img, &bit_vec);

    let pixel = img.get_pixel(0, 0);

    assert_eq!(pixel.data[1] % 2, bit_vec[1]);
}

#[cfg(test)]
#[test]
fn test_encode_blue_channel_lsb_set() {
    let mut img = image::open(&Path::new("/home/hugh/Pictures/scenery.jpg")).unwrap();
    let bit_vec = vec![0, 1, 1, 0, 1, 0, 0, 0];

    encode(&mut img, &bit_vec);

    let pixel = img.get_pixel(0, 0);

    assert_eq!(pixel.data[2] % 2 as u8, bit_vec[2]);
}

#[cfg(test)]
#[test]
fn test_full_byte_is_encoded() {
    let mut img = image::open(&Path::new("/home/hugh/Pictures/scenery.jpg")).unwrap();
    let mut encoded_bit_vec = Vec::new();
    let mut count = 0;

    let bit_vec = vec![0, 1, 1, 0, 1, 0, 0, 0];

    encode(&mut img, &bit_vec);

    'outer: for x_co_ord in 0..3 {
        let pixel = img.get_pixel(x_co_ord, 0);

        for channel in 0..3 {
            if count >= 8 {
                break 'outer;
            }

            encoded_bit_vec.push(pixel.data[channel] % 2);

            count += 1;
        }
    }

    assert_eq!(encoded_bit_vec, bit_vec);
}

#[cfg(test)]
#[test]
fn test_tile_image_length() {
    let img = image::open(&Path::new("/home/hugh/Pictures/colour.jpg")).unwrap();

    let vec = tile_image(&img);

    assert_eq!(vec.len(), 91200);
}

#[cfg(test)]
#[test]
fn test_tile_image_first_pixel() {
    let img = image::open(&Path::new("/home/hugh/Pictures/project.jpg")).unwrap();

    let pixel = img.get_pixel(0, 0);

    let vec = tile_image(&img);

    assert_eq!(vec[0], pixel);
}

#[cfg(test)]
#[test]
fn test_tile_image_random_pixel1() {
    let img = image::open(&Path::new("/home/hugh/Pictures/project.jpg")).unwrap();

    let pixel = img.get_pixel(15, 7);

    let vec = tile_image(&img);

    assert_eq!(vec[127], pixel);
}

#[cfg(test)]
#[test]
fn test_tile_image_random_pixel2() {
    let img = image::open(&Path::new("/home/hugh/Pictures/project.jpg")).unwrap();

    let pixel = img.get_pixel(40, 16);

    let vec = tile_image(&img);

    assert_eq!(vec[6720], pixel);
}

#[cfg(test)]
#[test]
fn test_tile_image_random_pixel3() {
    let img = image::open(&Path::new("/home/hugh/Pictures/project.jpg")).unwrap();

    let pixel = img.get_pixel(15, 7);

    let vec = tile_image(&img);

    assert_eq!(vec[127], pixel);
}

#[cfg(test)]
#[test]
fn test_tile_image_last_pixel() {
    let img = image::open(&Path::new("/home/hugh/Pictures/project.jpg")).unwrap();

    let pixel = img.get_pixel(399, 399);

    let vec = tile_image(&img);

    assert_eq!(vec[159999], pixel);
}
<|endoftext|>


<commit_before>use std::cmp;
use std::mem;
use std::fmt;
use std::collections::BTreeMap;
use mio;

use rocksdb::{DB,Writable,Options, WriteBatch};
use rocksdb::ffi::DBCFHandle;
use tempdir::TempDir;
use byteorder::{ByteOrder, BigEndian};
use spki_sexp;

use super::{Operation,OpResp};
use event_handler::EventHandler;

const REPLICATION_CREDIT : u64 = 1024;

#[derive(Debug)]
struct Forwarder {
    last_sent_downstream: Option<u64>,
    last_acked_downstream: Option<u64>,
    pending_operations: BTreeMap<u64, mio::Token>,
}

#[derive(Debug)]
struct Terminus {
    state: String,
}

#[derive(Debug)]
enum Role {
    Forwarder(Forwarder),
    Terminus(Terminus),
}

struct Log {
    dir: TempDir,
    db: DB,
    meta: DBCFHandle,
    data: DBCFHandle,
}

const META : &'static str = "meta";
const DATA : &'static str = "data";
const META_COMMITTED : &'static str = "committed";

impl Log {
    fn new() -> Log {
        let d = TempDir::new("rocksdb-log").expect("new log");
        info!("DB path: {:?}", d.path());
        let mut db = DB::open_default(&d.path().to_string_lossy()).expect("open db");
        let meta = db.create_cf(META, &Options::new()).expect("open meta cf");
        let data = db.create_cf(DATA, &Options::new()).expect("open data cf");

        Log { dir: d, db: db, meta: meta, data: data }
    }

    fn tokey(seq: u64) -> [u8; 8] {
        let mut buf : [u8; 8] = [0, 0, 0, 0, 0, 0, 0, 0];
        BigEndian::write_u64(&mut buf, seq);
        buf
    }

    fn fromkey(key: &[u8]) -> u64 {
        BigEndian::read_u64(&key)
    }

    pub fn seqno(&self) -> u64 {
        match self.db.get_cf(self.meta, &META_COMMITTED.as_bytes()) {
            Ok(Some(val)) => {
                let current = Self::fromkey(&val);
                let next = current + 1;
                debug!("Last exisiting current: {:?}; next: {:?}", current, next);
                next
            },
            Ok(None) => 0,
            Err(e) => panic!("Unexpected error getting commit point: {:?}", e),
        }
    }

    fn read(&self, seqno: u64) -> Option<Operation> {
        let key = Self::tokey(seqno);
        let ret = match self.db.get_cf(self.data, &key.as_ref()) {
            Ok(Some(val)) => Some(spki_sexp::from_bytes(&val).expect("decode operation")),
            Ok(None) => None,
            Err(e) => panic!("Unexpected error reading index: {:?}: {:?}", seqno, e),
        };
        debug!("Read: {:?} => {:?}", seqno, ret);
        ret
    }

    fn verify_sequential(&self, seqno: u64) -> bool {
        let current = self.seqno();
        if !(seqno == 0 || current <= seqno) {
            warn!("Hole in history: saw {}; have: {:?}", seqno, current);
            false
        } else {
            true
        }
    }

    fn insert_at(&mut self, seqno: u64, op: &Operation) {
        let data_bytes = spki_sexp::as_bytes(op).expect("encode operation");
        debug!("Checking {:?}", seqno);
        let key = Self::tokey(seqno);
        /* match self.db.get_cf(self.data, &key.as_ref()) {
            Ok(None) => (),
            Err(e) => panic!("Unexpected error reading index: {:?}: {:?}", seqno, e),
            Ok(_) => panic!("Unexpected entry at seqno: {:?}", seqno),
        }; */

        let mut batch = WriteBatch::new();
        batch.put_cf(self.data, &key.as_ref(), &data_bytes).expect("Persist operation");
        batch.put_cf(self.meta, META_COMMITTED.as_bytes(), &key.as_ref()).expect("Persist operation");
        self.db.write(batch).expect("Write batch");
        debug!("Wrote as {:?}/{:?}: {:?}", seqno, key, data_bytes.len());
    }
}

#[derive(Debug)]
pub struct ReplModel {
    next: Role,
    log: Log,
    current_epoch: u64,
}
impl fmt::Debug for Log {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("Log").field("dir", &self.dir.path()).finish()
    }
}

impl Role {
    fn process_operation(&mut self, channel: &mut EventHandler, epoch: u64, seqno: u64, op: Operation) {
        match self {
            &mut Role::Forwarder(ref mut f) => f.process_operation(channel, epoch, seqno, op),
            &mut Role::Terminus(ref mut t) => t.process_operation(channel, epoch, seqno, op),
        }
    }
    pub fn process_downstream_response(&mut self, reply: &OpResp) -> Option<mio::Token> {
        match self {
            &mut Role::Forwarder(ref mut f) => f.process_downstream_response(reply),
            _ => None,
        }
    }

    pub fn process_replication<F: FnMut(u64, Operation)>(&mut self, log: &Log, forward: F) -> bool {
        match self {
            &mut Role::Forwarder(ref mut f) => f.process_replication(log, forward),
            _ => false,
        }
    }
    fn has_pending(&self, token: mio::Token) -> bool {
        match self {
            &Role::Forwarder(ref f) => f.has_pending(token),
            _ => false,
        }
    }
    fn reset(&mut self) {
        match self {
            &mut Role::Forwarder(ref mut f) => f.reset(),
            _ => (),
        }
    }
}

impl Forwarder {
    fn new() -> Forwarder {
        Forwarder {
            last_sent_downstream: None,
            last_acked_downstream: None,
            pending_operations: BTreeMap::new(),
        }
    }

    fn process_operation(&mut self, channel: &mut EventHandler, epoch: u64, seqno: u64, op: Operation) {
        self.pending_operations.insert(seqno, channel.token());
    }

    fn process_downstream_response(&mut self, reply: &OpResp) -> Option<mio::Token> {
        match reply {
            &OpResp::Ok(epoch, seqno, _) | &OpResp::Err(epoch, seqno, _) => {
                self.last_acked_downstream = Some(seqno);
                if let Some(token) = self.pending_operations.remove(&seqno) {
                    info!("Found in-flight op {:?} for client token {:?}", seqno, token);
                    Some(token)
                } else {
                    warn!("Unexpected response for seqno: {:?}", seqno);
                    None
                }
            },
            &OpResp::HelloIWant(last_sent_downstream) => {
                info!("Downstream has {:?}", last_sent_downstream);
                // assert!(last_sent_downstream <= self.seqno());
                self.last_acked_downstream = Some(last_sent_downstream);
                self.last_sent_downstream = Some(last_sent_downstream);
                None
            },
        }
    }

    pub fn process_replication<F: FnMut(u64, Operation)>(&mut self, log: &Log, mut forward: F) -> bool {
        let mut changed = false;
        debug!("pre  Repl: Ours: {:?}; downstream sent: {:?}; acked: {:?}",
            log.seqno(), self.last_sent_downstream, self.last_acked_downstream);

        let mut changed = false;
        if let Some(send_next) = self.last_sent_downstream {
            if send_next < log.seqno() {
                let max_to_push_now = cmp::min(self.last_acked_downstream.unwrap_or(0) + REPLICATION_CREDIT, log.seqno());
                debug!("Window push {:?} - {:?}; waiting: {:?}", send_next, max_to_push_now, log.seqno() - max_to_push_now);
                for i in send_next..max_to_push_now {
                    if let Some(op) = log.read(i) {
                        debug!("Queueing seq:{:?}/{:?}; ds/seqno: {:?}", i, op, self.last_sent_downstream);
                        forward(i, op);
                        self.last_sent_downstream = Some(i+1);
                        changed = true
                    } else {
                        panic!("Attempted to replicate item not in log: {:?}", self);
                    }
                }
            }
            debug!("post Repl: Ours: {:?}; downstream sent: {:?}; acked: {:?}",
                log.seqno(), self.last_sent_downstream, self.last_acked_downstream);
        }
        changed
    }

    fn has_pending(&self, token: mio::Token) -> bool {
        self.pending_operations.values().all(|tok| tok != &token)
    }

    fn reset(&mut self) {
        self.last_sent_downstream = None
    }
}

impl Terminus {
    fn new() -> Terminus {
        Terminus {
            state: "".to_string()
        }
    }

    fn process_operation(&mut self, channel: &mut EventHandler, epoch: u64, seqno: u64, op: Operation) {
        info!("Terminus! {:?}/{:?}", seqno, op);
        let resp = match op {
            Operation::Set(s) => {
                self.state = s;
                OpResp::Ok(epoch, seqno, None)
            },
                Operation::Get => OpResp::Ok(epoch, seqno, Some(self.state.clone()))
        };
        channel.response(resp)
    }
}

impl ReplModel {
    pub fn new() -> ReplModel {
        ReplModel {
            log: Log::new(),
            current_epoch: Default::default(),
            next: Role::Terminus(Terminus::new()),
        }
    }

    pub fn seqno(&self) -> u64 {
        self.log.seqno()
    }

    fn next_seqno(&self) -> u64 {
        self.log.seqno()
    }

    pub fn process_operation(&mut self, channel: &mut EventHandler, seqno: Option<u64>, epoch: Option<u64>, op: Operation) {
        let seqno = seqno.unwrap_or_else(|| self.next_seqno());
        let epoch = epoch.unwrap_or_else(|| self.current_epoch);

       if epoch != self.current_epoch {
            warn!("Operation epoch ({}) differers from our last observed configuration: ({})",
                epoch, self.current_epoch);
            let resp = OpResp::Err(epoch, seqno, format!("BadEpoch: {}; last seen config: {}", epoch, self.current_epoch));
            channel.response(resp);
            return;
        }

        if !self.log.verify_sequential(seqno) {
            let resp = OpResp::Err(epoch, seqno, format!("Bad sequence number; saw: {:?}", seqno));
            channel.response(resp);
            return;
        }

        self.log.insert_at(seqno, &op);
        self.next.process_operation(channel, epoch, seqno, op)
    }

    pub fn process_downstream_response(&mut self, reply: &OpResp) -> Option<mio::Token> {
        self.next.process_downstream_response(reply)
    }

    pub fn process_replication<F: FnMut(u64, u64, Operation)>(&mut self, mut forward: F) -> bool {
        let epoch = self.current_epoch;
        self.next.process_replication(&self.log, |i, op| forward(epoch, i, op))
    }

    pub fn epoch_changed(&mut self, epoch: u64) {
        self.current_epoch = epoch;
    }

    pub fn has_pending(&self, token: mio::Token) -> bool {
        self.next.has_pending(token)
    }

    pub fn reset(&mut self) {
        self.next.reset()
    }

    pub fn set_has_downstream(&mut self, is_forwarder: bool) {
        // XXX: Replays?
        match (is_forwarder, &mut self.next) {
            (true, role @ &mut Role::Terminus(_)) => {
                let prev = mem::replace(role, Role::Forwarder(Forwarder::new()));
                info!("Switched to forwarding from {:?}", prev);
            },

            (false, role @ &mut Role::Forwarder(_)) => {
                let prev = mem::replace(role, Role::Terminus(Terminus::new()));
                info!("Switched to terminating from {:?}", prev);
            },
            _ => { info!("No change of config"); }
        }
    }
}
<commit_msg>Record a prepared watermark seperately.<commit_after>use std::cmp;
use std::mem;
use std::fmt;
use std::collections::BTreeMap;
use mio;

use rocksdb::{DB,Writable,Options, WriteBatch};
use rocksdb::ffi::DBCFHandle;
use tempdir::TempDir;
use byteorder::{ByteOrder, BigEndian};
use spki_sexp;

use super::{Operation,OpResp};
use event_handler::EventHandler;

const REPLICATION_CREDIT : u64 = 1024;

#[derive(Debug)]
struct Forwarder {
    last_sent_downstream: Option<u64>,
    last_acked_downstream: Option<u64>,
    pending_operations: BTreeMap<u64, mio::Token>,
}

#[derive(Debug)]
struct Terminus {
    state: String,
}

#[derive(Debug)]
enum Role {
    Forwarder(Forwarder),
    Terminus(Terminus),
}

struct Log {
    dir: TempDir,
    db: DB,
    meta: DBCFHandle,
    data: DBCFHandle,
}

const META : &'static str = "meta";
const DATA : &'static str = "data";
const META_PREPARED : &'static str = "prepared";
const META_COMMITTED : &'static str = "committed";

impl Log {
    fn new() -> Log {
        let d = TempDir::new("rocksdb-log").expect("new log");
        info!("DB path: {:?}", d.path());
        let mut db = DB::open_default(&d.path().to_string_lossy()).expect("open db");
        let meta = db.create_cf(META, &Options::new()).expect("open meta cf");
        let data = db.create_cf(DATA, &Options::new()).expect("open data cf");

        Log { dir: d, db: db, meta: meta, data: data }
    }

    fn tokey(seq: u64) -> [u8; 8] {
        let mut buf : [u8; 8] = [0, 0, 0, 0, 0, 0, 0, 0];
        BigEndian::write_u64(&mut buf, seq);
        buf
    }

    fn fromkey(key: &[u8]) -> u64 {
        BigEndian::read_u64(&key)
    }

    pub fn seqno(&self) -> u64 {
        self.read_seqno(META_PREPARED)
    }

    pub fn read_seqno(&self, name: &str) -> u64 {
        match self.db.get_cf(self.meta, name.as_bytes()) {
            Ok(Some(val)) => {
                let current = Self::fromkey(&val);
                let next = current + 1;
                debug!("Last exisiting {:?}: {:?}; next: {:?}", name, current, next);
                next
            },
            Ok(None) => 0,
            Err(e) => panic!("Unexpected error getting commit point: {:?}", e),
        }
    }

    fn read(&self, seqno: u64) -> Option<Operation> {
        let key = Self::tokey(seqno);
        let ret = match self.db.get_cf(self.data, &key.as_ref()) {
            Ok(Some(val)) => Some(spki_sexp::from_bytes(&val).expect("decode operation")),
            Ok(None) => None,
            Err(e) => panic!("Unexpected error reading index: {:?}: {:?}", seqno, e),
        };
        debug!("Read: {:?} => {:?}", seqno, ret);
        ret
    }

    fn verify_sequential(&self, seqno: u64) -> bool {
        let current = self.seqno();
        if !(seqno == 0 || current <= seqno) {
            warn!("Hole in history: saw {}; have: {:?}", seqno, current);
            false
        } else {
            true
        }
    }

    fn prepare(&mut self, seqno: u64, op: &Operation) {
        let data_bytes = spki_sexp::as_bytes(op).expect("encode operation");
        debug!("Prepare {:?}", seqno);
        let key = Self::tokey(seqno);
        /* match self.db.get_cf(self.data, &key.as_ref()) {
            Ok(None) => (),
            Err(e) => panic!("Unexpected error reading index: {:?}: {:?}", seqno, e),
            Ok(_) => panic!("Unexpected entry at seqno: {:?}", seqno),
        }; */

        let mut batch = WriteBatch::new();
        batch.put_cf(self.data, &key.as_ref(), &data_bytes).expect("Persist operation");
        batch.put_cf(self.meta, META_PREPARED.as_bytes(), &key.as_ref()).expect("Persist prepare point");
        self.db.write(batch).expect("Write batch");
        debug!("Watermarks: prepared: {:?}; committed: {:?}",
                self.read_seqno(META_PREPARED),
                self.read_seqno(META_COMMITTED));
    }

    fn commit_to(&mut self, seqno: u64) {
        debug!("Commit {:?}", seqno);
        let key = Self::tokey(seqno);

        match self.db.get_cf(self.meta, META_PREPARED.as_bytes()) {
            Ok(Some(val)) => {
                let prepared = Self::fromkey(&val);
                debug!("Committing: {:?}, prepared: {:?}", seqno, prepared);
                assert!(prepared <= seqno);
            },
            Ok(None) => panic!("Committing {:?} without having prepared?", seqno),
            Err(e) => panic!("Unexpected error reading index: {:?}: {:?}", seqno, e),
        };

        let mut batch = WriteBatch::new();
        batch.put_cf(self.meta, META_COMMITTED.as_bytes(), &key.as_ref()).expect("Persist commit point");
        self.db.write(batch).expect("Write batch");
        debug!("Watermarks: prepared: {:?}; committed: {:?}",
                self.read_seqno(META_PREPARED),
                self.read_seqno(META_COMMITTED));
    }
}

#[derive(Debug)]
pub struct ReplModel {
    next: Role,
    log: Log,
    current_epoch: u64,
}
impl fmt::Debug for Log {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("Log").field("dir", &self.dir.path()).finish()
    }
}

impl Role {
    fn process_operation(&mut self, channel: &mut EventHandler, epoch: u64, seqno: u64, op: Operation) {
        match self {
            &mut Role::Forwarder(ref mut f) => f.process_operation(channel, epoch, seqno, op),
            &mut Role::Terminus(ref mut t) => t.process_operation(channel, epoch, seqno, op),
        }
    }
    pub fn process_downstream_response(&mut self, reply: &OpResp) -> Option<mio::Token> {
        match self {
            &mut Role::Forwarder(ref mut f) => f.process_downstream_response(reply),
            _ => None,
        }
    }

    pub fn process_replication<F: FnMut(u64, Operation)>(&mut self, log: &Log, forward: F) -> bool {
        match self {
            &mut Role::Forwarder(ref mut f) => f.process_replication(log, forward),
            _ => false,
        }
    }
    fn has_pending(&self, token: mio::Token) -> bool {
        match self {
            &Role::Forwarder(ref f) => f.has_pending(token),
            _ => false,
        }
    }
    fn reset(&mut self) {
        match self {
            &mut Role::Forwarder(ref mut f) => f.reset(),
            _ => (),
        }
    }
}

impl Forwarder {
    fn new() -> Forwarder {
        Forwarder {
            last_sent_downstream: None,
            last_acked_downstream: None,
            pending_operations: BTreeMap::new(),
        }
    }

    fn process_operation(&mut self, channel: &mut EventHandler, epoch: u64, seqno: u64, op: Operation) {
        self.pending_operations.insert(seqno, channel.token());
    }

    fn process_downstream_response(&mut self, reply: &OpResp) -> Option<mio::Token> {
        match reply {
            &OpResp::Ok(epoch, seqno, _) | &OpResp::Err(epoch, seqno, _) => {
                self.last_acked_downstream = Some(seqno);
                if let Some(token) = self.pending_operations.remove(&seqno) {
                    info!("Found in-flight op {:?} for client token {:?}", seqno, token);
                    Some(token)
                } else {
                    warn!("Unexpected response for seqno: {:?}", seqno);
                    None
                }
            },
            &OpResp::HelloIWant(last_sent_downstream) => {
                info!("Downstream has {:?}", last_sent_downstream);
                // assert!(last_sent_downstream <= self.seqno());
                self.last_acked_downstream = Some(last_sent_downstream);
                self.last_sent_downstream = Some(last_sent_downstream);
                None
            },
        }
    }

    pub fn process_replication<F: FnMut(u64, Operation)>(&mut self, log: &Log, mut forward: F) -> bool {
        let mut changed = false;
        debug!("pre  Repl: Ours: {:?}; downstream sent: {:?}; acked: {:?}",
            log.seqno(), self.last_sent_downstream, self.last_acked_downstream);

        let mut changed = false;
        if let Some(send_next) = self.last_sent_downstream {
            if send_next < log.seqno() {
                let max_to_push_now = cmp::min(self.last_acked_downstream.unwrap_or(0) + REPLICATION_CREDIT, log.seqno());
                debug!("Window push {:?} - {:?}; waiting: {:?}", send_next, max_to_push_now, log.seqno() - max_to_push_now);
                for i in send_next..max_to_push_now {
                    if let Some(op) = log.read(i) {
                        debug!("Queueing seq:{:?}/{:?}; ds/seqno: {:?}", i, op, self.last_sent_downstream);
                        forward(i, op);
                        self.last_sent_downstream = Some(i+1);
                        changed = true
                    } else {
                        panic!("Attempted to replicate item not in log: {:?}", self);
                    }
                }
            }
            debug!("post Repl: Ours: {:?}; downstream sent: {:?}; acked: {:?}",
                log.seqno(), self.last_sent_downstream, self.last_acked_downstream);
        }
        changed
    }

    fn has_pending(&self, token: mio::Token) -> bool {
        self.pending_operations.values().all(|tok| tok != &token)
    }

    fn reset(&mut self) {
        self.last_sent_downstream = None
    }
}

impl Terminus {
    fn new() -> Terminus {
        Terminus {
            state: "".to_string()
        }
    }

    fn process_operation(&mut self, channel: &mut EventHandler, epoch: u64, seqno: u64, op: Operation) {
        info!("Terminus! {:?}/{:?}", seqno, op);
        let resp = match op {
            Operation::Set(s) => {
                self.state = s;
                OpResp::Ok(epoch, seqno, None)
            },
                Operation::Get => OpResp::Ok(epoch, seqno, Some(self.state.clone()))
        };
        channel.response(resp)
    }
}

impl ReplModel {
    pub fn new() -> ReplModel {
        ReplModel {
            log: Log::new(),
            current_epoch: Default::default(),
            next: Role::Terminus(Terminus::new()),
        }
    }

    pub fn seqno(&self) -> u64 {
        self.log.seqno()
    }

    fn next_seqno(&self) -> u64 {
        self.log.seqno()
    }

    pub fn process_operation(&mut self, channel: &mut EventHandler, seqno: Option<u64>, epoch: Option<u64>, op: Operation) {
        let seqno = seqno.unwrap_or_else(|| self.next_seqno());
        let epoch = epoch.unwrap_or_else(|| self.current_epoch);

       if epoch != self.current_epoch {
            warn!("Operation epoch ({}) differers from our last observed configuration: ({})",
                epoch, self.current_epoch);
            let resp = OpResp::Err(epoch, seqno, format!("BadEpoch: {}; last seen config: {}", epoch, self.current_epoch));
            channel.response(resp);
            return;
        }

        if !self.log.verify_sequential(seqno) {
            let resp = OpResp::Err(epoch, seqno, format!("Bad sequence number; saw: {:?}", seqno));
            channel.response(resp);
            return;
        }

        self.log.prepare(seqno, &op);
        self.log.commit_to(seqno);

        self.next.process_operation(channel, epoch, seqno, op)
    }

    pub fn process_downstream_response(&mut self, reply: &OpResp) -> Option<mio::Token> {
        self.next.process_downstream_response(reply)
    }

    pub fn process_replication<F: FnMut(u64, u64, Operation)>(&mut self, mut forward: F) -> bool {
        let epoch = self.current_epoch;
        self.next.process_replication(&self.log, |i, op| forward(epoch, i, op))
    }

    pub fn epoch_changed(&mut self, epoch: u64) {
        self.current_epoch = epoch;
    }

    pub fn has_pending(&self, token: mio::Token) -> bool {
        self.next.has_pending(token)
    }

    pub fn reset(&mut self) {
        self.next.reset()
    }

    pub fn set_has_downstream(&mut self, is_forwarder: bool) {
        // XXX: Replays?
        match (is_forwarder, &mut self.next) {
            (true, role @ &mut Role::Terminus(_)) => {
                let prev = mem::replace(role, Role::Forwarder(Forwarder::new()));
                info!("Switched to forwarding from {:?}", prev);
            },

            (false, role @ &mut Role::Forwarder(_)) => {
                let prev = mem::replace(role, Role::Terminus(Terminus::new()));
                info!("Switched to terminating from {:?}", prev);
            },
            _ => { info!("No change of config"); }
        }
    }
}
<|endoftext|>


<commit_before>
use hyper::server::Request as HyperRequest;
use hyper::method::Method;
use hyper::header::Headers;
use hyper::version::HttpVersion;
use std::collections::HashMap;
use typemap::TypeMap;

pub struct Request {
    method: Method,
    version: HttpVersion,
    headers: Headers,
    // only path part of this url
    path: String,
    // query string part of this url
    query_string: Option<String>,
    // if has body, keep it as raw here
    raw_body: Option<String>,
    // // params pair parsed from url query string
    // queries: Option<HashMap<String, String>>,
    // // params pair parsed from body
    // // if body is json, limit it to one level
    // body_params: Option<HashMap<String, String>>,
    // // combined params of queries and body_params
    // full_params: Option<HashMap<String, String>>,
    // ext key value pair
    ext: TypeMap

}

impl Request {
    pub fn new(method: Method, version: HttpVersion, headers: Headers, path: String, query_string: Option<String>) -> Request {
        // seperate path and query_string
        // let pathvec: Vec<&str> = pathstr.split('?').collect();
        // let path = pathvec[0].to_owned();
        // let mut query_string = None;

        // // if has query_string
        // if pathvec.len() > 1 {
        //     query_string = Some(pathvec[1].to_owned());
        // }

        Request {
            method: method,
            version: version,
            headers: headers,

            path: path,
            query_string: query_string,
            raw_body: None,
            // queries: None,
            // body_params: None,
            // full_params: None,
            ext: TypeMap::new()
        }

    }

    pub fn method(&self) -> &Method {
        &self.method
    }

    pub fn version(&self) -> &HttpVersion {
        &self.version
    }

    pub fn headers(&self) -> &Headers {
        &self.headers
    }

    pub fn path(&self) -> &String {
        &self.path
    }

    pub fn query_string(&self) -> &Option<String> {
        &self.query_string
    }

    pub fn raw_body(&self) -> &Option<String> {
        &self.raw_body
    }

    pub fn set_raw_body(&mut self, body: String) -> &mut Self {
        self.raw_body = Some(body);
        self
    }

    // pub fn query(&self) -> &Option<HashMap<String, String>> {
    //     &self.queries
    // }

    // pub fn body(&self) -> &Option<HashMap<String, String>> {
    //     &self.body_params
    // }

    // pub fn params(&self) -> &Option<HashMap<String, String>> {
    //     &self.full_params
    // }

    pub fn ext(&mut self) -> &TypeMap {
        &self.ext
    }

    pub fn ext_mut(&mut self) -> &mut TypeMap {
        &mut self.ext
    }
}

<commit_msg>make req.ext() mutable borrow as immutable.<commit_after>
use hyper::server::Request as HyperRequest;
use hyper::method::Method;
use hyper::header::Headers;
use hyper::version::HttpVersion;
use std::collections::HashMap;
use typemap::TypeMap;

pub struct Request {
    method: Method,
    version: HttpVersion,
    headers: Headers,
    // only path part of this url
    path: String,
    // query string part of this url
    query_string: Option<String>,
    // if has body, keep it as raw here
    raw_body: Option<String>,
    // // params pair parsed from url query string
    // queries: Option<HashMap<String, String>>,
    // // params pair parsed from body
    // // if body is json, limit it to one level
    // body_params: Option<HashMap<String, String>>,
    // // combined params of queries and body_params
    // full_params: Option<HashMap<String, String>>,
    // ext key value pair
    ext: TypeMap

}

impl Request {
    pub fn new(method: Method, version: HttpVersion, headers: Headers, path: String, query_string: Option<String>) -> Request {
        // seperate path and query_string
        // let pathvec: Vec<&str> = pathstr.split('?').collect();
        // let path = pathvec[0].to_owned();
        // let mut query_string = None;

        // // if has query_string
        // if pathvec.len() > 1 {
        //     query_string = Some(pathvec[1].to_owned());
        // }

        Request {
            method: method,
            version: version,
            headers: headers,

            path: path,
            query_string: query_string,
            raw_body: None,
            // queries: None,
            // body_params: None,
            // full_params: None,
            ext: TypeMap::new()
        }

    }

    pub fn method(&self) -> &Method {
        &self.method
    }

    pub fn version(&self) -> &HttpVersion {
        &self.version
    }

    pub fn headers(&self) -> &Headers {
        &self.headers
    }

    pub fn path(&self) -> &String {
        &self.path
    }

    pub fn query_string(&self) -> &Option<String> {
        &self.query_string
    }

    pub fn raw_body(&self) -> &Option<String> {
        &self.raw_body
    }

    pub fn set_raw_body(&mut self, body: String) -> &mut Self {
        self.raw_body = Some(body);
        self
    }

    // pub fn query(&self) -> &Option<HashMap<String, String>> {
    //     &self.queries
    // }

    // pub fn body(&self) -> &Option<HashMap<String, String>> {
    //     &self.body_params
    // }

    // pub fn params(&self) -> &Option<HashMap<String, String>> {
    //     &self.full_params
    // }

    pub fn ext(&self) -> &TypeMap {
        &self.ext
    }

    pub fn ext_mut(&mut self) -> &mut TypeMap {
        &mut self.ext
    }
}

<|endoftext|>


<commit_before>//! Utilities for the test suite.

use std::sync::Arc;
use std::fmt::Debug;
use rand::random;

use lift::lift0;
use signal::Signal;
use stream::Stream;

/// The identity function.
pub fn id<T>(t: T) -> T { t }

/// Trace equality of two signals.
pub fn signal_eq<T>(a: &Signal<T>, b: &Signal<T>) -> Signal<bool>
    where T: PartialEq + Clone + Send + Sync + 'static
{
    lift!(|a, b| a == b, a, b)
}

pub fn stream_eq<T>(a: &Stream<T>, b: &Stream<T>) -> Signal<bool>
    where T: PartialEq + Clone + Send + Sync + 'static + Debug,
{
    fn tag<T: Debug>(t: u64, a: T) -> Option<(u64, T)> {
        Some((t, a))
    }

    let tagger = lift0(random::<u64>);
    tagger.snapshot(a, tag)
        .merge(&tagger.snapshot(b, tag))
        .coalesce(|a, b| match (a, b) {
            (Some(a), Some(b)) => {
                if &a == &b { Some(a) } else { None }
            },
            _ => None,
        })
        .scan(true, |state, a| state && a.is_some())
}

/// A boxed function.
pub type ArcFn<A, B> = Arc<Box<Fn(A) -> B + Send + Sync + 'static>>;

/// Wrap a function into a constant signal of that function.
pub fn pure_fn<A, B, F: Fn(A) -> B + Send + Sync + 'static>(f: F) -> Signal<ArcFn<A, B>> {
    Signal::new(make_fn(f))
}

/// Box a function to erase its type.
pub fn make_fn<A, B, F: Fn(A) -> B + Send + Sync + 'static>(f: F) -> ArcFn<A, B> {
    Arc::new(Box::new(f))
}

/// Function composition on boxed functions.
pub fn comp<A, B, C>(f: ArcFn<B, C>, g: ArcFn<A, B>) -> ArcFn<A, C> {
    make_fn(move |a| f(g(a)))
}

/// Partially applied function composition.
pub fn partial_comp<A, B, C>(f: ArcFn<B, C>) -> ArcFn<ArcFn<A, B>, ArcFn<A, C>> {
    make_fn(move |g| comp(f.clone(), g))
}

/// Self-tests.
mod test {
    use signal::Signal;
    use stream::{ Stream, Sink };
    use super::{ stream_eq, signal_eq };

    #[test]
    fn signal_eq_const() {
        let a = Signal::new(3);
        let eq = signal_eq(&a, &a);
        assert!(eq.sample());
    }

    #[test]
    fn signal_eq_piecewise_const() {
        let sink = Sink::new();
        let sig = sink.stream().hold(4);
        let eq = signal_eq(&sig, &sig);
        for &a in &[1, 2, 3] {
            sink.send(a);
            assert!(eq.sample());
        }
    }

    #[test]
    fn signal_eq_algebraic() {
        let eq = signal_eq(&Stream::never().hold(3), &Signal::new(3));
        assert!(eq.sample());
    }

    #[test]
    fn stream_eq_never() {
        let eq = stream_eq(&Stream::<()>::never(), &Stream::never());
        assert!(eq.sample());
    }

    #[test]
    fn stream_eq_same_sink() {
        let sink = Sink::new();
        let eq = stream_eq(&sink.stream(), &sink.stream());
        for n in 0..20 { sink.send(n); }
        assert!(eq.sample());
    }

    #[test]
    fn stream_eq_algebraic() {
        fn f(n: i32) -> i32 { n + 3 }
        fn g(n: i32) -> i32 { 2 * n }
        fn h(n: i32) -> i32 { f(g(n)) }

        let sink = Sink::new();
        let eq = stream_eq(&sink.stream().map(g).map(f), &sink.stream().map(h));
        for n in 0..20 { sink.send(n); }
        assert!(eq.sample());
    }
}
<commit_msg>Document and tweak stream equality<commit_after>//! Utilities for the test suite.

use std::sync::Arc;
use std::fmt::Debug;
use rand::random;

use lift::lift0;
use signal::Signal;
use stream::Stream;

/// The identity function.
pub fn id<T>(t: T) -> T { t }

/// Trace equality of two signals.
pub fn signal_eq<T>(a: &Signal<T>, b: &Signal<T>) -> Signal<bool>
    where T: PartialEq + Clone + Send + Sync + 'static
{
    lift!(|a, b| a == b, a, b)
}

/// Trace equality of two streams.
pub fn stream_eq<T>(a: &Stream<T>, b: &Stream<T>) -> Signal<bool>
    where T: PartialEq + Clone + Send + Sync + 'static,
{
    fn tag<T>(t: u64, a: T) -> Option<(u64, T)> {
        Some((t, a))
    }

    let tagger = lift0(random::<u64>);
    tagger.snapshot(a, tag)
        .merge(&tagger.snapshot(b, tag))
        .coalesce(|a, b| match (a, b) {
            (Some(a), Some(b)) => {
                if &a == &b { Some(a) } else { None }
            },
            _ => None,
        })
        .scan(true, |state, a| state && a.is_some())
}

/// A boxed function.
pub type ArcFn<A, B> = Arc<Box<Fn(A) -> B + Send + Sync + 'static>>;

/// Wrap a function into a constant signal of that function.
pub fn pure_fn<A, B, F: Fn(A) -> B + Send + Sync + 'static>(f: F) -> Signal<ArcFn<A, B>> {
    Signal::new(make_fn(f))
}

/// Box a function to erase its type.
pub fn make_fn<A, B, F: Fn(A) -> B + Send + Sync + 'static>(f: F) -> ArcFn<A, B> {
    Arc::new(Box::new(f))
}

/// Function composition on boxed functions.
pub fn comp<A, B, C>(f: ArcFn<B, C>, g: ArcFn<A, B>) -> ArcFn<A, C> {
    make_fn(move |a| f(g(a)))
}

/// Partially applied function composition.
pub fn partial_comp<A, B, C>(f: ArcFn<B, C>) -> ArcFn<ArcFn<A, B>, ArcFn<A, C>> {
    make_fn(move |g| comp(f.clone(), g))
}

/// Self-tests.
mod test {
    use signal::Signal;
    use stream::{ Stream, Sink };
    use super::{ stream_eq, signal_eq };

    #[test]
    fn signal_eq_const() {
        let a = Signal::new(3);
        let eq = signal_eq(&a, &a);
        assert!(eq.sample());
    }

    #[test]
    fn signal_eq_piecewise_const() {
        let sink = Sink::new();
        let sig = sink.stream().hold(4);
        let eq = signal_eq(&sig, &sig);
        for &a in &[1, 2, 3] {
            sink.send(a);
            assert!(eq.sample());
        }
    }

    #[test]
    fn signal_eq_algebraic() {
        let eq = signal_eq(&Stream::never().hold(3), &Signal::new(3));
        assert!(eq.sample());
    }

    #[test]
    fn stream_eq_never() {
        let eq = stream_eq(&Stream::<()>::never(), &Stream::never());
        assert!(eq.sample());
    }

    #[test]
    fn stream_eq_same_sink() {
        let sink = Sink::new();
        let eq = stream_eq(&sink.stream(), &sink.stream());
        for n in 0..20 { sink.send(n); }
        assert!(eq.sample());
    }

    #[test]
    fn stream_eq_algebraic() {
        fn f(n: i32) -> i32 { n + 3 }
        fn g(n: i32) -> i32 { 2 * n }
        fn h(n: i32) -> i32 { f(g(n)) }

        let sink = Sink::new();
        let eq = stream_eq(&sink.stream().map(g).map(f), &sink.stream().map(h));
        for n in 0..20 { sink.send(n); }
        assert!(eq.sample());
    }
}
<|endoftext|>


<commit_before>// Copyright 2015 Jerome Rasky <jerome@rasky.co>
//
// Licensed under the Apache License, version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
//     <http://www.apache.org/licenses/LICENSE-2.0>
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. See the
// License for the specific language concerning governing permissions and
// limitations under the License.
use std::io::prelude::*;

use std::io::BufReader;
use std::sync::Arc;
use std::sync::mpsc::Sender;
use std::sync::atomic::{AtomicBool, Ordering};
use std::fs::File;
use std::borrow::Borrow;
use std::thread::JoinHandle;
use std::collections::{HashMap, VecDeque};
use std::rc::Rc;
use std::iter::FromIterator;

use std::env;
use std::io;
use std::thread;

use flx::{SearchBase, LineInfo};
use error::{StrError, StrResult};
use bis_c;

use types::*;
use constants::*;

pub fn start_threads(emit: Sender<Event>) -> StrResult<(JoinHandle<()>, Arc<AtomicBool>)> {
    // mask sigint
    trys!(bis_c::mask_sigint(), "Failed to mask SIGINT");

    // start the sigint thread
    let signal_emit = emit.clone();
    thread::spawn(move || {
        read_signals(signal_emit);
    });

    // start reading history
    let history_emit = emit.clone();
    thread::spawn(move || {
        read_history(history_emit);
    });

    // start the input thread
    let input_stop = Arc::new(AtomicBool::new(false));
    let stop = input_stop.clone();
    let input_thread = thread::spawn(|| read_input(emit, stop));

    Ok((input_thread, input_stop))
}

fn read_history(emit: Sender<Event>) {
    let history_path = match env::var("HISTFILE") {
        Ok(path) => path.into(),
        Err(env::VarError::NotPresent) => {
            debug!("History file not found, defaulting to ~/.bash_history");
            let mut home = env::home_dir().unwrap_or("".into());
            home.push(".bash_history");
            home
        },
        Err(e) => {
            panic!("Failed to get history file path: {}", e);
        }
    };

    trace!("History path: {:?}", history_path);

    let input_file = BufReader::new(trysp!(File::open(history_path),
                                           "Cauld not open history file"));
    let mut count = 0.0;
    let mut set: HashMap<Rc<String>, LineInfo> = HashMap::new();
    let mut short: VecDeque<Rc<String>> = VecDeque::new();

    for maybe_line in input_file.lines() {
        if let Ok(line) = maybe_line {
            let line = Rc::new(line);
            let item = set.entry(line.clone()).or_insert(LineInfo::new(line.as_str(), 0.0));
            let old_count = item.get_factor();
            let mut contains = false;
            for short_item in short.iter() {
                if line == *short_item {
                    contains = true;
                    break;
                }
            }

            if !contains {
                short.push_back(line.clone());
                while short.len() > 10 {
                    short.pop_front();
                }
            }
            item.set_factor(old_count + count);
            count += 1.0;
        }
    }

    // extract shortlist
    let mut recent = vec![];

    for item in short.into_iter().rev() {
        recent.push(Arc::new(item.as_ref().clone()));
    }

    // send off recent history
    trysp!(emit.send(Event::HistoryReady(recent)),
           "Failed to emit history ready signal");

    // create the search base
    let base = SearchBase::from_iter(set.into_iter().map(|(_, info)| info));

    // if this fails, we can't search anything
    trysp!(emit.send(Event::SearchReady(base)),
           "Failed to emit search ready signal");
}

fn read_input(emit: Sender<Event>, stop: Arc<AtomicBool>) {
    // this thread is joined on quit, so none of its sends should fail
    let mut escape = None;

    // escape sequence tree, assume ANSI
    let mut control: HashMap<String, Option<Event>> = HashMap::default();
    control.insert(format!("["), None);
    control.insert(format!("[A"), Some(Event::KeyUp));
    control.insert(format!("[B"), Some(Event::KeyDown));

    // read characters
    for maybe_chr in io::stdin().chars() {
        match maybe_chr {
            Err(_) => {
                error!("Failed to read input, quitting");
                trysp!(emit.send(Event::Quit(false)), "Failed to emit quit signal");
                break;
            }
            Ok(ESC) => {
                escape = Some(String::default());
                trace!("Begin escape sequence");
            }
            Ok(chr) => {
                match escape {
                    None => {
                        if chr.is_control() {
                            if chr == EOT {
                                // stop
                                trysp!(emit.send(Event::Quit(false)),
                                       "Failed to send quit event");
                            } else if chr == CTRL_U {
                                // clear query
                                trysp!(emit.send(Event::Clear), "Failed to send clear event");
                            } else if chr == CTRL_R {
                                // key up
                                trysp!(emit.send(Event::KeyDown), "Failed to send key up event");
                            } else if chr == CTRL_S {
                                // key down
                                trysp!(emit.send(Event::KeyUp), "Failed to send key down event");
                            } else if chr == '\n' {
                                // exit
                                trysp!(emit.send(Event::Quit(true)),
                                       "Failed to send quit signal");
                            } else if chr == ESC {
                                // escape sequence
                                escape = Some(format!(""));
                            } else if chr == BSPC || chr == DEL {
                                // backspace
                                trysp!(emit.send(Event::Backspace),
                                       "Failed to send backspace signal");
                            } else {
                                trace!("Unknown control character {:?}", chr);
                                trysp!(emit.send(Event::Bell), "Failed to send bell event");
                            }
                        } else {
                            trysp!(emit.send(Event::Input(chr)), "Failed to send character");
                        }
                    }
                    Some(mut seq) => {
                        seq.push(chr);
                        trace!("Sequence: {:?}", seq);
                        match control.get(&seq) {
                            None => {
                                // no possible escape sequence like this
                                trysp!(emit.send(Event::Bell), "Failed to send bell event");
                                trysp!(emit.send(Event::Input(chr)), "Failed to send character");
                                escape = None;
                            }
                            Some(&None) => {
                                // keep going
                                escape = Some(seq);
                            }
                            Some(&Some(ref event)) => {
                                // send the appropriate event
                                let cloned = trysp!(event.maybe_clone()
                                                         .ok_or(StrError::new("Event {:?} \
                                                                               could not be \
                                                                               cloned",
                                                                              None)),
                                                    "Failed to create event");
                                trysp!(emit.send(cloned), "Failed to send escape event");
                                escape = None;
                            }
                        }
                    }
                }
            }
        }

        // check for requested stop
        if stop.load(Ordering::Relaxed) {
            debug!("Input thread exiting");
            break;
        }
    }
}

fn read_signals(emit: Sender<Event>) {
    // wait for a sigint
    trysp!(bis_c::wait_sigint(), "Failed to wait for sigint");

    debug!("Caught sigint");

    // send a quit signal
    let _ = emit.send(Event::Quit(false));
    // might happen after events is closed, so don't fail
}

pub fn start_query(emit: Sender<Event>, base: Arc<SearchBase>, query: Arc<String>) {
    let result = base.query::<&String>(query.borrow(), MATCH_NUMBER);
    let _ = emit.send(Event::Match(result, query)).and_then(|_| {
        trace!("Finished query");
        Ok(())
    });
    // don't panic on fail send, events might be already closed
}
<commit_msg>Add some more trace output<commit_after>// Copyright 2015 Jerome Rasky <jerome@rasky.co>
//
// Licensed under the Apache License, version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
//     <http://www.apache.org/licenses/LICENSE-2.0>
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. See the
// License for the specific language concerning governing permissions and
// limitations under the License.
use std::io::prelude::*;

use std::io::BufReader;
use std::sync::Arc;
use std::sync::mpsc::Sender;
use std::sync::atomic::{AtomicBool, Ordering};
use std::fs::File;
use std::borrow::Borrow;
use std::thread::JoinHandle;
use std::collections::{HashMap, VecDeque};
use std::rc::Rc;
use std::iter::FromIterator;

use std::env;
use std::io;
use std::thread;

use flx::{SearchBase, LineInfo};
use error::{StrError, StrResult};
use bis_c;

use types::*;
use constants::*;

pub fn start_threads(emit: Sender<Event>) -> StrResult<(JoinHandle<()>, Arc<AtomicBool>)> {
    // mask sigint
    trys!(bis_c::mask_sigint(), "Failed to mask SIGINT");

    // start the sigint thread
    let signal_emit = emit.clone();
    thread::spawn(move || {
        read_signals(signal_emit);
    });

    // start reading history
    let history_emit = emit.clone();
    thread::spawn(move || {
        read_history(history_emit);
    });

    // start the input thread
    let input_stop = Arc::new(AtomicBool::new(false));
    let stop = input_stop.clone();
    let input_thread = thread::spawn(|| read_input(emit, stop));

    Ok((input_thread, input_stop))
}

fn read_history(emit: Sender<Event>) {
    let history_path = match env::var("HISTFILE") {
        Ok(path) => path.into(),
        Err(env::VarError::NotPresent) => {
            debug!("History file not found, defaulting to ~/.bash_history");
            let mut home = env::home_dir().unwrap_or("".into());
            home.push(".bash_history");
            home
        },
        Err(e) => {
            panic!("Failed to get history file path: {}", e);
        }
    };

    trace!("History path: {:?}", history_path);

    let input_file = BufReader::new(trysp!(File::open(history_path),
                                           "Cauld not open history file"));
    let mut count = 0.0;
    let mut set: HashMap<Rc<String>, LineInfo> = HashMap::new();
    let mut short: VecDeque<Rc<String>> = VecDeque::new();

    for maybe_line in input_file.lines() {
        if let Ok(line) = maybe_line {
            let line = Rc::new(line);
            let item = set.entry(line.clone()).or_insert(LineInfo::new(line.as_str(), 0.0));
            let old_count = item.get_factor();
            let mut contains = false;
            for short_item in short.iter() {
                if line == *short_item {
                    contains = true;
                    break;
                }
            }

            if !contains {
                short.push_back(line.clone());
                while short.len() > 10 {
                    short.pop_front();
                }
            }
            item.set_factor(old_count + count);
            count += 1.0;
        }
    }

    // extract shortlist
    let mut recent = vec![];

    for item in short.into_iter().rev() {
        recent.push(Arc::new(item.as_ref().clone()));
    }

    // send off recent history
    trysp!(emit.send(Event::HistoryReady(recent)),
           "Failed to emit history ready signal");

    // create the search base
    let base = SearchBase::from_iter(set.into_iter().map(|(_, info)| info));

    // if this fails, we can't search anything
    trysp!(emit.send(Event::SearchReady(base)),
           "Failed to emit search ready signal");
}

fn read_input(emit: Sender<Event>, stop: Arc<AtomicBool>) {
    // this thread is joined on quit, so none of its sends should fail
    let mut escape = None;

    // escape sequence tree, assume ANSI
    let mut control: HashMap<String, Option<Event>> = HashMap::default();
    control.insert(format!("["), None);
    control.insert(format!("[A"), Some(Event::KeyUp));
    control.insert(format!("[B"), Some(Event::KeyDown));

    // read characters
    for maybe_chr in io::stdin().chars() {
        match maybe_chr {
            Err(_) => {
                error!("Failed to read input, quitting");
                trysp!(emit.send(Event::Quit(false)), "Failed to emit quit signal");
                break;
            }
            Ok(ESC) => {
                escape = Some(String::default());
                trace!("Begin escape sequence");
            }
            Ok(chr) => {
                match escape {
                    None => {
                        if chr.is_control() {
                            if chr == EOT {
                                // stop
                                trace!("Got EOT");
                                trysp!(emit.send(Event::Quit(false)),
                                       "Failed to send quit event");
                            } else if chr == CTRL_U {
                                // clear query
                                trace!("Got CTRL_U");
                                trysp!(emit.send(Event::Clear), "Failed to send clear event");
                            } else if chr == CTRL_R {
                                // key up
                                trace!("Got CTRL_R");
                                trysp!(emit.send(Event::KeyDown), "Failed to send key up event");
                            } else if chr == CTRL_S {
                                // key down
                                trace!("Got CTRL_S");
                                trysp!(emit.send(Event::KeyUp), "Failed to send key down event");
                            } else if chr == '\n' {
                                // exit
                                trace!("Got newline");
                                trysp!(emit.send(Event::Quit(true)),
                                       "Failed to send quit signal");
                            } else if chr == ESC {
                                // escape sequence
                                trace!("Got ESC");
                                escape = Some(format!(""));
                            } else if chr == BSPC || chr == DEL {
                                // backspace
                                trace!("Got backspace");
                                trysp!(emit.send(Event::Backspace),
                                       "Failed to send backspace signal");
                            } else {
                                debug!("Unknown control character {:?}", chr);
                                trysp!(emit.send(Event::Bell), "Failed to send bell event");
                            }
                        } else {
                            trysp!(emit.send(Event::Input(chr)), "Failed to send character");
                        }
                    }
                    Some(mut seq) => {
                        seq.push(chr);
                        trace!("Sequence: {:?}", seq);
                        match control.get(&seq) {
                            None => {
                                // no possible escape sequence like this
                                trysp!(emit.send(Event::Bell), "Failed to send bell event");
                                trysp!(emit.send(Event::Input(chr)), "Failed to send character");
                                escape = None;
                            }
                            Some(&None) => {
                                // keep going
                                escape = Some(seq);
                            }
                            Some(&Some(ref event)) => {
                                // send the appropriate event
                                let cloned = trysp!(event.maybe_clone()
                                                         .ok_or(StrError::new("Event {:?} \
                                                                               could not be \
                                                                               cloned",
                                                                              None)),
                                                    "Failed to create event");
                                trysp!(emit.send(cloned), "Failed to send escape event");
                                escape = None;
                            }
                        }
                    }
                }
            }
        }

        // check for requested stop
        if stop.load(Ordering::Relaxed) {
            debug!("Input thread exiting");
            break;
        }
    }
}

fn read_signals(emit: Sender<Event>) {
    // wait for a sigint
    trysp!(bis_c::wait_sigint(), "Failed to wait for sigint");

    debug!("Caught sigint");

    // send a quit signal
    let _ = emit.send(Event::Quit(false));
    // might happen after events is closed, so don't fail
}

pub fn start_query(emit: Sender<Event>, base: Arc<SearchBase>, query: Arc<String>) {
    let result = base.query::<&String>(query.borrow(), MATCH_NUMBER);
    let _ = emit.send(Event::Match(result, query)).and_then(|_| {
        trace!("Finished query");
        Ok(())
    });
    // don't panic on fail send, events might be already closed
}
<|endoftext|>