<commit_before>/**
 *   Mles-utils to be used with Mles client or server.
 *   Copyright (C) 2017  Mles developers
 *
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
extern crate mles_utils;

use std::{thread, process, env};
use std::sync::mpsc::{Sender, Receiver};
use std::sync::mpsc::channel;
use std::net::TcpStream;
use std::net::SocketAddr;
use std::net::TcpListener;
use std::collections::HashMap;
use std::io::{Read, Error};
use std::io::Write;
use std::time::Duration;
use std::option::Option;
use std::str;
use mles_utils::*;

const HDRL: u64 = 4;
const KEYL: u64 = 8;

fn main() {
    let address = "0.0.0.0:8077";
    let listener = match TcpListener::bind(&address) {
        Ok(listener) => listener,
        Err(err) => {
            println!("Error: {}\n", err);
            process::exit(1);
        },
    };

    let keyval =match env::var("MLES_KEY") {
        Ok(val) => val,
        Err(_) => "".to_string(),
    };

    let mut spawned = HashMap::new();
    let option: Option<Duration> = Some(Duration::from_millis(50));
    let addr = listener.local_addr().unwrap();
    println!("Listening for connections on {}", addr);
    let (tx, rx) = channel();
    let (removedtx, removedrx) = channel();

    for socket in listener.incoming() {
        /* Check first has anybody removed channels */
        let mut remrx = true;
        while remrx {
            match removedrx.try_recv() {
                Ok(val) => {
                    let removed_channel: String = val;
                    println!("Removing unused channel {}", removed_channel.as_str());
                    spawned.remove(&removed_channel);
                    remrx = true;
                },
                Err(_) => { remrx = false; }
            }
        }
        /* 1. Read incoming msg channel
         * 2. If it does not exist, spawn new thread
         * 3. Send socket to thread
         */
        let socket = socket.unwrap();
        let stream = socket.try_clone().unwrap();
        let tuple = read_n(&stream, HDRL);
        let status = tuple.0;
        match status {
            Ok(0) => {
               continue;
            },
            Ok(_) => {},
            _ => {
               continue;
            },
        }
        let buf = tuple.1;
        if 0 == buf.len() {
            continue;
        }
        if read_hdr_type(buf.as_slice()) != 'M' as u32 {
            println!("Incorrect payload type");
            continue;
        }
        // read key
        let tuple = read_n(&stream, KEYL);
        let status = tuple.0;
        match status {
            Ok(0) => {
               continue;
            },
            Ok(_) => {},
            _ => {
               continue;
            },
        }
        // verify key
        let hkey;
        let key = read_key(tuple.1);
        if 0 == keyval.len() {
            let paddr = match stream.peer_addr() {
                Ok(paddr) => paddr,
                Err(_) => {
                    let addr = "0.0.0.0:0";
                    let addr = addr.parse::<SocketAddr>().unwrap();
                    addr
                }
            };
            hkey = do_hash(&paddr);
        }
        else {
            hkey = do_hash(&keyval);
        }
        if hkey != key {
            println!("Incorrect remote key");
            continue;
        }
        let tuple = read_n(&stream, read_hdr_len(buf.as_slice()) as u64);
        let status = tuple.0;
        match status {
            Ok(0) => {
               continue;
            },
            Ok(_) => {},
            _ => {
               continue;
            },
        }
        let buf = tuple.1;
        let decoded_msg = message_decode(buf.as_slice());
        if 0 == decoded_msg.channel.len() {
            continue;
        }
        let _val = socket.set_nodelay(true);
        let _val = socket.set_read_timeout(option);
        if !spawned.contains_key(decoded_msg.channel.as_str()) {
            let tx = tx.clone();
            let removedtx = removedtx.clone();
            let this_channel = decoded_msg.channel.clone();
            thread::spawn(move|| process_channel(tx, removedtx, this_channel));

            let thr_feed = rx.recv().unwrap();
            println!("Channel {}", decoded_msg.channel.as_str());
            spawned.insert(decoded_msg.channel.clone(), thr_feed);
        }
        let thr_socket = spawned.get_mut(&decoded_msg.channel).unwrap();
        thr_socket.send(socket).unwrap();
    }
}

fn process_channel(tx: Sender<Sender<TcpStream>>, removedtx: Sender<String>, this_channel: String ) {
    let mut cnt = 0;
    let (thr_tx, thr_rx): (Sender<TcpStream>, Receiver<TcpStream>) = channel();
    println!("Spawned: New channel created!");
    let mut users = HashMap::new();
    let mut messages: Vec<Vec<u8>> = Vec::new();
    tx.send(thr_tx.clone()).unwrap();
    loop {
        let mut removals = Vec::new();
        let mut newuser = true;
        while newuser {
            match thr_rx.try_recv() {
                Ok(val) => {
                    let mut thr: TcpStream = val;
                    cnt += 1;
                    println!("Adding user {}", cnt);
                    users.insert(cnt, thr.try_clone().unwrap());

                    /* If a new user, all push messages to her */
                    for buf in &messages {
                        thr.write(buf.as_slice()).unwrap();
                    }
                    newuser = true;
                },
                    Err(_) => { newuser = false; }
            }
        }
        for (user, thr_socket) in &users {
            let stream = thr_socket.try_clone().unwrap();
            let tuple = read_n(&stream, HDRL);
            let status = tuple.0;
            match status {
                Ok(0) => {
                    removals.push(user.clone());
                },
                    _ => {}
            }
            let mut buf = tuple.1;
            if 0 == buf.len() {
                continue;
            }
            if read_hdr_type(buf.as_slice()) != 'M' as u32 {
                continue;
            }
            let hdr_len = read_hdr_len(buf.as_slice()) as u64;
            if 0 == hdr_len {
                continue;
            }
            // read key
            let tuple = read_n(&stream, KEYL);
            let status = tuple.0;
            match status {
                Ok(0) => {
                    continue;
                },
                    Ok(_) => {},
                    _ => {
                        continue;
                    },
            }
            let key = tuple.1;
            //ignore key value for now
            buf.extend(key);
            let tuple = read_n(&stream, hdr_len);
            let status = tuple.0;
            match status {
                Ok(0) => {
                    removals.push(user.clone());
                },
                    _ => {}
            }
            let payload = tuple.1;
            if payload.len() != (hdr_len as usize) {
                continue;
            }
            buf.extend(payload);
            for (another_user, mut thr_sock) in &users {
                if user != another_user {
                    thr_sock.write(buf.as_slice()).unwrap();
                }
            }
            /* Add to local db */
            messages.push(buf);
        }
        for removal in &removals {
            println!("Removing user {}", removal);
            users.remove(removal);
        }
        if cnt > 0 && users.is_empty() {
            removedtx.send(this_channel).unwrap();
            break;
        }
    }
}

fn read_n<R>(reader: R, bytes_to_read: u64) -> (Result<usize, Error>, Vec<u8>)
where R: Read,
{
    let mut buf = vec![];
    let mut chunk = reader.take(bytes_to_read);
    let status = chunk.read_to_end(&mut buf);
    (status, buf)
}

<commit_msg>Peer support.<commit_after>/**
 *   Mles-utils to be used with Mles client or server.
 *   Copyright (C) 2017  Mles developers
 *
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
extern crate mles_utils;

use std::{thread, process, env};
use std::sync::mpsc::{Sender, Receiver};
use std::sync::mpsc::channel;
use std::net::TcpStream;
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::net::TcpListener;
use std::collections::HashMap;
use std::io::{Read, Error};
use std::io::Write;
use std::time::Duration;
use std::option::Option;
use std::str;
use mles_utils::*;

const HDRL: u64 = 4;
const KEYL: u64 = 8;

fn main() {
    let mut peer = "".to_string();
    for arg in env::args() {
        peer = arg;
        peer += ":8077";
    }
    let peer = match peer.parse::<SocketAddr>() {
        Ok(addr) => addr,
        Err(_) => {
            SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0)
        },
    };

    let address = "0.0.0.0:8077";
    let listener = match TcpListener::bind(&address) {
        Ok(listener) => listener,
        Err(err) => {
            println!("Error: {}\n", err);
            process::exit(1);
        },
    };

    let keyval =match env::var("MLES_KEY") {
        Ok(val) => val,
        Err(_) => "".to_string(),
    };

    let mut spawned = HashMap::new();
    let option: Option<Duration> = Some(Duration::from_millis(50));
    let addr = listener.local_addr().unwrap();
    println!("Listening for connections on {}", addr);
    let (tx, rx) = channel();
    let (removedtx, removedrx) = channel();

    for socket in listener.incoming() {
        /* Check first has anybody removed channels */
        let mut remrx = true;
        while remrx {
            match removedrx.try_recv() {
                Ok(val) => {
                    let removed_channel: String = val;
                    println!("Removing unused channel {}", removed_channel.as_str());
                    spawned.remove(&removed_channel);
                    remrx = true;
                },
                Err(_) => { remrx = false; }
            }
        }
        /* 1. Read incoming msg channel
         * 2. If it does not exist, spawn new thread
         * 3. Send socket to thread
         */
        let socket = socket.unwrap();
        let stream = socket.try_clone().unwrap();
        let tuple = read_n(&stream, HDRL);
        let status = tuple.0;
        match status {
            Ok(0) => {
               continue;
            },
            Ok(_) => {},
            _ => {
               continue;
            },
        }
        let buf = tuple.1;
        if 0 == buf.len() {
            continue;
        }
        if read_hdr_type(buf.as_slice()) != 'M' as u32 {
            println!("Incorrect payload type");
            continue;
        }
        // read key
        let tuple = read_n(&stream, KEYL);
        let status = tuple.0;
        match status {
            Ok(0) => {
               continue;
            },
            Ok(_) => {},
            _ => {
               continue;
            },
        }
        // verify key
        let hkey;
        let key = read_key(tuple.1);
        if 0 == keyval.len() {
            let paddr = match stream.peer_addr() {
                Ok(paddr) => paddr,
                Err(_) => {
                    let addr = "0.0.0.0:0";
                    let addr = addr.parse::<SocketAddr>().unwrap();
                    addr
                }
            };
            hkey = do_hash(&paddr);
        }
        else {
            hkey = do_hash(&keyval);
        }
        if hkey != key {
            println!("Incorrect remote key");
            continue;
        }
        let tuple = read_n(&stream, read_hdr_len(buf.as_slice()) as u64);
        let status = tuple.0;
        match status {
            Ok(0) => {
               continue;
            },
            Ok(_) => {},
            _ => {
               continue;
            },
        }
        let buf = tuple.1;
        let decoded_msg = message_decode(buf.as_slice());
        if 0 == decoded_msg.channel.len() {
            continue;
        }
        let _val = socket.set_nodelay(true);
        let _val = socket.set_read_timeout(option);
        if !spawned.contains_key(decoded_msg.channel.as_str()) {
            let tx = tx.clone();
            let removedtx = removedtx.clone();
            let this_channel = decoded_msg.channel.clone();
            thread::spawn(move|| process_channel(peer, tx, removedtx, this_channel));

            let thr_feed = rx.recv().unwrap();
            println!("Channel {}", decoded_msg.channel.as_str());
            spawned.insert(decoded_msg.channel.clone(), thr_feed);
        }
        let thr_socket = spawned.get_mut(&decoded_msg.channel).unwrap();
        thr_socket.send(socket).unwrap();
    }
}

fn process_channel(peer: SocketAddr, tx: Sender<Sender<TcpStream>>, removedtx: Sender<String>, this_channel: String ) {
    let mut cnt = 0;
    let (thr_tx, thr_rx): (Sender<TcpStream>, Receiver<TcpStream>) = channel();
    println!("Spawned: New channel created!");
    let mut users = HashMap::new();
    let mut messages: Vec<Vec<u8>> = Vec::new();
    tx.send(thr_tx.clone()).unwrap();

    if 0 != peer.port() {
        match TcpStream::connect(peer) {
            Ok(sock) => {
                let option: Option<Duration> = Some(Duration::from_millis(50));
                let _val = sock.set_nodelay(true);
                let _val = sock.set_read_timeout(option);
                cnt += 1;
                println!("Adding peer {}", cnt);
                users.insert(cnt, sock);
            },
            Err(_) => {
                println!("Could not connect to peer {}", peer);
            },
        }
    }

    loop {
        let mut removals = Vec::new();
        let mut newuser = true;
        while newuser {
            match thr_rx.try_recv() {
                Ok(val) => {
                    let mut thr: TcpStream = val;
                    cnt += 1;
                    println!("Adding user {}", cnt);
                    users.insert(cnt, thr.try_clone().unwrap());

                    /* If a new user, all push messages to her */
                    for buf in &messages {
                        thr.write(buf.as_slice()).unwrap();
                    }
                    newuser = true;
                },
                Err(_) => { newuser = false; }
            }
        }
        for (user, thr_socket) in &users {
            let stream = thr_socket.try_clone().unwrap();
            let tuple = read_n(&stream, HDRL);
            let status = tuple.0;
            match status {
                Ok(0) => {
                    removals.push(user.clone());
                },
                    _ => {}
            }
            let mut buf = tuple.1;
            if 0 == buf.len() {
                continue;
            }
            if read_hdr_type(buf.as_slice()) != 'M' as u32 {
                continue;
            }
            let hdr_len = read_hdr_len(buf.as_slice()) as u64;
            if 0 == hdr_len {
                continue;
            }
            // read key
            let tuple = read_n(&stream, KEYL);
            let status = tuple.0;
            match status {
                Ok(0) => {
                    continue;
                },
                    Ok(_) => {},
                    _ => {
                        continue;
                    },
            }
            let key = tuple.1;
            //ignore key value for now
            buf.extend(key);
            let tuple = read_n(&stream, hdr_len);
            let status = tuple.0;
            match status {
                Ok(0) => {
                    removals.push(user.clone());
                },
                    _ => {}
            }
            let payload = tuple.1;
            if payload.len() != (hdr_len as usize) {
                continue;
            }
            buf.extend(payload);
            for (another_user, mut thr_sock) in &users {
                if user != another_user {
                    thr_sock.write(buf.as_slice()).unwrap();
                }
            }
            /* Add to local db */
            messages.push(buf);
        }
        for removal in &removals {
            println!("Removing user {}", removal);
            users.remove(removal);
        }
        if cnt > 0 && users.is_empty() {
            removedtx.send(this_channel).unwrap();
            break;
        }
    }
}

fn read_n<R>(reader: R, bytes_to_read: u64) -> (Result<usize, Error>, Vec<u8>)
where R: Read,
{
    let mut buf = vec![];
    let mut chunk = reader.take(bytes_to_read);
    let status = chunk.read_to_end(&mut buf);
    (status, buf)
}

<|endoftext|>


<commit_before>use failure::*;
use std::env::var;
use std::fs;
use std::io::*;
use std::path::*;
use std::process::Command;
use zip::ZipArchive;

fn download_archive_windows(out_dir: &Path) -> Fallible<()> {
    if out_dir.join("libfftw3.dll").exists() && out_dir.join("libfftw3f.dll").exists() {
        return Ok(());
    }

    let archive = out_dir.join("fftw_windows.zip");
    if !archive.exists() {
        // Download
        let mut conn = ftp::FtpStream::connect("ftp.fftw.org:21")?;
        conn.login("anonymous", "anonymous")?;
        conn.cwd("pub/fftw")?;
        let buf = conn.simple_retr("fftw-3.3.5-dll64.zip")?.into_inner();
        // TODO calc checksum
        let mut f = fs::File::create(&archive)?;
        f.write(&buf)?;
    }
    let f = fs::File::open(&archive)?;
    let mut zip = ZipArchive::new(f)?;
    let target = var("TARGET").unwrap();
    for name in &["fftw3-3", "fftw3f-3"] {
        for ext in &["dll", "def"] {
            let filename = format!("lib{}.{}", name, ext);
            let mut zf = zip.by_name(&filename)?;
            let mut f = fs::File::create(out_dir.join(filename))?;
            copy(&mut zf, &mut f)?;
        }
        run(cc::windows_registry::find_tool(&target, "lib.exe")
            .unwrap()
            .to_command()
            .arg("/MACHINE:X64")
            .arg(format!("/DEF:lib{}.def", name))
            .arg(format!("/OUT:lib{}.lib", name))
            .current_dir(out_dir))
    }
    Ok(())
}

fn download_archive_unix(out_dir: &Path) -> Fallible<()> {
    // Build FFTW
    let root_dir = PathBuf::from(var("CARGO_MANIFEST_DIR").unwrap());
    let archive_dir = root_dir.join("fftw-3.3.8");
    build_fftw(&["--enable-single"], &archive_dir, &out_dir);
    build_fftw(&[], &archive_dir, &out_dir);
    Ok(())
}

fn build_fftw(flags: &[&str], src_dir: &Path, out_dir: &Path) {
    run(
        Command::new(fs::canonicalize(src_dir.join("configure")).unwrap())
            .arg("--with-pic")
            .arg("--enable-static")
            .arg("--disable-doc")
            .arg(format!("--prefix={}", out_dir.display()))
            .args(flags)
            .current_dir(&src_dir),
    );
    run(Command::new("make")
        .arg(format!("-j{}", var("NUM_JOBS").unwrap()))
        .current_dir(&src_dir));
    run(Command::new("make").arg("install").current_dir(&src_dir));
}

fn run(command: &mut Command) {
    println!("Running: {:?}", command);
    match command.status() {
        Ok(status) => {
            if !status.success() {
                panic!("`{:?}` failed: {}", command, status);
            }
        }
        Err(error) => {
            panic!("failed to execute `{:?}`: {}", command, error);
        }
    }
}

fn main() -> Fallible<()> {
    let out_dir = PathBuf::from(var("OUT_DIR").unwrap());
    if cfg!(target_os = "windows") {
        download_archive_windows(&out_dir)?;
        println!("cargo:rustc-link-search={}", out_dir.display());
        println!("cargo:rustc-link-lib=libfftw3-3");
        println!("cargo:rustc-link-lib=libfftw3f-3");
    } else {
        download_archive_unix(&out_dir)?;
        println!("cargo:rustc-link-search={}", out_dir.join("lib").display());
        println!("cargo:rustc-link-lib=static=fftw3");
        println!("cargo:rustc-link-lib=static=fftw3f");
    }
    Ok(())
}
<commit_msg>Check if generated archive exists for fast compile<commit_after>use failure::*;
use std::env::var;
use std::fs;
use std::io::*;
use std::path::*;
use std::process::Command;
use zip::ZipArchive;

fn download_archive_windows(out_dir: &Path) -> Fallible<()> {
    if out_dir.join("libfftw3.dll").exists() && out_dir.join("libfftw3f.dll").exists() {
        return Ok(());
    }

    let archive = out_dir.join("fftw_windows.zip");
    if !archive.exists() {
        // Download
        let mut conn = ftp::FtpStream::connect("ftp.fftw.org:21")?;
        conn.login("anonymous", "anonymous")?;
        conn.cwd("pub/fftw")?;
        let buf = conn.simple_retr("fftw-3.3.5-dll64.zip")?.into_inner();
        // TODO calc checksum
        let mut f = fs::File::create(&archive)?;
        f.write(&buf)?;
    }
    let f = fs::File::open(&archive)?;
    let mut zip = ZipArchive::new(f)?;
    let target = var("TARGET").unwrap();
    for name in &["fftw3-3", "fftw3f-3"] {
        for ext in &["dll", "def"] {
            let filename = format!("lib{}.{}", name, ext);
            let mut zf = zip.by_name(&filename)?;
            let mut f = fs::File::create(out_dir.join(filename))?;
            copy(&mut zf, &mut f)?;
        }
        run(cc::windows_registry::find_tool(&target, "lib.exe")
            .unwrap()
            .to_command()
            .arg("/MACHINE:X64")
            .arg(format!("/DEF:lib{}.def", name))
            .arg(format!("/OUT:lib{}.lib", name))
            .current_dir(out_dir))
    }
    Ok(())
}

fn build_unix(out_dir: &Path) -> Fallible<()> {
    let root_dir = PathBuf::from(var("CARGO_MANIFEST_DIR").unwrap());
    let archive_dir = root_dir.join("fftw-3.3.8");
    if !out_dir.join("lib/libfftw3.a").exists() {
        build_fftw(&[], &archive_dir, &out_dir);
    }
    if !out_dir.join("lib/libfftw3f.a").exists() {
        build_fftw(&["--enable-single"], &archive_dir, &out_dir);
    }
    Ok(())
}

fn build_fftw(flags: &[&str], src_dir: &Path, out_dir: &Path) {
    run(
        Command::new(fs::canonicalize(src_dir.join("configure")).unwrap())
            .arg("--with-pic")
            .arg("--enable-static")
            .arg("--disable-doc")
            .arg(format!("--prefix={}", out_dir.display()))
            .args(flags)
            .current_dir(&src_dir),
    );
    run(Command::new("make")
        .arg(format!("-j{}", var("NUM_JOBS").unwrap()))
        .current_dir(&src_dir));
    run(Command::new("make").arg("install").current_dir(&src_dir));
}

fn run(command: &mut Command) {
    println!("Running: {:?}", command);
    match command.status() {
        Ok(status) => {
            if !status.success() {
                panic!("`{:?}` failed: {}", command, status);
            }
        }
        Err(error) => {
            panic!("failed to execute `{:?}`: {}", command, error);
        }
    }
}

fn main() -> Fallible<()> {
    let out_dir = PathBuf::from(var("OUT_DIR").unwrap());
    if cfg!(target_os = "windows") {
        download_archive_windows(&out_dir)?;
        println!("cargo:rustc-link-search={}", out_dir.display());
        println!("cargo:rustc-link-lib=libfftw3-3");
        println!("cargo:rustc-link-lib=libfftw3f-3");
    } else {
        build_unix(&out_dir)?;
        println!("cargo:rustc-link-search={}", out_dir.join("lib").display());
        println!("cargo:rustc-link-lib=static=fftw3");
        println!("cargo:rustc-link-lib=static=fftw3f");
    }
    Ok(())
}
<|endoftext|>


<commit_before>use std::num;
use addressable::Addressable;

struct Area<ADDR> {
	base: ADDR,
	last: ADDR,
	handler: ~Addressable<ADDR>,
}

pub struct Memory<ADDR> {
	priv areas: ~[Area<ADDR>],
}

impl<ADDR> Memory<ADDR> {
	pub fn new () -> Memory<ADDR> {
		Memory { areas: ~[] }
	}

	pub fn add<T: Owned+Addressable<ADDR>> (&mut self, base: ADDR, last: ADDR, handler: ~T) {
		self.areas.push(Area { base: base, last: last, handler: handler as ~Addressable<ADDR> });
	}
}

impl<ADDR: Int> Addressable<ADDR> for Memory<ADDR> {
	pub fn get (&self, addr: ADDR) -> u8 {
		for self.areas.rev_iter().advance() |area| {
			if addr >= area.base && addr <= area.last {
				return area.handler.get(addr - area.base);
			}
		}
		fail!("memory: Read from unmapped memory area ($%x)", num::cast(addr));
	}

	pub fn set (&mut self, addr: ADDR, data: u8) {
		for self.areas.mut_rev_iter().advance() |area| {
			if addr >= area.base && addr <= area.last {
				area.handler.set(addr - area.base, data);
				return;
			}
		}
		fail!("memory: Write to unmapped memory area ($%x)", num::cast(addr));
	}
}

#[cfg(test)]
mod tests {
	use super::*;
	use addressable::Addressable;
	use ram::Ram;

	#[test]
	fn test_read_write () {
		let mut m: Memory<u16> = Memory::new();
		let ram1: Ram<u16> = Ram::new_sized(0x0400);
		let ram2: Ram<u16> = Ram::new_sized(0x0400);
		m.add(0x0000, 0x0400, ~ram1);
		m.add(0x0200, 0x0600, ~ram2);
		m.set(0x0300, 123);
		assert_eq!(m.get(0x0300), 123);
	}
}
<commit_msg>Updated trait name to match current Rust development<commit_after>use std::num;
use addressable::Addressable;

struct Area<ADDR> {
	base: ADDR,
	last: ADDR,
	handler: ~Addressable<ADDR>,
}

pub struct Memory<ADDR> {
	priv areas: ~[Area<ADDR>],
}

impl<ADDR> Memory<ADDR> {
	pub fn new () -> Memory<ADDR> {
		Memory { areas: ~[] }
	}

	pub fn add<T: Send+Addressable<ADDR>> (&mut self, base: ADDR, last: ADDR, handler: ~T) {
		self.areas.push(Area { base: base, last: last, handler: handler as ~Addressable<ADDR> });
	}
}

impl<ADDR: Int> Addressable<ADDR> for Memory<ADDR> {
	pub fn get (&self, addr: ADDR) -> u8 {
		for self.areas.rev_iter().advance() |area| {
			if addr >= area.base && addr <= area.last {
				return area.handler.get(addr - area.base);
			}
		}
		fail!("memory: Read from unmapped memory area ($%x)", num::cast(addr));
	}

	pub fn set (&mut self, addr: ADDR, data: u8) {
		for self.areas.mut_rev_iter().advance() |area| {
			if addr >= area.base && addr <= area.last {
				area.handler.set(addr - area.base, data);
				return;
			}
		}
		fail!("memory: Write to unmapped memory area ($%x)", num::cast(addr));
	}
}

#[cfg(test)]
mod tests {
	use super::*;
	use addressable::Addressable;
	use ram::Ram;

	#[test]
	fn test_read_write () {
		let mut m: Memory<u16> = Memory::new();
		let ram1: Ram<u16> = Ram::new_sized(0x0400);
		let ram2: Ram<u16> = Ram::new_sized(0x0400);
		m.add(0x0000, 0x0400, ~ram1);
		m.add(0x0200, 0x0600, ~ram2);
		m.set(0x0300, 123);
		assert_eq!(m.get(0x0300), 123);
	}
}
<|endoftext|>


<commit_before>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let graph = GraphMMap::<u32>::new(&filename);

        let nodes = graph.nodes();

        let mut src = vec![1.0; nodes / peers as usize];    // local rank accumulation
        let mut dst = vec![0.0; nodes];                     // local rank accumulation

        let mut start = time::precise_time_s();

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {

                    // /---- should look familiar! ----\
                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node];
                    }

                    let mut counter = 0u64;
                    for node in 0..src.len() {
                        let edges = graph.edges(index + peers * node);
                        let value = src[node] / edges.len() as f32;
                        for &b in edges {
                            dst[b as usize] += value;
                            counter += 1;
                        }
                    }
                    // \------ end familiar part ------/

                    let mut updates = 0u64;
                    for d in dst.iter() { if *d != 0.0 { updates += 1; } }

                    output.give_at(&iter, dst.drain_temp()
                                             .enumerate()
                                             .filter(|&(_,f)| f != 0.0)
                                             .map(|(u,f)| (u as u32, f)));

                    for _ in 0..graph.nodes() { dst.push(0.0); }

                    println!("iteration {:?}: {}s; {} edges; {} updates", iter, time::precise_time_s() - start, counter, updates);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        let local_index = index as usize % workers;

        let mut acc = vec![0.0; nodes / workers];

        ranks
        // .unary_notify(
        //     Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
        //     format!("Aggregation"),
        //     vec![],
        //     move |input, output, iterator| {
        //         while let Some((iter, data)) = input.pull() {
        //             iterator.notify_at(&iter);
        //             for (node, rank) in data.drain_temp() {
        //                 acc[node as usize / workers] += rank;
        //             }
        //         }
        //
        //         while let Some((item, _)) = iterator.next() {
        //
        //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
        //                                      .map(|(u,f)| (((u * workers + local_index) as u32), f)));
        //
        //             for _ in 0..(nodes/workers) { acc.push(0.0); }
        //             assert!(acc.len() == nodes/workers);
        //         }
        //     }
        // )
        .connect_loop(helper);
    }

    while root.step() { }
}
<commit_msg>off by one in array allocation<commit_after>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let graph = GraphMMap::<u32>::new(&filename);

        let nodes = graph.nodes();

        let mut src = vec![1.0; 1 + (nodes / peers as usize)];  // local rank accumulation
        let mut dst = vec![0.0; nodes];                         // local rank accumulation

        let mut start = time::precise_time_s();

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {

                    // /---- should look familiar! ----\
                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node];
                    }

                    let mut counter = 0u64;
                    for node in 0..src.len() {
                        let edges = graph.edges(index + peers * node);
                        let value = src[node] / edges.len() as f32;
                        for &b in edges {
                            dst[b as usize] += value;
                        }
                    }
                    // \------ end familiar part ------/

                    output.give_at(&iter, dst.drain_temp()
                                             .enumerate()
                                             .filter(|&(_,f)| f != 0.0)
                                             .map(|(u,f)| (u as u32, f)));

                    for _ in 0..graph.nodes() { dst.push(0.0); }

                    println!("iteration {:?}: {}s", iter, time::precise_time_s() - start);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        let local_index = index as usize % workers;

        let mut acc = vec![0.0; nodes / workers];

        ranks
        // .unary_notify(
        //     Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
        //     format!("Aggregation"),
        //     vec![],
        //     move |input, output, iterator| {
        //         while let Some((iter, data)) = input.pull() {
        //             iterator.notify_at(&iter);
        //             for (node, rank) in data.drain_temp() {
        //                 acc[node as usize / workers] += rank;
        //             }
        //         }
        //
        //         while let Some((item, _)) = iterator.next() {
        //
        //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
        //                                      .map(|(u,f)| (((u * workers + local_index) as u32), f)));
        //
        //             for _ in 0..(nodes/workers) { acc.push(0.0); }
        //             assert!(acc.len() == nodes/workers);
        //         }
        //     }
        // )
        .connect_loop(helper);
    }

    while root.step() { }
}
<|endoftext|>


<commit_before>use nom::character::complete::{multispace0, multispace1};
use std::fmt;
use std::str;

use column::Column;
use common::{column_identifier_no_alias};
use keywords::escape_if_keyword;

#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
pub enum OrderType {
    OrderAscending,
    OrderDescending,
}

impl fmt::Display for OrderType {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            OrderType::OrderAscending => write!(f, "ASC"),
            OrderType::OrderDescending => write!(f, "DESC"),
        }
    }
}

#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
pub struct OrderClause {
    pub columns: Vec<(Column, OrderType)>, // TODO(malte): can this be an arbitrary expr?
}

impl fmt::Display for OrderClause {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "ORDER BY ")?;
        write!(
            f,
            "{}",
            self.columns
                .iter()
                .map(|&(ref c, ref o)| format!("{} {}", escape_if_keyword(&c.name), o))
                .collect::<Vec<_>>()
                .join(", ")
        )
    }
}

named!(pub order_type<&[u8], OrderType>,
    alt!(
          map!(tag_no_case!("desc"), |_| OrderType::OrderDescending)
        | map!(tag_no_case!("asc"), |_| OrderType::OrderAscending)
    )
);

// Parse ORDER BY clause
named!(pub order_clause<&[u8], OrderClause>,
    do_parse!(
        multispace0 >>
        tag_no_case!("order by") >>
        multispace1 >>
        order_expr: many0!(
            do_parse!(
                fieldname: column_identifier_no_alias >>
                ordering: opt!(
                    do_parse!(
                        multispace0 >>
                        ordering: order_type >>
                        (ordering)
                    )
                ) >>
                opt!(
                    do_parse!(
                        multispace0 >>
                        tag!(",") >>
                        multispace0 >>
                        ()
                    )
                ) >>
                (fieldname, ordering.unwrap_or(OrderType::OrderAscending))
            )
        ) >>
        (OrderClause {
            columns: order_expr,
            // order: match ordering {
            //     None => OrderType::OrderAscending,
            //     Some(ref o) => o.clone(),
            // },
        })
    )
);

#[cfg(test)]
mod tests {
    use super::*;
    use select::selection;

    #[test]
    fn order_clause() {
        let qstring1 = "select * from users order by name desc\n";
        let qstring2 = "select * from users order by name asc, age desc\n";
        let qstring3 = "select * from users order by name\n";

        let expected_ord1 = OrderClause {
            columns: vec![("name".into(), OrderType::OrderDescending)],
        };
        let expected_ord2 = OrderClause {
            columns: vec![
                ("name".into(), OrderType::OrderAscending),
                ("age".into(), OrderType::OrderDescending),
            ],
        };
        let expected_ord3 = OrderClause {
            columns: vec![("name".into(), OrderType::OrderAscending)],
        };

        let res1 = selection(qstring1.as_bytes());
        let res2 = selection(qstring2.as_bytes());
        let res3 = selection(qstring3.as_bytes());
        assert_eq!(res1.unwrap().1.order, Some(expected_ord1));
        assert_eq!(res2.unwrap().1.order, Some(expected_ord2));
        assert_eq!(res3.unwrap().1.order, Some(expected_ord3));
    }
}
<commit_msg>Update `order.rs` to use nom v5 functions<commit_after>use nom::character::complete::{multispace0, multispace1};
use std::fmt;
use std::str;

use column::Column;
use common::{column_identifier_no_alias};
use keywords::escape_if_keyword;
use nom::IResult;
use nom::combinator::{map, opt};
use nom::branch::alt;
use nom::bytes::complete::{tag_no_case, tag};
use nom::sequence::{preceded, delimited, tuple};
use nom::multi::many0;

#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
pub enum OrderType {
    OrderAscending,
    OrderDescending,
}

impl fmt::Display for OrderType {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            OrderType::OrderAscending => write!(f, "ASC"),
            OrderType::OrderDescending => write!(f, "DESC"),
        }
    }
}

#[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)]
pub struct OrderClause {
    pub columns: Vec<(Column, OrderType)>, // TODO(malte): can this be an arbitrary expr?
}

impl fmt::Display for OrderClause {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "ORDER BY ")?;
        write!(
            f,
            "{}",
            self.columns
                .iter()
                .map(|&(ref c, ref o)| format!("{} {}", escape_if_keyword(&c.name), o))
                .collect::<Vec<_>>()
                .join(", ")
        )
    }
}

pub fn order_type(i: &[u8]) -> IResult<&[u8], OrderType> {
    alt((map(tag_no_case("desc"), |_| OrderType::OrderDescending),
          map(tag_no_case("asc"), |_| OrderType::OrderAscending)))(i)
}

fn order_expr(i: &[u8]) -> IResult<&[u8], (Column, OrderType)> {
    let (remaining_input, (field_name, ordering, _)) =
        tuple((column_identifier_no_alias, opt(preceded(multispace0, order_type)),
                opt(delimited(multispace0, tag(","), multispace0))))(i)?;

    Ok((remaining_input, (field_name, ordering.unwrap_or(OrderType::OrderAscending))))
}

// Parse ORDER BY clause
pub fn order_clause(i: &[u8]) -> IResult<&[u8], OrderClause> {
    let (remaining_input, (_, _, _, columns)) =
        tuple((multispace0, tag_no_case("order by"), multispace1, many0(order_expr)))(i)?;

    Ok((remaining_input, OrderClause { columns }))
}

#[cfg(test)]
mod tests {
    use super::*;
    use select::selection;

    #[test]
    fn order_clause() {
        let qstring1 = "select * from users order by name desc\n";
        let qstring2 = "select * from users order by name asc, age desc\n";
        let qstring3 = "select * from users order by name\n";

        let expected_ord1 = OrderClause {
            columns: vec![("name".into(), OrderType::OrderDescending)],
        };
        let expected_ord2 = OrderClause {
            columns: vec![
                ("name".into(), OrderType::OrderAscending),
                ("age".into(), OrderType::OrderDescending),
            ],
        };
        let expected_ord3 = OrderClause {
            columns: vec![("name".into(), OrderType::OrderAscending)],
        };

        let res1 = selection(qstring1.as_bytes());
        let res2 = selection(qstring2.as_bytes());
        let res3 = selection(qstring3.as_bytes());
        assert_eq!(res1.unwrap().1.order, Some(expected_ord1));
        assert_eq!(res2.unwrap().1.order, Some(expected_ord2));
        assert_eq!(res3.unwrap().1.order, Some(expected_ord3));
    }
}
<|endoftext|>


<commit_before>//! Utilities for defining a JSON parser.

use std::rc::Rc;
use std::cell::RefCell;

use serde_json::error;
use serde::ser::{ Serialize, Serializer };
use serde_json::value::Value as JSON;
use serde::de::{ Deserialize, Deserializer, Error };

/// Utility function: Make sure that we have consumed all the fields of an object.
pub fn check_no_more_fields(path: Path, json: JSON) -> Result<(), ParseError> {
    if let JSON::Object(obj) = json {
        if obj.len() == 0 {
            Ok(())
        } else {
            Err(ParseError::unknown_fields(obj.keys().cloned().collect(), &path))
        }
    } else {
        Ok(())
    }
}

/// A path in the JSON tree. Used for displaying error messages.
pub struct Path {
    buf: Rc<RefCell<String>>,
    len: usize,
}
impl Path {
    /// Create an empty Path.
    pub fn new() -> Self {
        Path {
            buf: Rc::new(RefCell::new(String::new())),
            len: 0,
        }
    }

    /// Push a suffix after a path.
    pub fn push<F, T>(&self, suffix: &str, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        let buf = self.buf.clone();
        let len;
        {
            let mut str = buf.borrow_mut();
            str.push_str(" > ");
            str.push_str(suffix);
            len = str.len();
        }
        let path = Path {
            buf: buf,
            len: len,
        };
        let result = cb(path);
        {
            let mut str = self.buf.borrow_mut();
            str.truncate(self.len)
        }
        result
    }
    pub fn to_string(&self) -> String {
        let mut str = self.buf.borrow().clone();
        str.truncate(self.len);
        str
    }
}

/// An error during parsing.
#[derive(Debug)]
pub enum ParseError {
    JSON(JSONError),
    MissingField {
        name: String,
        at: String,
    },
    UnknownFields {
        names: Vec<String>,
        at: String,
    },
    TypeError {
        name: String,
        at: String,
        expected: String,
    }
}

impl ParseError {
    pub fn missing_field(name: &str, at: &Path) -> Self {
        ParseError::MissingField {
            name: name.to_owned(),
            at: at.to_string(),
        }
    }
    pub fn type_error(name: &str, at: &Path, expected: &str) -> Self {
        ParseError::TypeError {
            name: name.to_owned(),
            at: at.to_string(),
            expected: expected.to_owned()
        }
    }
    pub fn unknown_fields(names: Vec<String>, at: &Path) -> Self {
        ParseError::UnknownFields {
            names: names,
            at: at.to_string()
        }
    }
    pub fn json(error: error::Error) -> Self {
        ParseError::JSON(JSONError(error))
    }
}

#[derive(Debug)]
pub struct JSONError(error::Error);

impl Serialize for JSONError {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: Serializer
    {
        serializer.visit_str(&format!("{:?}", self))
    }
}

pub trait Parse<T: Sized> {
    fn parse(path: Path, source: JSON) -> Result<T, ParseError>;
    fn take(path: Path, source: &mut JSON, field_name: &str) -> Result<T, ParseError> {
        if let JSON::Object(ref mut obj) = *source {
            if let Some(v) = obj.remove(field_name) {
                return Self::parse(path, v)
            }
        }
        Err(ParseError::missing_field(field_name, &path))
    }
    fn take_vec(path: Path, source: &mut JSON, field_name: &str) -> Result<Vec<T>, ParseError>
    {
        if let JSON::Object(ref mut obj) = *source {
            if let Some(ref mut json) = obj.remove(field_name) {
                if let JSON::Array(ref mut vec) = *json {
                    let mut result = Vec::with_capacity(vec.len());
                    for (json, i) in vec.drain(..).zip(0..) {
                        let json = json;
                        let parsed = try!(path.push(&format!("{}#{}", field_name, i),
                            |path| Self::parse(path, json)
                        ));
                        result.push(parsed);
                    }
                    return Ok(result)
                }
                return Err(ParseError::type_error(field_name, &path, "array"))
            }
        }
        Err(ParseError::missing_field(field_name, &path))
    }
}

impl<T> Parse<T> for T where T: Deserialize {
    fn parse(_: Path, source: JSON) -> Result<T, ParseError> {
        use serde_json;
        serde_json::from_value(source).map_err(ParseError::json)
    }
}
<commit_msg>Removing parse.rs<commit_after><|endoftext|>


<commit_before>//! Response type parsing.

use serde::de::DeserializeOwned;
use serde_json::{self, Value};

use std::io::{Cursor, Read};

use super::{HttpResponse, HttpResponseHead, SliceBody, ReadBody};
use error::*;

impl<B: ResponseBody> HttpResponse<B> {
    /// Convert this http response into either `Ok(T)` or an `Err(ApiError)`.
    ///
    /// # Examples
    ///
    /// Any type that implements `IsOk` can be parsed into a concrete response or an `ApiError`:
    ///
    /// ```no_run
    /// # extern crate serde_json;
    /// # extern crate elastic_responses;
    /// # use serde_json::*;
    /// # use elastic_responses::*;
    /// # use elastic_responses::error::*;
    /// # fn do_request() -> HttpResponseSlice<Vec<u8>> { unimplemented!() }
    /// # fn main() {
    /// // Send a request and read as a HttpResponse
    /// let http_response = do_request();
    ///
    /// let search_response = http_response.into_response::<GetResponseOf<Value>>();
    ///
    /// match search_response {
    ///     Ok(res) => {
    ///         // Do something with the SearchResponse
    ///     }
    ///     Err(ResponseError::Api(ApiError::IndexNotFound { index })) => {
    ///         // Do something with the missing index error
    ///     }
    ///     _ => {
    ///         // Some other error
    ///     }
    /// }
    /// # }
    /// ```
    pub fn into_response<T: IsOk + DeserializeOwned>(self) -> Result<T, ResponseError> {
        let maybe = T::is_ok(self.head, Unbuffered(self.body))?;

        match maybe.ok {
            true => {
                let ok = maybe.res.parse_ok()?;
                Ok(ok)
            }
            false => {
                let err = maybe.res.parse_err()?;
                Err(ResponseError::Api(err))
            }
        }
    }
}

/// A http response body that can be buffered into a json value.
pub trait ResponseBody where Self: Sized
{
    /// The type of a buffered response body.
    type Buffered: ResponseBody;

    /// Buffer the response body to a json value and return a new buffered representation.
    fn body(self) -> Result<(Value, Self::Buffered), ParseResponseError>;

    /// Parse the body as a success result.
    fn parse_ok<T: DeserializeOwned>(self) -> Result<T, ParseResponseError>;

    /// Parse the body as an API error.
    fn parse_err(self) -> Result<ApiError, ParseResponseError>;
}

impl<B: Read> ResponseBody for ReadBody<B> {
    type Buffered = ReadBody<Cursor<Vec<u8>>>;

    fn body(mut self) -> Result<(Value, Self::Buffered), ParseResponseError> {
        let mut buf = Vec::new();
        self.0.read_to_end(&mut buf)?;

        let body: Value = serde_json::from_reader(Cursor::new(&buf))?;

        Ok((body, ReadBody(Cursor::new(buf))))
    }

    fn parse_ok<T: DeserializeOwned>(self) -> Result<T, ParseResponseError> {
        serde_json::from_reader(self.0).map_err(|e| e.into())
    }

    fn parse_err(self) -> Result<ApiError, ParseResponseError> {
        serde_json::from_reader(self.0).map_err(|e| e.into())
    }
}

impl<B: AsRef<[u8]>> ResponseBody for SliceBody<B> {
    type Buffered = Self;

    fn body(self) -> Result<(Value, Self::Buffered), ParseResponseError> {
        let buf = self.0;

        let body: Value = serde_json::from_slice(buf.as_ref())?;

        Ok((body, SliceBody(buf)))
    }

    fn parse_ok<T: DeserializeOwned>(self) -> Result<T, ParseResponseError> {
        serde_json::from_slice(self.0.as_ref()).map_err(|e| e.into())
    }

    fn parse_err(self) -> Result<ApiError, ParseResponseError> {
        serde_json::from_slice(self.0.as_ref()).map_err(|e| e.into())
    }
}

/// Convert a response message into a either a success
/// or failure result.
pub trait IsOk
{
    /// Inspect the http response to determine whether or not it succeeded.
    fn is_ok<B: ResponseBody>(head: HttpResponseHead, body: Unbuffered<B>) -> Result<MaybeOkResponse<B>, ParseResponseError>;
}

impl IsOk for Value {
    fn is_ok<B: ResponseBody>(head: HttpResponseHead, body: Unbuffered<B>) -> Result<MaybeOkResponse<B>, ParseResponseError> {
        match head.status() {
            200...299 => Ok(MaybeOkResponse::ok(body)),
            _ => Ok(MaybeOkResponse::err(body)),
        }
    }
}

/// A response that might be successful or an `ApiError`.
pub struct MaybeOkResponse<B>
    where B: ResponseBody
{
    ok: bool,
    res: MaybeBufferedResponse<B>,
}

impl<B> MaybeOkResponse<B> where B: ResponseBody
{
    /// Create a new response that indicates where or not the
    /// body is successful or an `ApiError`.
    pub fn new<I>(ok: bool, res: I) -> Self
        where I: Into<MaybeBufferedResponse<B>>
    {
        MaybeOkResponse {
            ok: ok,
            res: res.into(),
        }
    }

    /// Create a response where the body is successful.
    pub fn ok<I>(res: I) -> Self
        where I: Into<MaybeBufferedResponse<B>>
    {
        Self::new(true, res)
    }

    /// Create a resposne where the body is an error.
    pub fn err<I>(res: I) -> Self
        where I: Into<MaybeBufferedResponse<B>>
    {
        Self::new(false, res)
    }
}

pub struct Unbuffered<B>(B);

impl<B: ResponseBody> Unbuffered<B> {
    /// Buffer the response body to a json value and return a new buffered representation.
    pub fn body(self) -> Result<(Value, Buffered<B>), ParseResponseError> {
        self.0.body().map(|(value, body)| (value, Buffered(body)))
    }
}

pub struct Buffered<B: ResponseBody>(B::Buffered);

/// A response body that may or may not have been buffered.
///
/// This type makes it possible to inspect the response body for
/// an error type before passing it along to be deserialised properly.
pub enum MaybeBufferedResponse<B>
    where B: ResponseBody
{
    Unbuffered(B),
    Buffered(B::Buffered),
}

impl<B> MaybeBufferedResponse<B>
    where B: ResponseBody
{
    fn parse_ok<T: DeserializeOwned>(self) -> Result<T, ParseResponseError> {
        match self {
            MaybeBufferedResponse::Unbuffered(b) => b.parse_ok(),
            MaybeBufferedResponse::Buffered(b) => b.parse_ok()
        }
    }

    fn parse_err(self) -> Result<ApiError, ParseResponseError> {
        match self {
            MaybeBufferedResponse::Unbuffered(b) => b.parse_err(),
            MaybeBufferedResponse::Buffered(b) => b.parse_err()
        }
    }
}

impl<B> From<Unbuffered<B>> for MaybeBufferedResponse<B>
    where B: ResponseBody
{
    fn from(value: Unbuffered<B>) -> Self {
        MaybeBufferedResponse::Unbuffered(value.0)
    }
}

impl<B> From<Buffered<B>> for MaybeBufferedResponse<B>
    where B: ResponseBody
{
    fn from(value: Buffered<B>) -> Self {
        MaybeBufferedResponse::Buffered(value.0)
    }
}
<commit_msg>buffer ReadBody as SliceBody<commit_after>//! Response type parsing.

use serde::de::DeserializeOwned;
use serde_json::{self, Value};

use std::io::{Cursor, Read};

use super::{HttpResponse, HttpResponseHead, SliceBody, ReadBody};
use error::*;

impl<B: ResponseBody> HttpResponse<B> {
    /// Convert this http response into either `Ok(T)` or an `Err(ApiError)`.
    ///
    /// # Examples
    ///
    /// Any type that implements `IsOk` can be parsed into a concrete response or an `ApiError`:
    ///
    /// ```no_run
    /// # extern crate serde_json;
    /// # extern crate elastic_responses;
    /// # use serde_json::*;
    /// # use elastic_responses::*;
    /// # use elastic_responses::error::*;
    /// # fn do_request() -> HttpResponseSlice<Vec<u8>> { unimplemented!() }
    /// # fn main() {
    /// // Send a document get request and read as a generic HttpResponse
    /// let http_response = do_request();
    ///
    /// let get_response = http_response.into_response::<GetResponseOf<Value>>();
    ///
    /// match get_response {
    ///     Ok(res) => {
    ///         // Do something with the GetResponse
    ///     }
    ///     Err(ResponseError::Api(ApiError::IndexNotFound { index })) => {
    ///         // Do something with the missing index error
    ///     }
    ///     _ => {
    ///         // Some other error
    ///     }
    /// }
    /// # }
    /// ```
    pub fn into_response<T: IsOk + DeserializeOwned>(self) -> Result<T, ResponseError> {
        let maybe = T::is_ok(self.head, Unbuffered(self.body))?;

        match maybe.ok {
            true => {
                let ok = maybe.res.parse_ok()?;
                Ok(ok)
            }
            false => {
                let err = maybe.res.parse_err()?;
                Err(ResponseError::Api(err))
            }
        }
    }
}

/// A http response body that can be buffered into a json value.
pub trait ResponseBody where Self: Sized
{
    /// The type of a buffered response body.
    type Buffered: ResponseBody;

    /// Buffer the response body to a json value and return a new buffered representation.
    fn body(self) -> Result<(Value, Self::Buffered), ParseResponseError>;

    /// Parse the body as a success result.
    fn parse_ok<T: DeserializeOwned>(self) -> Result<T, ParseResponseError>;

    /// Parse the body as an API error.
    fn parse_err(self) -> Result<ApiError, ParseResponseError>;
}

impl<B: Read> ResponseBody for ReadBody<B> {
    type Buffered = SliceBody<Vec<u8>>;

    fn body(mut self) -> Result<(Value, Self::Buffered), ParseResponseError> {
        let mut buf = Vec::new();
        self.0.read_to_end(&mut buf)?;

        let body: Value = serde_json::from_reader(Cursor::new(&buf))?;

        Ok((body, SliceBody(buf)))
    }

    fn parse_ok<T: DeserializeOwned>(self) -> Result<T, ParseResponseError> {
        serde_json::from_reader(self.0).map_err(|e| e.into())
    }

    fn parse_err(self) -> Result<ApiError, ParseResponseError> {
        serde_json::from_reader(self.0).map_err(|e| e.into())
    }
}

impl<B: AsRef<[u8]>> ResponseBody for SliceBody<B> {
    type Buffered = Self;

    fn body(self) -> Result<(Value, Self::Buffered), ParseResponseError> {
        let buf = self.0;

        let body: Value = serde_json::from_slice(buf.as_ref())?;

        Ok((body, SliceBody(buf)))
    }

    fn parse_ok<T: DeserializeOwned>(self) -> Result<T, ParseResponseError> {
        serde_json::from_slice(self.0.as_ref()).map_err(|e| e.into())
    }

    fn parse_err(self) -> Result<ApiError, ParseResponseError> {
        serde_json::from_slice(self.0.as_ref()).map_err(|e| e.into())
    }
}

/// Convert a response message into a either a success
/// or failure result.
pub trait IsOk
{
    /// Inspect the http response to determine whether or not it succeeded.
    fn is_ok<B: ResponseBody>(head: HttpResponseHead, body: Unbuffered<B>) -> Result<MaybeOkResponse<B>, ParseResponseError>;
}

impl IsOk for Value {
    fn is_ok<B: ResponseBody>(head: HttpResponseHead, body: Unbuffered<B>) -> Result<MaybeOkResponse<B>, ParseResponseError> {
        match head.status() {
            200...299 => Ok(MaybeOkResponse::ok(body)),
            _ => Ok(MaybeOkResponse::err(body)),
        }
    }
}

/// A response that might be successful or an `ApiError`.
pub struct MaybeOkResponse<B>
    where B: ResponseBody
{
    ok: bool,
    res: MaybeBufferedResponse<B>,
}

impl<B> MaybeOkResponse<B> where B: ResponseBody
{
    /// Create a new response that indicates where or not the
    /// body is successful or an `ApiError`.
    pub fn new<I>(ok: bool, res: I) -> Self
        where I: Into<MaybeBufferedResponse<B>>
    {
        MaybeOkResponse {
            ok: ok,
            res: res.into(),
        }
    }

    /// Create a response where the body is successful.
    pub fn ok<I>(res: I) -> Self
        where I: Into<MaybeBufferedResponse<B>>
    {
        Self::new(true, res)
    }

    /// Create a resposne where the body is an error.
    pub fn err<I>(res: I) -> Self
        where I: Into<MaybeBufferedResponse<B>>
    {
        Self::new(false, res)
    }
}

pub struct Unbuffered<B>(B);

impl<B: ResponseBody> Unbuffered<B> {
    /// Buffer the response body to a json value and return a new buffered representation.
    pub fn body(self) -> Result<(Value, Buffered<B>), ParseResponseError> {
        self.0.body().map(|(value, body)| (value, Buffered(body)))
    }
}

pub struct Buffered<B: ResponseBody>(B::Buffered);

/// A response body that may or may not have been buffered.
///
/// This type makes it possible to inspect the response body for
/// an error type before passing it along to be deserialised properly.
pub enum MaybeBufferedResponse<B>
    where B: ResponseBody
{
    Unbuffered(B),
    Buffered(B::Buffered),
}

impl<B> MaybeBufferedResponse<B>
    where B: ResponseBody
{
    fn parse_ok<T: DeserializeOwned>(self) -> Result<T, ParseResponseError> {
        match self {
            MaybeBufferedResponse::Unbuffered(b) => b.parse_ok(),
            MaybeBufferedResponse::Buffered(b) => b.parse_ok()
        }
    }

    fn parse_err(self) -> Result<ApiError, ParseResponseError> {
        match self {
            MaybeBufferedResponse::Unbuffered(b) => b.parse_err(),
            MaybeBufferedResponse::Buffered(b) => b.parse_err()
        }
    }
}

impl<B> From<Unbuffered<B>> for MaybeBufferedResponse<B>
    where B: ResponseBody
{
    fn from(value: Unbuffered<B>) -> Self {
        MaybeBufferedResponse::Unbuffered(value.0)
    }
}

impl<B> From<Buffered<B>> for MaybeBufferedResponse<B>
    where B: ResponseBody
{
    fn from(value: Buffered<B>) -> Self {
        MaybeBufferedResponse::Buffered(value.0)
    }
}
<|endoftext|>


<commit_before>//! Utilities for defining a JSON parser.

use util::Id;

use std::cell::RefCell;
use std::collections::{ HashMap, HashSet };
use std::error::Error as StdError;
use std::fmt::{ Display, Debug, Error as FmtError, Formatter };
use std::hash::Hash;
use std::rc::Rc;
use std::sync::Arc;

use serde_json::error;
use serde::ser::{ Serialize, Serializer };
use serde_json;
pub use serde_json::value::Value as JSON;
use serde::de::Error;

/// Utility function: Make sure that we have consumed all the fields of an object.
pub fn check_fields(path: Path, json: &JSON) -> Result<(), ParseError> {
    if let JSON::Object(ref obj) = *json {
        if obj.is_empty() {
            Ok(())
        } else {
            Err(ParseError::unknown_fields(obj.keys().cloned().collect(), &path))
        }
    } else {
        Ok(())
    }
}

/// A path in the JSON tree. Used for displaying error messages.
#[derive(Clone, Debug)]
pub struct Path {
    buf: Rc<RefCell<String>>,
    len: usize,
}
impl Path {
    /// Create an empty Path.
    pub fn new() -> Self {
        Path {
            buf: Rc::new(RefCell::new(String::new())),
            len: 0,
        }
    }

    /// Push a suffix after a path.
    pub fn push_str<F, T>(&self, suffix: &str, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        let buf = self.buf.clone();
        let len;
        {
            let mut str = buf.borrow_mut();
            str.push_str(suffix);
            len = str.len();
        }
        let path = Path {
            buf: buf,
            len: len,
        };
        let result = cb(path);
        {
            let mut str = self.buf.borrow_mut();
            str.truncate(self.len)
        }
        result
    }

    pub fn push_index<F, T>(&self, index: usize, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        self.push_str(&format!("[{}]", index), cb)
    }

    pub fn push<F, T>(&self, suffix: &str, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        self.push_str(&format!(".{}", suffix), cb)
    }

    pub fn to_string(&self) -> String {
        let mut str = self.buf.borrow().clone();
        str.truncate(self.len);
        str
    }
}

/// An error during parsing.
#[derive(Debug)]
pub enum ParseError {
    JSON(JSONError),
    MissingField {
        name: String,
        at: String,
    },
    UnknownFields {
        names: Vec<String>,
        at: String,
    },
    TypeError {
        name: String,
        at: String,
        expected: String,
    },
    EmptyObject {
        at: String
    },
    UnknownConstant {
        at: String,
        constant: String,
    }
}

impl Display for ParseError {
    fn fmt(&self, formatter: &mut Formatter) -> Result<(), FmtError> {
        (self as &Debug).fmt(formatter)
    }
}
impl StdError for ParseError {
    fn description(&self) -> &str {
        "Error while parsing to JSON"
    }
}

impl ParseError {
    pub fn missing_field(name: &str, at: &Path) -> Self {
        ParseError::MissingField {
            name: name.to_owned(),
            at: at.to_string(),
        }
    }
    pub fn type_error(name: &str, at: &Path, expected: &str) -> Self {
        ParseError::TypeError {
            name: name.to_owned(),
            at: at.to_string(),
            expected: expected.to_owned()
        }
    }
    pub fn unknown_fields(names: Vec<String>, at: &Path) -> Self {
        ParseError::UnknownFields {
            names: names,
            at: at.to_string()
        }
    }
    pub fn unknown_constant(value: &str, at: &Path) -> Self {
        ParseError::UnknownConstant {
            constant: value.to_owned(),
            at: at.to_string()
        }
    }
    pub fn empty_object(at: &Path) -> Self {
        ParseError::EmptyObject {
            at: at.to_string(),
        }
    }
    pub fn json(error: error::Error) -> Self {
        ParseError::JSON(JSONError(error))
    }
}

#[derive(Debug)]
pub struct JSONError(error::Error);

impl Serialize for JSONError {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: Serializer
    {
        serializer.serialize_str(&format!("{:?}", self))
    }
}

/// An object that knows how to parse values from JSON into type T.
///
/// The JSON object is expected to be consumed along the way. A successful parse will
/// typically leave an empty JSON object.
pub trait Parser<T: Sized> {
    fn description() -> String;
    fn from_str(source: &str) -> Result<T, ParseError> {
        Self::from_str_at(Path::new(), source)
    }
    fn from_str_at(path: Path, source: &str) -> Result<T, ParseError> {
        match serde_json::from_str(source) {
            Err(err) => Err(ParseError::json(err)),
            Ok(mut json) => Self::parse(path, &mut json)
        }
    }

    /// Parse a single value from JSON, consuming as much as necessary from JSON.
    fn parse(path: Path, source: &mut JSON) -> Result<T, ParseError>;

    /// Parse a field from JSON, consuming it.
    fn take(path: Path, source: &mut JSON, field_name: &str) -> Result<T, ParseError> {
        match Self::take_opt(path.clone(), source, field_name) {
            Some(result) => result,
            None => Err(ParseError::missing_field(field_name, &path))
        }
    }

    /// Parse a field from JSON, consuming it.
    fn take_opt(path: Path, source: &mut JSON, field_name: &str) -> Option<Result<T, ParseError>> {
        if let JSON::Object(ref mut obj) = *source {
            if let Some(mut v) = obj.remove(field_name) {
                Some(Self::parse(path, &mut v))
            } else {
                None
            }
        } else {
            Some(Err(ParseError::type_error(field_name, &path, "object")))
        }
    }

    /// Parse a field containing an array from JSON, consuming the field.
    fn take_vec_opt(path: Path, source: &mut JSON, field_name: &str) -> Option<Result<Vec<T>, ParseError>>
    {
        if let JSON::Object(ref mut obj) = *source {
            if let Some(ref mut json) = obj.remove(field_name) {
                if let JSON::Array(ref mut vec) = *json {
                    let mut result = Vec::with_capacity(vec.len());
                    for (json, i) in vec.iter_mut().zip(0..) {
                        match path.push(&format!("[{}]", i),
                            |path| Self::parse(path, json)
                        ) {
                            Err(error) => return Some(Err(error)),
                            Ok(parsed) => result.push(parsed)
                        }
                    }
                    Some(Ok(result))
                } else {
                    Some(Err(ParseError::type_error(field_name, &path, "array")))
                }
            } else {
                None
            }
        } else {
            Some(Err(ParseError::missing_field(field_name, &path)))
        }
    }

    fn take_vec(path: Path, source: &mut JSON, field_name: &str) -> Result<Vec<T>, ParseError> {
        match Self::take_vec_opt(path.clone(), source, field_name) {
            Some(result) => result,
            None => Err(ParseError::missing_field(field_name, &path))
        }
    }
}

impl Parser<f64> for f64 {
    fn description() -> String {
        "Number".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::I64(val) => Ok(val as f64),
            JSON::F64(val) => Ok(val),
            JSON::U64(val) => Ok(val as f64),
            _ => Err(ParseError::type_error("as float", &path, "number"))
        }
    }
}

impl Parser<bool> for bool {
    fn description() -> String {
        "bool".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::Bool(ref b) => Ok(*b),
            JSON::U64(0) | JSON::I64(0) => Ok(false),
            JSON::U64(1) | JSON::I64(1) => Ok(true),
            JSON::String(ref str) if str == "true" => Ok(true),
            JSON::String(ref str) if str == "false" => Ok(false),
            _ => Err(ParseError::type_error("as bool", &path, "boolean"))
        }
    }
}

impl Parser<u8> for u8 {
    fn description() -> String {
        "byte".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match source.as_u64() {
            None => Err(ParseError::type_error("as byte", &path, "positive integer")),
            Some(ref val) if *val > u8::max_value() as u64 =>
                Err(ParseError::type_error("as byte", &path, "positive integer")),
            Some(ref val) => Ok(*val as u8)
        }
    }
}

impl<T> Parser<Vec<T>> for Vec<T> where T: Parser<T> {
    fn description() -> String {
        format!("Array<{}>", T::description())
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        // Otherwise, parse as an actual array.
        match *source {
            JSON::Array(ref mut array) => {
                let mut result = Vec::with_capacity(array.len());
                for (source, i) in array.iter_mut ().zip(0..) {
                    let value = try!(path.push_index(i, |path| T::parse(path, source)));
                    result.push(value)
                }
                Ok(result)
            }
            JSON::Null => {
                // Accept `null` as an empty array.
                Ok(vec![])
            }
            _ => {
                // Attempt to promote the value to an array.
                let single = try!(path.push_str("", |path| T::parse(path, source)));
                Ok(vec![single])
            }
        }
    }
}

/*
impl<T, U> Parser<(T, U)> for (T, U) where T: Parser<T>, U: Parser<U> {
    fn description() -> String {
        format!("({}, {})", T::description(), U::description())
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::Array(ref mut array) if array.len() == 2 => {
                let mut right = array.pop().unwrap(); // We just checked that length == 2
                let mut left = array.pop().unwrap(); // We just checked that length == 2
                let left_parsed = try!(path.push(&T::description() as &str, |path| {T::parse(path, &mut left)}));
                let right_parsed = try!(path.push(&U::description() as &str, |path| {U::parse(path, &mut right)}));
                Ok((left_parsed, right_parsed))
            }
            _ => Err(ParseError::type_error("pair of values", &path, "array"))
        }
    }
}
*/
impl Parser<String> for String {
    fn description() -> String {
        "String".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match source.as_string() {
            Some(str) => Ok(str.to_owned()),
            None => Err(ParseError::type_error("string", &path, "string"))
        }
    }
}

impl<T> Parser<Arc<T>> for Arc<T> where T: Parser<T> {
    fn description() -> String {
        T::description()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        Ok(Arc::new(try!(T::parse(path, source))))
    }
}

pub trait ToJSON {
    fn to_json(&self) -> JSON;
}

impl ToJSON for String {
    fn to_json(&self) -> JSON {
        JSON::String(self.clone())
    }
}

impl ToJSON for bool {
    fn to_json(&self) -> JSON {
        JSON::Bool(*self)
    }
}

impl ToJSON for f64 {
    fn to_json(&self) -> JSON {
        JSON::F64(*self)
    }
}

impl ToJSON for usize {
    fn to_json(&self) -> JSON {
        JSON::U64(*self as u64)
    }
}

impl ToJSON for JSON {
    fn to_json(&self) -> JSON {
        self.clone()
    }
}

impl<T> ToJSON for HashSet<T> where T: ToJSON + Eq + Hash {
    fn to_json(&self) -> JSON {
        JSON::Array((*self).iter().map(T::to_json).collect())
    }
}

impl<T> ToJSON for HashMap<String, T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|(k, v)| (k.clone(), T::to_json(v))).collect())
    }
}

impl<T> ToJSON for Vec<T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Array(self.iter().map(|x| x.to_json()).collect())
    }
}

impl<'a, T> ToJSON for Vec<(&'a str, T)> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|&(ref k, ref v)| {
            ((*k).to_owned(), v.to_json())
        }).collect())
    }
}

impl <'a> ToJSON for &'a str {
    fn to_json(&self) -> JSON {
        JSON::String((*self).to_owned())
    }
}

impl<'a, T> ToJSON for &'a T where T: ToJSON {
    fn to_json(&self) -> JSON {
        (**self).to_json()
    }
}

impl<K, T, V> ToJSON for HashMap<Id<K>, Result<T, V>> where T: ToJSON, V: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|(k, result)| {
            let k = k.to_string();
            let result = match *result {
                Ok(ref ok) => ok.to_json(),
                Err(ref err) => vec![("Error", err)].to_json()
            };
            (k, result)
        }).collect())
    }
}

impl<T> ToJSON for Option<T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        match *self {
            None => JSON::Null,
            Some(ref result) => result.to_json()
        }
    }
}

impl ToJSON for () {
    fn to_json(&self) -> JSON {
        JSON::Null
    }
}

/*
impl<T> Parser<T> for T where T: Deserialize {
    fn parse(_: Path, source: &mut JSON) -> Result<T, ParseError> {
        use serde_json;
        serde_json::from_value(source.clone()).map_err(ParseError::json)
    }
}
*/
<commit_msg>Better deserialization error messages<commit_after>//! Utilities for defining a JSON parser.

use util::Id;

use std::cell::RefCell;
use std::collections::{ HashMap, HashSet };
use std::error::Error as StdError;
use std::fmt::{ Display, Debug, Error as FmtError, Formatter };
use std::hash::Hash;
use std::rc::Rc;
use std::sync::Arc;

use serde_json::error;
use serde::ser::{ Serialize, Serializer };
use serde_json;
pub use serde_json::value::Value as JSON;
use serde::de::Error;

/// Utility function: Make sure that we have consumed all the fields of an object.
pub fn check_fields(path: Path, json: &JSON) -> Result<(), ParseError> {
    if let JSON::Object(ref obj) = *json {
        if obj.is_empty() {
            Ok(())
        } else {
            Err(ParseError::unknown_fields(obj.keys().cloned().collect(), &path))
        }
    } else {
        Ok(())
    }
}

/// A path in the JSON tree. Used for displaying error messages.
#[derive(Clone, Debug)]
pub struct Path {
    buf: Rc<RefCell<String>>,
    len: usize,
}
impl Path {
    /// Create an empty Path.
    pub fn new() -> Self {
        Path {
            buf: Rc::new(RefCell::new(String::new())),
            len: 0,
        }
    }

    /// Push a suffix after a path.
    pub fn push_str<F, T>(&self, suffix: &str, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        let buf = self.buf.clone();
        let len;
        {
            let mut str = buf.borrow_mut();
            str.push_str(suffix);
            len = str.len();
        }
        let path = Path {
            buf: buf,
            len: len,
        };
        let result = cb(path);
        {
            let mut str = self.buf.borrow_mut();
            str.truncate(self.len)
        }
        result
    }

    pub fn push_index<F, T>(&self, index: usize, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        self.push_str(&format!("[{}]", index), cb)
    }

    pub fn push<F, T>(&self, suffix: &str, cb: F) -> T
        where F: FnOnce(Path) -> T
    {
        self.push_str(&format!(".{}", suffix), cb)
    }

    pub fn to_string(&self) -> String {
        let mut str = self.buf.borrow().clone();
        str.truncate(self.len);
        str
    }
}

/// An error during parsing.
#[derive(Debug)]
pub enum ParseError {
    JSON(JSONError),
    MissingField {
        name: String,
        at: String,
    },
    UnknownFields {
        names: Vec<String>,
        at: String,
    },
    TypeError {
        name: String,
        at: String,
        expected: String,
    },
    EmptyObject {
        at: String
    },
    UnknownConstant {
        at: String,
        constant: String,
    }
}

impl Display for ParseError {
    fn fmt(&self, formatter: &mut Formatter) -> Result<(), FmtError> {
        (self as &Debug).fmt(formatter)
    }
}
impl StdError for ParseError {
    fn description(&self) -> &str {
        "Error while parsing to JSON"
    }
}

impl ParseError {
    pub fn missing_field(name: &str, at: &Path) -> Self {
        ParseError::MissingField {
            name: name.to_owned(),
            at: at.to_string(),
        }
    }
    pub fn type_error(name: &str, at: &Path, expected: &str) -> Self {
        ParseError::TypeError {
            name: name.to_owned(),
            at: at.to_string(),
            expected: expected.to_owned()
        }
    }
    pub fn unknown_fields(names: Vec<String>, at: &Path) -> Self {
        ParseError::UnknownFields {
            names: names,
            at: at.to_string()
        }
    }
    pub fn unknown_constant(value: &str, at: &Path) -> Self {
        ParseError::UnknownConstant {
            constant: value.to_owned(),
            at: at.to_string()
        }
    }
    pub fn empty_object(at: &Path) -> Self {
        ParseError::EmptyObject {
            at: at.to_string(),
        }
    }
    pub fn json(error: error::Error) -> Self {
        ParseError::JSON(JSONError(error))
    }
}

#[derive(Debug)]
pub struct JSONError(error::Error);

impl Serialize for JSONError {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: Serializer
    {
        serializer.serialize_str(&format!("{:?}", self))
    }
}

/// An object that knows how to parse values from JSON into type T.
///
/// The JSON object is expected to be consumed along the way. A successful parse will
/// typically leave an empty JSON object.
pub trait Parser<T: Sized> {
    fn description() -> String;
    fn from_str(source: &str) -> Result<T, ParseError> {
        Self::from_str_at(Path::new(), source)
    }
    fn from_str_at(path: Path, source: &str) -> Result<T, ParseError> {
        match serde_json::from_str(source) {
            Err(err) => Err(ParseError::json(err)),
            Ok(mut json) => Self::parse(path, &mut json)
        }
    }

    /// Parse a single value from JSON, consuming as much as necessary from JSON.
    fn parse(path: Path, source: &mut JSON) -> Result<T, ParseError>;

    /// Parse a field from JSON, consuming it.
    fn take(path: Path, source: &mut JSON, field_name: &str) -> Result<T, ParseError> {
        match Self::take_opt(path.clone(), source, field_name) {
            Some(result) => result,
            None => Err(ParseError::missing_field(field_name, &path))
        }
    }

    /// Parse a field from JSON, consuming it.
    fn take_opt(path: Path, source: &mut JSON, field_name: &str) -> Option<Result<T, ParseError>> {
        if let JSON::Object(ref mut obj) = *source {
            if let Some(mut v) = obj.remove(field_name) {
                Some(Self::parse(path, &mut v))
            } else {
                None
            }
        } else {
            Some(Err(ParseError::type_error(field_name, &path, "object")))
        }
    }

    /// Parse a field containing an array from JSON, consuming the field.
    fn take_vec_opt(path: Path, source: &mut JSON, field_name: &str) -> Option<Result<Vec<T>, ParseError>>
    {
        if let JSON::Object(ref mut obj) = *source {
            if let Some(ref mut json) = obj.remove(field_name) {
                if let JSON::Array(ref mut vec) = *json {
                    let mut result = Vec::with_capacity(vec.len());
                    for (json, i) in vec.iter_mut().zip(0..) {
                        match path.push_index(i, |path| Self::parse(path, json)) {
                            Err(error) => return Some(Err(error)),
                            Ok(parsed) => result.push(parsed)
                        }
                    }
                    Some(Ok(result))
                } else {
                    Some(Err(ParseError::type_error(field_name, &path, "array")))
                }
            } else {
                None
            }
        } else {
            Some(Err(ParseError::missing_field(field_name, &path)))
        }
    }

    fn take_vec(path: Path, source: &mut JSON, field_name: &str) -> Result<Vec<T>, ParseError> {
        match Self::take_vec_opt(path.clone(), source, field_name) {
            Some(result) => result,
            None => Err(ParseError::missing_field(field_name, &path))
        }
    }
}

impl Parser<f64> for f64 {
    fn description() -> String {
        "Number".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::I64(val) => Ok(val as f64),
            JSON::F64(val) => Ok(val),
            JSON::U64(val) => Ok(val as f64),
            _ => Err(ParseError::type_error("as float", &path, "number"))
        }
    }
}

impl Parser<bool> for bool {
    fn description() -> String {
        "bool".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::Bool(ref b) => Ok(*b),
            JSON::U64(0) | JSON::I64(0) => Ok(false),
            JSON::U64(1) | JSON::I64(1) => Ok(true),
            JSON::String(ref str) if str == "true" => Ok(true),
            JSON::String(ref str) if str == "false" => Ok(false),
            _ => Err(ParseError::type_error("as bool", &path, "boolean"))
        }
    }
}

impl Parser<u8> for u8 {
    fn description() -> String {
        "byte".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match source.as_u64() {
            None => Err(ParseError::type_error("as byte", &path, "positive integer")),
            Some(ref val) if *val > u8::max_value() as u64 =>
                Err(ParseError::type_error("as byte", &path, "positive integer")),
            Some(ref val) => Ok(*val as u8)
        }
    }
}

impl<T> Parser<Vec<T>> for Vec<T> where T: Parser<T> {
    fn description() -> String {
        format!("Array<{}>", T::description())
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        // Otherwise, parse as an actual array.
        match *source {
            JSON::Array(ref mut array) => {
                let mut result = Vec::with_capacity(array.len());
                for (source, i) in array.iter_mut ().zip(0..) {
                    let value = try!(path.push_index(i, |path| T::parse(path, source)));
                    result.push(value)
                }
                Ok(result)
            }
            JSON::Null => {
                // Accept `null` as an empty array.
                Ok(vec![])
            }
            _ => {
                // Attempt to promote the value to an array.
                let single = try!(path.push_str("", |path| T::parse(path, source)));
                Ok(vec![single])
            }
        }
    }
}

/*
impl<T, U> Parser<(T, U)> for (T, U) where T: Parser<T>, U: Parser<U> {
    fn description() -> String {
        format!("({}, {})", T::description(), U::description())
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match *source {
            JSON::Array(ref mut array) if array.len() == 2 => {
                let mut right = array.pop().unwrap(); // We just checked that length == 2
                let mut left = array.pop().unwrap(); // We just checked that length == 2
                let left_parsed = try!(path.push(&T::description() as &str, |path| {T::parse(path, &mut left)}));
                let right_parsed = try!(path.push(&U::description() as &str, |path| {U::parse(path, &mut right)}));
                Ok((left_parsed, right_parsed))
            }
            _ => Err(ParseError::type_error("pair of values", &path, "array"))
        }
    }
}
*/
impl Parser<String> for String {
    fn description() -> String {
        "String".to_owned()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        match source.as_string() {
            Some(str) => Ok(str.to_owned()),
            None => Err(ParseError::type_error("string", &path, "string"))
        }
    }
}

impl<T> Parser<Arc<T>> for Arc<T> where T: Parser<T> {
    fn description() -> String {
        T::description()
    }
    fn parse(path: Path, source: &mut JSON) -> Result<Self, ParseError> {
        Ok(Arc::new(try!(T::parse(path, source))))
    }
}

pub trait ToJSON {
    fn to_json(&self) -> JSON;
}

impl ToJSON for String {
    fn to_json(&self) -> JSON {
        JSON::String(self.clone())
    }
}

impl ToJSON for bool {
    fn to_json(&self) -> JSON {
        JSON::Bool(*self)
    }
}

impl ToJSON for f64 {
    fn to_json(&self) -> JSON {
        JSON::F64(*self)
    }
}

impl ToJSON for usize {
    fn to_json(&self) -> JSON {
        JSON::U64(*self as u64)
    }
}

impl ToJSON for JSON {
    fn to_json(&self) -> JSON {
        self.clone()
    }
}

impl<T> ToJSON for HashSet<T> where T: ToJSON + Eq + Hash {
    fn to_json(&self) -> JSON {
        JSON::Array((*self).iter().map(T::to_json).collect())
    }
}

impl<T> ToJSON for HashMap<String, T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|(k, v)| (k.clone(), T::to_json(v))).collect())
    }
}

impl<T> ToJSON for Vec<T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Array(self.iter().map(|x| x.to_json()).collect())
    }
}

impl<'a, T> ToJSON for Vec<(&'a str, T)> where T: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|&(ref k, ref v)| {
            ((*k).to_owned(), v.to_json())
        }).collect())
    }
}

impl <'a> ToJSON for &'a str {
    fn to_json(&self) -> JSON {
        JSON::String((*self).to_owned())
    }
}

impl<'a, T> ToJSON for &'a T where T: ToJSON {
    fn to_json(&self) -> JSON {
        (**self).to_json()
    }
}

impl<K, T, V> ToJSON for HashMap<Id<K>, Result<T, V>> where T: ToJSON, V: ToJSON {
    fn to_json(&self) -> JSON {
        JSON::Object(self.iter().map(|(k, result)| {
            let k = k.to_string();
            let result = match *result {
                Ok(ref ok) => ok.to_json(),
                Err(ref err) => vec![("Error", err)].to_json()
            };
            (k, result)
        }).collect())
    }
}

impl<T> ToJSON for Option<T> where T: ToJSON {
    fn to_json(&self) -> JSON {
        match *self {
            None => JSON::Null,
            Some(ref result) => result.to_json()
        }
    }
}

impl ToJSON for () {
    fn to_json(&self) -> JSON {
        JSON::Null
    }
}

/*
impl<T> Parser<T> for T where T: Deserialize {
    fn parse(_: Path, source: &mut JSON) -> Result<T, ParseError> {
        use serde_json;
        serde_json::from_value(source.clone()).map_err(ParseError::json)
    }
}
*/
<|endoftext|>