<commit_before>use prodbg_api::read_write::{Reader, Writer};
use prodbg_api::backend::{CBackendCallbacks};
use plugins::PluginHandler;
use reader_wrapper::{ReaderWrapper, WriterWrapper};
use backend_plugin::{BackendHandle, BackendPlugins};
use libc::{c_void};

#[derive(PartialEq, Eq, Clone, Copy, Debug)]
pub struct SessionHandle(pub u64);

///! Session is a major part of ProDBG. There can be several sessions active at the same time
///! and each session has exactly one backend. There are only communication internally in a session
///! sessions can't (at least now) not talk to eachother.
///!
///! A backend can have several views at the same time. Data can be sent between the backend and
///| views using the PDReader/PDWriter APIs (prodbg::Writer prodbg::Reader in Rust) and this is the
///| only way for views and backends to talk with each other. There are several reasons for this
///| approach:
///!
///| 1. No "hacks" on trying to share memory. Plugins can be over a socket/webview/etc.
///! 2. Views and backends makes no assumetions on the inner workings of the others.
///! 3. Backends and views can post messages which anyone can decide to (optionally) act on.
///!
pub struct Session {
    pub handle: SessionHandle,
    pub reader: Reader,

    current_writer: usize,
    writers: [Writer; 2],
    action: i32,

    backend: Option<BackendHandle>,
}

///! Connection options for Remote connections. Currently just one Ip adderss
///!
pub struct ConnectionSettings<'a> {
    pub address: &'a str,
}

impl Session {
    pub fn new(handle: SessionHandle) -> Session {
        Session {
            handle: handle,
            writers: [
                WriterWrapper::create_writer(),
                WriterWrapper::create_writer(),
            ],
            reader: ReaderWrapper::create_reader(),
            action: 0,
            current_writer: 0,
            backend: None,
        }
    }

    pub fn get_current_writer(&mut self) -> &mut Writer {
        &mut self.writers[self.current_writer]
    }

    pub fn start_remote(_plugin_handler: &PluginHandler, _settings: &ConnectionSettings) {}

    pub fn start_local(_: &str, _: usize) {}

    pub fn set_backend(&mut self, backend: Option<BackendHandle>) {
        self.backend = backend
    }

    pub fn action_step(&mut self) {
        println!("do step");
        self.action = 4;
    }

    pub fn send_menu_id(&mut self, menu_id: u32, backend_plugins: &mut BackendPlugins) {
        if let Some(backend) = backend_plugins.get_backend(self.backend) {
            if menu_id >= backend.menu_id_offset && (menu_id < backend.menu_id_offset + 1000) {
                let writer = self.get_current_writer();
                //println!("Write event, writer {}", );
                writer.event_begin(35); // Menu event, TODO: Fix hard-coded value
                writer.write_u32("menu_id", menu_id - backend.menu_id_offset);
                writer.event_end();
            }
        }
    }

    // The way this code works is to allow the view plugins to have "two rounds" of updates.
    // That is to allow the view plugins to send things that other view plugins can listen
    // to and not only get data from the backend.
    pub fn update(&mut self, backend_plugins: &mut BackendPlugins) {
        // swap the writers
        let c_writer = self.current_writer;
        let n_writer = (self.current_writer + 1) & 1;

        ReaderWrapper::init_from_writer(&mut self.reader, &self.writers[c_writer]);
        ReaderWrapper::reset_writer(&mut self.writers[n_writer]);

        if let Some(backend) = backend_plugins.get_backend(self.backend) {
            unsafe {
                let plugin_funcs = backend.plugin_type.plugin_funcs as *mut CBackendCallbacks;
                ((*plugin_funcs).update.unwrap())(backend.plugin_data,
                                                  self.action,
                                                  self.reader.api as *mut c_void,
                                                  self.writers[n_writer].api as *mut c_void);
            }
        }

        self.action = 0;
        self.current_writer = n_writer;

        //ReaderWrapper::init_from_writer(&mut self.reader, &self.writers[1]);
        //ReaderWrapper::reset_writer(&mut self.writers[0]);
    }
}

///
/// Sessions handler
///
pub struct Sessions {
    instances: Vec<Session>,
    current: usize,
    session_counter: SessionHandle,
}

impl Sessions {
    pub fn new() -> Sessions {
        Sessions {
            instances: Vec::new(),
            current: 0,
            session_counter: SessionHandle(0),
        }
    }

    pub fn create_instance(&mut self) -> SessionHandle {
        let s = Session::new(self.session_counter);
        let handle = s.handle;
        self.instances.push(s);
        self.session_counter.0 += 1;
        handle
    }

    pub fn update(&mut self, backend_plugins: &mut BackendPlugins) {
        for session in self.instances.iter_mut() {
            session.update(backend_plugins);
        }
    }

    pub fn get_current(&mut self) -> &mut Session {
        let current = self.current;
        &mut self.instances[current]
    }

    pub fn get_session(&mut self, handle: SessionHandle) -> Option<&mut Session> {
        for i in 0..self.instances.len() {
            if self.instances[i].handle == handle {
                return Some(&mut self.instances[i]);
            }
        }

        None
    }
}

#[cfg(test)]
mod tests {
    use core::reader_wrapper::{ReaderWrapper};
    use super::*;

    #[test]
    fn create_session() {
        let _session = Session::new();
    }

    #[test]
    fn write_simple_event() {
        let mut session = Session::new();

        session.writers[0].event_begin(0x44);
        session.writers[0].event_end();

        ReaderWrapper::init_from_writer(&mut session.reader, &session.writers[0]);

        assert_eq!(session.reader.get_event().unwrap(), 0x44);
    }
}

<commit_msg>Support for calling step_over, run and break<commit_after>use prodbg_api::read_write::{Reader, Writer};
use prodbg_api::backend::{CBackendCallbacks};
use plugins::PluginHandler;
use reader_wrapper::{ReaderWrapper, WriterWrapper};
use backend_plugin::{BackendHandle, BackendPlugins};
use libc::{c_void};
use prodbg_api::events::*;

#[derive(PartialEq, Eq, Clone, Copy, Debug)]
pub struct SessionHandle(pub u64);

///! Session is a major part of ProDBG. There can be several sessions active at the same time
///! and each session has exactly one backend. There are only communication internally in a session
///! sessions can't (at least now) not talk to eachother.
///!
///! A backend can have several views at the same time. Data can be sent between the backend and
///| views using the PDReader/PDWriter APIs (prodbg::Writer prodbg::Reader in Rust) and this is the
///| only way for views and backends to talk with each other. There are several reasons for this
///| approach:
///!
///| 1. No "hacks" on trying to share memory. Plugins can be over a socket/webview/etc.
///! 2. Views and backends makes no assumetions on the inner workings of the others.
///! 3. Backends and views can post messages which anyone can decide to (optionally) act on.
///!
pub struct Session {
    pub handle: SessionHandle,
    pub reader: Reader,

    current_writer: usize,
    writers: [Writer; 2],
    action: i32,

    backend: Option<BackendHandle>,
}

///! Connection options for Remote connections. Currently just one Ip adderss
///!
pub struct ConnectionSettings<'a> {
    pub address: &'a str,
}

impl Session {
    pub fn new(handle: SessionHandle) -> Session {
        Session {
            handle: handle,
            writers: [
                WriterWrapper::create_writer(),
                WriterWrapper::create_writer(),
            ],
            reader: ReaderWrapper::create_reader(),
            action: 0,
            current_writer: 0,
            backend: None,
        }
    }

    pub fn get_current_writer(&mut self) -> &mut Writer {
        &mut self.writers[self.current_writer]
    }

    pub fn start_remote(_plugin_handler: &PluginHandler, _settings: &ConnectionSettings) {}

    pub fn start_local(_: &str, _: usize) {}

    pub fn set_backend(&mut self, backend: Option<BackendHandle>) {
        self.backend = backend
    }

    pub fn action_step(&mut self) {
        println!("do step");
        self.action = ACTION_STEP;
    }

    pub fn action_run(&mut self) {
        println!("do run");
        self.action = ACTION_RUN;
    }

    pub fn action_step_over(&mut self) {
        println!("do step over");
        self.action = ACTION_STEP_OVER;
    }

    pub fn action_break(&mut self) {
        println!("do break");
        self.action = ACTION_BREAK;
    }

    pub fn send_menu_id(&mut self, menu_id: u32, backend_plugins: &mut BackendPlugins) {
        if let Some(backend) = backend_plugins.get_backend(self.backend) {
            if menu_id >= backend.menu_id_offset && (menu_id < backend.menu_id_offset + 1000) {
                let writer = self.get_current_writer();
                //println!("Write event, writer {}", );
                writer.event_begin(35); // Menu event, TODO: Fix hard-coded value
                writer.write_u32("menu_id", menu_id - backend.menu_id_offset);
                writer.event_end();
            }
        }
    }

    // The way this code works is to allow the view plugins to have "two rounds" of updates.
    // That is to allow the view plugins to send things that other view plugins can listen
    // to and not only get data from the backend.
    pub fn update(&mut self, backend_plugins: &mut BackendPlugins) {
        // swap the writers
        let c_writer = self.current_writer;
        let n_writer = (self.current_writer + 1) & 1;

        ReaderWrapper::init_from_writer(&mut self.reader, &self.writers[c_writer]);
        ReaderWrapper::reset_writer(&mut self.writers[n_writer]);

        if let Some(backend) = backend_plugins.get_backend(self.backend) {
            unsafe {
                let plugin_funcs = backend.plugin_type.plugin_funcs as *mut CBackendCallbacks;
                ((*plugin_funcs).update.unwrap())(backend.plugin_data,
                                                  self.action,
                                                  self.reader.api as *mut c_void,
                                                  self.writers[n_writer].api as *mut c_void);
            }
        }

        self.action = 0;
        self.current_writer = n_writer;

        //ReaderWrapper::init_from_writer(&mut self.reader, &self.writers[1]);
        //ReaderWrapper::reset_writer(&mut self.writers[0]);
    }
}

///
/// Sessions handler
///
pub struct Sessions {
    instances: Vec<Session>,
    current: usize,
    session_counter: SessionHandle,
}

impl Sessions {
    pub fn new() -> Sessions {
        Sessions {
            instances: Vec::new(),
            current: 0,
            session_counter: SessionHandle(0),
        }
    }

    pub fn create_instance(&mut self) -> SessionHandle {
        let s = Session::new(self.session_counter);
        let handle = s.handle;
        self.instances.push(s);
        self.session_counter.0 += 1;
        handle
    }

    pub fn update(&mut self, backend_plugins: &mut BackendPlugins) {
        for session in self.instances.iter_mut() {
            session.update(backend_plugins);
        }
    }

    pub fn get_current(&mut self) -> &mut Session {
        let current = self.current;
        &mut self.instances[current]
    }

    pub fn get_session(&mut self, handle: SessionHandle) -> Option<&mut Session> {
        for i in 0..self.instances.len() {
            if self.instances[i].handle == handle {
                return Some(&mut self.instances[i]);
            }
        }

        None
    }
}

#[cfg(test)]
mod tests {
    use core::reader_wrapper::{ReaderWrapper};
    use super::*;

    #[test]
    fn create_session() {
        let _session = Session::new();
    }

    #[test]
    fn write_simple_event() {
        let mut session = Session::new();

        session.writers[0].event_begin(0x44);
        session.writers[0].event_end();

        ReaderWrapper::init_from_writer(&mut session.reader, &session.writers[0]);

        assert_eq!(session.reader.get_event().unwrap(), 0x44);
    }
}

<|endoftext|>


<commit_before>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

// static USAGE: &'static str = "
// Usage: pagerank <source> <workers>
// ";

static USAGE: &'static str = "
Usage: scc <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let graph = GraphMMap::<u32>::new(&filename);

        let nodes = graph.nodes();

        let mut src = vec![1.0; nodes / peers as usize];    // local rank accumulation
        let mut dst = vec![0.0; nodes];                     // local rank accumulation

        let mut start = time::precise_time_s();

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {
                    // /---- should look familiar! ----\
                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node];
                    }

                    for node in 0..src.len() {
                        let edges = graph.edges(index + peers * node);
                        let value = src[node] / edges.len() as f32;
                        for &b in edges {
                            dst[b as usize] += value;
                        }
                    }
                    // \------ end familiar part ------/
                    output.give_at(&iter, dst.drain_temp()
                                             .enumerate()
                                             .filter(|&(_,f)| f != 0.0)
                                             .map(|(u,f)| (u as u32, f)));

                    // dst.resize(graph.nodes(), 0.0);
                    for _ in 0..graph.nodes() { dst.push(0.0); }

                    println!("{}s", time::precise_time_s() - start);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        let local_index = index as usize % workers;

        let mut acc = vec![0.0; nodes / workers];

        ranks
        .unary_notify(
            Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
            format!("Aggregation"),
            vec![],
            move |input, output, iterator| {
                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        acc[node as usize / workers] += rank;
                    }
                }

                while let Some((item, _)) = iterator.next() {

                    let mut count = 0;
                    for x in &acc { if *x != 0.0 { count += 1; } }
                    println!("about to send {} non-zero elements: ({} bytes)", count, 8 * count);

                    output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
                                             .map(|(u,f)| (((u * workers + local_index) as u32), f)));

                    for _ in 0..(nodes/workers) { acc.push(0.0); }
                    assert!(acc.len() == nodes/workers);
                }
            }
        )
        .connect_loop(helper);
    }

    while root.step() { }
}
<commit_msg>diagnostics<commit_after>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

// static USAGE: &'static str = "
// Usage: pagerank <source> <workers>
// ";

static USAGE: &'static str = "
Usage: scc <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let graph = GraphMMap::<u32>::new(&filename);

        let nodes = graph.nodes();

        let mut src = vec![1.0; nodes / peers as usize];    // local rank accumulation
        let mut dst = vec![0.0; nodes];                     // local rank accumulation

        let mut start = time::precise_time_s();

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {
                    // /---- should look familiar! ----\
                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node];
                    }

                    for node in 0..src.len() {
                        let edges = graph.edges(index + peers * node);
                        let value = src[node] / edges.len() as f32;
                        for &b in edges {
                            dst[b as usize] += value;
                        }
                    }
                    // \------ end familiar part ------/
                    output.give_at(&iter, dst.drain_temp()
                                             .enumerate()
                                             .filter(|&(_,f)| f != 0.0)
                                             .map(|(u,f)| (u as u32, f)));

                    // dst.resize(graph.nodes(), 0.0);
                    for _ in 0..graph.nodes() { dst.push(0.0); }

                    println!("{}s", time::precise_time_s() - start);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        let local_index = index as usize % workers;

        let mut acc = vec![0.0; nodes / workers];

        ranks
        .unary_notify(
            Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
            format!("Aggregation"),
            vec![],
            move |input, output, iterator| {
                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        acc[node as usize / workers] += rank;
                    }
                }

                while let Some((item, _)) = iterator.next() {

                    let mut count = 0;
                    for x in &acc { if *x != 0.0 { count += 1; } }
                    println!("about to send {} non-zero elements of {}: ({} bytes)", count, acc.len(), 8 * count);

                    output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
                                             .map(|(u,f)| (((u * workers + local_index) as u32), f)));

                    for _ in 0..(nodes/workers) { acc.push(0.0); }
                    assert!(acc.len() == nodes/workers);
                }
            }
        )
        .connect_loop(helper);
    }

    while root.step() { }
}
<|endoftext|>


<commit_before>//! Tidy check to ensure that unstable features are all in order.
//!
//! This check will ensure properties like:
//!
//! * All stability attributes look reasonably well formed.
//! * The set of library features is disjoint from the set of language features.
//! * Library features have at most one stability level.
//! * Library features have at most one `since` value.
//! * All unstable lang features have tests to ensure they are actually unstable.

use std::collections::HashMap;
use std::fmt;
use std::fs::{self, File};
use std::io::prelude::*;
use std::path::Path;

use regex::{Regex, escape};

mod version;
use self::version::Version;

const FEATURE_GROUP_START_PREFIX: &str = "// feature group start:";
const FEATURE_GROUP_END_PREFIX: &str = "// feature group end";

#[derive(Debug, PartialEq, Clone)]
pub enum Status {
    Stable,
    Removed,
    Unstable,
}

impl fmt::Display for Status {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let as_str = match *self {
            Status::Stable => "stable",
            Status::Unstable => "unstable",
            Status::Removed => "removed",
        };
        fmt::Display::fmt(as_str, f)
    }
}

#[derive(Debug, Clone)]
pub struct Feature {
    pub level: Status,
    pub since: Option<Version>,
    pub has_gate_test: bool,
    pub tracking_issue: Option<u32>,
    pub group: Option<String>,
}

pub type Features = HashMap<String, Feature>;

pub fn check(path: &Path, bad: &mut bool, quiet: bool) {
    let mut features = collect_lang_features(path, bad);
    assert!(!features.is_empty());

    let lib_features = get_and_check_lib_features(path, bad, &features);
    assert!(!lib_features.is_empty());

    let mut contents = String::new();

    super::walk_many(&[&path.join("test/ui"),
                       &path.join("test/ui-fulldeps"),
                       &path.join("test/compile-fail")],
                     &mut |path| super::filter_dirs(path),
                     &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        let filen_underscore = filename.replace('-',"_").replace(".rs","");
        let filename_is_gate_test = test_filen_gate(&filen_underscore, &mut features);

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        for (i, line) in contents.lines().enumerate() {
            let mut err = |msg: &str| {
                tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
            };

            let gate_test_str = "gate-test-";

            let feature_name = match line.find(gate_test_str) {
                Some(i) => {
                    line[i+gate_test_str.len()..].splitn(2, ' ').next().unwrap()
                },
                None => continue,
            };
            match features.get_mut(feature_name) {
                Some(f) => {
                    if filename_is_gate_test {
                        err(&format!("The file is already marked as gate test \
                                      through its name, no need for a \
                                      'gate-test-{}' comment",
                                     feature_name));
                    }
                    f.has_gate_test = true;
                }
                None => {
                    err(&format!("gate-test test found referencing a nonexistent feature '{}'",
                                 feature_name));
                }
            }
        }
    });

    // Only check the number of lang features.
    // Obligatory testing for library features is dumb.
    let gate_untested = features.iter()
                                .filter(|&(_, f)| f.level == Status::Unstable)
                                .filter(|&(_, f)| !f.has_gate_test)
                                .collect::<Vec<_>>();

    for &(name, _) in gate_untested.iter() {
        println!("Expected a gate test for the feature '{}'.", name);
        println!("Hint: create a failing test file named 'feature-gate-{}.rs'\
                \n      in the 'ui' test suite, with its failures due to\
                \n      missing usage of #![feature({})].", name, name);
        println!("Hint: If you already have such a test and don't want to rename it,\
                \n      you can also add a // gate-test-{} line to the test file.",
                 name);
    }

    if !gate_untested.is_empty() {
        tidy_error!(bad, "Found {} features without a gate test.", gate_untested.len());
    }

    if *bad {
        return;
    }
    if quiet {
        println!("* {} features", features.len());
        return;
    }

    let mut lines = Vec::new();
    for (name, feature) in features.iter() {
        lines.push(format!("{:<32} {:<8} {:<12} {:<8}",
                           name,
                           "lang",
                           feature.level,
                           feature.since.as_ref().map_or("None".to_owned(),
                                                         |since| since.to_string())));
    }
    for (name, feature) in lib_features {
        lines.push(format!("{:<32} {:<8} {:<12} {:<8}",
                           name,
                           "lib",
                           feature.level,
                           feature.since.as_ref().map_or("None".to_owned(),
                                                         |since| since.to_string())));
    }

    lines.sort();
    for line in lines {
        println!("* {}", line);
    }
}

fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> {
    let r = Regex::new(&format!(r#"{} *= *"([^"]*)""#, escape(attr)))
        .expect("malformed regex for find_attr_val");
    r.captures(line)
        .and_then(|c| c.get(1))
        .map(|m| m.as_str())
}

#[test]
fn test_find_attr_val() {
    let s = r#"#[unstable(feature = "checked_duration_since", issue = "58402")]"#;
    assert_eq!(find_attr_val(s, "feature"), Some("checked_duration_since"));
    assert_eq!(find_attr_val(s, "issue"), Some("58402"));
    assert_eq!(find_attr_val(s, "since"), None);
}

fn test_filen_gate(filen_underscore: &str, features: &mut Features) -> bool {
    if filen_underscore.starts_with("feature_gate") {
        for (n, f) in features.iter_mut() {
            if filen_underscore == format!("feature_gate_{}", n) {
                f.has_gate_test = true;
                return true;
            }
        }
    }
    return false;
}

pub fn collect_lang_features(base_src_path: &Path, bad: &mut bool) -> Features {
    let contents = t!(fs::read_to_string(base_src_path.join("libsyntax/feature_gate.rs")));

    // We allow rustc-internal features to omit a tracking issue.
    // To make tidy accept omitting a tracking issue, group the list of features
    // without one inside `// no tracking issue START` and `// no tracking issue END`.
    let mut next_feature_omits_tracking_issue = false;

    let mut next_feature_group = None;

    contents.lines().zip(1..)
        .filter_map(|(line, line_number)| {
            let line = line.trim();

            // Within START and END, the tracking issue can be omitted.
            match line {
                "// no tracking issue START" => {
                    next_feature_omits_tracking_issue = true;
                    return None;
                }
                "// no tracking issue END" => {
                    next_feature_omits_tracking_issue = false;
                    return None;
                }
                _ => {}
            }

            if line.starts_with(FEATURE_GROUP_START_PREFIX) {
                let group = line.trim_start_matches(FEATURE_GROUP_START_PREFIX).trim();
                next_feature_group = Some(group.to_owned());
                return None;
            } else if line.starts_with(FEATURE_GROUP_END_PREFIX) {
                next_feature_group = None;
                return None;
            }

            let mut parts = line.split(',');
            let level = match parts.next().map(|l| l.trim().trim_start_matches('(')) {
                Some("active") => Status::Unstable,
                Some("removed") => Status::Removed,
                Some("accepted") => Status::Stable,
                _ => return None,
            };
            let name = parts.next().unwrap().trim();
            let since_str = parts.next().unwrap().trim().trim_matches('"');
            let since = match since_str.parse() {
                Ok(since) => Some(since),
                Err(err) => {
                    tidy_error!(
                        bad,
                        "libsyntax/feature_gate.rs:{}: failed to parse since: {} ({})",
                        line_number,
                        since_str,
                        err,
                    );
                    None
                }
            };
            let issue_str = parts.next().unwrap().trim();
            let tracking_issue = if issue_str.starts_with("None") {
                if level == Status::Unstable && !next_feature_omits_tracking_issue {
                    *bad = true;
                    tidy_error!(
                        bad,
                        "libsyntax/feature_gate.rs:{}: no tracking issue for feature {}",
                        line_number,
                        name,
                    );
                }
                None
            } else {
                let s = issue_str.split('(').nth(1).unwrap().split(')').nth(0).unwrap();
                Some(s.parse().unwrap())
            };
            Some((name.to_owned(),
                Feature {
                    level,
                    since,
                    has_gate_test: false,
                    tracking_issue,
                    group: next_feature_group.clone(),
                }))
        })
        .collect()
}

pub fn collect_lib_features(base_src_path: &Path) -> Features {
    let mut lib_features = Features::new();

    // This library feature is defined in the `compiler_builtins` crate, which
    // has been moved out-of-tree. Now it can no longer be auto-discovered by
    // `tidy`, because we need to filter out its (submodule) directory. Manually
    // add it to the set of known library features so we can still generate docs.
    lib_features.insert("compiler_builtins_lib".to_owned(), Feature {
        level: Status::Unstable,
        since: None,
        has_gate_test: false,
        tracking_issue: None,
        group: None,
    });

    map_lib_features(base_src_path,
                     &mut |res, _, _| {
        if let Ok((name, feature)) = res {
            if lib_features.contains_key(name) {
                return;
            }
            lib_features.insert(name.to_owned(), feature);
        }
    });
   lib_features
}

fn get_and_check_lib_features(base_src_path: &Path,
                              bad: &mut bool,
                              lang_features: &Features) -> Features {
    let mut lib_features = Features::new();
    map_lib_features(base_src_path,
                     &mut |res, file, line| {
            match res {
                Ok((name, f)) => {
                    let mut check_features = |f: &Feature, list: &Features, display: &str| {
                        if let Some(ref s) = list.get(name) {
                            if f.tracking_issue != s.tracking_issue {
                                tidy_error!(bad,
                                            "{}:{}: mismatches the `issue` in {}",
                                            file.display(),
                                            line,
                                            display);
                            }
                        }
                    };
                    check_features(&f, &lang_features, "corresponding lang feature");
                    check_features(&f, &lib_features, "previous");
                    lib_features.insert(name.to_owned(), f);
                },
                Err(msg) => {
                    tidy_error!(bad, "{}:{}: {}", file.display(), line, msg);
                },
            }

    });
    lib_features
}

fn map_lib_features(base_src_path: &Path,
                    mf: &mut dyn FnMut(Result<(&str, Feature), &str>, &Path, usize)) {
    let mut contents = String::new();
    super::walk(base_src_path,
                &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
                &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        let mut becoming_feature: Option<(String, Feature)> = None;
        for (i, line) in contents.lines().enumerate() {
            macro_rules! err {
                ($msg:expr) => {{
                    mf(Err($msg), file, i + 1);
                    continue;
                }};
            };
            if let Some((ref name, ref mut f)) = becoming_feature {
                if f.tracking_issue.is_none() {
                    f.tracking_issue = find_attr_val(line, "issue")
                    .map(|s| s.parse().unwrap());
                }
                if line.ends_with(']') {
                    mf(Ok((name, f.clone())), file, i + 1);
                } else if !line.ends_with(',') && !line.ends_with('\\') {
                    // We need to bail here because we might have missed the
                    // end of a stability attribute above because the ']'
                    // might not have been at the end of the line.
                    // We could then get into the very unfortunate situation that
                    // we continue parsing the file assuming the current stability
                    // attribute has not ended, and ignoring possible feature
                    // attributes in the process.
                    err!("malformed stability attribute");
                } else {
                    continue;
                }
            }
            becoming_feature = None;
            if line.contains("rustc_const_unstable(") {
                // `const fn` features are handled specially.
                let feature_name = match find_attr_val(line, "feature") {
                    Some(name) => name,
                    None => err!("malformed stability attribute"),
                };
                let feature = Feature {
                    level: Status::Unstable,
                    since: None,
                    has_gate_test: false,
                    // FIXME(#57563): #57563 is now used as a common tracking issue,
                    // although we would like to have specific tracking issues for each
                    // `rustc_const_unstable` in the future.
                    tracking_issue: Some(57563),
                    group: None,
                };
                mf(Ok((feature_name, feature)), file, i + 1);
                continue;
            }
            let level = if line.contains("[unstable(") {
                Status::Unstable
            } else if line.contains("[stable(") {
                Status::Stable
            } else {
                continue;
            };
            let feature_name = match find_attr_val(line, "feature") {
                Some(name) => name,
                None => err!("malformed stability attribute"),
            };
            let since = match find_attr_val(line, "since").map(|x| x.parse()) {
                Some(Ok(since)) => Some(since),
                Some(Err(_err)) => {
                    err!("malformed since attribute");
                },
                None if level == Status::Stable => {
                    err!("malformed stability attribute");
                }
                None => None,
            };
            let tracking_issue = find_attr_val(line, "issue").map(|s| s.parse().unwrap());

            let feature = Feature {
                level,
                since,
                has_gate_test: false,
                tracking_issue,
                group: None,
            };
            if line.contains(']') {
                mf(Ok((feature_name, feature)), file, i + 1);
            } else {
                becoming_feature = Some((feature_name.to_owned(), feature));
            }
        }
    });
}
<commit_msg>Ensure language features in group are sorted by since<commit_after>//! Tidy check to ensure that unstable features are all in order.
//!
//! This check will ensure properties like:
//!
//! * All stability attributes look reasonably well formed.
//! * The set of library features is disjoint from the set of language features.
//! * Library features have at most one stability level.
//! * Library features have at most one `since` value.
//! * All unstable lang features have tests to ensure they are actually unstable.
//! * Language features in a group are sorted by `since` value.

use std::collections::HashMap;
use std::fmt;
use std::fs::{self, File};
use std::io::prelude::*;
use std::path::Path;

use regex::{Regex, escape};

mod version;
use self::version::Version;

const FEATURE_GROUP_START_PREFIX: &str = "// feature group start:";
const FEATURE_GROUP_END_PREFIX: &str = "// feature group end";

#[derive(Debug, PartialEq, Clone)]
pub enum Status {
    Stable,
    Removed,
    Unstable,
}

impl fmt::Display for Status {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let as_str = match *self {
            Status::Stable => "stable",
            Status::Unstable => "unstable",
            Status::Removed => "removed",
        };
        fmt::Display::fmt(as_str, f)
    }
}

#[derive(Debug, Clone)]
pub struct Feature {
    pub level: Status,
    pub since: Option<Version>,
    pub has_gate_test: bool,
    pub tracking_issue: Option<u32>,
    pub group: Option<String>,
}

pub type Features = HashMap<String, Feature>;

pub fn check(path: &Path, bad: &mut bool, quiet: bool) {
    let mut features = collect_lang_features(path, bad);
    assert!(!features.is_empty());

    let lib_features = get_and_check_lib_features(path, bad, &features);
    assert!(!lib_features.is_empty());

    let mut contents = String::new();

    super::walk_many(&[&path.join("test/ui"),
                       &path.join("test/ui-fulldeps"),
                       &path.join("test/compile-fail")],
                     &mut |path| super::filter_dirs(path),
                     &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        let filen_underscore = filename.replace('-',"_").replace(".rs","");
        let filename_is_gate_test = test_filen_gate(&filen_underscore, &mut features);

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        for (i, line) in contents.lines().enumerate() {
            let mut err = |msg: &str| {
                tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
            };

            let gate_test_str = "gate-test-";

            let feature_name = match line.find(gate_test_str) {
                Some(i) => {
                    line[i+gate_test_str.len()..].splitn(2, ' ').next().unwrap()
                },
                None => continue,
            };
            match features.get_mut(feature_name) {
                Some(f) => {
                    if filename_is_gate_test {
                        err(&format!("The file is already marked as gate test \
                                      through its name, no need for a \
                                      'gate-test-{}' comment",
                                     feature_name));
                    }
                    f.has_gate_test = true;
                }
                None => {
                    err(&format!("gate-test test found referencing a nonexistent feature '{}'",
                                 feature_name));
                }
            }
        }
    });

    // Only check the number of lang features.
    // Obligatory testing for library features is dumb.
    let gate_untested = features.iter()
                                .filter(|&(_, f)| f.level == Status::Unstable)
                                .filter(|&(_, f)| !f.has_gate_test)
                                .collect::<Vec<_>>();

    for &(name, _) in gate_untested.iter() {
        println!("Expected a gate test for the feature '{}'.", name);
        println!("Hint: create a failing test file named 'feature-gate-{}.rs'\
                \n      in the 'ui' test suite, with its failures due to\
                \n      missing usage of #![feature({})].", name, name);
        println!("Hint: If you already have such a test and don't want to rename it,\
                \n      you can also add a // gate-test-{} line to the test file.",
                 name);
    }

    if !gate_untested.is_empty() {
        tidy_error!(bad, "Found {} features without a gate test.", gate_untested.len());
    }

    if *bad {
        return;
    }
    if quiet {
        println!("* {} features", features.len());
        return;
    }

    let mut lines = Vec::new();
    for (name, feature) in features.iter() {
        lines.push(format!("{:<32} {:<8} {:<12} {:<8}",
                           name,
                           "lang",
                           feature.level,
                           feature.since.as_ref().map_or("None".to_owned(),
                                                         |since| since.to_string())));
    }
    for (name, feature) in lib_features {
        lines.push(format!("{:<32} {:<8} {:<12} {:<8}",
                           name,
                           "lib",
                           feature.level,
                           feature.since.as_ref().map_or("None".to_owned(),
                                                         |since| since.to_string())));
    }

    lines.sort();
    for line in lines {
        println!("* {}", line);
    }
}

fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> {
    let r = Regex::new(&format!(r#"{} *= *"([^"]*)""#, escape(attr)))
        .expect("malformed regex for find_attr_val");
    r.captures(line)
        .and_then(|c| c.get(1))
        .map(|m| m.as_str())
}

#[test]
fn test_find_attr_val() {
    let s = r#"#[unstable(feature = "checked_duration_since", issue = "58402")]"#;
    assert_eq!(find_attr_val(s, "feature"), Some("checked_duration_since"));
    assert_eq!(find_attr_val(s, "issue"), Some("58402"));
    assert_eq!(find_attr_val(s, "since"), None);
}

fn test_filen_gate(filen_underscore: &str, features: &mut Features) -> bool {
    if filen_underscore.starts_with("feature_gate") {
        for (n, f) in features.iter_mut() {
            if filen_underscore == format!("feature_gate_{}", n) {
                f.has_gate_test = true;
                return true;
            }
        }
    }
    return false;
}

pub fn collect_lang_features(base_src_path: &Path, bad: &mut bool) -> Features {
    let contents = t!(fs::read_to_string(base_src_path.join("libsyntax/feature_gate.rs")));

    // We allow rustc-internal features to omit a tracking issue.
    // To make tidy accept omitting a tracking issue, group the list of features
    // without one inside `// no tracking issue START` and `// no tracking issue END`.
    let mut next_feature_omits_tracking_issue = false;

    let mut next_feature_group = None;
    let mut prev_since = None;

    contents.lines().zip(1..)
        .filter_map(|(line, line_number)| {
            let line = line.trim();

            // Within START and END, the tracking issue can be omitted.
            match line {
                "// no tracking issue START" => {
                    next_feature_omits_tracking_issue = true;
                    return None;
                }
                "// no tracking issue END" => {
                    next_feature_omits_tracking_issue = false;
                    return None;
                }
                _ => {}
            }

            if line.starts_with(FEATURE_GROUP_START_PREFIX) {
                let group = line.trim_start_matches(FEATURE_GROUP_START_PREFIX).trim();
                next_feature_group = Some(group.to_owned());
                prev_since = None;
                return None;
            } else if line.starts_with(FEATURE_GROUP_END_PREFIX) {
                next_feature_group = None;
                prev_since = None;
                return None;
            }

            let mut parts = line.split(',');
            let level = match parts.next().map(|l| l.trim().trim_start_matches('(')) {
                Some("active") => Status::Unstable,
                Some("removed") => Status::Removed,
                Some("accepted") => Status::Stable,
                _ => return None,
            };
            let name = parts.next().unwrap().trim();

            let since_str = parts.next().unwrap().trim().trim_matches('"');
            let since = match since_str.parse() {
                Ok(since) => Some(since),
                Err(err) => {
                    tidy_error!(
                        bad,
                        "libsyntax/feature_gate.rs:{}: failed to parse since: {} ({})",
                        line_number,
                        since_str,
                        err,
                    );
                    None
                }
            };
            if next_feature_group.is_some() {
                if prev_since > since {
                    tidy_error!(
                        bad,
                        "libsyntax/feature_gate.rs:{}: feature {} is not sorted by since",
                        line_number,
                        name,
                    );
                }
                prev_since = since.clone();
            }

            let issue_str = parts.next().unwrap().trim();
            let tracking_issue = if issue_str.starts_with("None") {
                if level == Status::Unstable && !next_feature_omits_tracking_issue {
                    *bad = true;
                    tidy_error!(
                        bad,
                        "libsyntax/feature_gate.rs:{}: no tracking issue for feature {}",
                        line_number,
                        name,
                    );
                }
                None
            } else {
                let s = issue_str.split('(').nth(1).unwrap().split(')').nth(0).unwrap();
                Some(s.parse().unwrap())
            };
            Some((name.to_owned(),
                Feature {
                    level,
                    since,
                    has_gate_test: false,
                    tracking_issue,
                    group: next_feature_group.clone(),
                }))
        })
        .collect()
}

pub fn collect_lib_features(base_src_path: &Path) -> Features {
    let mut lib_features = Features::new();

    // This library feature is defined in the `compiler_builtins` crate, which
    // has been moved out-of-tree. Now it can no longer be auto-discovered by
    // `tidy`, because we need to filter out its (submodule) directory. Manually
    // add it to the set of known library features so we can still generate docs.
    lib_features.insert("compiler_builtins_lib".to_owned(), Feature {
        level: Status::Unstable,
        since: None,
        has_gate_test: false,
        tracking_issue: None,
        group: None,
    });

    map_lib_features(base_src_path,
                     &mut |res, _, _| {
        if let Ok((name, feature)) = res {
            if lib_features.contains_key(name) {
                return;
            }
            lib_features.insert(name.to_owned(), feature);
        }
    });
   lib_features
}

fn get_and_check_lib_features(base_src_path: &Path,
                              bad: &mut bool,
                              lang_features: &Features) -> Features {
    let mut lib_features = Features::new();
    map_lib_features(base_src_path,
                     &mut |res, file, line| {
            match res {
                Ok((name, f)) => {
                    let mut check_features = |f: &Feature, list: &Features, display: &str| {
                        if let Some(ref s) = list.get(name) {
                            if f.tracking_issue != s.tracking_issue {
                                tidy_error!(bad,
                                            "{}:{}: mismatches the `issue` in {}",
                                            file.display(),
                                            line,
                                            display);
                            }
                        }
                    };
                    check_features(&f, &lang_features, "corresponding lang feature");
                    check_features(&f, &lib_features, "previous");
                    lib_features.insert(name.to_owned(), f);
                },
                Err(msg) => {
                    tidy_error!(bad, "{}:{}: {}", file.display(), line, msg);
                },
            }

    });
    lib_features
}

fn map_lib_features(base_src_path: &Path,
                    mf: &mut dyn FnMut(Result<(&str, Feature), &str>, &Path, usize)) {
    let mut contents = String::new();
    super::walk(base_src_path,
                &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
                &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if !filename.ends_with(".rs") || filename == "features.rs" ||
           filename == "diagnostic_list.rs" {
            return;
        }

        contents.truncate(0);
        t!(t!(File::open(&file), &file).read_to_string(&mut contents));

        let mut becoming_feature: Option<(String, Feature)> = None;
        for (i, line) in contents.lines().enumerate() {
            macro_rules! err {
                ($msg:expr) => {{
                    mf(Err($msg), file, i + 1);
                    continue;
                }};
            };
            if let Some((ref name, ref mut f)) = becoming_feature {
                if f.tracking_issue.is_none() {
                    f.tracking_issue = find_attr_val(line, "issue")
                    .map(|s| s.parse().unwrap());
                }
                if line.ends_with(']') {
                    mf(Ok((name, f.clone())), file, i + 1);
                } else if !line.ends_with(',') && !line.ends_with('\\') {
                    // We need to bail here because we might have missed the
                    // end of a stability attribute above because the ']'
                    // might not have been at the end of the line.
                    // We could then get into the very unfortunate situation that
                    // we continue parsing the file assuming the current stability
                    // attribute has not ended, and ignoring possible feature
                    // attributes in the process.
                    err!("malformed stability attribute");
                } else {
                    continue;
                }
            }
            becoming_feature = None;
            if line.contains("rustc_const_unstable(") {
                // `const fn` features are handled specially.
                let feature_name = match find_attr_val(line, "feature") {
                    Some(name) => name,
                    None => err!("malformed stability attribute"),
                };
                let feature = Feature {
                    level: Status::Unstable,
                    since: None,
                    has_gate_test: false,
                    // FIXME(#57563): #57563 is now used as a common tracking issue,
                    // although we would like to have specific tracking issues for each
                    // `rustc_const_unstable` in the future.
                    tracking_issue: Some(57563),
                    group: None,
                };
                mf(Ok((feature_name, feature)), file, i + 1);
                continue;
            }
            let level = if line.contains("[unstable(") {
                Status::Unstable
            } else if line.contains("[stable(") {
                Status::Stable
            } else {
                continue;
            };
            let feature_name = match find_attr_val(line, "feature") {
                Some(name) => name,
                None => err!("malformed stability attribute"),
            };
            let since = match find_attr_val(line, "since").map(|x| x.parse()) {
                Some(Ok(since)) => Some(since),
                Some(Err(_err)) => {
                    err!("malformed since attribute");
                },
                None if level == Status::Stable => {
                    err!("malformed stability attribute");
                }
                None => None,
            };
            let tracking_issue = find_attr_val(line, "issue").map(|s| s.parse().unwrap());

            let feature = Feature {
                level,
                since,
                has_gate_test: false,
                tracking_issue,
                group: None,
            };
            if line.contains(']') {
                mf(Ok((feature_name, feature)), file, i + 1);
            } else {
                becoming_feature = Some((feature_name.to_owned(), feature));
            }
        }
    });
}
<|endoftext|>


<commit_before>use std;
use std::collections::VecDeque;
use std::io::{Read, Write};
use std::process::{Command, Child, ChildStdin, ChildStdout, ExitStatus};
use std::sync::{mpsc, Arc, Mutex, Semaphore};
use std::thread;

use byteorder::{ReadBytesExt, WriteBytesExt, NativeEndian};

#[derive(Clone)]
pub struct Evaluator {
    queue: Arc<Mutex<VecDeque<Request>>>,
    has_work: Arc<Semaphore>
}

enum Request {
    Terminate,
    Restart,
    Work {
        code: String,
        timeout: usize,
        reporter: mpsc::Sender<Output>
    }
}

struct Output {
    success: bool,
    output: String
}

impl Evaluator {
    fn new() -> Self {
        Evaluator {
            queue: Arc::new(Mutex::new(VecDeque::new())),
            has_work: Arc::new(Semaphore::new(0))
        }
    }

    fn send_request(&self, req: Request) {
        self.queue.lock().unwrap().push_back(req);
        self.has_work.release();
    }

    #[cfg(not(unix))]
    pub fn eval(&self, _: &str, _: usize) -> Result<String, String> {
        Err("not implemented".to_owned())
    }

    #[cfg(unix)]
    pub fn eval(&self, code: &str, timeout: usize) -> Result<String, String> {
        let (tx, rx) = mpsc::channel();
        self.send_request(Request::Work {
            code: code.to_owned(),
            timeout: timeout,
            reporter: tx
        });

        let result = match rx.recv() {
            Ok(x) => x,
            Err(x) => {
                println!("couldn't receive result: {:?}", x);
                Output {
                    success: false,
                    output: "something bad happened".to_owned()
                }
            }
        };

        // this basically controls whether the output is prefixed if it's in a channel
        if result.success { Ok(result.output) } else { Err(result.output) }
    }

    pub fn restart(&self) {
        self.send_request(Request::Restart);
    }

    pub fn terminate(&self) {
        self.send_request(Request::Terminate);
    }
}

fn worker_evaluate(stdin: &mut ChildStdin,
                   stdout: Arc<Mutex<ChildStdout>>,
                   work: &Request)
                   -> Result<(), ()> {
    macro_rules! try_io {
        ($x:expr, $repr:expr) => {
            match $x {
                Err(x) => {
                    println!("couldn't communicate with child (1): {:?}", x);
                    match $repr.send(Output { success: false, output: "couldn't communicate with child (1)".to_owned() }) {
                        Ok(_) => (),
                        Err(x) => println!("couldn't report error (1): {:?}", x)
                    };
                    return Err(());
                },
                Ok(x) => x
            }
        }
    };

    macro_rules! try_io2 {
        ($x:expr, $repr:expr) => {
            match $x {
                Err(x) => {
                    println!("couldn't communicate with child (2): {:?}", x);
                    match $repr.send(Err(Output { success: false, output: "couldn't communicate with child (2)".to_owned() })) {
                        Ok(_) => (),
                        Err(x) => println!("couldn't report error (2): {:?}", x)
                    };
                    return;
                },
                Ok(x) => x
            }
        }
    };

    if let Request::Work { ref code, timeout, ref reporter } = *work {
        try_io!(stdin.write_i32::<NativeEndian>((timeout * 1000) as i32),
                reporter);
        let bytes = code.as_bytes();
        try_io!(stdin.write_i32::<NativeEndian>(bytes.len() as i32),
                reporter);
        try_io!(stdin.write_all(bytes), reporter);
        try_io!(stdin.flush(), reporter);

        let (tx, rx) = mpsc::channel();

        {
            // wait for response
            let tx = tx.clone();
            thread::spawn(move || {
                let mut stdout = stdout.lock().unwrap();
                let success = try_io2!(stdout.read_u8(), tx) == 1;
                let result_len = try_io2!(stdout.read_i32::<NativeEndian>(), tx);
                if result_len > 1024 * 1024 {
                    match tx.send(Ok(Output {
                        success: false,
                        output: "response from child too large".to_owned()
                    })) {
                        _ => (),
                    };
                    return;
                }
                let mut result_bytes = vec![0u8; result_len as usize];
                try_io2!(stdout.read_exact(&mut result_bytes), tx);
                match tx.send(Ok(Output {
                    success: success,
                    output: String::from_utf8_lossy(&result_bytes).into_owned()
                })) {
                    _ => (),
                };
            });
        }

        {
            // timeout
            let timeout = (timeout as f64 * 1.5) as u64;
            thread::spawn(move || {
                thread::sleep(::std::time::Duration::new(timeout, 0));
                match tx.send(Err(Output {
                    success: false,
                    output: "timed out waiting for evaluator response".to_owned()
                })) {
                    _ => (),
                };
            });
        }
        let mut err = false;

        // this one receives from the two threads spawned above
        // Ok means we got a message through the channel
        // Err means both threads above died
        let result = match rx.recv() {
            Ok(x) => x,
            Err(_) => {
                err = true;
                Ok(Output {
                    success: false,
                    output: "couldn't receive result from communicator thread".to_owned()
                })
            }
        };

        // this one is the result from the threads
        // Ok means we got the response from the sandboxed evaluator
        // Err means we timed out on Rust end (if the evaluator timed out, it'll be
        // Ok(Output { success: false, output: "timed out" }) or something like that
        let result = match result {
            Ok(x) => x,
            Err(x) => {
                err = true;
                x
            }
        };

        // we'll just ignore this error, not going to restart child
        match reporter.send(result) {
            Ok(_) => (),
            Err(x) => println!("couldn't return result: {:?}", x),
        };

        // returning Err here will make the loop in worker(..) break
        // restarting the child
        if err { Err(()) } else { Ok(()) }
    } else {
        println!("worker_evaluate got something other than Request::Work");
        Ok(())
    }
}

fn worker<F>(childfn: F, queue: Arc<Mutex<VecDeque<Request>>>, has_work: Arc<Semaphore>)
    where F: Fn() -> Child + Send + 'static {
    let mut terminate;
    loop {
        let mut evaluator = childfn();
        println!("started persistent child pid {}", evaluator.id());
        let mut stdin = evaluator.stdin.take().unwrap();
        let stdout = Arc::new(Mutex::new(evaluator.stdout.take().unwrap()));
        loop {
            has_work.acquire();
            let mut rvec = queue.lock().unwrap();
            let work = rvec.pop_front();
            std::mem::drop(rvec);

            if let Some(work) = work {
                match work {
                    Request::Terminate => {
                        println!("requested to terminate persistent child pid {}",
                                 evaluator.id());
                        terminate = true;
                        break;
                    }
                    Request::Restart => {
                        println!("requested to restart persistent child pid {}",
                                 evaluator.id());
                        terminate = false;
                        break;
                    }
                    Request::Work { .. } => {
                        let res = worker_evaluate(&mut stdin, stdout.clone(), &work);
                        match res {
                            Ok(_) => (),
                            Err(_) => {
                                terminate = false;
                                break;
                            }
                        };
                    }
                }
            }
        }
        let pid = evaluator.id();
        println!("killing persistent child pid {}", pid);
        match sudo_kill(pid) {
            Err(x) => println!("failed to kill {}: {}", pid, x),
            Ok(x) => println!("kill result: {:?}", x),
        };
        // we only do this here because Child::kill does waitpid, to reap the process
        match evaluator.kill() {
            _ => (),
        };
        if terminate {
            break;
        }
    }
}

fn sudo_kill(pid: u32) -> Result<ExitStatus, String> {
    try!(Command::new("sudo")
             .args(&["kill", "-KILL"])
             .arg(format!("{}", pid))
             .spawn()
             .map_err(|x| format!("couldn't spawn sudo kill: {:?}", x)))
        .wait()
        .map_err(|x| format!("couldn't SIGKILL: {:?}", x))
}

pub fn new<F>(childfn: F) -> Evaluator
    where F: Fn() -> Child + Send + 'static {
    let ret = Evaluator::new();
    let (queue, has_work) = (ret.queue.clone(), ret.has_work.clone());
    thread::spawn(move || {
        worker(childfn, queue, has_work);
    });
    ret
}
<commit_msg>Restart child if output too large too<commit_after>use std;
use std::collections::VecDeque;
use std::io::{Read, Write};
use std::process::{Command, Child, ChildStdin, ChildStdout, ExitStatus};
use std::sync::{mpsc, Arc, Mutex, Semaphore};
use std::thread;

use byteorder::{ReadBytesExt, WriteBytesExt, NativeEndian};

#[derive(Clone)]
pub struct Evaluator {
    queue: Arc<Mutex<VecDeque<Request>>>,
    has_work: Arc<Semaphore>
}

enum Request {
    Terminate,
    Restart,
    Work {
        code: String,
        timeout: usize,
        reporter: mpsc::Sender<Output>
    }
}

#[derive(PartialEq)]
enum Status {
    EvalbotError,
    ChildError,
    Success
}

struct Output {
    success: Status,
    output: String
}

impl Evaluator {
    fn new() -> Self {
        Evaluator {
            queue: Arc::new(Mutex::new(VecDeque::new())),
            has_work: Arc::new(Semaphore::new(0))
        }
    }

    fn send_request(&self, req: Request) {
        self.queue.lock().unwrap().push_back(req);
        self.has_work.release();
    }

    #[cfg(not(unix))]
    pub fn eval(&self, _: &str, _: usize) -> Result<String, String> {
        Err("not implemented".to_owned())
    }

    #[cfg(unix)]
    pub fn eval(&self, code: &str, timeout: usize) -> Result<String, String> {
        let (tx, rx) = mpsc::channel();
        self.send_request(Request::Work {
            code: code.to_owned(),
            timeout: timeout,
            reporter: tx
        });

        let result = match rx.recv() {
            Ok(x) => x,
            Err(x) => {
                println!("couldn't receive result: {:?}", x);
                Output {
                    success: Status::EvalbotError,
                    output: "something bad happened".to_owned()
                }
            }
        };

        // this basically controls whether the output is prefixed if it's in a channel
        if result.success == Status::Success { Ok(result.output) } else { Err(result.output) }
    }

    pub fn restart(&self) {
        self.send_request(Request::Restart);
    }

    pub fn terminate(&self) {
        self.send_request(Request::Terminate);
    }
}

fn worker_evaluate(stdin: &mut ChildStdin,
                   stdout: Arc<Mutex<ChildStdout>>,
                   work: &Request)
                   -> Result<(), ()> {
    macro_rules! try_io {
        ($x:expr, $repr:expr) => {
            match $x {
                Err(x) => {
                    println!("couldn't communicate with child (1): {:?}", x);
                    match $repr.send(Output { success: Status::EvalbotError, output: "couldn't communicate with child (1)".to_owned() }) {
                        Ok(_) => (),
                        Err(x) => println!("couldn't report error (1): {:?}", x)
                    };
                    return Err(());
                },
                Ok(x) => x
            }
        }
    };

    macro_rules! try_io2 {
        ($x:expr, $repr:expr) => {
            match $x {
                Err(x) => {
                    println!("couldn't communicate with child (2): {:?}", x);
                    match $repr.send(Output { success: Status::EvalbotError, output: "couldn't communicate with child (2)".to_owned() }) {
                        Ok(_) => (),
                        Err(x) => println!("couldn't report error (2): {:?}", x)
                    };
                    return;
                },
                Ok(x) => x
            }
        }
    };

    if let Request::Work { ref code, timeout, ref reporter } = *work {
        try_io!(stdin.write_i32::<NativeEndian>((timeout * 1000) as i32),
                reporter);
        let bytes = code.as_bytes();
        try_io!(stdin.write_i32::<NativeEndian>(bytes.len() as i32),
                reporter);
        try_io!(stdin.write_all(bytes), reporter);
        try_io!(stdin.flush(), reporter);

        let (tx, rx) = mpsc::channel();

        {
            // wait for response
            let tx = tx.clone();
            thread::spawn(move || {
                let mut stdout = stdout.lock().unwrap();
                let success = try_io2!(stdout.read_u8(), tx) == 1;
                let result_len = try_io2!(stdout.read_i32::<NativeEndian>(), tx);
                if result_len > 1024 * 1024 {
                    match tx.send(Output {
                        success: Status::EvalbotError,
                        output: "response from child too large".to_owned()
                    }) {
                        _ => (),
                    };
                    return;
                }
                let mut result_bytes = vec![0u8; result_len as usize];
                try_io2!(stdout.read_exact(&mut result_bytes), tx);
                match tx.send(Output {
                    success: if success { Status::Success } else { Status::ChildError },
                    output: String::from_utf8_lossy(&result_bytes).into_owned()
                }) {
                    _ => (),
                };
            });
        }

        {
            // timeout
            let timeout = (timeout as f64 * 1.5) as u64;
            thread::spawn(move || {
                thread::sleep(::std::time::Duration::new(timeout, 0));
                match tx.send(Output {
                    success: Status::EvalbotError,
                    output: "timed out waiting for evaluator response".to_owned()
                }) {
                    _ => (),
                };
            });
        }
        let mut err = false;

        // this one receives from the two threads spawned above
        // Ok means we got a message through the channel
        // Err means both threads above died
        let result = match rx.recv() {
            Ok(x) => x,
            Err(_) => {
                err = true;
                Output {
                    success: Status::EvalbotError,
                    output: "couldn't receive result from communicator thread".to_owned()
                }
            }
        };

        if result.success == Status::EvalbotError {
            err = true;
        }

        // we'll just ignore this error, not going to restart child
        match reporter.send(result) {
            Ok(_) => (),
            Err(x) => println!("couldn't return result: {:?}", x),
        };

        // returning Err here will make the loop in worker(..) break
        // restarting the child
        if err { Err(()) } else { Ok(()) }
    } else {
        println!("worker_evaluate got something other than Request::Work");
        Ok(())
    }
}

fn worker<F>(childfn: F, queue: Arc<Mutex<VecDeque<Request>>>, has_work: Arc<Semaphore>)
    where F: Fn() -> Child + Send + 'static {
    let mut terminate;
    loop {
        let mut evaluator = childfn();
        println!("started persistent child pid {}", evaluator.id());
        let mut stdin = evaluator.stdin.take().unwrap();
        let stdout = Arc::new(Mutex::new(evaluator.stdout.take().unwrap()));
        loop {
            has_work.acquire();
            let mut rvec = queue.lock().unwrap();
            let work = rvec.pop_front();
            std::mem::drop(rvec);

            if let Some(work) = work {
                match work {
                    Request::Terminate => {
                        println!("requested to terminate persistent child pid {}",
                                 evaluator.id());
                        terminate = true;
                        break;
                    }
                    Request::Restart => {
                        println!("requested to restart persistent child pid {}",
                                 evaluator.id());
                        terminate = false;
                        break;
                    }
                    Request::Work { .. } => {
                        let res = worker_evaluate(&mut stdin, stdout.clone(), &work);
                        match res {
                            Ok(_) => (),
                            Err(_) => {
                                terminate = false;
                                break;
                            }
                        };
                    }
                }
            }
        }
        let pid = evaluator.id();
        println!("killing persistent child pid {}", pid);
        match sudo_kill(pid) {
            Err(x) => println!("failed to kill {}: {}", pid, x),
            Ok(x) => println!("kill result: {:?}", x),
        };
        // we only do this here because Child::kill does waitpid, to reap the process
        match evaluator.kill() {
            _ => (),
        };
        if terminate {
            break;
        }
    }
}

fn sudo_kill(pid: u32) -> Result<ExitStatus, String> {
    try!(Command::new("sudo")
             .args(&["kill", "-KILL"])
             .arg(format!("{}", pid))
             .spawn()
             .map_err(|x| format!("couldn't spawn sudo kill: {:?}", x)))
        .wait()
        .map_err(|x| format!("couldn't SIGKILL: {:?}", x))
}

pub fn new<F>(childfn: F) -> Evaluator
    where F: Fn() -> Child + Send + 'static {
    let ret = Evaluator::new();
    let (queue, has_work) = (ret.queue.clone(), ret.has_work.clone());
    thread::spawn(move || {
        worker(childfn, queue, has_work);
    });
    ret
}
<|endoftext|>


<commit_before>use crate::cmd::run::{Db, Options};
use crate::{err, r};
use ql2::query::QueryType;
use ql2::term::TermType;
use serde::{ser, Serialize, Serializer};
use serde_json::value::{Number, Value};
use std::collections::{HashMap, VecDeque};
use std::{fmt, str};

#[derive(Debug, Clone)]
pub(crate) enum Datum {
    Null,
    Bool(bool),
    Number(Number),
    String(String),
    Array(Vec<Datum>),
    Object(HashMap<String, Datum>),
}

impl Default for Datum {
    fn default() -> Self {
        Self::Null
    }
}

impl Serialize for Datum {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        match self {
            Self::Null => serializer.serialize_none(),
            Self::Bool(boolean) => boolean.serialize(serializer),
            Self::Number(num) => num.serialize(serializer),
            Self::String(string) => string.serialize(serializer),
            Self::Array(arr) => (TermType::MakeArray as i32, arr).serialize(serializer),
            Self::Object(map) => map.serialize(serializer),
        }
    }
}

impl From<Value> for Datum {
    fn from(value: Value) -> Self {
        match value {
            Value::Null => Self::Null,
            Value::Bool(boolean) => Self::Bool(boolean),
            Value::Number(num) => Self::Number(num),
            Value::String(string) => Self::String(string),
            Value::Array(arr) => Self::Array(arr.into_iter().map(Into::into).collect()),
            Value::Object(map) => Self::Object(
                map.into_iter()
                    .map(|(key, value)| (key, value.into()))
                    .collect(),
            ),
        }
    }
}

/// The query that will be sent to RethinkDB
#[derive(Debug, Clone, Default)]
pub struct Query {
    typ: TermType,
    datum: Option<Datum>,
    args: VecDeque<Query>,
    opts: Option<Datum>,
    change_feed: bool,
}

impl Query {
    #[doc(hidden)]
    pub fn new(typ: TermType) -> Self {
        Self {
            typ,
            ..Default::default()
        }
    }

    #[doc(hidden)]
    pub fn with_parent(mut self, parent: Query) -> Self {
        self.change_feed = parent.change_feed;
        self.args.push_front(parent);
        self
    }

    #[doc(hidden)]
    pub fn with_arg<T>(mut self, arg: T) -> Self
    where
        T: Into<Query>,
    {
        self.args.push_back(arg.into());
        self
    }

    #[doc(hidden)]
    pub fn with_opts<T>(mut self, opts: T) -> Self
    where
        T: Serialize,
    {
        let opts = serde_json::to_value(&opts)
            // it's safe to unwrap here because we only use opts
            // types that are derived in this crate and we know
            // those don't return errors
            .unwrap()
            .into();
        self.opts = Some(opts);
        self
    }

    pub(crate) fn mark_change_feed(mut self) -> Self {
        self.change_feed = true;
        self
    }

    pub(crate) fn change_feed(&self) -> bool {
        self.change_feed
    }
}

impl From<Datum> for Query {
    fn from(datum: Datum) -> Self {
        Self {
            typ: TermType::Datum,
            datum: Some(datum),
            ..Default::default()
        }
    }
}

impl Serialize for Query {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        match self.typ {
            TermType::Datum => self.datum.serialize(serializer),
            _ => {
                let typ = self.typ as i32;
                match &self.opts {
                    Some(map) => (typ, &self.args, map).serialize(serializer),
                    None => (typ, &self.args).serialize(serializer),
                }
            }
        }
    }
}

#[derive(Debug, Clone)]
pub(crate) struct Payload<'a>(
    pub(crate) QueryType,
    pub(crate) Option<Query>,
    pub(crate) Options<'a>,
);

impl Serialize for Payload<'_> {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let Payload(typ, qry, opts) = self;
        let typ = *typ as i32;
        match qry {
            Some(query) => (typ, query, opts).serialize(serializer),
            None => (typ,).serialize(serializer),
        }
    }
}

impl Payload<'_> {
    pub(crate) fn to_bytes(&self) -> Result<Vec<u8>, err::Error> {
        Ok(serde_json::to_vec(self)?)
    }
}

// for debugging purposes only
impl fmt::Display for Payload<'_> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        // print the serialised string if we can
        if let Ok(payload) = self.to_bytes() {
            if let Ok(payload) = str::from_utf8(&payload) {
                return write!(f, "{}", payload);
            }
        }
        // otherwise just print the debug form
        write!(f, "{:?}", self)
    }
}

impl Serialize for Db<'_> {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let Self(name) = self;
        r.db(*name).serialize(serializer)
    }
}
<commit_msg>fix changefeeds<commit_after>use crate::cmd::run::{Db, Options};
use crate::{err, r};
use ql2::query::QueryType;
use ql2::term::TermType;
use serde::{Serialize, Serializer};
use serde_json::value::{Number, Value};
use std::collections::{HashMap, VecDeque};
use std::{fmt, str};

#[derive(Debug, Clone)]
pub(crate) enum Datum {
    Null,
    Bool(bool),
    Number(Number),
    String(String),
    Array(Vec<Datum>),
    Object(HashMap<String, Datum>),
}

impl Default for Datum {
    fn default() -> Self {
        Self::Null
    }
}

impl Serialize for Datum {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        match self {
            Self::Null => serializer.serialize_none(),
            Self::Bool(boolean) => boolean.serialize(serializer),
            Self::Number(num) => num.serialize(serializer),
            Self::String(string) => string.serialize(serializer),
            Self::Array(arr) => (TermType::MakeArray as i32, arr).serialize(serializer),
            Self::Object(map) => map.serialize(serializer),
        }
    }
}

impl From<Value> for Datum {
    fn from(value: Value) -> Self {
        match value {
            Value::Null => Self::Null,
            Value::Bool(boolean) => Self::Bool(boolean),
            Value::Number(num) => Self::Number(num),
            Value::String(string) => Self::String(string),
            Value::Array(arr) => Self::Array(arr.into_iter().map(Into::into).collect()),
            Value::Object(map) => Self::Object(
                map.into_iter()
                    .map(|(key, value)| (key, value.into()))
                    .collect(),
            ),
        }
    }
}

/// The query that will be sent to RethinkDB
#[derive(Debug, Clone, Default)]
pub struct Query {
    typ: TermType,
    datum: Option<Datum>,
    args: VecDeque<Query>,
    opts: Option<Datum>,
    change_feed: bool,
}

impl Query {
    #[doc(hidden)]
    pub fn new(typ: TermType) -> Self {
        Self {
            typ,
            ..Default::default()
        }
    }

    #[doc(hidden)]
    pub fn with_parent(mut self, parent: Query) -> Self {
        self.change_feed = self.change_feed || parent.change_feed;
        self.args.push_front(parent);
        self
    }

    #[doc(hidden)]
    pub fn with_arg<T>(mut self, arg: T) -> Self
    where
        T: Into<Query>,
    {
        self.args.push_back(arg.into());
        self
    }

    #[doc(hidden)]
    pub fn with_opts<T>(mut self, opts: T) -> Self
    where
        T: Serialize,
    {
        let opts = serde_json::to_value(&opts)
            // it's safe to unwrap here because we only use opts
            // types that are derived in this crate and we know
            // those don't return errors
            .unwrap()
            .into();
        self.opts = Some(opts);
        self
    }

    pub(crate) fn mark_change_feed(mut self) -> Self {
        self.change_feed = true;
        self
    }

    pub(crate) fn change_feed(&self) -> bool {
        self.change_feed
    }
}

impl From<Datum> for Query {
    fn from(datum: Datum) -> Self {
        Self {
            typ: TermType::Datum,
            datum: Some(datum),
            ..Default::default()
        }
    }
}

impl Serialize for Query {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        match self.typ {
            TermType::Datum => self.datum.serialize(serializer),
            _ => {
                let typ = self.typ as i32;
                match &self.opts {
                    Some(map) => (typ, &self.args, map).serialize(serializer),
                    None => (typ, &self.args).serialize(serializer),
                }
            }
        }
    }
}

#[derive(Debug, Clone)]
pub(crate) struct Payload<'a>(
    pub(crate) QueryType,
    pub(crate) Option<Query>,
    pub(crate) Options<'a>,
);

impl Serialize for Payload<'_> {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let Payload(typ, qry, opts) = self;
        let typ = *typ as i32;
        match qry {
            Some(query) => (typ, query, opts).serialize(serializer),
            None => (typ,).serialize(serializer),
        }
    }
}

impl Payload<'_> {
    pub(crate) fn to_bytes(&self) -> Result<Vec<u8>, err::Error> {
        Ok(serde_json::to_vec(self)?)
    }
}

// for debugging purposes only
impl fmt::Display for Payload<'_> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        // print the serialised string if we can
        if let Ok(payload) = self.to_bytes() {
            if let Ok(payload) = str::from_utf8(&payload) {
                return write!(f, "{}", payload);
            }
        }
        // otherwise just print the debug form
        write!(f, "{:?}", self)
    }
}

impl Serialize for Db<'_> {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let Self(name) = self;
        r.db(*name).serialize(serializer)
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate askama;
#[macro_use]
extern crate serde_json;

use askama::Template;
use serde_json::Value;

#[derive(Template)]
#[template(path = "filters.html")]
struct TestTemplate {
    strvar: String,
}

#[test]
fn filter_escape() {
    let s = TestTemplate {
        strvar: "// my <html> is \"unsafe\" & should be 'escaped'".to_string(),
    };
    assert_eq!(
        s.render().unwrap(),
        "// my <html> is "unsafe" & \
         should be 'escaped'"
    );
}

#[derive(Template)]
#[template(path = "format.html", escape = "none")]
struct FormatTemplate<'a> {
    var: &'a str,
}

#[test]
fn filter_format() {
    let t = FormatTemplate { var: "formatted" };
    assert_eq!(t.render().unwrap(), "\"formatted\"");
}

#[derive(Template)]
#[template(source = "{{ s|myfilter }}", ext = "txt")]
struct MyFilterTemplate<'a> {
    s: &'a str,
}

mod filters {
    pub fn myfilter(s: &str) -> ::askama::Result<String> {
        Ok(s.replace("oo", "aa").to_string())
    }
    // for test_nested_filter_ref
    pub fn mytrim(s: &::std::fmt::Display) -> ::askama::Result<String> {
        let s = format!("{}", s);
        Ok(s.trim().to_owned())
    }
}

#[test]
fn test_my_filter() {
    let t = MyFilterTemplate { s: "foo" };
    assert_eq!(t.render().unwrap(), "faa");
}

#[derive(Template)]
#[template(path = "filters_join.html")]
struct JoinTemplate<'a> {
    s: &'a [&'a str],
}

#[test]
fn test_join() {
    let t = JoinTemplate {
        s: &["foo", "bar", "bazz"],
    };
    assert_eq!(t.render().unwrap(), "foo, bar, bazz");
}

#[derive(Template)]
#[template(path = "filters_join.html")]
struct VecJoinTemplate {
    s: Vec<String>,
}

#[test]
fn test_vec_join() {
    let t = VecJoinTemplate {
        s: vec!["foo".into(), "bar".into(), "bazz".into()],
    };
    assert_eq!(t.render().unwrap(), "foo, bar, bazz");
}

#[derive(Template)]
#[template(path = "json.html")]
struct JsonTemplate<'a> {
    foo: &'a str,
    bar: &'a Value,
}

#[test]
fn test_json() {
    let val = json!({"arr": [ "one", 2, true, null ]});
    let t = JsonTemplate {
        foo: "a",
        bar: &val,
    };
    // Note: the json filter lacks a way to specify initial indentation
    assert_eq!(
        t.render().unwrap(),
        r#"{
  "foo": "a",
  "bar": {
  "arr": [
    "one",
    2,
    true,
    null
  ]
}
}"#
    );
}

#[derive(Template)]
#[template(source = "{{ x|mytrim|safe }}", ext = "html")]
struct NestedFilterTemplate {
    x: String,
}

#[test]
fn test_nested_filter_ref() {
    let t = NestedFilterTemplate {
        x: " floo & bar".to_string(),
    };
    assert_eq!(t.render().unwrap(), "floo & bar");
}

#[derive(Template)]
#[template(source = "{% let p = baz.print(foo.as_ref()) %}{{ p|upper }}", ext = "html")]
struct FilterLetFilterTemplate {
    foo: String,
    baz: Baz,
}

struct Baz {}

impl Baz {
    fn print(&self, s: &str) -> String {
        s.trim().to_owned()
    }
}

#[test]
fn test_filter_let_filter() {
    let t = FilterLetFilterTemplate {
        foo: " bar ".to_owned(),
        baz: Baz {},
    };
    assert_eq!(t.render().unwrap(), "BAR");
}
<commit_msg>Add test for 'truncate' filter<commit_after>#[macro_use]
extern crate askama;
#[macro_use]
extern crate serde_json;

use askama::Template;
use serde_json::Value;

#[derive(Template)]
#[template(path = "filters.html")]
struct TestTemplate {
    strvar: String,
}

#[test]
fn filter_escape() {
    let s = TestTemplate {
        strvar: "// my <html> is \"unsafe\" & should be 'escaped'".to_string(),
    };
    assert_eq!(
        s.render().unwrap(),
        "// my <html> is "unsafe" & \
         should be 'escaped'"
    );
}

#[derive(Template)]
#[template(path = "format.html", escape = "none")]
struct FormatTemplate<'a> {
    var: &'a str,
}

#[test]
fn filter_format() {
    let t = FormatTemplate { var: "formatted" };
    assert_eq!(t.render().unwrap(), "\"formatted\"");
}

#[derive(Template)]
#[template(source = "{{ s|myfilter }}", ext = "txt")]
struct MyFilterTemplate<'a> {
    s: &'a str,
}

mod filters {
    pub fn myfilter(s: &str) -> ::askama::Result<String> {
        Ok(s.replace("oo", "aa").to_string())
    }
    // for test_nested_filter_ref
    pub fn mytrim(s: &::std::fmt::Display) -> ::askama::Result<String> {
        let s = format!("{}", s);
        Ok(s.trim().to_owned())
    }
}

#[test]
fn test_my_filter() {
    let t = MyFilterTemplate { s: "foo" };
    assert_eq!(t.render().unwrap(), "faa");
}

#[derive(Template)]
#[template(path = "filters_join.html")]
struct JoinTemplate<'a> {
    s: &'a [&'a str],
}

#[test]
fn test_join() {
    let t = JoinTemplate {
        s: &["foo", "bar", "bazz"],
    };
    assert_eq!(t.render().unwrap(), "foo, bar, bazz");
}

#[derive(Template)]
#[template(path = "filters_join.html")]
struct VecJoinTemplate {
    s: Vec<String>,
}

#[test]
fn test_vec_join() {
    let t = VecJoinTemplate {
        s: vec!["foo".into(), "bar".into(), "bazz".into()],
    };
    assert_eq!(t.render().unwrap(), "foo, bar, bazz");
}

#[derive(Template)]
#[template(path = "json.html")]
struct JsonTemplate<'a> {
    foo: &'a str,
    bar: &'a Value,
}

#[test]
fn test_json() {
    let val = json!({"arr": [ "one", 2, true, null ]});
    let t = JsonTemplate {
        foo: "a",
        bar: &val,
    };
    // Note: the json filter lacks a way to specify initial indentation
    assert_eq!(
        t.render().unwrap(),
        r#"{
  "foo": "a",
  "bar": {
  "arr": [
    "one",
    2,
    true,
    null
  ]
}
}"#
    );
}

#[derive(Template)]
#[template(source = "{{ x|mytrim|safe }}", ext = "html")]
struct NestedFilterTemplate {
    x: String,
}

#[test]
fn test_nested_filter_ref() {
    let t = NestedFilterTemplate {
        x: " floo & bar".to_string(),
    };
    assert_eq!(t.render().unwrap(), "floo & bar");
}

#[derive(Template)]
#[template(source = "{% let p = baz.print(foo.as_ref()) %}{{ p|upper }}", ext = "html")]
struct FilterLetFilterTemplate {
    foo: String,
    baz: Baz,
}

struct Baz {}

impl Baz {
    fn print(&self, s: &str) -> String {
        s.trim().to_owned()
    }
}

#[test]
fn test_filter_let_filter() {
    let t = FilterLetFilterTemplate {
        foo: " bar ".to_owned(),
        baz: Baz {},
    };
    assert_eq!(t.render().unwrap(), "BAR");
}

#[derive(Template)]
#[template(source = "{{ foo|truncate(10) }}{{ foo|truncate(5) }}", ext = "txt", print = "code")]
struct TruncateFilter {
    foo: String,
}

#[test]
fn test_filter_truncate() {
    let t = TruncateFilter {
        foo: "alpha bar".into(),
    };
    assert_eq!(t.render().unwrap(), "alpha baralpha...");
}
<|endoftext|>