<commit_before>extern crate gtk;
extern crate glib;
extern crate gstreamer as gst;

#[cfg(any(feature = "profiling-listener", feature = "profiling-tracker"))]
use chrono::Utc;

use std::rc::Rc;
use std::cell::RefCell;

use std::path::PathBuf;

use std::sync::mpsc::{channel, Receiver};

use gtk::prelude::*;

use media::{Context, ContextMessage};
use media::ContextMessage::*;

use super::{AudioController, InfoController, VideoController};

#[derive(Clone, PartialEq)]
pub enum ControllerState {
    EOS,
    Paused,
    Playing,
    Stopped,
}

const LISTENER_PERIOD: u32 = 250; // 250 ms (4 Hz)
const TRACKER_PERIOD:  u32 = 17;  //  17 ms (60 Hz)

pub struct MainController {
    window: gtk::ApplicationWindow,
    header_bar: gtk::HeaderBar,
    play_pause_btn: gtk::ToolButton,

    video_ctrl: VideoController,
    info_ctrl: InfoController,
    audio_ctrl: AudioController,

    context: Option<Context>,
    state: ControllerState,
    duration: Option<u64>, // duration is accurately known at eos
    seeking: bool,

    this_opt: Option<Rc<RefCell<MainController>>>,
    keep_going: bool,
    listener_src: Option<glib::SourceId>,
    tracker_src: Option<glib::SourceId>,
}

impl MainController {
    pub fn new(builder: gtk::Builder) -> Rc<RefCell<Self>> {
        let this = Rc::new(RefCell::new(MainController {
            window: builder.get_object("application-window").unwrap(),
            header_bar: builder.get_object("header-bar").unwrap(),
            play_pause_btn: builder.get_object("play_pause-toolbutton").unwrap(),

            video_ctrl: VideoController::new(&builder),
            info_ctrl: InfoController::new(&builder),
            audio_ctrl: AudioController::new(&builder),

            context: None,
            state: ControllerState::Stopped,
            duration: None,
            seeking: false,

            this_opt: None,
            keep_going: true,
            listener_src: None,
            tracker_src: None,
        }));

        {
            let mut this_mut = this.borrow_mut();

            let this_rc = Rc::clone(&this);
            this_mut.this_opt = Some(this_rc);

            this_mut.window.connect_delete_event(|_, _| {
                gtk::main_quit();
                Inhibit(false)
            });
            this_mut.window.set_titlebar(&this_mut.header_bar);

            let this_rc = Rc::clone(&this);
            this_mut.play_pause_btn.connect_clicked(move |_| {
                this_rc.borrow_mut().play_pause();
            });

            // TODO: add key bindings to seek by steps
            // play/pause, etc.

            this_mut.video_ctrl.register_callbacks(&this);
            this_mut.info_ctrl.register_callbacks(&this);
            this_mut.audio_ctrl.register_callbacks(&this);
        }

        let open_btn: gtk::Button = builder.get_object("open-btn").unwrap();
        let this_rc = Rc::clone(&this);
        open_btn.connect_clicked(move |_| {
            this_rc.borrow_mut().select_media();
        });

        this
    }

    pub fn show_all(&self) {
        self.window.show_all();
    }

    fn play_pause(&mut self) {
        let context =
            match self.context.take() {
                Some(context) => context,
                None => {
                    self.select_media();
                    return;
                },
            };

        if self.state != ControllerState::EOS {
            match context.get_state() {
                gst::State::Paused => {
                    self.register_tracker();
                    self.play_pause_btn.set_icon_name("media-playback-pause");
                    self.state = ControllerState::Playing;
                    context.play().unwrap();
                    self.context = Some(context);
                }
                gst::State::Playing => {
                    context.pause().unwrap();
                    self.play_pause_btn.set_icon_name("media-playback-start");
                    self.remove_tracker();
                    self.state = ControllerState::Paused;
                    self.context = Some(context);
                },
                state => {
                    println!("Can't play/pause in state {:?}", state);
                    self.context = Some(context);
                },
            };
        } else {
            // Restart the stream from the begining
            self.context = Some(context);
            self.seek(0, true); // accurate (slow)
        }
    }

    fn stop(&mut self) {
        if let Some(context) = self.context.as_mut() {
            context.stop();
        };

        // remove callbacks in order to avoid conflict on borrowing of self
        self.remove_listener();
        self.remove_tracker();

        self.audio_ctrl.cleanup();

        self.play_pause_btn.set_icon_name("media-playback-start");

        self.state = ControllerState::Stopped;
        self.duration = None;
    }

    pub fn seek(&mut self, position: u64, accurate: bool) {
        if self.state != ControllerState::Stopped {
            self.seeking = true;

            // update position even though the stream
            // is not sync yet for the user to notice
            // the seek request in being handled
            self.info_ctrl.seek(position);
            self.audio_ctrl.seek(position, &self.state);

            self.context.as_ref()
                .expect("MainController::seek no context")
                .seek(position, accurate);

            if self.state != ControllerState::Playing {
                self.context.as_ref()
                    .expect("MainController::seek no context")
                    .play().unwrap();
                self.register_tracker();
                self.play_pause_btn.set_icon_name("media-playback-pause");
                self.state = ControllerState::Playing;
            }
        }
    }

    fn select_media(&mut self) {
        self.stop();

        let file_dlg = gtk::FileChooserDialog::new(
            Some("Open a media file"),
            Some(&self.window),
            gtk::FileChooserAction::Open,
        );
        // Note: couldn't find equivalents for STOCK_OK
        file_dlg.add_button("Open", gtk::ResponseType::Ok.into());

        if file_dlg.run() == gtk::ResponseType::Ok.into() {
            self.open_media(file_dlg.get_filename().unwrap());
        }

        file_dlg.close();
    }

    fn handle_eos(&mut self) {
        self.play_pause_btn.set_icon_name("media-playback-start");
        self.state = ControllerState::EOS;
    }

    fn remove_listener(&mut self) {
        if let Some(source_id) = self.listener_src.take() {
            glib::source_remove(source_id);
        }
    }

    fn register_listener(&mut self,
        timeout: u32,
        ui_rx: Receiver<ContextMessage>,
    ) {
        if self.listener_src.is_some() {
            return;
        }

        let this_rc = Rc::clone(self.this_opt.as_ref().unwrap());

        self.listener_src = Some(gtk::timeout_add(timeout, move || {
            let mut keep_going = true;

            for message in ui_rx.try_iter() {
                match message {
                    AsyncDone => {
                        this_rc.borrow_mut().seeking = false;
                    },
                    InitDone => {
                        let mut this_mut = this_rc.borrow_mut();

                        let context = this_mut.context.take()
                            .expect("MainController: InitDone but no context available");

                        this_mut.header_bar.set_subtitle(
                            Some(context.file_name.as_str())
                        );

                        this_mut.info_ctrl.new_media(&context);
                        this_mut.video_ctrl.new_media(&context);
                        this_mut.audio_ctrl.new_media(&context);

                        this_mut.context = Some(context);

                        this_mut.state = ControllerState::Paused;
                    },
                    Eos => {
                        let mut this_mut = this_rc.borrow_mut();
                        let position = this_mut.context.as_mut()
                            .expect("MainController::listener no context while getting position")
                            .get_position();

                        this_mut.info_ctrl.update_duration(position);
                        this_mut.duration = Some(position);

                        this_mut.info_ctrl.tick(position);
                        this_mut.audio_ctrl.tick();

                        this_mut.handle_eos();

                        // Remove listener and tracker.
                        // Note: tracker will be register again in case of
                        // a seek. Listener is of no use anymore because
                        // the context won't send any more Eos nor AsyncDone
                        // after an EOS.
                        keep_going = false;
                    },
                    FailedToOpenMedia => {
                        eprintln!("ERROR: failed to open media");

                        let mut this_mut = this_rc.borrow_mut();

                        this_mut.context = None;
                        this_mut.keep_going = false;
                        keep_going = false;
                    },
                };

                if !keep_going { break; }
            }

            if !keep_going {
                let mut this_mut = this_rc.borrow_mut();
                this_mut.listener_src = None;
                this_mut.tracker_src = None;
            }

            glib::Continue(keep_going)
        }));
    }

    fn remove_tracker(&mut self) {
        if let Some(source_id) = self.tracker_src.take() {
            glib::source_remove(source_id);
        }
    }

    fn register_tracker(&mut self) {
        if self.tracker_src.is_some() {
            return;
        }

        let this_rc = Rc::clone(self.this_opt.as_ref().unwrap());

        self.tracker_src = Some(gtk::timeout_add(TRACKER_PERIOD, move || {
            #[cfg(feature = "profiling-tracker")]
            let start = Utc::now();

            let mut keep_going = true;

            let mut this_mut = this_rc.borrow_mut();

            #[cfg(feature = "profiling-tracker")]
            let before_tick = Utc::now();

            let position = this_mut.context.as_mut()
                .expect("MainController::tracker no context while getting position")
                .get_position();

            if !this_mut.seeking {
                this_mut.info_ctrl.tick(position);
                this_mut.audio_ctrl.tick();
            }

            if let Some(duration) = this_mut.duration {
                if position >= duration {
                    if !this_mut.seeking {
                        // this check is necessary as EOS is not sent
                        // in case of a seek after EOS
                        this_mut.handle_eos();
                        this_mut.tracker_src = None;
                        keep_going = false;
                    }
                } else if this_mut.seeking {
                    // this check is necessary as AsyncDone is not sent
                    // in case of a seek after EOS
                    this_mut.seeking = false;
                }
            }

            #[cfg(feature = "profiling-tracker")]
            let end = Utc::now();

            #[cfg(feature = "profiling-tracker")]
            println!("tracker,{},{},{}",
                start.time().format("%H:%M:%S%.6f"),
                before_tick.time().format("%H:%M:%S%.6f"),
                end.time().format("%H:%M:%S%.6f"),
            );

            glib::Continue(this_mut.keep_going && keep_going)
        }));
    }

    fn open_media(&mut self, filepath: PathBuf) {
        assert_eq!(self.listener_src, None);

        self.info_ctrl.cleanup();
        self.audio_ctrl.cleanup();
        self.header_bar.set_subtitle("");

        let (ctx_tx, ui_rx) = channel();

        self.seeking = false;
        self.keep_going = true;
        self.register_listener(LISTENER_PERIOD, ui_rx);

        match Context::new(
            filepath,
            self.audio_ctrl.get_dbl_buffer_mtx(),
            self.video_ctrl.video_box.clone(),
            ctx_tx
        ) {
            Ok(context) => {
                self.context = Some(context);
            },
            Err(error) => eprintln!("Error opening media: {}", error),
        };
    }
}
<commit_msg>Switch to play after a seek only if EOS or Ready We want to keep the behaviour in Paused state which allows seeking without starting playback.<commit_after>extern crate gtk;
extern crate glib;
extern crate gstreamer as gst;

#[cfg(any(feature = "profiling-listener", feature = "profiling-tracker"))]
use chrono::Utc;

use std::rc::Rc;
use std::cell::RefCell;

use std::path::PathBuf;

use std::sync::mpsc::{channel, Receiver};

use gtk::prelude::*;

use media::{Context, ContextMessage};
use media::ContextMessage::*;

use super::{AudioController, InfoController, VideoController};

#[derive(Clone, PartialEq)]
pub enum ControllerState {
    EOS,
    Ready,
    Paused,
    Playing,
    Stopped,
}

const LISTENER_PERIOD: u32 = 250; // 250 ms (4 Hz)
const TRACKER_PERIOD:  u32 = 17;  //  17 ms (60 Hz)

pub struct MainController {
    window: gtk::ApplicationWindow,
    header_bar: gtk::HeaderBar,
    play_pause_btn: gtk::ToolButton,

    video_ctrl: VideoController,
    info_ctrl: InfoController,
    audio_ctrl: AudioController,

    context: Option<Context>,
    state: ControllerState,
    duration: Option<u64>, // duration is accurately known at eos
    seeking: bool,

    this_opt: Option<Rc<RefCell<MainController>>>,
    keep_going: bool,
    listener_src: Option<glib::SourceId>,
    tracker_src: Option<glib::SourceId>,
}

impl MainController {
    pub fn new(builder: gtk::Builder) -> Rc<RefCell<Self>> {
        let this = Rc::new(RefCell::new(MainController {
            window: builder.get_object("application-window").unwrap(),
            header_bar: builder.get_object("header-bar").unwrap(),
            play_pause_btn: builder.get_object("play_pause-toolbutton").unwrap(),

            video_ctrl: VideoController::new(&builder),
            info_ctrl: InfoController::new(&builder),
            audio_ctrl: AudioController::new(&builder),

            context: None,
            state: ControllerState::Stopped,
            duration: None,
            seeking: false,

            this_opt: None,
            keep_going: true,
            listener_src: None,
            tracker_src: None,
        }));

        {
            let mut this_mut = this.borrow_mut();

            let this_rc = Rc::clone(&this);
            this_mut.this_opt = Some(this_rc);

            this_mut.window.connect_delete_event(|_, _| {
                gtk::main_quit();
                Inhibit(false)
            });
            this_mut.window.set_titlebar(&this_mut.header_bar);

            let this_rc = Rc::clone(&this);
            this_mut.play_pause_btn.connect_clicked(move |_| {
                this_rc.borrow_mut().play_pause();
            });

            // TODO: add key bindings to seek by steps
            // play/pause, etc.

            this_mut.video_ctrl.register_callbacks(&this);
            this_mut.info_ctrl.register_callbacks(&this);
            this_mut.audio_ctrl.register_callbacks(&this);
        }

        let open_btn: gtk::Button = builder.get_object("open-btn").unwrap();
        let this_rc = Rc::clone(&this);
        open_btn.connect_clicked(move |_| {
            this_rc.borrow_mut().select_media();
        });

        this
    }

    pub fn show_all(&self) {
        self.window.show_all();
    }

    fn play_pause(&mut self) {
        let context =
            match self.context.take() {
                Some(context) => context,
                None => {
                    self.select_media();
                    return;
                },
            };

        if self.state != ControllerState::EOS {
            match context.get_state() {
                gst::State::Paused => {
                    self.register_tracker();
                    self.play_pause_btn.set_icon_name("media-playback-pause");
                    self.state = ControllerState::Playing;
                    context.play().unwrap();
                    self.context = Some(context);
                }
                gst::State::Playing => {
                    context.pause().unwrap();
                    self.play_pause_btn.set_icon_name("media-playback-start");
                    self.remove_tracker();
                    self.state = ControllerState::Paused;
                    self.context = Some(context);
                },
                state => {
                    println!("Can't play/pause in state {:?}", state);
                    self.context = Some(context);
                },
            };
        } else {
            // Restart the stream from the begining
            self.context = Some(context);
            self.seek(0, true); // accurate (slow)
        }
    }

    fn stop(&mut self) {
        if let Some(context) = self.context.as_mut() {
            context.stop();
        };

        // remove callbacks in order to avoid conflict on borrowing of self
        self.remove_listener();
        self.remove_tracker();

        self.audio_ctrl.cleanup();

        self.play_pause_btn.set_icon_name("media-playback-start");

        self.state = ControllerState::Stopped;
        self.duration = None;
    }

    pub fn seek(&mut self, position: u64, accurate: bool) {
        if self.state != ControllerState::Stopped {
            self.seeking = true;

            // update position even though the stream
            // is not sync yet for the user to notice
            // the seek request in being handled
            self.info_ctrl.seek(position);
            self.audio_ctrl.seek(position, &self.state);

            self.context.as_ref()
                .expect("MainController::seek no context")
                .seek(position, accurate);

            if self.state == ControllerState::EOS
            || self.state == ControllerState::Ready
            {
                if self.state == ControllerState::Ready {
                    self.context.as_ref()
                        .expect("MainController::seek no context")
                        .play().unwrap();
                }
                self.register_tracker();
                self.play_pause_btn.set_icon_name("media-playback-pause");
                self.state = ControllerState::Playing;
            }
        }
    }

    fn select_media(&mut self) {
        self.stop();

        let file_dlg = gtk::FileChooserDialog::new(
            Some("Open a media file"),
            Some(&self.window),
            gtk::FileChooserAction::Open,
        );
        // Note: couldn't find equivalents for STOCK_OK
        file_dlg.add_button("Open", gtk::ResponseType::Ok.into());

        if file_dlg.run() == gtk::ResponseType::Ok.into() {
            self.open_media(file_dlg.get_filename().unwrap());
        }

        file_dlg.close();
    }

    fn handle_eos(&mut self) {
        self.play_pause_btn.set_icon_name("media-playback-start");
        self.state = ControllerState::EOS;
    }

    fn remove_listener(&mut self) {
        if let Some(source_id) = self.listener_src.take() {
            glib::source_remove(source_id);
        }
    }

    fn register_listener(&mut self,
        timeout: u32,
        ui_rx: Receiver<ContextMessage>,
    ) {
        if self.listener_src.is_some() {
            return;
        }

        let this_rc = Rc::clone(self.this_opt.as_ref().unwrap());

        self.listener_src = Some(gtk::timeout_add(timeout, move || {
            let mut keep_going = true;

            for message in ui_rx.try_iter() {
                match message {
                    AsyncDone => {
                        this_rc.borrow_mut().seeking = false;
                    },
                    InitDone => {
                        let mut this_mut = this_rc.borrow_mut();

                        let context = this_mut.context.take()
                            .expect("MainController: InitDone but no context available");

                        this_mut.header_bar.set_subtitle(
                            Some(context.file_name.as_str())
                        );

                        this_mut.info_ctrl.new_media(&context);
                        this_mut.video_ctrl.new_media(&context);
                        this_mut.audio_ctrl.new_media(&context);

                        this_mut.context = Some(context);

                        this_mut.state = ControllerState::Ready;
                    },
                    Eos => {
                        let mut this_mut = this_rc.borrow_mut();
                        let position = this_mut.context.as_mut()
                            .expect("MainController::listener no context while getting position")
                            .get_position();

                        this_mut.info_ctrl.update_duration(position);
                        this_mut.duration = Some(position);

                        this_mut.info_ctrl.tick(position);
                        this_mut.audio_ctrl.tick();

                        this_mut.handle_eos();

                        // Remove listener and tracker.
                        // Note: tracker will be register again in case of
                        // a seek. Listener is of no use anymore because
                        // the context won't send any more Eos nor AsyncDone
                        // after an EOS.
                        keep_going = false;
                    },
                    FailedToOpenMedia => {
                        eprintln!("ERROR: failed to open media");

                        let mut this_mut = this_rc.borrow_mut();

                        this_mut.context = None;
                        this_mut.keep_going = false;
                        keep_going = false;
                    },
                };

                if !keep_going { break; }
            }

            if !keep_going {
                let mut this_mut = this_rc.borrow_mut();
                this_mut.listener_src = None;
                this_mut.tracker_src = None;
            }

            glib::Continue(keep_going)
        }));
    }

    fn remove_tracker(&mut self) {
        if let Some(source_id) = self.tracker_src.take() {
            glib::source_remove(source_id);
        }
    }

    fn register_tracker(&mut self) {
        if self.tracker_src.is_some() {
            return;
        }

        let this_rc = Rc::clone(self.this_opt.as_ref().unwrap());

        self.tracker_src = Some(gtk::timeout_add(TRACKER_PERIOD, move || {
            #[cfg(feature = "profiling-tracker")]
            let start = Utc::now();

            let mut keep_going = true;

            let mut this_mut = this_rc.borrow_mut();

            #[cfg(feature = "profiling-tracker")]
            let before_tick = Utc::now();

            let position = this_mut.context.as_mut()
                .expect("MainController::tracker no context while getting position")
                .get_position();

            if !this_mut.seeking {
                this_mut.info_ctrl.tick(position);
                this_mut.audio_ctrl.tick();
            }

            if let Some(duration) = this_mut.duration {
                if position >= duration {
                    if !this_mut.seeking {
                        // this check is necessary as EOS is not sent
                        // in case of a seek after EOS
                        this_mut.handle_eos();
                        this_mut.tracker_src = None;
                        keep_going = false;
                    }
                } else if this_mut.seeking {
                    // this check is necessary as AsyncDone is not sent
                    // in case of a seek after EOS
                    this_mut.seeking = false;
                }
            }

            #[cfg(feature = "profiling-tracker")]
            let end = Utc::now();

            #[cfg(feature = "profiling-tracker")]
            println!("tracker,{},{},{}",
                start.time().format("%H:%M:%S%.6f"),
                before_tick.time().format("%H:%M:%S%.6f"),
                end.time().format("%H:%M:%S%.6f"),
            );

            glib::Continue(this_mut.keep_going && keep_going)
        }));
    }

    fn open_media(&mut self, filepath: PathBuf) {
        assert_eq!(self.listener_src, None);

        self.info_ctrl.cleanup();
        self.audio_ctrl.cleanup();
        self.header_bar.set_subtitle("");

        let (ctx_tx, ui_rx) = channel();

        self.seeking = false;
        self.keep_going = true;
        self.register_listener(LISTENER_PERIOD, ui_rx);

        match Context::new(
            filepath,
            self.audio_ctrl.get_dbl_buffer_mtx(),
            self.video_ctrl.video_box.clone(),
            ctx_tx
        ) {
            Ok(context) => {
                self.context = Some(context);
            },
            Err(error) => eprintln!("Error opening media: {}", error),
        };
    }
}
<|endoftext|>


<commit_before>//! Various algorithms for graph pruning - removing unnecessary vertices/edges.

use asm::SEQUENCES;
use collections::HmGIR;
use collections::girs::edges::Edge;
use collections::graphs::pt_graph::{EdgeIndex, Node, NodeIndex, PtGraph};
use compress::{compress_node, encode_fasta_symbol};
use prelude::{EdgeWeight, K1_SIZE, K_SIZE};
use slices::{BasicSlice, NodeSlice};

use petgraph::EdgeDirection;

use std::collections::hash_map::Entry;
use std::iter;
use std::slice;

/// Describes prunable structure in the sense of genome assembly
pub trait Prunable: Clean {
    /// Remove all input and output dead paths
    fn remove_dead_paths(&mut self);
}

/// A trait for keeping the graph clean.
/// It keeps simple functions used for basic graph cleanups
pub trait Clean {
    /// Remove vertives without any edges.
    fn remove_single_vertices(&mut self);
    /// Remove edges with weight below threshold.
    fn remove_weak_edges(&mut self, threshold: EdgeWeight);
}

impl Prunable for PtGraph {
    fn remove_dead_paths(&mut self) {
        info!("Starting graph pruning");
        loop {
            debug!("Detected {} input/output vertices",
                   Externals::new(self.raw_nodes().iter().enumerate()).count());
            let mut to_remove: Vec<EdgeIndex> = vec![];
            // analyze found input/output vertices
            for v in Externals::new(self.raw_nodes().iter().enumerate()) {
                // sort into output and input paths
                match v {
                    VertexType::Input(v_) => {
                        // decide whether or not vertex is in the dead path
                        if let Some(dead_path) = check_dead_path(self,
                                                                 v_,
                                                                 EdgeDirection::Incoming,
                                                                 EdgeDirection::Outgoing) {
                            to_remove.extend(dead_path);
                        }
                    }
                    VertexType::Output(v_) => {
                        if let Some(dead_path) = check_dead_path(self,
                                                                 v_,
                                                                 EdgeDirection::Outgoing,
                                                                 EdgeDirection::Incoming) {
                            to_remove.extend(dead_path);
                        }
                    }
                }
            }
            // if there are no dead paths left pruning is done
            if to_remove.is_empty() {
                info!("Graph is pruned");
                return;
            }
            // reverse sort edge indices such that removal won't cause any troubles with swapped
            // edge indices (see `petgraph`'s explanation of `remove_edge`)
            to_remove.sort_by(|a, b| b.cmp(a));
            remove_paths(self, to_remove.as_slice());
        }
    }
}

impl Clean for PtGraph {
    fn remove_single_vertices(&mut self) {
        self.retain_nodes(|g, n| {
            if let None = g.neighbors_undirected(n).next() {
                false
            }
            else {
                true
            }
        });
    }

    fn remove_weak_edges(&mut self, threshold: EdgeWeight) {
        self.retain_edges(|g, e| unwrap!(g.edge_weight(e)).1 >= threshold);
        self.remove_single_vertices();
    }
}

impl Clean for HmGIR {
    fn remove_single_vertices(&mut self) {
        let mut keys_to_remove: Vec<NodeSlice> = self.iter()
            .filter(|&(_, val)| val.is_empty())
            .map(|(key, _)| *key)
            .collect();
        keys_to_remove = keys_to_remove.into_iter()
            .filter(|x| !has_incoming_edges(self, x))
            .collect();
        for key in keys_to_remove {
            self.remove(&key);
        }
    }

    fn remove_weak_edges(&mut self, threshold: EdgeWeight) {
        for edges in self.values_mut() {
            *edges = edges.iter()
                .cloned()
                .filter(|&x| x.1 >= threshold)
                .collect::<Vec<Edge>>()
                .into_boxed_slice();
        }
        self.remove_single_vertices();
    }
}

/// Utility function which gets us every possible incoming edge.
/// because of memory savings we do not hold an array of incoming edges,
/// instead we will exploit the idea behind sequencing genome, namely
/// common bytes for each sequence.
/// WARNING: this may or may not be optimal if we follow the fasta standard
/// but should be sufficiently faster for just 5 characters we use at the moment
fn has_incoming_edges(gir: &mut HmGIR, node: &NodeSlice) -> bool {
    let mut output = false;
    // copy current sequence to register
    let mut vec = node.byte_name();
    // shift the register one character to the right
    vec.truncate(K1_SIZE - 1);
    vec.insert(0, b'A');
    let mut v = Vec::new();
    compress_node(&vec, &mut v);
    SEQUENCES.write()[0] = v.into_boxed_slice();
    let mask = 0b00111111u8;
    // try to bruteforce by inserting all possible characters: ACTGN
    let tmp_ns = NodeSlice::new(0);
    for chr in &[b'A', b'C', b'T', b'G'] {
        {
            let mut s = SEQUENCES.write();
            s[0][0] &= mask;
            s[0][0] |= encode_fasta_symbol(*chr, 0u8) << 6;
        }
        // dummy read slice used to check if we can find it in the gir
        if let Entry::Occupied(e) = gir.entry(tmp_ns) {
            // if we got any hits check if our vertex is in the outgoing
            if let Some(_) = e.get().iter().find(|&x| x.0 == node.offset()) {
                output = true;
                break;
            }
        }
    }
    output
}

enum VertexType<T> {
    Input(T),
    Output(T),
}

/// Iterator yielding vertices which either have no incoming or outgoing edges.
struct Externals<'a> {
    iter: iter::Enumerate<slice::Iter<'a, Node>>,
}

impl<'a> Externals<'a> {
    fn new(iter_: iter::Enumerate<slice::Iter<'a, Node>>) -> Externals {
        Externals { iter: iter_ }
    }
}

impl<'a> Iterator for Externals<'a> {
    type Item = VertexType<NodeIndex>;
    fn next(&mut self) -> Option<VertexType<NodeIndex>> {
        loop {
            match self.iter.next() {
                None => return None,
                Some((index, node)) => {
                    if node.next_edge(EdgeDirection::Incoming) == EdgeIndex::end() {
                        return Some(VertexType::Input(NodeIndex::new(index)));
                    }
                    else if node.next_edge(EdgeDirection::Outgoing) == EdgeIndex::end() {
                        return Some(VertexType::Output(NodeIndex::new(index)));
                    }
                    else {
                        continue;
                    }
                }
            }
        }
    }
}

/// Remove dead input path.
fn remove_paths(graph: &mut PtGraph, to_remove: &[EdgeIndex]) {
    debug!("Removing {} dead paths", to_remove.len());
    for e in to_remove.iter() {
        graph.remove_edge(*e);
    }
    graph.remove_single_vertices();
}

/// Check if vertex initializes a dead path.
fn check_dead_path(graph: &PtGraph, vertex: NodeIndex, first_direction: EdgeDirection,
                   second_direction: EdgeDirection)
                   -> Option<Vec<EdgeIndex>> {
    let mut output_vec = vec![];
    let mut current_vertex = vertex;
    let mut cnt = 0;
    loop {
        cnt += 1;
        if cnt >= 2 * (K_SIZE) {
            // this path is not dead
            return None;
        }
        // this line lets us check outgoing once, without the need to iterate twice
        let next_edge = graph.first_edge(current_vertex, second_direction);
        if let Some(e) = next_edge {
            // add vertex to path
            output_vec.push(e);
            // move to the next vertex in path
            current_vertex = unwrap!(graph.edge_endpoints(e)).1;
        }
        // if vertex has no outgoing edges
        else {
            return Some(output_vec);
        }
        if let Some(_) = graph.neighbors_directed(current_vertex, first_direction).nth(2) {
            return Some(output_vec);
        }
    }
}

#[cfg(test)]
mod tests {
    #![allow(unused_variables)]
    pub use ::collections::graphs::pt_graph::{PtGraph, EdgeIndex};
    pub use ::slices::EdgeSlice;
    pub use super::*;

    describe! pr {
        before_each {
            let mut graph: PtGraph = PtGraph::default();
        }

        describe! empty_graph {
            it "prunes single graph" {
                assert_eq!(graph.node_count(), 0);
                assert_eq!(graph.edge_count(), 0);
                graph.remove_weak_edges(10);
                assert_eq!(graph.node_count(), 0);
                assert_eq!(graph.edge_count(), 0);
            }
        }

        describe! with_nodes {
            before_each {
                let x = graph.add_node(());
                let y = graph.add_node(());
                let z = graph.add_node(());
                assert_eq!(graph.node_count(), 3);
            }

            describe! remove_weak_edges {
                it "prunes single weak edge" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    graph.add_edge(y, z, (EdgeSlice::default(), 1));
                    assert_eq!(graph.edge_count(), 2);
                    graph.remove_weak_edges(10);
                    assert_eq!(graph.node_count(), 2);
                    assert_eq!(graph.edge_count(), 1);
                }

                it "prunes single weak edge and no nodes" {
                    let w = graph.add_node(());
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    graph.add_edge(y, z, (EdgeSlice::default(), 1));
                    graph.add_edge(z, w, (EdgeSlice::default(), 100));
                    assert_eq!(graph.edge_count(), 3);
                    graph.remove_weak_edges(10);
                    assert_eq!(graph.node_count(), 4);
                    assert_eq!(graph.edge_count(), 2);
                }

                it "prunes strong edges" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    graph.add_edge(y, z, (EdgeSlice::default(), 100));
                    assert_eq!(graph.edge_count(), 2);
                    graph.remove_weak_edges(10);
                    assert_eq!(graph.node_count(), 3);
                    assert_eq!(graph.edge_count(), 2);
                }

                it "prunes cycle" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 1));
                    graph.add_edge(y, z, (EdgeSlice::default(), 1));
                    graph.add_edge(z, x, (EdgeSlice::default(), 1));
                    assert_eq!(graph.edge_count(), 3);
                    graph.remove_weak_edges(10);
                    assert_eq!(graph.node_count(), 0);
                    assert_eq!(graph.edge_count(), 0);
                }
            }

            describe! remove_single_vertices {
                it "doesn't remove vertices" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    graph.add_edge(y, z, (EdgeSlice::default(), 1));
                    assert_eq!(graph.edge_count(), 2);
                    graph.remove_single_vertices();
                    assert_eq!(graph.node_count(), 3);
                    assert_eq!(graph.edge_count(), 2);
                }

                it "removes one vertex" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    assert_eq!(graph.edge_count(), 1);
                    graph.remove_single_vertices();
                    assert_eq!(graph.node_count(), 2);
                    assert_eq!(graph.edge_count(), 1);
                }

                it "removes two vertices" {
                    graph.add_edge(x, x, (EdgeSlice::default(), 100));
                    assert_eq!(graph.edge_count(), 1);
                    graph.remove_single_vertices();
                    assert_eq!(graph.node_count(), 1);
                    assert_eq!(graph.edge_count(), 1);
                }

                it "removes all vertices" {
                    assert_eq!(graph.edge_count(), 0);
                    graph.remove_single_vertices();
                    assert_eq!(graph.node_count(), 0);
                    assert_eq!(graph.edge_count(), 0);
                }

                it "removes after removal of edge" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    graph.add_edge(y, z, (EdgeSlice::default(), 1));
                    assert_eq!(graph.edge_count(), 2);
                    assert_eq!(graph.node_count(), 3);
                    graph.remove_edge(EdgeIndex::new(1));
                    assert_eq!(graph.edge_count(), 1);
                    assert_eq!(graph.node_count(), 3);
                    graph.remove_single_vertices();
                    assert_eq!(graph.node_count(), 2);
                    assert_eq!(graph.edge_count(), 1);
                }
            }
        }
    }
}
<commit_msg>Fix clippy warnings<commit_after>//! Various algorithms for graph pruning - removing unnecessary vertices/edges.

use asm::SEQUENCES;
use collections::HmGIR;
use collections::girs::edges::Edge;
use collections::graphs::pt_graph::{EdgeIndex, Node, NodeIndex, PtGraph};
use compress::{compress_node, encode_fasta_symbol};
use prelude::{EdgeWeight, K1_SIZE, K_SIZE};
use slices::{BasicSlice, NodeSlice};

use petgraph::EdgeDirection;

use std::collections::hash_map::Entry;
use std::iter;
use std::slice;

/// Describes prunable structure in the sense of genome assembly
pub trait Prunable: Clean {
    /// Remove all input and output dead paths
    fn remove_dead_paths(&mut self);
}

/// A trait for keeping the graph clean.
/// It keeps simple functions used for basic graph cleanups
pub trait Clean {
    /// Remove vertives without any edges.
    fn remove_single_vertices(&mut self);
    /// Remove edges with weight below threshold.
    fn remove_weak_edges(&mut self, threshold: EdgeWeight);
}

impl Prunable for PtGraph {
    fn remove_dead_paths(&mut self) {
        info!("Starting graph pruning");
        loop {
            debug!("Detected {} input/output vertices",
                   Externals::new(self.raw_nodes().iter().enumerate()).count());
            let mut to_remove: Vec<EdgeIndex> = vec![];
            // analyze found input/output vertices
            for v in Externals::new(self.raw_nodes().iter().enumerate()) {
                // sort into output and input paths
                match v {
                    VertexType::Input(v_) => {
                        // decide whether or not vertex is in the dead path
                        if let Some(dead_path) = check_dead_path(self,
                                                                 v_,
                                                                 EdgeDirection::Incoming,
                                                                 EdgeDirection::Outgoing) {
                            to_remove.extend(dead_path);
                        }
                    }
                    VertexType::Output(v_) => {
                        if let Some(dead_path) = check_dead_path(self,
                                                                 v_,
                                                                 EdgeDirection::Outgoing,
                                                                 EdgeDirection::Incoming) {
                            to_remove.extend(dead_path);
                        }
                    }
                }
            }
            // if there are no dead paths left pruning is done
            if to_remove.is_empty() {
                info!("Graph is pruned");
                return;
            }
            // reverse sort edge indices such that removal won't cause any troubles with swapped
            // edge indices (see `petgraph`'s explanation of `remove_edge`)
            to_remove.sort_by(|a, b| b.cmp(a));
            remove_paths(self, to_remove.as_slice());
        }
    }
}

impl Clean for PtGraph {
    fn remove_single_vertices(&mut self) {
        self.retain_nodes(|g, n| g.neighbors_undirected(n).next().is_some());
    }

    fn remove_weak_edges(&mut self, threshold: EdgeWeight) {
        self.retain_edges(|g, e| unwrap!(g.edge_weight(e)).1 >= threshold);
        self.remove_single_vertices();
    }
}

impl Clean for HmGIR {
    fn remove_single_vertices(&mut self) {
        let mut keys_to_remove: Vec<NodeSlice> = self.iter()
            .filter(|&(_, val)| val.is_empty())
            .map(|(key, _)| *key)
            .collect();
        keys_to_remove = keys_to_remove.into_iter()
            .filter(|x| !has_incoming_edges(self, x))
            .collect();
        for key in keys_to_remove {
            self.remove(&key);
        }
    }

    fn remove_weak_edges(&mut self, threshold: EdgeWeight) {
        for edges in self.values_mut() {
            *edges = edges.iter()
                .cloned()
                .filter(|&x| x.1 >= threshold)
                .collect::<Vec<Edge>>()
                .into_boxed_slice();
        }
        self.remove_single_vertices();
    }
}

/// Utility function which gets us every possible incoming edge.
/// because of memory savings we do not hold an array of incoming edges,
/// instead we will exploit the idea behind sequencing genome, namely
/// common bytes for each sequence.
/// WARNING: this may or may not be optimal if we follow the fasta standard
/// but should be sufficiently faster for just 5 characters we use at the moment
fn has_incoming_edges(gir: &mut HmGIR, node: &NodeSlice) -> bool {
    let mut output = false;
    // copy current sequence to register
    let mut vec = node.byte_name();
    // shift the register one character to the right
    vec.truncate(K1_SIZE - 1);
    vec.insert(0, b'A');
    let mut v = Vec::new();
    compress_node(&vec, &mut v);
    SEQUENCES.write()[0] = v.into_boxed_slice();
    let mask = 0b00111111u8;
    // try to bruteforce by inserting all possible characters: ACTGN
    let tmp_ns = NodeSlice::new(0);
    for chr in &[b'A', b'C', b'T', b'G'] {
        {
            let mut s = SEQUENCES.write();
            s[0][0] &= mask;
            s[0][0] |= encode_fasta_symbol(*chr, 0u8) << 6;
        }
        // dummy read slice used to check if we can find it in the gir
        if let Entry::Occupied(e) = gir.entry(tmp_ns) {
            // if we got any hits check if our vertex is in the outgoing
            if e.get().iter().any(|&x| x.0 == node.offset()) {
                output = true;
                break;
            }
        }
    }
    output
}

enum VertexType<T> {
    Input(T),
    Output(T),
}

/// Iterator yielding vertices which either have no incoming or outgoing edges.
struct Externals<'a> {
    iter: iter::Enumerate<slice::Iter<'a, Node>>,
}

impl<'a> Externals<'a> {
    fn new(iter_: iter::Enumerate<slice::Iter<'a, Node>>) -> Externals {
        Externals { iter: iter_ }
    }
}

impl<'a> Iterator for Externals<'a> {
    type Item = VertexType<NodeIndex>;
    fn next(&mut self) -> Option<VertexType<NodeIndex>> {
        loop {
            match self.iter.next() {
                None => return None,
                Some((index, node)) => {
                    if node.next_edge(EdgeDirection::Incoming) == EdgeIndex::end() {
                        return Some(VertexType::Input(NodeIndex::new(index)));
                    }
                    else if node.next_edge(EdgeDirection::Outgoing) == EdgeIndex::end() {
                        return Some(VertexType::Output(NodeIndex::new(index)));
                    }
                    else {
                        continue;
                    }
                }
            }
        }
    }
}

/// Remove dead input path.
fn remove_paths(graph: &mut PtGraph, to_remove: &[EdgeIndex]) {
    debug!("Removing {} dead paths", to_remove.len());
    for e in to_remove.iter() {
        graph.remove_edge(*e);
    }
    graph.remove_single_vertices();
}

/// Check if vertex initializes a dead path.
fn check_dead_path(graph: &PtGraph, vertex: NodeIndex, first_direction: EdgeDirection,
                   second_direction: EdgeDirection)
                   -> Option<Vec<EdgeIndex>> {
    let mut output_vec = vec![];
    let mut current_vertex = vertex;
    let mut cnt = 0;
    loop {
        cnt += 1;
        if cnt >= 2 * (K_SIZE) {
            // this path is not dead
            return None;
        }
        // this line lets us check outgoing once, without the need to iterate twice
        let next_edge = graph.first_edge(current_vertex, second_direction);
        if let Some(e) = next_edge {
            // add vertex to path
            output_vec.push(e);
            // move to the next vertex in path
            current_vertex = unwrap!(graph.edge_endpoints(e)).1;
        }
        // if vertex has no outgoing edges
        else {
            return Some(output_vec);
        }
        if graph.neighbors_directed(current_vertex, first_direction).nth(2).is_some() {
            return Some(output_vec);
        }
    }
}

#[cfg(test)]
mod tests {
    #![allow(unused_variables)]
    pub use ::collections::graphs::pt_graph::{PtGraph, EdgeIndex};
    pub use ::slices::EdgeSlice;
    pub use super::*;

    describe! pr {
        before_each {
            let mut graph: PtGraph = PtGraph::default();
        }

        describe! empty_graph {
            it "prunes single graph" {
                assert_eq!(graph.node_count(), 0);
                assert_eq!(graph.edge_count(), 0);
                graph.remove_weak_edges(10);
                assert_eq!(graph.node_count(), 0);
                assert_eq!(graph.edge_count(), 0);
            }
        }

        describe! with_nodes {
            before_each {
                let x = graph.add_node(());
                let y = graph.add_node(());
                let z = graph.add_node(());
                assert_eq!(graph.node_count(), 3);
            }

            describe! remove_weak_edges {
                it "prunes single weak edge" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    graph.add_edge(y, z, (EdgeSlice::default(), 1));
                    assert_eq!(graph.edge_count(), 2);
                    graph.remove_weak_edges(10);
                    assert_eq!(graph.node_count(), 2);
                    assert_eq!(graph.edge_count(), 1);
                }

                it "prunes single weak edge and no nodes" {
                    let w = graph.add_node(());
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    graph.add_edge(y, z, (EdgeSlice::default(), 1));
                    graph.add_edge(z, w, (EdgeSlice::default(), 100));
                    assert_eq!(graph.edge_count(), 3);
                    graph.remove_weak_edges(10);
                    assert_eq!(graph.node_count(), 4);
                    assert_eq!(graph.edge_count(), 2);
                }

                it "prunes strong edges" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    graph.add_edge(y, z, (EdgeSlice::default(), 100));
                    assert_eq!(graph.edge_count(), 2);
                    graph.remove_weak_edges(10);
                    assert_eq!(graph.node_count(), 3);
                    assert_eq!(graph.edge_count(), 2);
                }

                it "prunes cycle" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 1));
                    graph.add_edge(y, z, (EdgeSlice::default(), 1));
                    graph.add_edge(z, x, (EdgeSlice::default(), 1));
                    assert_eq!(graph.edge_count(), 3);
                    graph.remove_weak_edges(10);
                    assert_eq!(graph.node_count(), 0);
                    assert_eq!(graph.edge_count(), 0);
                }
            }

            describe! remove_single_vertices {
                it "doesn't remove vertices" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    graph.add_edge(y, z, (EdgeSlice::default(), 1));
                    assert_eq!(graph.edge_count(), 2);
                    graph.remove_single_vertices();
                    assert_eq!(graph.node_count(), 3);
                    assert_eq!(graph.edge_count(), 2);
                }

                it "removes one vertex" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    assert_eq!(graph.edge_count(), 1);
                    graph.remove_single_vertices();
                    assert_eq!(graph.node_count(), 2);
                    assert_eq!(graph.edge_count(), 1);
                }

                it "removes two vertices" {
                    graph.add_edge(x, x, (EdgeSlice::default(), 100));
                    assert_eq!(graph.edge_count(), 1);
                    graph.remove_single_vertices();
                    assert_eq!(graph.node_count(), 1);
                    assert_eq!(graph.edge_count(), 1);
                }

                it "removes all vertices" {
                    assert_eq!(graph.edge_count(), 0);
                    graph.remove_single_vertices();
                    assert_eq!(graph.node_count(), 0);
                    assert_eq!(graph.edge_count(), 0);
                }

                it "removes after removal of edge" {
                    graph.add_edge(x, y, (EdgeSlice::default(), 100));
                    graph.add_edge(y, z, (EdgeSlice::default(), 1));
                    assert_eq!(graph.edge_count(), 2);
                    assert_eq!(graph.node_count(), 3);
                    graph.remove_edge(EdgeIndex::new(1));
                    assert_eq!(graph.edge_count(), 1);
                    assert_eq!(graph.node_count(), 3);
                    graph.remove_single_vertices();
                    assert_eq!(graph.node_count(), 2);
                    assert_eq!(graph.edge_count(), 1);
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>use super::{make_test, TestOptions};
use rustc_span::edition::DEFAULT_EDITION;

#[test]
fn make_test_basic() {
    //basic use: wraps with `fn main`, adds `#![allow(unused)]`
    let opts = TestOptions::default();
    let input = "assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 2));
}

#[test]
fn make_test_crate_name_no_use() {
    // If you give a crate name but *don't* use it within the test, it won't bother inserting
    // the `extern crate` statement.
    let opts = TestOptions::default();
    let input = "assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 2));
}

#[test]
fn make_test_crate_name() {
    // If you give a crate name and use it within the test, it will insert an `extern crate`
    // statement before `fn main`.
    let opts = TestOptions::default();
    let input = "use asdf::qwop;
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
extern crate asdf;
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 3));
}

#[test]
fn make_test_no_crate_inject() {
    // Even if you do use the crate within the test, setting `opts.no_crate_inject` will skip
    // adding it anyway.
    let opts = TestOptions { no_crate_inject: true, display_warnings: false, attrs: vec![] };
    let input = "use asdf::qwop;
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 2));
}

#[test]
fn make_test_ignore_std() {
    // Even if you include a crate name, and use it in the doctest, we still won't include an
    // `extern crate` statement if the crate is "std" -- that's included already by the
    // compiler!
    let opts = TestOptions::default();
    let input = "use std::*;
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
fn main() {
use std::*;
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, Some("std"), false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 2));
}

#[test]
fn make_test_manual_extern_crate() {
    // When you manually include an `extern crate` statement in your doctest, `make_test`
    // assumes you've included one for your own crate too.
    let opts = TestOptions::default();
    let input = "extern crate asdf;
use asdf::qwop;
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
extern crate asdf;
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 2));
}

#[test]
fn make_test_manual_extern_crate_with_macro_use() {
    let opts = TestOptions::default();
    let input = "#[macro_use] extern crate asdf;
use asdf::qwop;
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
#[macro_use] extern crate asdf;
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 2));
}

#[test]
fn make_test_opts_attrs() {
    // If you supplied some doctest attributes with `#![doc(test(attr(...)))]`, it will use
    // those instead of the stock `#![allow(unused)]`.
    let mut opts = TestOptions::default();
    opts.attrs.push("feature(sick_rad)".to_string());
    let input = "use asdf::qwop;
assert_eq!(2+2, 4);";
    let expected = "#![feature(sick_rad)]
extern crate asdf;
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 3));

    // Adding more will also bump the returned line offset.
    opts.attrs.push("feature(hella_dope)".to_string());
    let expected = "#![feature(sick_rad)]
#![feature(hella_dope)]
extern crate asdf;
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 4));
}

#[test]
fn make_test_crate_attrs() {
    // Including inner attributes in your doctest will apply them to the whole "crate", pasting
    // them outside the generated main function.
    let opts = TestOptions::default();
    let input = "#![feature(sick_rad)]
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
#![feature(sick_rad)]
fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 2));
}

#[test]
fn make_test_with_main() {
    // Including your own `fn main` wrapper lets the test use it verbatim.
    let opts = TestOptions::default();
    let input = "fn main() {
    assert_eq!(2+2, 4);
}";
    let expected = "#![allow(unused)]
fn main() {
    assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 1));
}

#[test]
fn make_test_fake_main() {
    // ... but putting it in a comment will still provide a wrapper.
    let opts = TestOptions::default();
    let input = "//Ceci n'est pas une `fn main`
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
//Ceci n'est pas une `fn main`
fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 2));
}

#[test]
fn make_test_dont_insert_main() {
    // Even with that, if you set `dont_insert_main`, it won't create the `fn main` wrapper.
    let opts = TestOptions::default();
    let input = "//Ceci n'est pas une `fn main`
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
//Ceci n'est pas une `fn main`
assert_eq!(2+2, 4);"
        .to_string();
    let output = make_test(input, None, true, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 1));
}

#[test]
fn make_test_display_warnings() {
    // If the user is asking to display doctest warnings, suppress the default `allow(unused)`.
    let mut opts = TestOptions::default();
    opts.display_warnings = true;
    let input = "assert_eq!(2+2, 4);";
    let expected = "fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();
    let output = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 1));
}

#[test]
fn make_test_issues_21299_33731() {
    let opts = TestOptions::default();

    let input = "// fn main
assert_eq!(2+2, 4);";

    let expected = "#![allow(unused)]
// fn main
fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();

    let output = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 2));

    let input = "extern crate hella_qwop;
assert_eq!(asdf::foo, 4);";

    let expected = "#![allow(unused)]
extern crate hella_qwop;
extern crate asdf;
fn main() {
assert_eq!(asdf::foo, 4);
}"
    .to_string();

    let output = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 3));
}

#[test]
fn make_test_main_in_macro() {
    let opts = TestOptions::default();
    let input = "#[macro_use] extern crate my_crate;
test_wrapper! {
    fn main() {}
}";
    let expected = "#![allow(unused)]
#[macro_use] extern crate my_crate;
test_wrapper! {
    fn main() {}
}"
    .to_string();

    let output = make_test(input, Some("my_crate"), false, &opts, DEFAULT_EDITION);
    assert_eq!(output, (expected, 1));
}
<commit_msg>Update doctest tests<commit_after>use super::{make_test, TestOptions};
use rustc_span::edition::DEFAULT_EDITION;

#[test]
fn make_test_basic() {
    //basic use: wraps with `fn main`, adds `#![allow(unused)]`
    let opts = TestOptions::default();
    let input = "assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 2));
}

#[test]
fn make_test_crate_name_no_use() {
    // If you give a crate name but *don't* use it within the test, it won't bother inserting
    // the `extern crate` statement.
    let opts = TestOptions::default();
    let input = "assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 2));
}

#[test]
fn make_test_crate_name() {
    // If you give a crate name and use it within the test, it will insert an `extern crate`
    // statement before `fn main`.
    let opts = TestOptions::default();
    let input = "use asdf::qwop;
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
extern crate asdf;
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 3));
}

#[test]
fn make_test_no_crate_inject() {
    // Even if you do use the crate within the test, setting `opts.no_crate_inject` will skip
    // adding it anyway.
    let opts = TestOptions { no_crate_inject: true, display_warnings: false, attrs: vec![] };
    let input = "use asdf::qwop;
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 2));
}

#[test]
fn make_test_ignore_std() {
    // Even if you include a crate name, and use it in the doctest, we still won't include an
    // `extern crate` statement if the crate is "std" -- that's included already by the
    // compiler!
    let opts = TestOptions::default();
    let input = "use std::*;
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
fn main() {
use std::*;
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, Some("std"), false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 2));
}

#[test]
fn make_test_manual_extern_crate() {
    // When you manually include an `extern crate` statement in your doctest, `make_test`
    // assumes you've included one for your own crate too.
    let opts = TestOptions::default();
    let input = "extern crate asdf;
use asdf::qwop;
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
extern crate asdf;
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 2));
}

#[test]
fn make_test_manual_extern_crate_with_macro_use() {
    let opts = TestOptions::default();
    let input = "#[macro_use] extern crate asdf;
use asdf::qwop;
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
#[macro_use] extern crate asdf;
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 2));
}

#[test]
fn make_test_opts_attrs() {
    // If you supplied some doctest attributes with `#![doc(test(attr(...)))]`, it will use
    // those instead of the stock `#![allow(unused)]`.
    let mut opts = TestOptions::default();
    opts.attrs.push("feature(sick_rad)".to_string());
    let input = "use asdf::qwop;
assert_eq!(2+2, 4);";
    let expected = "#![feature(sick_rad)]
extern crate asdf;
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 3));

    // Adding more will also bump the returned line offset.
    opts.attrs.push("feature(hella_dope)".to_string());
    let expected = "#![feature(sick_rad)]
#![feature(hella_dope)]
extern crate asdf;
fn main() {
use asdf::qwop;
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 4));
}

#[test]
fn make_test_crate_attrs() {
    // Including inner attributes in your doctest will apply them to the whole "crate", pasting
    // them outside the generated main function.
    let opts = TestOptions::default();
    let input = "#![feature(sick_rad)]
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
#![feature(sick_rad)]
fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 2));
}

#[test]
fn make_test_with_main() {
    // Including your own `fn main` wrapper lets the test use it verbatim.
    let opts = TestOptions::default();
    let input = "fn main() {
    assert_eq!(2+2, 4);
}";
    let expected = "#![allow(unused)]
fn main() {
    assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 1));
}

#[test]
fn make_test_fake_main() {
    // ... but putting it in a comment will still provide a wrapper.
    let opts = TestOptions::default();
    let input = "//Ceci n'est pas une `fn main`
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
//Ceci n'est pas une `fn main`
fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 2));
}

#[test]
fn make_test_dont_insert_main() {
    // Even with that, if you set `dont_insert_main`, it won't create the `fn main` wrapper.
    let opts = TestOptions::default();
    let input = "//Ceci n'est pas une `fn main`
assert_eq!(2+2, 4);";
    let expected = "#![allow(unused)]
//Ceci n'est pas une `fn main`
assert_eq!(2+2, 4);"
        .to_string();
    let (output, len, _) = make_test(input, None, true, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 1));
}

#[test]
fn make_test_display_warnings() {
    // If the user is asking to display doctest warnings, suppress the default `allow(unused)`.
    let mut opts = TestOptions::default();
    opts.display_warnings = true;
    let input = "assert_eq!(2+2, 4);";
    let expected = "fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();
    let (output, len, _) = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 1));
}

#[test]
fn make_test_issues_21299_33731() {
    let opts = TestOptions::default();

    let input = "// fn main
assert_eq!(2+2, 4);";

    let expected = "#![allow(unused)]
// fn main
fn main() {
assert_eq!(2+2, 4);
}"
    .to_string();

    let (output, len, _) = make_test(input, None, false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 2));

    let input = "extern crate hella_qwop;
assert_eq!(asdf::foo, 4);";

    let expected = "#![allow(unused)]
extern crate hella_qwop;
extern crate asdf;
fn main() {
assert_eq!(asdf::foo, 4);
}"
    .to_string();

    let (output, len, _) = make_test(input, Some("asdf"), false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 3));
}

#[test]
fn make_test_main_in_macro() {
    let opts = TestOptions::default();
    let input = "#[macro_use] extern crate my_crate;
test_wrapper! {
    fn main() {}
}";
    let expected = "#![allow(unused)]
#[macro_use] extern crate my_crate;
test_wrapper! {
    fn main() {}
}"
    .to_string();

    let (output, len, _) = make_test(input, Some("my_crate"), false, &opts, DEFAULT_EDITION);
    assert_eq!((output, len), (expected, 1));
}
<|endoftext|>


<commit_before>/*
    TODO:
    1. Copy downloaded files to other places. (Completed)
    2. Verify the downloaded package.
    3. Make it support repos someday.
*/
extern crate hyper;
use self::hyper::Client;
use std::fs::File;
use std::io::{ Read, Write };
use std::path::Path;

macro_rules! custom_try {
    ($x:expr) => (match $x {
        Ok(x) => x,
        Err(why) => panic!("An error occured during package downloading. {}", why),
    });
}

pub fn download_file(url: &str, output: &Path) {
    let client = Client::new();
    let response = custom_try!(client.get(url).send());
    let mut target = custom_try!(File::create(output));

    for byte in response.bytes() {
        target.write(&[byte.unwrap()]);
    }
}<commit_msg>Follow code style<commit_after>/*
    TODO:
    1. Copy downloaded files to other places. (Completed)
    2. Verify the downloaded package.
    3. Make it support repos someday.
*/
extern crate hyper;

use self::hyper::Client;
use std::fs::File;
use std::io::{ Read, Write };
use std::path::Path;

macro_rules! custom_try {
    ($x:expr) => (match $x {
        Ok(x) => x,
        Err(why) => panic!("An error occured during package downloading. {}", why),
    });
}

pub fn download_file(url: &str, output: &Path) {
    let client = Client::new();
    let response = custom_try!(client.get(url).send());
    let mut target = custom_try!(File::create(output));

    for byte in response.bytes() {
        target.write(&[byte.unwrap()]);
    }
}<|endoftext|>


<commit_before>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # Tagged stream
//!
//! A tagged stream recursively applies tags to regions of an input stream. A stream consisting of a single tag forms a tree,
//! so this stream type is useful for representing the parsed represention of an input.
//!
//! The simplest form of tagged stream is one that's just a set of untagged symbols. This can be created using the `TaggedStream::from_reader`
//! method.
//!
//! ```
//! # use concordance::*;
//! #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
//! enum SomeTag {
//!     Identifier
//! };
//!
//! let simple_tagged = TaggedStream::from_reader(&mut "Hello, world".read_symbols());
//! let with_tag      = simple_tagged.with_tags(vec![(0..5, SomeTag::Identifier)].iter().cloned());
//! ```
//!

// TODO: a possibly better way to design this would be to make it so that there's an underlying stream we read from when generating tag
// symbols so that we don't need to store the contents of the stream in memory

use std::slice::Iter;
use std::ops::Index;
use std::ops::Range;

use super::symbol_reader::*;
use super::tokenizer::*;
use super::symbol_range_dfa::*;

///
/// Represents a symbol in a tagged stream.
///
/// A symbol can either be a sequence of untagged base symbols, or it can be a tag (which itself can contain any number of tagged symbol)
///
#[derive(Clone, PartialEq, Eq, Debug)]
pub enum TagSymbol<Base: Clone+Ord, Tag: Clone+Ord> {
    /// An untagged element in this stream
    Untagged(Base),

    /// A tagged region of the stream
    Tagged(Tag, TaggedStream<Base, Tag>)
}

///
/// Represents a stream of tagged symbols
///
#[derive(Clone, PartialEq, Eq, Debug)]
pub struct TaggedStream<Base: Clone+Ord, Tag: Clone+Ord> {
    /// The data in this stream
    data: Vec<TagSymbol<Base, Tag>>
}

impl<Base: Ord+Clone, Tag: Ord+Clone> TaggedStream<Base, Tag> {
    ///
    /// Creates a basic tagged stream from a source of the base symbol
    ///
    pub fn from_reader(reader: &mut SymbolReader<Base>) -> TaggedStream<Base, Tag> {
        // Read from the reader into an array
        let mut symbols = vec![];

        while let Some(next_symbol) = reader.next_symbol() {
            symbols.push(TagSymbol::Untagged(next_symbol));
        }

        // Generate a simple tagged stream from the result
        TaggedStream { data: symbols }
    }

    ///
    /// The number of symbols in this stream
    ///
    pub fn len(&self) -> usize {
        self.data.len()
    }

    ///
    /// Replaces a range in this stream with a tag
    ///
    pub fn tag(&mut self, tag: Tag, range: Range<usize>) {
        // Create a tag to replace the range
        let replaced_symbols    = self.data[range.clone()].to_vec();
        let tag_symbol          = TagSymbol::Tagged(tag, TaggedStream { data: replaced_symbols });

        // Draining seems to be for reading a range but does double duty for deleting a range?
        // I don't think rust has a way to replace a range in a vector, or at least not one that's easy to find in the docs.
        self.data.drain(range.clone());

        // Draining then inserting is inefficient compared to flat out replacing items :-/ This may be possible but the vec docs aren't very easy to read
        self.data.insert(range.start, tag_symbol);
    }

    ///
    /// Creates a tag symbol by tagging a particular range within this stream
    ///
    pub fn tag_range(&self, range: Range<usize>, tag: Tag) -> TagSymbol<Base, Tag> {
        let tag_data = self.data[range].to_vec();

        TagSymbol::Tagged(tag, TaggedStream { data: tag_data })
    }

    ///
    /// Applies tags to the specified ranges in this stream
    ///
    /// The tag ranges must be in ascending order and must not overlap
    ///
    pub fn with_tags<I>(&self, tags: I) -> TaggedStream<Base, Tag>
        where I : Iterator<Item=(Range<usize>, Tag)> {
        // The data that will make up the new stream
        let mut new_stream = vec![];

        // The last processed range (we haven't processed any range yet, so this starts at 0..0)
        let mut last_range = 0..0;

        // Build up the result from the tag list and the last range
        for tag in tags {
            // Fetch the information for this tag and check ordering
            let (next_range, next_tag)  = tag;
            let end                     = last_range.end;

            if end > next_range.start {
                panic!("Tags for with_tags must be sorted in order");
            }

            // Sometimes need to leave some data from this stream untagged
            if end < next_range.start {
                new_stream.extend(self.data[end..next_range.start].iter().cloned());
            }

            // Push the tag
            new_stream.push(self.tag_range(next_range.clone(), next_tag));

            // Update state
            last_range = next_range;
        }

        // Append anything left in the stream
        if last_range.end < self.data.len() {
            new_stream.extend(self.data[last_range.end..self.data.len()].iter().cloned());
        }

        // Final result
        TaggedStream { data: new_stream }
    }

    ///
    /// Runs the current values of this tagged stream through a tokenizer and tags anything it matches
    ///
    /// This takes a mapping function: the idea is that
    ///
    pub fn tokenize<DfaSymbol: Ord, MapFn>(&self, dfa: &SymbolRangeDfa<DfaSymbol, Tag>, map_symbol: MapFn) -> TaggedStream<Base, Tag>
        where MapFn: Fn(TagSymbol<Base, Tag>) -> Option<DfaSymbol> {
        unimplemented!();
    }
}

impl<Base: Clone+Ord, Tag: Clone+Ord> Index<usize> for TaggedStream<Base, Tag> {
    type Output = TagSymbol<Base, Tag>;

    fn index(&self, index: usize) -> &TagSymbol<Base, Tag> {
        &self.data[index]
    }
}

impl<'a, Base: Clone+Ord, Tag: Clone+Ord> SymbolSource<'a, TagSymbol<Base, Tag>> for &'a TaggedStream<Base, Tag> {
    type SymbolReader = Iter<'a, TagSymbol<Base, Tag>>;

    /// Returns a new object that can read the symbols from this one
    fn read_symbols(self) -> Self::SymbolReader {
        self.data.read_symbols()
    }
}

#[cfg(test)]
mod test {
    use super::super::*;

    #[test]
    fn can_tag_range() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let mut tagged: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());

        tagged.tag(Tags::Hello, 0..5);
        tagged.tag(Tags::World, 1..6);

        assert!(tagged.len() == 2);

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('H'));
            assert!(stream[1] == TagSymbol::Untagged('e'));
            assert!(stream[2] == TagSymbol::Untagged('l'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('o'));
        } else {
            assert!(false);
        }

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[1] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('W'));
            assert!(stream[1] == TagSymbol::Untagged('o'));
            assert!(stream[2] == TagSymbol::Untagged('r'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn can_tag_everything_with_tags() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..5, Tags::Hello), (5..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 2);

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('H'));
            assert!(stream[1] == TagSymbol::Untagged('e'));
            assert!(stream[2] == TagSymbol::Untagged('l'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('o'));
        } else {
            assert!(false);
        }

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[1] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('W'));
            assert!(stream[1] == TagSymbol::Untagged('o'));
            assert!(stream[2] == TagSymbol::Untagged('r'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn with_tags_preserves_middle() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..4, Tags::Hello), (6..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 4);
        assert!(tagged[1] == TagSymbol::Untagged('o'));
        assert!(tagged[2] == TagSymbol::Untagged('W'));

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 4);
            assert!(stream[0] == TagSymbol::Untagged('H'));
            assert!(stream[1] == TagSymbol::Untagged('e'));
            assert!(stream[2] == TagSymbol::Untagged('l'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
        } else {
            assert!(false);
        }

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[3] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 4);
            assert!(stream[0] == TagSymbol::Untagged('o'));
            assert!(stream[1] == TagSymbol::Untagged('r'));
            assert!(stream[2] == TagSymbol::Untagged('l'));
            assert!(stream[3] == TagSymbol::Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn with_tags_preserves_end() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..5, Tags::Hello)].iter().cloned());

        assert!(tagged.len() == 6);

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('H'));
            assert!(stream[1] == TagSymbol::Untagged('e'));
            assert!(stream[2] == TagSymbol::Untagged('l'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('o'));
        } else {
            assert!(false);
        }

        assert!(tagged[1] == TagSymbol::Untagged('W'));
        assert!(tagged[2] == TagSymbol::Untagged('o'));
        assert!(tagged[3] == TagSymbol::Untagged('r'));
        assert!(tagged[4] == TagSymbol::Untagged('l'));
        assert!(tagged[5] == TagSymbol::Untagged('d'));
    }

    #[test]
    fn with_tags_preserves_start() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(5..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 6);

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[5] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('W'));
            assert!(stream[1] == TagSymbol::Untagged('o'));
            assert!(stream[2] == TagSymbol::Untagged('r'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('d'));
        } else {
            assert!(false);
        }

        assert!(tagged[0] == TagSymbol::Untagged('H'));
        assert!(tagged[1] == TagSymbol::Untagged('e'));
        assert!(tagged[2] == TagSymbol::Untagged('l'));
        assert!(tagged[3] == TagSymbol::Untagged('l'));
        assert!(tagged[4] == TagSymbol::Untagged('o'));
    }
}
<commit_msg>Think I might make it so that every symbol has to be mapped to something<commit_after>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # Tagged stream
//!
//! A tagged stream recursively applies tags to regions of an input stream. A stream consisting of a single tag forms a tree,
//! so this stream type is useful for representing the parsed represention of an input.
//!
//! The simplest form of tagged stream is one that's just a set of untagged symbols. This can be created using the `TaggedStream::from_reader`
//! method.
//!
//! ```
//! # use concordance::*;
//! #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
//! enum SomeTag {
//!     Identifier
//! };
//!
//! let simple_tagged = TaggedStream::from_reader(&mut "Hello, world".read_symbols());
//! let with_tag      = simple_tagged.with_tags(vec![(0..5, SomeTag::Identifier)].iter().cloned());
//! ```
//!

// TODO: a possibly better way to design this would be to make it so that there's an underlying stream we read from when generating tag
// symbols so that we don't need to store the contents of the stream in memory

use std::slice::Iter;
use std::ops::Index;
use std::ops::Range;

use super::symbol_reader::*;
use super::tokenizer::*;
use super::symbol_range_dfa::*;

///
/// Represents a symbol in a tagged stream.
///
/// A symbol can either be a sequence of untagged base symbols, or it can be a tag (which itself can contain any number of tagged symbol)
///
#[derive(Clone, PartialEq, Eq, Debug)]
pub enum TagSymbol<Base: Clone+Ord, Tag: Clone+Ord> {
    /// An untagged element in this stream
    Untagged(Base),

    /// A tagged region of the stream
    Tagged(Tag, TaggedStream<Base, Tag>)
}

///
/// Represents a stream of tagged symbols
///
#[derive(Clone, PartialEq, Eq, Debug)]
pub struct TaggedStream<Base: Clone+Ord, Tag: Clone+Ord> {
    /// The data in this stream
    data: Vec<TagSymbol<Base, Tag>>
}

impl<Base: Ord+Clone, Tag: Ord+Clone> TaggedStream<Base, Tag> {
    ///
    /// Creates a basic tagged stream from a source of the base symbol
    ///
    pub fn from_reader(reader: &mut SymbolReader<Base>) -> TaggedStream<Base, Tag> {
        // Read from the reader into an array
        let mut symbols = vec![];

        while let Some(next_symbol) = reader.next_symbol() {
            symbols.push(TagSymbol::Untagged(next_symbol));
        }

        // Generate a simple tagged stream from the result
        TaggedStream { data: symbols }
    }

    ///
    /// The number of symbols in this stream
    ///
    pub fn len(&self) -> usize {
        self.data.len()
    }

    ///
    /// Replaces a range in this stream with a tag
    ///
    pub fn tag(&mut self, tag: Tag, range: Range<usize>) {
        // Create a tag to replace the range
        let replaced_symbols    = self.data[range.clone()].to_vec();
        let tag_symbol          = TagSymbol::Tagged(tag, TaggedStream { data: replaced_symbols });

        // Draining seems to be for reading a range but does double duty for deleting a range?
        // I don't think rust has a way to replace a range in a vector, or at least not one that's easy to find in the docs.
        self.data.drain(range.clone());

        // Draining then inserting is inefficient compared to flat out replacing items :-/ This may be possible but the vec docs aren't very easy to read
        self.data.insert(range.start, tag_symbol);
    }

    ///
    /// Creates a tag symbol by tagging a particular range within this stream
    ///
    pub fn tag_range(&self, range: Range<usize>, tag: Tag) -> TagSymbol<Base, Tag> {
        let tag_data = self.data[range].to_vec();

        TagSymbol::Tagged(tag, TaggedStream { data: tag_data })
    }

    ///
    /// Applies tags to the specified ranges in this stream
    ///
    /// The tag ranges must be in ascending order and must not overlap
    ///
    pub fn with_tags<I>(&self, tags: I) -> TaggedStream<Base, Tag>
        where I : Iterator<Item=(Range<usize>, Tag)> {
        // The data that will make up the new stream
        let mut new_stream = vec![];

        // The last processed range (we haven't processed any range yet, so this starts at 0..0)
        let mut last_range = 0..0;

        // Build up the result from the tag list and the last range
        for tag in tags {
            // Fetch the information for this tag and check ordering
            let (next_range, next_tag)  = tag;
            let end                     = last_range.end;

            if end > next_range.start {
                panic!("Tags for with_tags must be sorted in order");
            }

            // Sometimes need to leave some data from this stream untagged
            if end < next_range.start {
                new_stream.extend(self.data[end..next_range.start].iter().cloned());
            }

            // Push the tag
            new_stream.push(self.tag_range(next_range.clone(), next_tag));

            // Update state
            last_range = next_range;
        }

        // Append anything left in the stream
        if last_range.end < self.data.len() {
            new_stream.extend(self.data[last_range.end..self.data.len()].iter().cloned());
        }

        // Final result
        TaggedStream { data: new_stream }
    }

    ///
    /// Runs the current values of this tagged stream through a tokenizer and tags anything it matches
    ///
    /// This takes a mapping function to describe how symbols in this stream map to symbols in the DFA. Note that every symbol
    /// (tagged or untagged) must be mapped to a DFA symbol, so if only tagged or untagged symbols are being used it's necessary
    /// to decide how the other symbols are mapped (eg, to an unused symbol)
    ///
    pub fn tokenize<DfaSymbol: Ord, MapFn>(&self, dfa: &SymbolRangeDfa<DfaSymbol, Tag>, map_symbol: MapFn) -> TaggedStream<Base, Tag>
        where MapFn: Fn(TagSymbol<Base, Tag>) -> DfaSymbol {
        unimplemented!();
    }
}

impl<Base: Clone+Ord, Tag: Clone+Ord> Index<usize> for TaggedStream<Base, Tag> {
    type Output = TagSymbol<Base, Tag>;

    fn index(&self, index: usize) -> &TagSymbol<Base, Tag> {
        &self.data[index]
    }
}

impl<'a, Base: Clone+Ord, Tag: Clone+Ord> SymbolSource<'a, TagSymbol<Base, Tag>> for &'a TaggedStream<Base, Tag> {
    type SymbolReader = Iter<'a, TagSymbol<Base, Tag>>;

    /// Returns a new object that can read the symbols from this one
    fn read_symbols(self) -> Self::SymbolReader {
        self.data.read_symbols()
    }
}

#[cfg(test)]
mod test {
    use super::super::*;

    #[test]
    fn can_tag_range() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let mut tagged: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());

        tagged.tag(Tags::Hello, 0..5);
        tagged.tag(Tags::World, 1..6);

        assert!(tagged.len() == 2);

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('H'));
            assert!(stream[1] == TagSymbol::Untagged('e'));
            assert!(stream[2] == TagSymbol::Untagged('l'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('o'));
        } else {
            assert!(false);
        }

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[1] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('W'));
            assert!(stream[1] == TagSymbol::Untagged('o'));
            assert!(stream[2] == TagSymbol::Untagged('r'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn can_tag_everything_with_tags() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..5, Tags::Hello), (5..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 2);

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('H'));
            assert!(stream[1] == TagSymbol::Untagged('e'));
            assert!(stream[2] == TagSymbol::Untagged('l'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('o'));
        } else {
            assert!(false);
        }

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[1] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('W'));
            assert!(stream[1] == TagSymbol::Untagged('o'));
            assert!(stream[2] == TagSymbol::Untagged('r'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn with_tags_preserves_middle() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..4, Tags::Hello), (6..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 4);
        assert!(tagged[1] == TagSymbol::Untagged('o'));
        assert!(tagged[2] == TagSymbol::Untagged('W'));

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 4);
            assert!(stream[0] == TagSymbol::Untagged('H'));
            assert!(stream[1] == TagSymbol::Untagged('e'));
            assert!(stream[2] == TagSymbol::Untagged('l'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
        } else {
            assert!(false);
        }

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[3] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 4);
            assert!(stream[0] == TagSymbol::Untagged('o'));
            assert!(stream[1] == TagSymbol::Untagged('r'));
            assert!(stream[2] == TagSymbol::Untagged('l'));
            assert!(stream[3] == TagSymbol::Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn with_tags_preserves_end() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..5, Tags::Hello)].iter().cloned());

        assert!(tagged.len() == 6);

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('H'));
            assert!(stream[1] == TagSymbol::Untagged('e'));
            assert!(stream[2] == TagSymbol::Untagged('l'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('o'));
        } else {
            assert!(false);
        }

        assert!(tagged[1] == TagSymbol::Untagged('W'));
        assert!(tagged[2] == TagSymbol::Untagged('o'));
        assert!(tagged[3] == TagSymbol::Untagged('r'));
        assert!(tagged[4] == TagSymbol::Untagged('l'));
        assert!(tagged[5] == TagSymbol::Untagged('d'));
    }

    #[test]
    fn with_tags_preserves_start() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(5..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 6);

        if let TagSymbol::Tagged(ref tag, ref stream) = tagged[5] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == TagSymbol::Untagged('W'));
            assert!(stream[1] == TagSymbol::Untagged('o'));
            assert!(stream[2] == TagSymbol::Untagged('r'));
            assert!(stream[3] == TagSymbol::Untagged('l'));
            assert!(stream[4] == TagSymbol::Untagged('d'));
        } else {
            assert!(false);
        }

        assert!(tagged[0] == TagSymbol::Untagged('H'));
        assert!(tagged[1] == TagSymbol::Untagged('e'));
        assert!(tagged[2] == TagSymbol::Untagged('l'));
        assert!(tagged[3] == TagSymbol::Untagged('l'));
        assert!(tagged[4] == TagSymbol::Untagged('o'));
    }
}
<|endoftext|>