<commit_before>mod log;
use capnp;
use rand;
use raft_capnp::{append_entries, append_entries_reply,
                 request_vote, request_vote_reply};
use rpc::{RpcError};
use rpc::server::{RpcObject, RpcServer};
use std::net::{SocketAddr};
use std::time::{Duration, Instant};
use std::thread;
use std::sync::{Arc, Mutex};
use std::sync::mpsc::{channel, Sender, Receiver};

use self::log::{Log, MemoryLog, Entry};
use std::time;
use std::io::Error as IoError;
use rand::distributions::{IndependentSample, Range};

// Constants
// TODO: Many of these should be overwritable by Config
const ELECTION_TIMEOUT_MIN: u64 = 150; // min election timeout wait value in m.s.
const ELECTION_TIMEOUT_MAX: u64 = 300; // min election timeout wait value in m.s.
const HEARTBEAT_INTERVAL: u64    = 75; // time between hearbeats

pub struct Config {
    cluster: Vec<SocketAddr>,
    leader: SocketAddr,
    addr: SocketAddr,
    heartbeat_timeout: Duration,
}

impl Config {
    pub fn new (cluster: Vec<SocketAddr>, leader: SocketAddr,
                addr: SocketAddr, heartbeat_timeout: Duration) -> Config {
        Config {
            cluster: cluster,
            leader: leader,
            addr: addr,
            heartbeat_timeout: heartbeat_timeout,
        }
    }

    // TODO eventually implement
    // pub fn fromFile (file: String) -> Config {
    //
    // }
}

// States that each machine can be in!
#[derive(PartialEq)]
pub enum State {
    CANDIDATE,
    LEADER,
    FOLLOWER,
}

enum RpcType {
    APPEND_ENTRIES,
    REQUEST_VOTE,
}

struct AppendEntriesMessage {
    term: u64,
    leader_id: u64,
    prev_log_index: u64,
    prev_log_term: u64,
    entries: Entry,
    leader_commit: u64,
}

struct AppendEntriesReply {
    term: u64,
    commit_index: u64,
    peer_addr: SocketAddr, // or some other identifying field
    success: u64,
}

struct RequestVoteMessage {
    term: u64,
    candidate_id: u64,
    last_log_index: u64,
    las_log_term: u64,
}

struct RequestVoteReply {
    term: u64,
    vote_granted: u64,
}

struct ClientAppendRequest {
    entry: Entry,
}

enum PeerThreadMessage {
    AppendEntries (AppendEntriesMessage),
    RequestVote (RequestVoteMessage),
}

enum MainThreadMessage {
    AppendEntriesReply (AppendEntriesReply),
    RequestVoteReply (RequestVoteReply),
    ClientAppendRequest (ClientAppendRequest),
}

pub struct PeerHandle {
    to_peer: Sender<PeerThreadMessage>,
}

pub struct Peer {
    addr: SocketAddr,
    commit_index: u64,
    pending_entries: Vec<Entry>,
    to_main: Sender<MainThreadMessage>,
    from_main: Receiver<PeerThreadMessage>,
}

impl Peer {
    fn start (addr: SocketAddr, to_main: Sender<MainThreadMessage>) -> PeerHandle {
        let (to_peer, from_main) = channel();
        //let (to_main, from_peer) = channel();
        let commit_index = 0;

        thread::spawn(move || {
            let peer = Peer {
                addr: addr,
                commit_index: commit_index,
                pending_entries: vec![],
                to_main: to_main,
                from_main: from_main,
            };
            peer.main();
        });

        PeerHandle {
            to_peer: to_peer,
            //from_peer: from_peer,
        }
    }

    pub fn send_append_entries (&mut self, entry: AppendEntriesMessage) {
        // TODO (syd)
        // 1. Construct an empty append_entries rpc
        // 2. Copy in entries from |ro_log| if peer commit index is behind.
        unimplemented!();
    }

    pub fn send_request_vote (&self, vote: RequestVoteMessage) {
        unimplemented!()
    }

    // Main loop for this machine to push to Peers.
    fn main (mut self) {
        loop {
            match self.from_main.recv().unwrap() { // If recv fails, we in deep shit already, so just unwrap
                PeerThreadMessage::AppendEntries(entry) => self.send_append_entries(entry),
                PeerThreadMessage::RequestVote(vote) => self.send_request_vote(vote)
            }
        }
    }
}

// Store's the state that the server is currently in along with the current_term
// and current_id. This fields should all share a lock.
struct ServerState {
    current_state: State,
    current_term: u64,
    current_id: u64
}

pub struct Server {
    state: Arc<Mutex<ServerState>>,
    log: Arc<Mutex<Log>>,
    peers: Vec<PeerHandle>,
    addr: SocketAddr,
    last_leader_contact: Instant,
    voted_for: Option<SocketAddr>,
    last_heartbeat: Instant,
}

///
/// Starts up a new raft server with the given config.
/// This is mostly just a bootstrapper for now. It probably won't end up in the public API
/// As of right now this function does not return. It just runs the state machine as long as it's
/// in a live thread.
///
pub fn start_server (config: Config) -> ! {
    let (tx, rx) = channel();
    let mut server = Server::new(config, tx).unwrap();

    // The server starts up in a follower state. Set a timeout to become a candidate.
    loop {
        match server.state.current_state {
            State::FOLLOWER => {
                let btwn = Range::new(ELECTION_TIMEOUT_MIN, ELECTION_TIMEOUT_MAX);
                let mut range = rand::thread_rng();
                let timeout = Duration::from_millis(btwn.ind_sample(&mut range));
                thread::sleep(timeout);

                let now = Instant::now();
                let last_leader_contact = server.last_leader_contact;
                if now.duration_since(last_leader_contact) >= timeout {
                    // we have not heard from the leader for one timeout duration.
                    // start an election
                    server.start_election();
                }
            },
            State::CANDIDATE => {
                server.start_election();
                unimplemented!()
            },
            State::LEADER => {
                let heartbeat_wait = Duration::from_millis(HEARTBEAT_INTERVAL);
                let duration_since_last_heartbeat = Instant::now().duration_since(server.last_heartbeat);
                // TODO (sydli) : use checked_sub here (possible underflow)
                let next_heartbeat = heartbeat_wait - duration_since_last_heartbeat;
                // Timed wait on leader message pipe.
                let message = match rx.recv_timeout(next_heartbeat) {
                    Ok(message) => message,
                    // If we timed out, just send a heartbeat.
                    Err(e) => {
                        server.send_append_entries();
                        continue;
                    },
                };
                match message {
                    MainThreadMessage::AppendEntriesReply(m) => {
                        server.aggregate_append_entries_reply(m);
                        server.update_commit_index();
                    },
                    MainThreadMessage::ClientAppendRequest(m) => {
                        server.send_append_entries();
                    },
                    _ => unimplemented!(),
                };
            },
        } // end match server.state
    } // end loop
}

impl Server {
    fn new (config: Config, tx: Sender<MainThreadMessage>) -> Result<Server, IoError> {
        let addr = config.addr;
        // 1. Start RPC request handlers
        let append_entries_handler: Box<RpcObject> = Box::new(AppendEntriesHandler {});
        let services = vec![
            (1, append_entries_handler),
        ];
        let mut server = RpcServer::new_with_services(services);
        try!(
            server.bind((config.addr.ip(), config.addr.port()))
            .and_then(|_| {
                server.repl()
            })
        );

        // 2. Start peer threads.
        let peers = config.cluster.into_iter()
            .filter(|x| *x != addr) // filter all computers that aren't me
            .map(|x| Peer::start(x, tx.clone()))
            .collect::<Vec<PeerHandle>>();

        let log = Arc::new(Mutex::new(MemoryLog::new()));

        let state = Arc::new(Mutex::new(ServerState {
            current_state: State::FOLLOWER,
            current_term: 0,
            current_id: 0
        }));

        // 3. Construct server state object.
        Ok(Server {
            state: state,
            log: log,
            peers: peers,
            addr: addr,
            voted_for: None,
            last_leader_contact: Instant::now(),
            last_heartbeat: Instant::now()
        })
    }

    ///
    /// Write |entry| to the log and try to replicate it across all servers.
    /// This function is non-blocking; it simply forwards AppendEntries messages
    /// to all known peers.
    ///
    fn send_append_entries(&mut self) {
        self.last_heartbeat = Instant::now();
    }

    ///
    /// Aggregate peer's append entries response into server state
    ///
    fn aggregate_append_entries_reply(&self, message: AppendEntriesReply) {
        unimplemented!();
    }

    ///
    /// Update commit_index count to the most recent log entry with quorum.
    ///
    fn update_commit_index(&self) {
        unimplemented!();
    }

    ///
    /// Starts a new election by requesting votes from all peers
    /// If the election is successful then the server transitions into the LEADER state
    /// Otherwise the leader stays in the candidate state.
    /// The assumption is that the caller will start another election
    ///
    fn start_election(&mut self) {
        debug_assert!(self.state.current_state == State::CANDIDATE);
        // tell each peer to send out RequestToVote RPCs
        //let
        unimplemented!();
    }
}

struct AppendEntriesHandler {}

impl RpcObject for AppendEntriesHandler {
    fn handle_rpc (&self, params: capnp::any_pointer::Reader, result: capnp::any_pointer::Builder)
        -> Result<(), RpcError>
    {
        // TODO: If this is a valid rpc from the leader, update last_leader_contact

        // TODO: If we're in the CANDIDATE state and this leader's term is >= our current term
        //

        // TODO (syd) implement
        // 1. Ensure prevLogIndex matches our own.
        // 2. Append outstanding log entries.
        // 3. Construct reply.
        params.get_as::<append_entries::Reader>()
           .map(|append_entries| {
               result.init_as::<append_entries_reply::Builder>().set_success(true);
           })
           .map_err(RpcError::Capnp)
    }
}
<commit_msg>WIP: Leader Election<commit_after>mod log;
use capnp;
use rand;
use raft_capnp::{append_entries, append_entries_reply,
                 request_vote, request_vote_reply};
use rpc::{RpcError};
use rpc::server::{RpcObject, RpcServer};
use std::net::{SocketAddr};
use std::time::{Duration, Instant};
use std::thread;
use std::sync::{Arc, Mutex, MutexGuard};
use std::sync::mpsc::{channel, Sender, Receiver};

use self::log::{Log, MemoryLog, Entry};
use std::time;
use std::io::Error as IoError;
use rand::distributions::{IndependentSample, Range};
use std::collections::HashMap;

// Constants
// TODO: Many of these should be overwritable by Config
const ELECTION_TIMEOUT_MIN: u64 = 150; // min election timeout wait value in m.s.
const ELECTION_TIMEOUT_MAX: u64 = 300; // min election timeout wait value in m.s.
const HEARTBEAT_INTERVAL: u64    = 75; // time between hearbeats

pub struct Config {
    // Each server has a unique 64bit integer id that and a socket address
    // These mappings MUST be identical for eaah server in the cluster
    cluster: HashMap<u64, SocketAddr>,
    leader: u64,
    me: (u64, SocketAddr),
    heartbeat_timeout: Duration,
}

impl Config {
    pub fn new (cluster: HashMap<u64, SocketAddr>, leader: u64, my_id: u64,
                my_addr: SocketAddr, heartbeat_timeout: Duration) -> Config {
        Config {
            cluster: cluster,
            leader: leader,
            me: (my_id, my_addr),
            heartbeat_timeout: heartbeat_timeout,
        }
    }

    // TODO eventually implement
    // pub fn fromFile (file: String) -> Config {
    //
    // }
}

// States that each machine can be in!
#[derive(PartialEq)]
pub enum State {
    CANDIDATE,
    LEADER,
    FOLLOWER,
}

enum RpcType {
    APPEND_ENTRIES,
    REQUEST_VOTE,
}

//#[derive(Clone)]
struct AppendEntriesMessage {
    term: u64,
    leader_id: u64,
    prev_log_index: u64,
    prev_log_term: u64,
    entries: Entry,
    leader_commit: u64,
}

struct AppendEntriesReply {
    term: u64,
    commit_index: u64,
    peer_addr: SocketAddr, // or some other identifying field
    success: u64,
}

#[derive(Copy, Clone)]
struct RequestVoteMessage {
    term: u64,
    candidate_id: u64,
    last_log_index: u64,
    last_log_term: u64,
}

struct RequestVoteReply {
    term: u64,
    vote_granted: u64,
}

struct ClientAppendRequest {
    entry: Entry,
}

enum PeerThreadMessage {
    AppendEntries (AppendEntriesMessage),
    RequestVote (RequestVoteMessage),
}

enum MainThreadMessage {
    AppendEntriesReply (AppendEntriesReply),
    RequestVoteReply (RequestVoteReply),
    ClientAppendRequest (ClientAppendRequest),
}

pub struct PeerHandle {
    to_peer: Sender<PeerThreadMessage>,
}

pub struct Peer {
    addr: SocketAddr,
    commit_index: u64,
    pending_entries: Vec<Entry>,
    to_main: Sender<MainThreadMessage>,
    from_main: Receiver<PeerThreadMessage>,
}

impl Peer {
    fn start (addr: SocketAddr, to_main: Sender<MainThreadMessage>) -> PeerHandle {
        let (to_peer, from_main) = channel();
        //let (to_main, from_peer) = channel();
        let commit_index = 0;

        thread::spawn(move || {
            let peer = Peer {
                addr: addr,
                commit_index: commit_index,
                pending_entries: vec![],
                to_main: to_main,
                from_main: from_main,
            };
            peer.main();
        });

        PeerHandle {
            to_peer: to_peer,
            //from_peer: from_peer,
        }
    }

    pub fn send_append_entries (&mut self, entry: AppendEntriesMessage) {
        // TODO (syd)
        // 1. Construct an empty append_entries rpc
        // 2. Copy in entries from |ro_log| if peer commit index is behind.
        unimplemented!();
    }

    pub fn send_request_vote (&self, vote: RequestVoteMessage) {
        unimplemented!()
    }

    // Main loop for this machine to push to Peers.
    fn main (mut self) {
        loop {
            match self.from_main.recv().unwrap() { // If recv fails, we in deep shit already, so just unwrap
                PeerThreadMessage::AppendEntries(entry) => self.send_append_entries(entry),
                PeerThreadMessage::RequestVote(vote) => self.send_request_vote(vote)
            }
        }
    }
}

// Store's the state that the server is currently in along with the current_term
// and current_id. This fields should all share a lock.
struct ServerState {
    // TODO: state and term must be persisted to disk
    current_state: State,
    current_term: u64,
    current_index: u64,
    last_leader_contact: Instant,
    voted_for: Option<u64>
}

// TODO: RW locks?
pub struct Server {
    state: Arc<Mutex<ServerState>>,
    log: Arc<Mutex<Log>>,
    peers: Vec<PeerHandle>,
    me: (u64, SocketAddr),
    last_heartbeat: Instant,
}

///
/// Starts up a new raft server with the given config.
/// This is mostly just a bootstrapper for now. It probably won't end up in the public API
/// As of right now this function does not return. It just runs the state machine as long as it's
/// in a live thread.
///
pub fn start_server (config: Config) -> ! {
    let (tx, rx) = channel();
    let mut server = Server::new(config, tx).unwrap();

    // NB: This thread handles all state changes EXCEPT for those that move us back into the
    // follower state from either the candidate or leader state. Those are both handled in the
    // AppendEntriesRpcHandler

    // The server starts up in a follower state. Set a timeout to become a candidate.
    loop {
        let mut state = server.state.lock().unwrap(); // panics if any other threads have panicked.
        match state.current_state {
            State::FOLLOWER => {
                let btwn = Range::new(ELECTION_TIMEOUT_MIN, ELECTION_TIMEOUT_MAX);
                let mut range = rand::thread_rng();
                // TODO(jason): Subtract time since last_leader_contact from wait time
                let timeout = Duration::from_millis(btwn.ind_sample(&mut range));
                drop(state);
                thread::sleep(timeout);
                state = server.state.lock().unwrap();

                let now = Instant::now();
                let last_leader_contact = state.last_leader_contact;
                if now.duration_since(last_leader_contact) >= timeout {
                    // we have not heard from the leader for one timeout duration.
                    // start an election
                    server.start_election(state);
                }
            },
            State::CANDIDATE => {
                server.start_election(state);
                unimplemented!()
            },
            State::LEADER => {
                let heartbeat_wait = Duration::from_millis(HEARTBEAT_INTERVAL);
                let duration_since_last_heartbeat = Instant::now().duration_since(server.last_heartbeat);
                // TODO (sydli) : use checked_sub here (possible underflow)
                let next_heartbeat = heartbeat_wait - duration_since_last_heartbeat;
                // Timed wait on leader message pipe.
                let message = match rx.recv_timeout(next_heartbeat) {
                    Ok(message) => message,
                    // If we timed out, just send a heartbeat.
                    Err(e) => {
                        server.send_append_entries();
                        continue;
                    },
                };
                match message {
                    MainThreadMessage::AppendEntriesReply(m) => {
                        server.aggregate_append_entries_reply(m);
                        server.update_commit_index();
                    },
                    MainThreadMessage::ClientAppendRequest(m) => {
                        server.send_append_entries();
                    },
                    _ => unimplemented!(),
                };
            },
        } // end match server.state
    } // end loop
}

impl Server {
    fn new (config: Config, tx: Sender<MainThreadMessage>) -> Result<Server, IoError> {
        let me = config.me;
        // 1. Start RPC request handlers
        let append_entries_handler: Box<RpcObject> = Box::new(AppendEntriesHandler {});
        let services = vec![
            (1, append_entries_handler),
        ];
        let mut server = RpcServer::new_with_services(services);
        try!(
            server.bind((config.me.1.ip(), config.me.1.port()))
            .and_then(|_| {
                server.repl()
            })
        );

        // TODO: Should peers be a map of id->PeerHandle. I.E. Will we ever need to send a message
        // to a specific peer?

        // 2. Start peer threads.
        let peers = config.cluster.into_iter()
            .filter(|&(id, addr)| id != me.0) // filter all computers that aren't me
            .map(|(id, addr)| Peer::start(addr, tx.clone()))
            .collect::<Vec<PeerHandle>>();

        let log = Arc::new(Mutex::new(MemoryLog::new()));

        let state = Arc::new(Mutex::new(ServerState {
            current_state: State::FOLLOWER,
            current_term: 0,
            current_index: 0,
            voted_for: None,
            last_leader_contact: Instant::now()
        }));

        // 3. Construct server state object.
        Ok(Server {
            state: state,
            log: log,
            peers: peers,
            me: me,
            last_heartbeat: Instant::now()
        })
    }

    ///
    /// Write |entry| to the log and try to replicate it across all servers.
    /// This function is non-blocking; it simply forwards AppendEntries messages
    /// to all known peers.
    ///
    fn send_append_entries(&/*mut*/ self) {
        unimplemented!();
        //self.last_heartbeat = Instant::now();
    }

    ///
    /// Aggregate peer's append entries response into server state
    ///
    fn aggregate_append_entries_reply(&self, message: AppendEntriesReply) {
        unimplemented!();
    }

    ///
    /// Update commit_index count to the most recent log entry with quorum.
    ///
    fn update_commit_index(&self) {
        unimplemented!();
    }

    ///
    /// Starts a new election by requesting votes from all peers
    /// If the election is successful then the server transitions into the LEADER state
    /// Otherwise the leader stays in the candidate state.
    /// The assumption is that the caller will start another election
    ///
    fn start_election(&self, mut state: MutexGuard<ServerState>) {
        { // state lock scope
            // move the lock into this scope
            let mut locked_state = state;
            // transition to the candidate state
            let last_log_index = locked_state.current_index;
            let last_log_term = locked_state.current_term;
            locked_state.current_state = State::CANDIDATE;
            locked_state.current_term += 1;
            locked_state.current_index = 0;
            locked_state.voted_for = Some(self.me.0); // vote for ourselves
            // tell each peer to send out RequestToVote RPCs
            let request_vote_message = RequestVoteMessage {
                term: locked_state.current_term,
                candidate_id: self.me.0,
                last_log_index: last_log_index,
                last_log_term: last_log_term
            };
            let ref peers = self.peers;
            for peer in peers {
                peer.to_peer.send(PeerThreadMessage::RequestVote(request_vote_message))
                    .unwrap(); // panic if the peer thread is down
            }
        }
        unimplemented!();
    }
}

struct AppendEntriesHandler {}

impl RpcObject for AppendEntriesHandler {
    fn handle_rpc (&self, params: capnp::any_pointer::Reader, result: capnp::any_pointer::Builder)
        -> Result<(), RpcError>
    {
        // TODO: If this is a valid rpc from the leader, update last_leader_contact

        // TODO: If we're in the CANDIDATE state and this leader's term is >= our current term
        //

        // TODO (syd) implement
        // 1. Ensure prevLogIndex matches our own.
        // 2. Append outstanding log entries.
        // 3. Construct reply.
        params.get_as::<append_entries::Reader>()
           .map(|append_entries| {
               result.init_as::<append_entries_reply::Builder>().set_success(true);
           })
           .map_err(RpcError::Capnp)
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use servo::BrowserId;
use super::{BrowserState, DeadBrowserState};

#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub enum TabState {
    Alive(BrowserState),
    Dead(DeadBrowserState),
}

impl TabState {
    fn is_alive(&self) -> bool {
        match *self {
            TabState::Alive(_) => true,
            TabState::Dead(_) => false,
        }
    }
    fn is_fg(&self) -> bool {
        self.ref_browser().map(|b| !b.background).unwrap_or(false)
    }
    fn is_bg(&self) -> bool {
        self.ref_browser().map(|b| b.background).unwrap_or(false)
    }
    fn ref_browser(&self) -> Result<&BrowserState, &'static str> {
        match *self {
            TabState::Alive(ref x) => Ok(x),
            TabState::Dead(_) => Err("Dead browser"),
        }
    }
    fn mut_browser(&mut self) -> Result<&mut BrowserState, &'static str> {
        match *self {
            TabState::Alive(ref mut x) => Ok(x),
            TabState::Dead(_) => Err("Dead browser"),
        }
    }
    fn kill(&mut self) -> Result<(), &'static str> {
        if !self.is_alive() {
            return Err("Already dead");
        }
        let id = self.ref_browser().unwrap().id;
        let tab = TabState::Dead(DeadBrowserState { id });
        *self = tab;
        Ok(())
    }
    fn foreground(&mut self) -> Result<(), &'static str> {
        match *self {
            TabState::Alive(ref mut browser) if browser.background => {
                browser.background = false;
                Ok(())
            }
            TabState::Alive(_) => {
                Err("Already foreground")
            },
            TabState::Dead(_) => {
                Err("Dead browser")
            }
        }
    }
    fn background(&mut self) -> Result<(), &'static str> {
        match *self {
            TabState::Alive(ref mut browser) if !browser.background => {
                browser.background = true;
                Ok(())
            }
            TabState::Alive(_) => {
                Err("Already background")
            },
            TabState::Dead(_) => {
                Err("Dead browser")
            }
        }
    }
}

#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub struct TabsState(Vec<TabState>);

impl TabsState {

    pub fn new() -> TabsState {
        TabsState(Vec::new())
    }

    pub fn has_more_than_one(&self) -> bool {
        self.0.iter().filter(|tab| tab.is_alive()).count() > 1
    }

    pub fn kill_fg(&mut self) -> Result<BrowserId, &'static str> {
        let fg_idx = self.0.iter().position(TabState::is_fg).ok_or("No foreground tab")?;
        if self.can_select_next()? {
            self.select_next()?;
        } else if self.can_select_prev()? {
            self.select_prev()?;
        } else {
            return Err("No background tab to select");
        }
        let id = self.0[fg_idx].ref_browser()?.id;
        self.0[fg_idx].kill()?;
        Ok(id)
    }

    pub fn can_select_next(&self) -> Result<bool,&'static str> {
        let fg_idx = self.0.iter().position(TabState::is_fg).ok_or("No foreground tab")?;
        Ok(self.0.iter().skip(fg_idx + 1).any(TabState::is_bg))
    }

    pub fn select_next(&mut self) -> Result<(), &'static str> {
        let fg_idx = self.0.iter()
            .position(|tab| tab.is_fg())
            .ok_or("No foreground tab")?;
        let next_idx = self.0.iter()
            .enumerate()
            .skip(fg_idx + 1)
            .find(|&(_, tab)| tab.is_bg())
            .map(|(idx, _)| idx)
            .ok_or("No tab to select")?;
        self.0[fg_idx].background()?;
        self.0[next_idx].foreground()?;
        Ok(())
    }

    pub fn can_select_prev(&self) -> Result<bool, &'static str> {
        let fg_idx = self.0.iter().position(TabState::is_fg).ok_or("No foreground tab")?;
        Ok(self.0.iter().rev().skip(self.0.len() - fg_idx).any(TabState::is_bg))
    }

    pub fn select_prev(&mut self) -> Result<(), &'static str> {
        let fg_idx = self.0.iter()
            .position(TabState::is_fg)
            .ok_or("No foreground tab")?;
        let prev_idx = self.0.iter()
            .enumerate()
            .rev().skip(self.0.len() - fg_idx)
            .find(|&(_, tab)| tab.is_bg())
            .map(|(idx, _)| idx)
            .ok_or("No tab to select")?;
        self.0[fg_idx].background()?;
        self.0[prev_idx].foreground()?;
        Ok(())
    }

    pub fn select_first(&mut self) -> Result<(), &'static str> {
        let fg_idx = self.0.iter()
            .position(TabState::is_fg)
            .ok_or("No foreground tab")?;
        let first_idx = self.0.iter()
            .position(TabState::is_bg)
            .ok_or("No tab to select")?;
        self.0[fg_idx].background()?;
        self.0[first_idx].foreground()?;
        Ok(())
    }

    pub fn select_last(&mut self) -> Result<(), &'static str> {
        let fg_idx = self.0.iter()
            .position(TabState::is_fg)
            .ok_or("No foreground tab")?;
        let last_idx = self.0.iter().enumerate().rev()
            .find(|&(_, tab)| tab.is_bg())
            .map(&|(idx, _)| idx)
            .ok_or("No tab to select")?;
        self.0[fg_idx].background()?;
        self.0[last_idx].foreground()?;
        Ok(())
    }

    pub fn can_select_nth(&self, index: usize) -> bool {
        self.0.iter()
            .filter(|tab| tab.is_alive())
            .nth(index)
            .is_some()
    }

    pub fn select_nth(&mut self, index: usize) -> Result<(), &'static str> {
        let fg_idx = self.0.iter()
            .position(TabState::is_fg)
            .ok_or("No foreground tab")?;
        let nth_idx = self.0.iter()
            .enumerate()
            .filter(|&(_, tab)| tab.is_alive())
            .nth(index)
            .map(|(index, _)| index)
            .ok_or("No tab to select")?;
        self.0[fg_idx].background()?;
        self.0[nth_idx].foreground()?;
        Ok(())
    }

    pub fn append_new(&mut self, mut browser: BrowserState) -> Result<(), &'static str> {
        if self.0.len() == 0 {
            browser.background = false;
            self.0.push(TabState::Alive(browser));
            Ok(())
        } else if !browser.background {
            browser.background = true;
            self.0.push(TabState::Alive(browser));
            self.select_last()
        } else {
            self.0.push(TabState::Alive(browser));
            Ok(())
        }
    }

    pub fn find_browser(&mut self, id: &BrowserId) -> Option<&mut BrowserState> {
        self.0.iter_mut()
            .filter_map(|tab| tab.mut_browser().ok())
            .find(|b| b.id == *id)
    }

    pub fn find_browser_at(&self, idx: usize) -> Option<&BrowserState> {
        self.0.iter()
            .nth(idx)
            .and_then(|tab| tab.ref_browser().ok())
    }

    pub fn index_to_alive_index(&self, idx: usize) -> Option<usize> {
        // We need to also check tab == found_tab as we don't want
        // to discard a dead tab. The user might want to know the
        // position of what used to be an alive tab
        self.0.iter().nth(idx).and_then(|found_tab| {
            self.0.iter()
                .filter(|&tab| tab.is_alive() || tab == found_tab)
                .position(|tab| tab == found_tab)
        })
    }

    pub fn ref_fg_browser(&self) -> Result<&BrowserState, &'static str> {
        self.0.iter()
            .find(|tab| tab.is_fg())
            .ok_or("No foreground tab")
            .and_then(|tab| tab.ref_browser())
    }

    pub fn mut_fg_browser(&mut self) -> Result<&mut BrowserState, &'static str> {
        self.0.iter_mut()
            .find(|tab| tab.is_fg())
            .ok_or("No foreground tab")
            .and_then(|tab| tab.mut_browser())
    }

    pub fn fg_browser_index(&self) -> Result<usize, &'static str> {
        self.0.iter()
            .position(|tab| tab.is_fg())
            .ok_or("No foreground tab")
    }

    pub fn alive_browsers<'a>(&self) -> Vec<&BrowserState> {
        self.0.iter().filter_map(|tab| tab.ref_browser().ok()).collect()
    }
}
<commit_msg>useless tabs<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use servo::BrowserId;
use super::{BrowserState, DeadBrowserState};

#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub enum TabState {
    Alive(BrowserState),
    Dead(DeadBrowserState),
}

impl TabState {
    fn is_alive(&self) -> bool {
        match *self {
            TabState::Alive(_) => true,
            TabState::Dead(_) => false,
        }
    }
    fn is_fg(&self) -> bool {
        self.ref_browser().map(|b| !b.background).unwrap_or(false)
    }
    fn is_bg(&self) -> bool {
        self.ref_browser().map(|b| b.background).unwrap_or(false)
    }
    fn ref_browser(&self) -> Result<&BrowserState, &'static str> {
        match *self {
            TabState::Alive(ref x) => Ok(x),
            TabState::Dead(_) => Err("Dead browser"),
        }
    }
    fn mut_browser(&mut self) -> Result<&mut BrowserState, &'static str> {
        match *self {
            TabState::Alive(ref mut x) => Ok(x),
            TabState::Dead(_) => Err("Dead browser"),
        }
    }
    fn kill(&mut self) -> Result<(), &'static str> {
        if !self.is_alive() {
            return Err("Already dead");
        }
        let id = self.ref_browser().unwrap().id;
        let tab = TabState::Dead(DeadBrowserState { id });
        *self = tab;
        Ok(())
    }
    fn foreground(&mut self) -> Result<(), &'static str> {
        match *self {
            TabState::Alive(ref mut browser) if browser.background => {
                browser.background = false;
                Ok(())
            }
            TabState::Alive(_) => {
                Err("Already foreground")
            },
            TabState::Dead(_) => {
                Err("Dead browser")
            }
        }
    }
    fn background(&mut self) -> Result<(), &'static str> {
        match *self {
            TabState::Alive(ref mut browser) if !browser.background => {
                browser.background = true;
                Ok(())
            }
            TabState::Alive(_) => {
                Err("Already background")
            },
            TabState::Dead(_) => {
                Err("Dead browser")
            }
        }
    }
}

#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub struct TabsState(Vec<TabState>);

impl TabsState {

    pub fn new() -> TabsState {
        TabsState(Vec::new())
    }

    pub fn has_more_than_one(&self) -> bool {
        self.0.iter().filter(|tab| tab.is_alive()).count() > 1
    }

    pub fn kill_fg(&mut self) -> Result<BrowserId, &'static str> {
        let fg_idx = self.0.iter().position(TabState::is_fg).ok_or("No foreground tab")?;
        if self.can_select_next()? {
            self.select_next()?;
        } else if self.can_select_prev()? {
            self.select_prev()?;
        } else {
            return Err("No background tab to select");
        }
        let id = self.0[fg_idx].ref_browser()?.id;
        self.0[fg_idx].kill()?;
        Ok(id)
    }

    pub fn can_select_next(&self) -> Result<bool,&'static str> {
        let fg_idx = self.0.iter().position(TabState::is_fg).ok_or("No foreground tab")?;
        Ok(self.0.iter().skip(fg_idx + 1).any(TabState::is_bg))
    }

    pub fn select_next(&mut self) -> Result<(), &'static str> {
        let fg_idx = self.0.iter()
            .position(|tab| tab.is_fg())
            .ok_or("No foreground tab")?;
        let next_idx = self.0.iter()
            .enumerate()
            .skip(fg_idx + 1)
            .find(|&(_, tab)| tab.is_bg())
            .map(|(idx, _)| idx)
            .ok_or("No tab to select")?;
        self.0[fg_idx].background()?;
        self.0[next_idx].foreground()?;
        Ok(())
    }

    pub fn can_select_prev(&self) -> Result<bool, &'static str> {
        let fg_idx = self.0.iter().position(TabState::is_fg).ok_or("No foreground tab")?;
        Ok(self.0.iter().rev().skip(self.0.len() - fg_idx).any(TabState::is_bg))
    }

    pub fn select_prev(&mut self) -> Result<(), &'static str> {
        let fg_idx = self.0.iter()
            .position(TabState::is_fg)
            .ok_or("No foreground tab")?;
        let prev_idx = self.0.iter()
            .enumerate()
            .rev().skip(self.0.len() - fg_idx)
            .find(|&(_, tab)| tab.is_bg())
            .map(|(idx, _)| idx)
            .ok_or("No tab to select")?;
        self.0[fg_idx].background()?;
        self.0[prev_idx].foreground()?;
        Ok(())
    }

    pub fn select_first(&mut self) -> Result<(), &'static str> {
        let fg_idx = self.0.iter()
            .position(TabState::is_fg)
            .ok_or("No foreground tab")?;
        let first_idx = self.0.iter()
            .position(TabState::is_bg)
            .ok_or("No tab to select")?;
        self.0[fg_idx].background()?;
        self.0[first_idx].foreground()?;
        Ok(())
    }

    pub fn select_last(&mut self) -> Result<(), &'static str> {
        let fg_idx = self.0.iter()
            .position(TabState::is_fg)
            .ok_or("No foreground tab")?;
        let last_idx = self.0.iter().enumerate().rev()
            .find(|&(_, tab)| tab.is_bg())
            .map(&|(idx, _)| idx)
            .ok_or("No tab to select")?;
        self.0[fg_idx].background()?;
        self.0[last_idx].foreground()?;
        Ok(())
    }

    pub fn can_select_nth(&self, index: usize) -> bool {
        self.0.iter()
            .filter(|tab| tab.is_alive())
            .nth(index)
            .is_some()
    }

    pub fn select_nth(&mut self, index: usize) -> Result<(), &'static str> {
        let fg_idx = self.0.iter()
            .position(TabState::is_fg)
            .ok_or("No foreground tab")?;
        let nth_idx = self.0.iter()
            .enumerate()
            .filter(|&(_, tab)| tab.is_alive())
            .nth(index)
            .map(|(index, _)| index)
            .ok_or("No tab to select")?;
        self.0[fg_idx].background()?;
        self.0[nth_idx].foreground()?;
        Ok(())
    }

    pub fn append_new(&mut self, mut browser: BrowserState) -> Result<(), &'static str> {
        if self.0.len() == 0 {
            browser.background = false;
            self.0.push(TabState::Alive(browser));
            Ok(())
        } else if !browser.background {
            browser.background = true;
            self.0.push(TabState::Alive(browser));
            self.select_last()
        } else {
            self.0.push(TabState::Alive(browser));
            Ok(())
        }
    }

    pub fn find_browser(&mut self, id: &BrowserId) -> Option<&mut BrowserState> {
        self.0.iter_mut()
            .filter_map(|tab| tab.mut_browser().ok())
            .find(|b| b.id == *id)
    }

    pub fn find_browser_at(&self, idx: usize) -> Option<&BrowserState> {
        self.0.iter()
            .nth(idx)
            .and_then(|tab| tab.ref_browser().ok())
    }

    pub fn index_to_alive_index(&self, idx: usize) -> Option<usize> {
        // We need to also check tab == found_tab as we don't want
        // to discard a dead tab. The user might want to know the
        // position of what used to be an alive tab
        self.0.iter().nth(idx).and_then(|found_tab| {
            self.0.iter()
                .filter(|&tab| tab.is_alive() || tab == found_tab)
                .position(|tab| tab == found_tab)
        })
    }

    pub fn ref_fg_browser(&self) -> Result<&BrowserState, &'static str> {
        self.0.iter()
            .find(|tab| tab.is_fg())
            .ok_or("No foreground tab")
            .and_then(|tab| tab.ref_browser())
    }

    pub fn mut_fg_browser(&mut self) -> Result<&mut BrowserState, &'static str> {
        self.0.iter_mut()
            .find(|tab| tab.is_fg())
            .ok_or("No foreground tab")
            .and_then(|tab| tab.mut_browser())
    }

    pub fn fg_browser_index(&self) -> Result<usize, &'static str> {
        self.0.iter()
            .position(|tab| tab.is_fg())
            .ok_or("No foreground tab")
    }

    pub fn alive_browsers<'a>(&self) -> Vec<&BrowserState> {
        self.0.iter().filter_map(|tab| tab.ref_browser().ok()).collect()
    }
}
<|endoftext|>


<commit_before>use super::Blob;
use failure::Error;
use futures::Future;
use postgres::Connection;
use rusoto_core::region::Region;
use rusoto_credential::DefaultCredentialsProvider;
use rusoto_s3::{GetObjectRequest, PutObjectRequest, S3Client, S3};
use std::convert::TryInto;
use std::io::Read;
use time::Timespec;
use log::{error, warn};

pub(crate) struct S3Backend<'a> {
    client: S3Client,
    bucket: &'a str,
}

impl<'a> S3Backend<'a> {
    pub(crate) fn new(client: S3Client, bucket: &'a str) -> Self {
        Self { client, bucket }
    }

    pub(super) fn get(&self, path: &str) -> Result<Blob, Error> {
        let res = self
            .client
            .get_object(GetObjectRequest {
                bucket: self.bucket.to_string(),
                key: path.into(),
                ..Default::default()
            })
            .sync()?;

        let mut b = res.body.unwrap().into_blocking_read();
        let mut content = Vec::with_capacity(
            res.content_length
                .and_then(|l| l.try_into().ok())
                .unwrap_or(0),
        );
        b.read_to_end(&mut content).unwrap();

        let date_updated = parse_timespec(&res.last_modified.unwrap())?;

        Ok(Blob {
            path: path.into(),
            mime: res.content_type.unwrap(),
            date_updated,
            content,
        })
    }

    pub(super) fn store_batch(&self, batch: &[Blob]) -> Result<(), Error> {
        let mut rt = tokio::runtime::Runtime::new().unwrap();
        let mut attempts = 0;

        loop {
            let mut futures = Vec::new();
            for blob in batch {
                futures.push(
                    self.client
                        .put_object(PutObjectRequest {
                            bucket: self.bucket.to_string(),
                            key: blob.path.clone(),
                            body: Some(blob.content.clone().into()),
                            content_type: Some(blob.mime.clone()),
                            ..Default::default()
                        })
                        .inspect(|_| {
                            crate::web::metrics::UPLOADED_FILES_TOTAL.inc_by(1);
                        }),
                );
            }
            attempts += 1;

            match rt.block_on(::futures::future::join_all(futures)) {
                // this batch was successful, start another batch if there are still more files
                Ok(_) => break,
                Err(err) => {
                    error!("failed to upload to s3: {:?}", err);
                    // if a futures error occurs, retry the batch
                    if attempts > 2 {
                        panic!("failed to upload 3 times, exiting");
                    }
                }
            }
        }
        Ok(())
    }
}

#[cfg(not(test))]
const TIME_FMT: &str = "%a, %d %b %Y %H:%M:%S %Z";
#[cfg(test)]
pub(crate) const TIME_FMT: &str = "%a, %d %b %Y %H:%M:%S %Z";

fn parse_timespec(raw: &str) -> Result<Timespec, Error> {
    Ok(time::strptime(raw, TIME_FMT)?.to_timespec())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::test::TestEnvironment;

    fn assert_s3_404(env: &TestEnvironment, path: &'static str) {
        use rusoto_core::RusotoError;
        use rusoto_s3::GetObjectError;

        let s3 = env.s3().not_found(path);
        let backend = S3Backend::new(s3.client, s3.bucket);
        let err = backend.get(path).unwrap_err();
        let status = match err
            .downcast_ref::<RusotoError<GetObjectError>>()
            .expect("wanted GetObject")
        {
            RusotoError::Unknown(http) => http.status,
            _ => panic!("wrong error"),
        };
        assert_eq!(status, 404);
    }

    #[test]
    fn test_parse_timespec() {
        crate::test::wrapper(|_| {
            // Test valid conversions
            assert_eq!(
                parse_timespec("Thu, 1 Jan 1970 00:00:00 GMT")?,
                Timespec::new(0, 0)
            );
            assert_eq!(
                parse_timespec("Mon, 16 Apr 2018 04:33:50 GMT")?,
                Timespec::new(1523853230, 0)
            );

            // Test invalid conversion
            assert!(parse_timespec("foo").is_err());

            Ok(())
        })
    }

    #[test]
    fn test_path_get() {
        crate::test::wrapper(|env| {
            let blob = Blob {
                path: "dir/foo.txt".into(),
                mime: "text/plain".into(),
                date_updated: Timespec::new(42, 0),
                content: "Hello world!".into(),
            };

            // Add a test file to the database
            let s3 = env.s3().upload(blob.clone());

            let backend = S3Backend::new(s3.client, &s3.bucket);

            // Test that the proper file was returned
            assert_eq!(blob, backend.get("dir/foo.txt")?);

            // Test that other files are not returned
            assert_s3_404(&env, "dir/bar.txt");
            assert_s3_404(&env, "foo.txt");

            Ok(())
        });
    }
}

pub(crate) static S3_BUCKET_NAME: &str = "rust-docs-rs";

pub(crate) fn s3_client() -> Option<S3Client> {
    // If AWS keys aren't configured, then presume we should use the DB exclusively
    // for file storage.
    if std::env::var_os("AWS_ACCESS_KEY_ID").is_none() && std::env::var_os("FORCE_S3").is_none() {
        return None;
    }
    let creds = match DefaultCredentialsProvider::new() {
        Ok(creds) => creds,
        Err(err) => {
            warn!("failed to retrieve AWS credentials: {}", err);
            return None;
        }
    };
    Some(S3Client::new_with(
        rusoto_core::request::HttpClient::new().unwrap(),
        creds,
        std::env::var("S3_ENDPOINT")
            .ok()
            .map(|e| Region::Custom {
                name: std::env::var("S3_REGION").unwrap_or_else(|_| "us-west-1".to_owned()),
                endpoint: e,
            })
            .unwrap_or(Region::UsWest1),
    ))
}

pub fn move_to_s3(conn: &Connection, n: usize) -> Result<usize, Error> {
    let trans = conn.transaction()?;
    let client = s3_client().expect("configured s3");

    let rows = trans.query(
        &format!(
            "SELECT path, mime, content FROM files WHERE content != E'in-s3' LIMIT {}",
            n
        ),
        &[],
    )?;
    let count = rows.len();

    let mut rt = ::tokio::runtime::Runtime::new().unwrap();
    let mut futures = Vec::new();
    for row in &rows {
        let path: String = row.get(0);
        let mime: String = row.get(1);
        let content: Vec<u8> = row.get(2);
        let path_1 = path.clone();
        futures.push(
            client
                .put_object(PutObjectRequest {
                    bucket: S3_BUCKET_NAME.into(),
                    key: path.clone(),
                    body: Some(content.into()),
                    content_type: Some(mime),
                    ..Default::default()
                })
                .map(move |_| path_1)
                .map_err(move |e| panic!("failed to upload to {}: {:?}", path, e)),
        );
    }

    match rt.block_on(::futures::future::join_all(futures)) {
        Ok(paths) => {
            let statement = trans.prepare("DELETE FROM files WHERE path = $1").unwrap();
            for path in paths {
                statement.execute(&[&path]).unwrap();
            }
        }
        Err(e) => {
            panic!("results err: {:?}", e);
        }
    }

    trans.commit()?;

    Ok(count)
}
<commit_msg>Remove code duplication<commit_after>use super::Blob;
use failure::Error;
use futures::Future;
use postgres::Connection;
use rusoto_core::region::Region;
use rusoto_credential::DefaultCredentialsProvider;
use rusoto_s3::{GetObjectRequest, PutObjectRequest, S3Client, S3};
use std::convert::TryInto;
use std::io::Read;
use time::Timespec;
use log::{error, warn};

pub(crate) struct S3Backend<'a> {
    client: S3Client,
    bucket: &'a str,
}

impl<'a> S3Backend<'a> {
    pub(crate) fn new(client: S3Client, bucket: &'a str) -> Self {
        Self { client, bucket }
    }

    pub(super) fn get(&self, path: &str) -> Result<Blob, Error> {
        let res = self
            .client
            .get_object(GetObjectRequest {
                bucket: self.bucket.to_string(),
                key: path.into(),
                ..Default::default()
            })
            .sync()?;

        let mut b = res.body.unwrap().into_blocking_read();
        let mut content = Vec::with_capacity(
            res.content_length
                .and_then(|l| l.try_into().ok())
                .unwrap_or(0),
        );
        b.read_to_end(&mut content).unwrap();

        let date_updated = parse_timespec(&res.last_modified.unwrap())?;

        Ok(Blob {
            path: path.into(),
            mime: res.content_type.unwrap(),
            date_updated,
            content,
        })
    }

    pub(super) fn store_batch(&self, batch: &[Blob]) -> Result<(), Error> {
        let mut rt = tokio::runtime::Runtime::new().unwrap();
        let mut attempts = 0;

        loop {
            let mut futures = Vec::new();
            for blob in batch {
                futures.push(
                    self.client
                        .put_object(PutObjectRequest {
                            bucket: self.bucket.to_string(),
                            key: blob.path.clone(),
                            body: Some(blob.content.clone().into()),
                            content_type: Some(blob.mime.clone()),
                            ..Default::default()
                        })
                        .inspect(|_| {
                            crate::web::metrics::UPLOADED_FILES_TOTAL.inc_by(1);
                        }),
                );
            }
            attempts += 1;

            match rt.block_on(::futures::future::join_all(futures)) {
                // this batch was successful, start another batch if there are still more files
                Ok(_) => break,
                Err(err) => {
                    error!("failed to upload to s3: {:?}", err);
                    // if a futures error occurs, retry the batch
                    if attempts > 2 {
                        panic!("failed to upload 3 times, exiting");
                    }
                }
            }
        }
        Ok(())
    }
}

// public for testing
pub(crate) const TIME_FMT: &str = "%a, %d %b %Y %H:%M:%S %Z";

fn parse_timespec(raw: &str) -> Result<Timespec, Error> {
    Ok(time::strptime(raw, TIME_FMT)?.to_timespec())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::test::TestEnvironment;

    fn assert_s3_404(env: &TestEnvironment, path: &'static str) {
        use rusoto_core::RusotoError;
        use rusoto_s3::GetObjectError;

        let s3 = env.s3().not_found(path);
        let backend = S3Backend::new(s3.client, s3.bucket);
        let err = backend.get(path).unwrap_err();
        let status = match err
            .downcast_ref::<RusotoError<GetObjectError>>()
            .expect("wanted GetObject")
        {
            RusotoError::Unknown(http) => http.status,
            _ => panic!("wrong error"),
        };
        assert_eq!(status, 404);
    }

    #[test]
    fn test_parse_timespec() {
        crate::test::wrapper(|_| {
            // Test valid conversions
            assert_eq!(
                parse_timespec("Thu, 1 Jan 1970 00:00:00 GMT")?,
                Timespec::new(0, 0)
            );
            assert_eq!(
                parse_timespec("Mon, 16 Apr 2018 04:33:50 GMT")?,
                Timespec::new(1523853230, 0)
            );

            // Test invalid conversion
            assert!(parse_timespec("foo").is_err());

            Ok(())
        })
    }

    #[test]
    fn test_path_get() {
        crate::test::wrapper(|env| {
            let blob = Blob {
                path: "dir/foo.txt".into(),
                mime: "text/plain".into(),
                date_updated: Timespec::new(42, 0),
                content: "Hello world!".into(),
            };

            // Add a test file to the database
            let s3 = env.s3().upload(blob.clone());

            let backend = S3Backend::new(s3.client, &s3.bucket);

            // Test that the proper file was returned
            assert_eq!(blob, backend.get("dir/foo.txt")?);

            // Test that other files are not returned
            assert_s3_404(&env, "dir/bar.txt");
            assert_s3_404(&env, "foo.txt");

            Ok(())
        });
    }
}

pub(crate) static S3_BUCKET_NAME: &str = "rust-docs-rs";

pub(crate) fn s3_client() -> Option<S3Client> {
    // If AWS keys aren't configured, then presume we should use the DB exclusively
    // for file storage.
    if std::env::var_os("AWS_ACCESS_KEY_ID").is_none() && std::env::var_os("FORCE_S3").is_none() {
        return None;
    }
    let creds = match DefaultCredentialsProvider::new() {
        Ok(creds) => creds,
        Err(err) => {
            warn!("failed to retrieve AWS credentials: {}", err);
            return None;
        }
    };
    Some(S3Client::new_with(
        rusoto_core::request::HttpClient::new().unwrap(),
        creds,
        std::env::var("S3_ENDPOINT")
            .ok()
            .map(|e| Region::Custom {
                name: std::env::var("S3_REGION").unwrap_or_else(|_| "us-west-1".to_owned()),
                endpoint: e,
            })
            .unwrap_or(Region::UsWest1),
    ))
}

pub fn move_to_s3(conn: &Connection, n: usize) -> Result<usize, Error> {
    let trans = conn.transaction()?;
    let client = s3_client().expect("configured s3");

    let rows = trans.query(
        &format!(
            "SELECT path, mime, content FROM files WHERE content != E'in-s3' LIMIT {}",
            n
        ),
        &[],
    )?;
    let count = rows.len();

    let mut rt = ::tokio::runtime::Runtime::new().unwrap();
    let mut futures = Vec::new();
    for row in &rows {
        let path: String = row.get(0);
        let mime: String = row.get(1);
        let content: Vec<u8> = row.get(2);
        let path_1 = path.clone();
        futures.push(
            client
                .put_object(PutObjectRequest {
                    bucket: S3_BUCKET_NAME.into(),
                    key: path.clone(),
                    body: Some(content.into()),
                    content_type: Some(mime),
                    ..Default::default()
                })
                .map(move |_| path_1)
                .map_err(move |e| panic!("failed to upload to {}: {:?}", path, e)),
        );
    }

    match rt.block_on(::futures::future::join_all(futures)) {
        Ok(paths) => {
            let statement = trans.prepare("DELETE FROM files WHERE path = $1").unwrap();
            for path in paths {
                statement.execute(&[&path]).unwrap();
            }
        }
        Err(e) => {
            panic!("results err: {:?}", e);
        }
    }

    trans.commit()?;

    Ok(count)
}
<|endoftext|>


<commit_before>use std::fmt::Debug;
use std::hash::Hash;
use std::rc::Rc;
use std::mem::replace;

use adapton_syntax::* ;
use adapton_sigs::* ;

pub trait ListT<A:Adapton,Hd> : Debug+Hash+PartialEq+Eq+Clone {
    type List : Debug+Hash+PartialEq+Eq+Clone ;

    fn nil  (&mut A) -> Self::List ;
    fn cons (&mut A, Hd, Self::List) -> Self::List ;
    fn rc   (&mut A, Rc<Self::List>) -> Self::List ;

    // requisite "adaptonic" constructors: `name` and `art`:
    fn name (&mut A, A::Name, Self::List) -> Self::List ;
    fn art  (&mut A, Art<Self::List,A::Loc>) -> Self::List ;

    // ** Want/need to express a negative product here among Nil, Cons and Name!
    // fn elim<Res,Nil,Cons,Name> (self:&Self, &mut A, &Self::List, Nil, Cons, Name) -> Res
    //     where Nil:FnOnce(&mut A) -> Res
    //     ,    Cons:FnOnce(&mut A, &Hd, &Self::List) -> Res
    //     ,    Name:FnOnce(&mut A, &A::Name, &Self::List) -> Res ;

    // Derived from `cons` and `nil` above:
    fn singleton (st:&mut A, hd:Hd) -> Self::List {
        let nil = Self::nil(st);
        Self::cons(st, hd, nil)
    }

    fn elim<Res,Nil,Cons,Name> (&mut A, &Self::List, Nil, Cons, Name) -> Res
        where Nil:FnOnce(&mut A) -> Res
        ,    Cons:FnOnce(&mut A, &Hd, &Self::List) -> Res
        ,    Name:FnOnce(&mut A, &A::Name, &Self::List) -> Res ;

    // Derived from `elim` above:
    fn is_empty (st:&mut A, list:&Self::List) -> bool {
        Self::elim(st, &list, |_|true, |_,_,_|false, |_,_,_|false)
    }
}

#[allow(dead_code)]
pub fn list_merge<A:Adapton,X:Ord+Clone,L:ListT<A,X>>
    (st:&mut A,
     xo:Option<&X>, // xo is useful for merging binary trees; does 3-way merge on (xo, hd(l1), hd(l2)).
     n1:Option<&A::Name>, l1:&L::List,
     n2:Option<&A::Name>, l2:&L::List  ) -> L::List
{
    L::elim(st, l1,
            |_| l2.clone(),
            |st,h1,t1|
            L::elim(st, l2,
                    |st| L::nil(st),
                    |st, h2, t2|
                    if match xo {
                        None => false,
                        Some(x) => ( x <= h1 && x <= h2 ) }
                    {
                        let rest = list_merge::<A,X,L>(st, None, n1, l1, n2, l2);
                        L::cons(st, xo.unwrap().clone(), rest)
                    }
                    else if h1 <= h2 {
                        //TODO: Nominal memoization with n1:
                        let rest = list_merge::<A,X,L>(st, xo, None, t1, n2, l2);
                        L::cons(st, (*h1).clone(), rest)
                    }
                    else {
                        //TODO: Nominal memoization with n2:
                        let rest = list_merge::<A,X,L>(st, xo, n1, l1, None, t2);
                        L::cons(st, (*h2).clone(), rest)
                    },
                    |st, m2, t2| list_merge::<A,X,L>(st, xo, n1, l1, Some(m2), t2)
                    ),
            |st,n1,t1| list_merge::<A,X,L>(st, xo, Some(n1), t1, n2, l2)
            )
}

pub fn list_merge_sort<A:Adapton,X:Ord+Hash+Clone,L:ListT<A,X>,T:TreeT<A,X,X>>
    (st:&mut A, list:&L::List) -> L::List
{
    let tree = tree_of_list::<A,X,T,L>(st, list);
    T::fold_up (st, tree,
                |st, x| // Nil:
                L::singleton(st, x),
                |st, x, left, right| // Bin:
                list_merge::<A,X,L>(st, Some(&x), None, &left, None, &right)
                )
}

// Questions:
//  - Should `Name`s always be passed by reference?
//  - Do these Fn argss for fold need to be passed in `Rc<Box<_>>`s ?

pub trait TreeT<A:Adapton,Leaf,Bin:Hash> : Debug+Hash+PartialEq+Eq+Clone {
    type Tree : Debug+Hash+PartialEq+Eq+Clone ;

    fn nil  (&mut A) -> Self::Tree ;
    fn leaf (&mut A, Leaf) -> Self::Tree ;
    fn bin  (&mut A, Bin, Self::Tree, Self::Tree) -> Self::Tree ;

    // requisite "adaptonic" constructors: `name` and `art`:
    fn name (&mut A, A::Name, Self::Tree, Self::Tree) -> Self::Tree ;
    fn art  (&mut A, Art<Self::Tree,A::Loc>) -> Self::Tree ;

    fn fold<Res,LeafC,BinC> (&mut A, Self::Tree, Res, LeafC, BinC) -> Res
        where LeafC:Fn(&mut A, Res, Leaf) -> Res
        ,      BinC:Fn(&mut A, Res, Bin ) -> Res ;

    fn fold_up<Res,LeafC,BinC> (&mut A, Self::Tree, LeafC, BinC) -> Res
        where LeafC:Fn(&mut A, Leaf) -> Res
        ,      BinC:Fn(&mut A, Bin, Res, Res ) -> Res ;

    // TODO: Add derived operations (max, min, sum, etc.) for fold_up.
}

#[derive(Debug,PartialEq,Eq,Hash,Clone)]
enum List<A:Adapton,Hd> {
    Nil,
    Cons(Hd,      Box<List<A,Hd>>),
    Rc(           Rc<List<A,Hd>>),
    Name(A::Name, Box<List<A,Hd>>),
    Art(          Art<List<A,Hd>, A::Loc>),
}

// TODO: Why Does Adapton have to implement all of these?
//       It's not actually *contained* within the List structure; it cannot be ecountered there.
//       It's only ever present in a negative position (as a function parameter).
impl<A:Adapton+Debug+Hash+PartialEq+Eq+Clone,Hd:Debug+Hash+PartialEq+Eq+Clone> ListT<A,Hd> for List<A,Hd> {
    type List = List<A,Hd>;

    fn nil  (_:&mut A)                             -> Self::List { List::Nil }
    fn cons (_:&mut A, hd:Hd, tl:Self::List)       -> Self::List { List::Cons(hd,Box::new(tl)) }
    fn name (_:&mut A, nm:A::Name, tl:Self::List)  -> Self::List { List::Name(nm, Box::new(tl)) }
    fn rc   (_:&mut A, rc:Rc<List<A,Hd>>)          -> Self::List { List::Rc(rc) }
    fn art  (_:&mut A, art:Art<List<A,Hd>,A::Loc>) -> Self::List { List::Art(art) }

    // fn elim<Res,Nil,Cons,Name> (self:&Self, st:&mut A, list:&Self::List, nilf:Nil, consf:Cons, namef:Name) -> Res
    //     where Nil:FnOnce(&mut A) -> Res
    //     ,    Cons:FnOnce(&mut A, &Hd, &Self::List) -> Res
    //     ,    Name:FnOnce(&mut A, &A::Name, &Self::List) -> Res
    // {
    //     match *list {
    //         List::Nil => nilf(st),
    //         List::Cons(ref hd, ref tl) => consf(st, hd, &*tl),
    //         List::Name(ref nm, ref tl) => namef(st, nm, &*tl),
    //         List::Rc(ref rc) => self.elim(st, &*rc, nilf, consf, namef),
    //         List::Art(ref art) => {
    //             let list = st.force(art);
    //             self.elim(st, &list, nilf, consf, namef)
    //         }
    //     }
    // }

    fn elim<Res,Nil,Cons,Name> (st:&mut A, list:&Self::List, nilf:Nil, consf:Cons, namef:Name) -> Res
        where Nil:FnOnce(&mut A) -> Res
        ,    Cons:FnOnce(&mut A, &Hd, &Self::List) -> Res
        ,    Name:FnOnce(&mut A, &A::Name, &Self::List) -> Res
    {
        match *list {
            List::Nil => nilf(st),
            List::Cons(ref hd, ref tl) => consf(st, hd, &*tl),
            List::Name(ref nm, ref tl) => namef(st, nm, &*tl),
            List::Rc(ref rc) => Self::elim(st, &*rc, nilf, consf, namef),
            List::Art(ref art) => {
                let list = st.force(art);
                Self::elim(st, &list, nilf, consf, namef)
            }
        }
    }

}

// https://doc.rust-lang.org/book/macros.html
//
// macro_rules! o_O {
//     (
//         $(
//             $x:expr; [ $( $y:expr ),* ]
//          );*
//     ) => {
//         &[ $($( $x + $y ),*),* ]
//     }
// }
//
// fn main() {
//     let a: &[i32]
//         = o_O!(10; [1, 2, 3];
//                20; [4, 5, 6]);
//
//     assert_eq!(a, [11, 12, 13, 24, 25, 26]);
// }

// TODO: Need to gensym a variable for each argument below:
//
// macro_rules! thunk {
//     ( $f:ident , $st:expr , $( $arg:expr ),* ) => {
//         let fval = Rc::new(Box::new(
//             |st, args|{
//                 let ($( $arg ),*) = args ;
//                 f( st, $( $arg ),* )
//             })) ;
//         ($st).thunk (ArtId::Eager, prog_pt!(f), fval, $( $arg ),* )
//     }}

fn tree_of_list_rec_memo <A:Adapton, X:Hash+Clone, T:TreeT<A,X,X>, L:ListT<A,X>>
    (st:&mut A, list:&L::List, left_tree:&T::Tree, left_tree_lev:u32, parent_lev:u32) ->
    (T::Tree, L::List)
{
    let t = st.thunk (ArtId::Eager,
                      prog_pt!(tree_of_list_rec),
                      Rc::new(Box::new(|st, args|{
                          let (list, left_tree, left_tree_lev, parent_lev) = args ;
                          tree_of_list_rec::<A,X,T,L>
                              (st, list, left_tree, left_tree_lev, parent_lev)
                      })),
                      (list, left_tree, left_tree_lev, parent_lev)
                      ) ;
    st.force( &t )
}

fn tree_of_list_rec <A:Adapton, X:Hash+Clone, T:TreeT<A,X,X>, L:ListT<A,X>>
    (st:&mut A, list:&L::List, left_tree:&T::Tree, left_tree_lev:u32, parent_lev:u32)
     -> (T::Tree, L::List)
{
    L::elim (
        st, &list,
        /* Nil */  |st| ( T::nil(st), L::nil(st) ),
        /* Cons */ |st, hd, rest| {
            let lev_hd = (1 + (my_hash(hd).leading_zeros())) as u32 ;
            if left_tree_lev <= lev_hd && lev_hd <= parent_lev {
                let nil = T::nil(st) ;
                let (right_tree, rest) = tree_of_list_rec::<A,X,T,L> ( st, rest, &nil, 0 as u32, lev_hd ) ;
                let tree = T::bin ( st, hd.clone(), left_tree.clone(), right_tree ) ;
                tree_of_list_rec::<A,X,T,L> ( st, &rest, &tree, lev_hd, parent_lev )
            }
            else {
                let rest = L::cons(st, hd.clone(), rest.clone()) ;
                (left_tree.clone(), rest)
            }},
        /* Name */ |st, nm, rest| {
            let lev_nm = (1 + 64 + (my_hash(nm).leading_zeros())) as u32 ;
            if left_tree_lev <= lev_nm && lev_nm <= parent_lev {
                let nil = T::nil(st) ;
                let (right_tree, rest) = tree_of_list_rec_memo::<A,X,T,L> ( st, rest, &nil, 0 as u32, lev_nm ) ;
                // TODO: Place left_ and right_ trees into articulations (not Boxes), named by name.
                let tree = T::name( st, nm.clone(), left_tree.clone(), right_tree ) ;
                tree_of_list_rec_memo::<A,X,T,L> ( st, &rest, &tree, lev_nm, parent_lev )
            }
            else {
                let rest = L::name(st, nm.clone(), rest.clone()) ;
                (left_tree.clone(), rest)
            }}
        )
}

pub fn tree_of_list <A:Adapton, X:Hash+Clone, T:TreeT<A,X,X>, L:ListT<A,X>>
    (st:&mut A, list:&L::List) -> T::Tree
{
    let nil = T::nil(st) ;
    let (tree, rest) = tree_of_list_rec::<A,X,T,L> (st, list, &nil, 0 as u32, u32::max_value()) ;
    assert!( L::is_empty( st, &rest ) );
    tree
}
<commit_msg>dead comments<commit_after>use std::fmt::Debug;
use std::hash::Hash;
use std::rc::Rc;
use std::mem::replace;

use adapton_syntax::* ;
use adapton_sigs::* ;

pub trait ListT<A:Adapton,Hd> : Debug+Hash+PartialEq+Eq+Clone {
    type List : Debug+Hash+PartialEq+Eq+Clone ;

    fn nil  (&mut A) -> Self::List ;
    fn cons (&mut A, Hd, Self::List) -> Self::List ;
    fn rc   (&mut A, Rc<Self::List>) -> Self::List ;

    // requisite "adaptonic" constructors: `name` and `art`:
    fn name (&mut A, A::Name, Self::List) -> Self::List ;
    fn art  (&mut A, Art<Self::List,A::Loc>) -> Self::List ;

    fn elim<Res,Nil,Cons,Name> (&mut A, &Self::List, Nil, Cons, Name) -> Res
        where Nil:FnOnce(&mut A) -> Res
        ,    Cons:FnOnce(&mut A, &Hd, &Self::List) -> Res
        ,    Name:FnOnce(&mut A, &A::Name, &Self::List) -> Res ;

    // Derived from `cons` and `nil` above:
    fn singleton (st:&mut A, hd:Hd) -> Self::List {
        let nil = Self::nil(st);
        Self::cons(st, hd, nil)
    }

    // Derived from `elim` above:
    fn is_empty (st:&mut A, list:&Self::List) -> bool {
        Self::elim(st, &list, |_|true, |_,_,_|false, |_,_,_|false)
    }
}

#[allow(dead_code)]
pub fn list_merge<A:Adapton,X:Ord+Clone,L:ListT<A,X>>
    (st:&mut A,
     xo:Option<&X>, // xo is useful for merging binary trees; does 3-way merge on (xo, hd(l1), hd(l2)).
     n1:Option<&A::Name>, l1:&L::List,
     n2:Option<&A::Name>, l2:&L::List  ) -> L::List
{
    L::elim(st, l1,
            |_| l2.clone(),
            |st,h1,t1|
            L::elim(st, l2,
                    |st| L::nil(st),
                    |st, h2, t2|
                    if match xo {
                        None => false,
                        Some(x) => ( x <= h1 && x <= h2 ) }
                    {
                        let rest = list_merge::<A,X,L>(st, None, n1, l1, n2, l2);
                        L::cons(st, xo.unwrap().clone(), rest)
                    }
                    else if h1 <= h2 {
                        //TODO: Nominal memoization with n1:
                        let rest = list_merge::<A,X,L>(st, xo, None, t1, n2, l2);
                        L::cons(st, (*h1).clone(), rest)
                    }
                    else {
                        //TODO: Nominal memoization with n2:
                        let rest = list_merge::<A,X,L>(st, xo, n1, l1, None, t2);
                        L::cons(st, (*h2).clone(), rest)
                    },
                    |st, m2, t2| list_merge::<A,X,L>(st, xo, n1, l1, Some(m2), t2)
                    ),
            |st,n1,t1| list_merge::<A,X,L>(st, xo, Some(n1), t1, n2, l2)
            )
}

pub fn list_merge_sort<A:Adapton,X:Ord+Hash+Clone,L:ListT<A,X>,T:TreeT<A,X,X>>
    (st:&mut A, list:&L::List) -> L::List
{
    let tree = tree_of_list::<A,X,T,L>(st, list);
    T::fold_up (st, tree,
                |st, x| // Nil:
                L::singleton(st, x),
                |st, x, left, right| // Bin:
                list_merge::<A,X,L>(st, Some(&x), None, &left, None, &right)
                )
}

// Questions:
//  - Should `Name`s always be passed by reference?
//  - Do these Fn argss for fold need to be passed in `Rc<Box<_>>`s ?

pub trait TreeT<A:Adapton,Leaf,Bin:Hash> : Debug+Hash+PartialEq+Eq+Clone {
    type Tree : Debug+Hash+PartialEq+Eq+Clone ;

    fn nil  (&mut A) -> Self::Tree ;
    fn leaf (&mut A, Leaf) -> Self::Tree ;
    fn bin  (&mut A, Bin, Self::Tree, Self::Tree) -> Self::Tree ;

    // requisite "adaptonic" constructors: `name` and `art`:
    fn name (&mut A, A::Name, Self::Tree, Self::Tree) -> Self::Tree ;
    fn art  (&mut A, Art<Self::Tree,A::Loc>) -> Self::Tree ;

    fn fold<Res,LeafC,BinC> (&mut A, Self::Tree, Res, LeafC, BinC) -> Res
        where LeafC:Fn(&mut A, Res, Leaf) -> Res
        ,      BinC:Fn(&mut A, Res, Bin ) -> Res ;

    fn fold_up<Res,LeafC,BinC> (&mut A, Self::Tree, LeafC, BinC) -> Res
        where LeafC:Fn(&mut A, Leaf) -> Res
        ,      BinC:Fn(&mut A, Bin, Res, Res ) -> Res ;

    // TODO: Add derived operations (max, min, sum, etc.) for fold_up.
}

#[derive(Debug,PartialEq,Eq,Hash,Clone)]
enum List<A:Adapton,Hd> {
    Nil,
    Cons(Hd,      Box<List<A,Hd>>),
    Rc(           Rc<List<A,Hd>>),
    Name(A::Name, Box<List<A,Hd>>),
    Art(          Art<List<A,Hd>, A::Loc>),
}

// TODO: Why Does Adapton have to implement all of these?
//       It's not actually *contained* within the List structure; it cannot be ecountered there.
//       It's only ever present in a negative position (as a function parameter).
impl<A:Adapton+Debug+Hash+PartialEq+Eq+Clone,Hd:Debug+Hash+PartialEq+Eq+Clone> ListT<A,Hd> for List<A,Hd> {
    type List = List<A,Hd>;

    fn nil  (_:&mut A)                             -> Self::List { List::Nil }
    fn cons (_:&mut A, hd:Hd, tl:Self::List)       -> Self::List { List::Cons(hd,Box::new(tl)) }
    fn name (_:&mut A, nm:A::Name, tl:Self::List)  -> Self::List { List::Name(nm, Box::new(tl)) }
    fn rc   (_:&mut A, rc:Rc<List<A,Hd>>)          -> Self::List { List::Rc(rc) }
    fn art  (_:&mut A, art:Art<List<A,Hd>,A::Loc>) -> Self::List { List::Art(art) }

    fn elim<Res,Nil,Cons,Name> (st:&mut A, list:&Self::List, nilf:Nil, consf:Cons, namef:Name) -> Res
        where Nil:FnOnce(&mut A) -> Res
        ,    Cons:FnOnce(&mut A, &Hd, &Self::List) -> Res
        ,    Name:FnOnce(&mut A, &A::Name, &Self::List) -> Res
    {
        match *list {
            List::Nil => nilf(st),
            List::Cons(ref hd, ref tl) => consf(st, hd, &*tl),
            List::Name(ref nm, ref tl) => namef(st, nm, &*tl),
            List::Rc(ref rc) => Self::elim(st, &*rc, nilf, consf, namef),
            List::Art(ref art) => {
                let list = st.force(art);
                Self::elim(st, &list, nilf, consf, namef)
            }
        }
    }

}

// https://doc.rust-lang.org/book/macros.html
//
// macro_rules! o_O {
//     (
//         $(
//             $x:expr; [ $( $y:expr ),* ]
//          );*
//     ) => {
//         &[ $($( $x + $y ),*),* ]
//     }
// }
//
// fn main() {
//     let a: &[i32]
//         = o_O!(10; [1, 2, 3];
//                20; [4, 5, 6]);
//
//     assert_eq!(a, [11, 12, 13, 24, 25, 26]);
// }

// TODO: Need to gensym a variable for each argument below:
//
// macro_rules! thunk {
//     ( $f:ident , $st:expr , $( $arg:expr ),* ) => {
//         let fval = Rc::new(Box::new(
//             |st, args|{
//                 let ($( $arg ),*) = args ;
//                 f( st, $( $arg ),* )
//             })) ;
//         ($st).thunk (ArtId::Eager, prog_pt!(f), fval, $( $arg ),* )
//     }}

fn tree_of_list_rec_memo <A:Adapton, X:Hash+Clone, T:TreeT<A,X,X>, L:ListT<A,X>>
    (st:&mut A, list:&L::List, left_tree:&T::Tree, left_tree_lev:u32, parent_lev:u32) ->
    (T::Tree, L::List)
{
    let t = st.thunk (ArtId::Eager,
                      prog_pt!(tree_of_list_rec),
                      Rc::new(Box::new(|st, args|{
                          let (list, left_tree, left_tree_lev, parent_lev) = args ;
                          tree_of_list_rec::<A,X,T,L>
                              (st, list, left_tree, left_tree_lev, parent_lev)
                      })),
                      (list, left_tree, left_tree_lev, parent_lev)
                      ) ;
    st.force( &t )
}

fn tree_of_list_rec <A:Adapton, X:Hash+Clone, T:TreeT<A,X,X>, L:ListT<A,X>>
    (st:&mut A, list:&L::List, left_tree:&T::Tree, left_tree_lev:u32, parent_lev:u32)
     -> (T::Tree, L::List)
{
    L::elim (
        st, &list,
        /* Nil */  |st| ( T::nil(st), L::nil(st) ),
        /* Cons */ |st, hd, rest| {
            let lev_hd = (1 + (my_hash(hd).leading_zeros())) as u32 ;
            if left_tree_lev <= lev_hd && lev_hd <= parent_lev {
                let nil = T::nil(st) ;
                let (right_tree, rest) = tree_of_list_rec::<A,X,T,L> ( st, rest, &nil, 0 as u32, lev_hd ) ;
                let tree = T::bin ( st, hd.clone(), left_tree.clone(), right_tree ) ;
                tree_of_list_rec::<A,X,T,L> ( st, &rest, &tree, lev_hd, parent_lev )
            }
            else {
                let rest = L::cons(st, hd.clone(), rest.clone()) ;
                (left_tree.clone(), rest)
            }},
        /* Name */ |st, nm, rest| {
            let lev_nm = (1 + 64 + (my_hash(nm).leading_zeros())) as u32 ;
            if left_tree_lev <= lev_nm && lev_nm <= parent_lev {
                let nil = T::nil(st) ;
                let (right_tree, rest) = tree_of_list_rec_memo::<A,X,T,L> ( st, rest, &nil, 0 as u32, lev_nm ) ;
                // TODO: Place left_ and right_ trees into articulations (not Boxes), named by name.
                let tree = T::name( st, nm.clone(), left_tree.clone(), right_tree ) ;
                tree_of_list_rec_memo::<A,X,T,L> ( st, &rest, &tree, lev_nm, parent_lev )
            }
            else {
                let rest = L::name(st, nm.clone(), rest.clone()) ;
                (left_tree.clone(), rest)
            }}
        )
}

pub fn tree_of_list <A:Adapton, X:Hash+Clone, T:TreeT<A,X,X>, L:ListT<A,X>>
    (st:&mut A, list:&L::List) -> T::Tree
{
    let nil = T::nil(st) ;
    let (tree, rest) = tree_of_list_rec::<A,X,T,L> (st, list, &nil, 0 as u32, u32::max_value()) ;
    assert!( L::is_empty( st, &rest ) );
    tree
}
<|endoftext|>


<commit_before>//! A simple adaptive threadpool that returns a oneshot future.

use std::{
    collections::VecDeque,
    sync::{Arc, Once},
    time::{Duration, Instant},
};

use parking_lot::{Condvar, Mutex};

use crate::{debug_delay, Lazy, OneShot, Result};

static BLOCKING_QUEUE: Lazy<Queue, fn() -> Queue> = Lazy::new(Default::default);
static IO_QUEUE: Lazy<Queue, fn() -> Queue> = Lazy::new(Default::default);
static SNAPSHOT_QUEUE: Lazy<Queue, fn() -> Queue> = Lazy::new(Default::default);

type Work = Box<dyn FnOnce() + Send + 'static>;

#[derive(Default)]
struct Queue {
    cv: Condvar,
    mu: Mutex<VecDeque<Work>>,
}

#[allow(unsafe_code)]
unsafe impl Send for Queue {}

impl Queue {
    fn recv_timeout(&self, duration: Duration) -> Option<Work> {
        let mut queue = self.mu.lock();

        let cutoff = Instant::now() + duration;

        while queue.is_empty() {
            let res = self.cv.wait_until(&mut queue, cutoff);
            if res.timed_out() {
                break;
            }
        }

        queue.pop_front()
    }

    fn send(&self, work: Work) -> usize {
        let mut queue = self.mu.lock();
        queue.push_back(work);

        let len = queue.len();

        // having held the mutex makes this linearized
        // with the notify below.
        drop(queue);

        self.cv.notify_all();

        len
    }

    fn perform_work(&self) {
        let wait_limit = Duration::from_millis(100);

        loop {
            let guard = crate::pin();
            guard.flush();
            drop(guard);
            debug_delay();
            let task_res = self.recv_timeout(wait_limit);

            if let Some(task) = task_res {
                //println!("starting work");
                (task)();
                //println!("done work");
            }
        }
    }
}

/// Spawn a function on the threadpool.
pub fn spawn<F, R>(work: F) -> Result<OneShot<R>>
where
    F: FnOnce() -> R + Send + 'static,
    R: Send + 'static + Sized,
{
    spawn_to(work, &BLOCKING_QUEUE)
}

#[cfg(all(
    not(miri),
    any(
        windows,
        target_os = "linux",
        target_os = "macos",
        target_os = "dragonfly",
        target_os = "freebsd",
        target_os = "openbsd",
        target_os = "netbsd",
    )
))]
fn spawn_to<F, R>(work: F, queue: &'static Queue) -> Result<OneShot<R>>
where
    F: FnOnce() -> R + Send + 'static,
    R: Send + 'static + Sized,
{
    static START_THREADS: Once = Once::new();

    START_THREADS.call_once(|| {
        std::thread::Builder::new()
            .name("sled-io-thread".into())
            .spawn(|| IO_QUEUE.perform_work())
            .expect("failed to spawn critical IO thread");

        std::thread::Builder::new()
            .name("sled-blocking-thread".into())
            .spawn(|| BLOCKING_QUEUE.perform_work())
            .expect("failed to spawn critical blocking thread");

        std::thread::Builder::new()
            .name("sled-snapshot-thread".into())
            .spawn(|| SNAPSHOT_QUEUE.perform_work())
            .expect("failed to spawn critical snapshot thread");
    });

    let (promise_filler, promise) = OneShot::pair();
    let task = move || {
        promise_filler.fill((work)());
    };

    queue.send(Box::new(task));

    Ok(promise)
}

#[cfg(any(
    miri,
    not(any(
        windows,
        target_os = "linux",
        target_os = "macos",
        target_os = "dragonfly",
        target_os = "freebsd",
        target_os = "openbsd",
        target_os = "netbsd",
    ))
))]
pub fn spawn_to<F, R>(work: F, _: &'static Queue) -> Result<OneShot<R>>
where
    F: FnOnce() -> R + Send + 'static,
    R: Send + 'static,
{
    // Polyfill for platforms other than those we explicitly trust to
    // perform threaded work on. Just execute a task without involving threads.
    let (promise_filler, promise) = OneShot::pair();
    promise_filler.fill((work)());
    Ok(promise)
}

pub fn truncate(
    config: crate::RunningConfig,
    at: u64,
) -> Result<OneShot<Result<()>>> {
    spawn_to(
        move || {
            log::debug!("truncating file to length {}", at);
            config
                .file
                .set_len(at)
                .and_then(|_| config.file.sync_all())
                .map_err(|e| e.into())
        },
        &IO_QUEUE,
    )
}

pub fn take_fuzzy_snapshot(
    pc: crate::pagecache::PageCache,
) -> Result<OneShot<()>> {
    spawn_to(
        move || {
            if let Err(e) = pc.take_fuzzy_snapshot() {
                log::error!("failed to write snapshot: {:?}", e);
            }
        },
        &SNAPSHOT_QUEUE,
    )
}

pub(crate) fn write_to_log(
    iobuf: Arc<crate::pagecache::iobuf::IoBuf>,
    iobufs: Arc<crate::pagecache::iobuf::IoBufs>,
) -> Result<OneShot<()>> {
    spawn_to(
        move || {
            let lsn = iobuf.lsn;
            if let Err(e) = iobufs.write_to_log(iobuf) {
                log::error!(
                    "hit error while writing iobuf with lsn {}: {:?}",
                    lsn,
                    e
                );

                // store error before notifying so that waiting threads will see
                // it
                iobufs.config.set_global_error(e);

                let intervals = iobufs.intervals.lock();

                // having held the mutex makes this linearized
                // with the notify below.
                drop(intervals);

                let _notified = iobufs.interval_updated.notify_all();
            }
        },
        &IO_QUEUE,
    )
}
<commit_msg>Remove commented-out code<commit_after>//! A simple adaptive threadpool that returns a oneshot future.

use std::{
    collections::VecDeque,
    sync::{Arc, Once},
    time::{Duration, Instant},
};

use parking_lot::{Condvar, Mutex};

use crate::{debug_delay, Lazy, OneShot, Result};

static BLOCKING_QUEUE: Lazy<Queue, fn() -> Queue> = Lazy::new(Default::default);
static IO_QUEUE: Lazy<Queue, fn() -> Queue> = Lazy::new(Default::default);
static SNAPSHOT_QUEUE: Lazy<Queue, fn() -> Queue> = Lazy::new(Default::default);

type Work = Box<dyn FnOnce() + Send + 'static>;

#[derive(Default)]
struct Queue {
    cv: Condvar,
    mu: Mutex<VecDeque<Work>>,
}

#[allow(unsafe_code)]
unsafe impl Send for Queue {}

impl Queue {
    fn recv_timeout(&self, duration: Duration) -> Option<Work> {
        let mut queue = self.mu.lock();

        let cutoff = Instant::now() + duration;

        while queue.is_empty() {
            let res = self.cv.wait_until(&mut queue, cutoff);
            if res.timed_out() {
                break;
            }
        }

        queue.pop_front()
    }

    fn send(&self, work: Work) -> usize {
        let mut queue = self.mu.lock();
        queue.push_back(work);

        let len = queue.len();

        // having held the mutex makes this linearized
        // with the notify below.
        drop(queue);

        self.cv.notify_all();

        len
    }

    fn perform_work(&self) {
        let wait_limit = Duration::from_millis(100);

        loop {
            let guard = crate::pin();
            guard.flush();
            drop(guard);
            debug_delay();
            let task_res = self.recv_timeout(wait_limit);

            if let Some(task) = task_res {
                (task)();
            }
        }
    }
}

/// Spawn a function on the threadpool.
pub fn spawn<F, R>(work: F) -> Result<OneShot<R>>
where
    F: FnOnce() -> R + Send + 'static,
    R: Send + 'static + Sized,
{
    spawn_to(work, &BLOCKING_QUEUE)
}

#[cfg(all(
    not(miri),
    any(
        windows,
        target_os = "linux",
        target_os = "macos",
        target_os = "dragonfly",
        target_os = "freebsd",
        target_os = "openbsd",
        target_os = "netbsd",
    )
))]
fn spawn_to<F, R>(work: F, queue: &'static Queue) -> Result<OneShot<R>>
where
    F: FnOnce() -> R + Send + 'static,
    R: Send + 'static + Sized,
{
    static START_THREADS: Once = Once::new();

    START_THREADS.call_once(|| {
        std::thread::Builder::new()
            .name("sled-io-thread".into())
            .spawn(|| IO_QUEUE.perform_work())
            .expect("failed to spawn critical IO thread");

        std::thread::Builder::new()
            .name("sled-blocking-thread".into())
            .spawn(|| BLOCKING_QUEUE.perform_work())
            .expect("failed to spawn critical blocking thread");

        std::thread::Builder::new()
            .name("sled-snapshot-thread".into())
            .spawn(|| SNAPSHOT_QUEUE.perform_work())
            .expect("failed to spawn critical snapshot thread");
    });

    let (promise_filler, promise) = OneShot::pair();
    let task = move || {
        promise_filler.fill((work)());
    };

    queue.send(Box::new(task));

    Ok(promise)
}

#[cfg(any(
    miri,
    not(any(
        windows,
        target_os = "linux",
        target_os = "macos",
        target_os = "dragonfly",
        target_os = "freebsd",
        target_os = "openbsd",
        target_os = "netbsd",
    ))
))]
pub fn spawn_to<F, R>(work: F, _: &'static Queue) -> Result<OneShot<R>>
where
    F: FnOnce() -> R + Send + 'static,
    R: Send + 'static,
{
    // Polyfill for platforms other than those we explicitly trust to
    // perform threaded work on. Just execute a task without involving threads.
    let (promise_filler, promise) = OneShot::pair();
    promise_filler.fill((work)());
    Ok(promise)
}

pub fn truncate(
    config: crate::RunningConfig,
    at: u64,
) -> Result<OneShot<Result<()>>> {
    spawn_to(
        move || {
            log::debug!("truncating file to length {}", at);
            config
                .file
                .set_len(at)
                .and_then(|_| config.file.sync_all())
                .map_err(|e| e.into())
        },
        &IO_QUEUE,
    )
}

pub fn take_fuzzy_snapshot(
    pc: crate::pagecache::PageCache,
) -> Result<OneShot<()>> {
    spawn_to(
        move || {
            if let Err(e) = pc.take_fuzzy_snapshot() {
                log::error!("failed to write snapshot: {:?}", e);
            }
        },
        &SNAPSHOT_QUEUE,
    )
}

pub(crate) fn write_to_log(
    iobuf: Arc<crate::pagecache::iobuf::IoBuf>,
    iobufs: Arc<crate::pagecache::iobuf::IoBufs>,
) -> Result<OneShot<()>> {
    spawn_to(
        move || {
            let lsn = iobuf.lsn;
            if let Err(e) = iobufs.write_to_log(iobuf) {
                log::error!(
                    "hit error while writing iobuf with lsn {}: {:?}",
                    lsn,
                    e
                );

                // store error before notifying so that waiting threads will see
                // it
                iobufs.config.set_global_error(e);

                let intervals = iobufs.intervals.lock();

                // having held the mutex makes this linearized
                // with the notify below.
                drop(intervals);

                let _notified = iobufs.interval_updated.notify_all();
            }
        },
        &IO_QUEUE,
    )
}
<|endoftext|>