<commit_before>use std::collections::{RingBuf, Deque};

use message::IrcMessage;
use event::{
    IrcEvent,
    IrcEventMessage,
};

pub trait MessageResponder {
    fn on_message(&mut self, message: &IrcMessage) -> Vec<IrcMessage>;

    fn finished(&self) -> bool { false }
}

pub trait MessageWatcher {
    fn on_message(&mut self, message: &IrcMessage);

    /// If true, the `MessageWatcher` should be removed from the watcher set
    fn finished(&self) -> bool;
}

pub trait EventWatcher{
    fn on_event(&mut self, message: &IrcEvent);

    /// If true, the `EventWatcher` should be removed from the watcher set
    fn is_finished(&self) -> bool;

    fn get_name(&self) -> &'static str;
}

pub trait Bundler {
    fn on_message(&mut self, message: &IrcMessage) -> Vec<IrcEvent>;

    /// If true, the `Bundler` should be removed from the bundler set
    fn is_finished(&mut self) -> bool;

    fn get_name(&self) -> &'static str;
}

pub trait BundlerTrigger {
	fn on_message(&mut self, message: &IrcMessage) -> Vec<Box<Bundler+Send>>;
}

pub struct BundlerManager {
    // Unfinished watchers currently attached to the stream
    event_watchers: RingBuf<Box<EventWatcher+Send>>,

    // Active event bundlers.
    event_bundlers: RingBuf<Box<Bundler+Send>>,

    // Bundler triggers.  They create Bundlers.
    bundler_triggers: Vec<Box<BundlerTrigger+Send>>,
}

impl BundlerManager {
    pub fn new() -> BundlerManager {
        BundlerManager {
            event_watchers: RingBuf::new(),
            event_bundlers: RingBuf::new(),
            bundler_triggers: Vec::new(),
        }
    }

    // Do we really need +Send here?
    pub fn add_watcher(&mut self, watcher: Box<EventWatcher+Send>) {
        self.event_watchers.push(watcher);
    }

    pub fn add_bundler(&mut self, bundler: Box<Bundler+Send>) {
        self.event_bundlers.push(bundler);
    }

    pub fn add_bundler_trigger(&mut self, bundler: Box<BundlerTrigger+Send>) {
        self.bundler_triggers.push(bundler);
    }

    pub fn on_message(&mut self, message: &IrcMessage) -> Vec<IrcEvent> {
        let mut outgoing_events: Vec<IrcEvent> = Vec::new();

        for new_bundler in bundler_trigger_impl(&mut self.bundler_triggers, message).into_iter() {
            self.event_bundlers.push(new_bundler);
        }

        for event in bundler_accept_impl(&mut self.event_bundlers, message).into_iter() {
            outgoing_events.push(event);
        }

        outgoing_events.push(IrcEventMessage(message.clone()));

        for event in outgoing_events.iter() {
            for watcher in watcher_accept_impl(&mut self.event_watchers, event).into_iter() {
                drop(watcher);
            }
        }

        outgoing_events
    }
}

fn bundler_trigger_impl(triggers: &mut Vec<Box<BundlerTrigger+Send>>,
                       message: &IrcMessage
                      ) -> Vec<Box<Bundler+Send>> {

    let mut activating: Vec<Box<Bundler+Send>> = Vec::new();
    for trigger in triggers.iter_mut() {
        let new_bundlers = trigger.on_message(message);
        activating.reserve_additional(new_bundlers.len());
        for bundler in new_bundlers.into_iter() {
            activating.push(bundler);
        }
    }
    activating
}

fn watcher_accept_impl(buf: &mut RingBuf<Box<EventWatcher+Send>>,
                       event: &IrcEvent
                      ) -> Vec<Box<EventWatcher+Send>> {
    let mut keep_watchers: RingBuf<Box<EventWatcher+Send>> = RingBuf::new();
    let mut finished_watchers: Vec<Box<EventWatcher+Send>> = Vec::new();

    loop {
        match buf.pop_front() {
            Some(mut watcher) => {
                watcher.on_event(event);
                if watcher.is_finished() {
                    finished_watchers.push(watcher);
                } else {
                    keep_watchers.push(watcher);
                }
            },
            None => break
        }
    }
    loop {
        match keep_watchers.pop_front() {
            Some(watcher) => buf.push(watcher),
            None => break
        }
    }
    finished_watchers
}

fn bundler_accept_impl(buf: &mut RingBuf<Box<Bundler+Send>>,
                       message: &IrcMessage
                      ) -> Vec<IrcEvent> {

    let mut keep_bundlers: RingBuf<Box<Bundler+Send>> = RingBuf::new();
    let mut emit_events: Vec<IrcEvent> = Vec::new();

    loop {
        match buf.pop_front() {
            Some(mut bundler) => {
                for event in bundler.on_message(message).into_iter() {
                    emit_events.push(event);
                }
                if !bundler.is_finished() {
                    keep_bundlers.push(bundler);
                }
            },
            None => break
        }
    }
    loop {
        match keep_bundlers.pop_front() {
            Some(watcher) => buf.push(watcher),
            None => break
        }
    }
    emit_events
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::{IoResult, BufReader};

    use message::IrcMessage;
    use watchers::{
        WhoBundlerTrigger,
        JoinBundlerTrigger,
    };
    use event::{
        IrcEventJoinBundle,
        IrcEventWhoBundle,
    };

    const TEST_DATA: &'static [u8] = include_bin!("../../testdata/watcher.txt");

    fn unsafe_to_irc_message(line_res: IoResult<String>) -> IrcMessage {
        let line = match line_res {
            Ok(line) => line,
            Err(err) => panic!("err: {}", err)
        };
        let totrim: &[_] = &['\n', '\r'];
        match IrcMessage::from_str(line.as_slice().trim_right_chars(totrim)) {
            Ok(message) => message,
            Err(err) => panic!("err: {}", err)
        }
    }

    #[test]
    fn test_bundle_watcher() {
        let mut reader = BufReader::new(TEST_DATA);
        let mut bunman = BundlerManager::new();
        bunman.add_bundler_trigger(box JoinBundlerTrigger::new());
        bunman.add_bundler_trigger(box WhoBundlerTrigger::new());
        let mut events = Vec::new();

        for msg in reader.lines().map(unsafe_to_irc_message) {
            events.extend(bunman.on_message(&msg).into_iter());
        }

        let mut join_bundles = 0u;
        let mut who_bundles = 0u;

        for event in events.into_iter() {
            if let IrcEventJoinBundle(_) = event {
                join_bundles += 1;
            }
            if let IrcEventWhoBundle(_) = event {
                who_bundles += 1
            }
        }
        assert_eq!(join_bundles, 1);
        assert_eq!(who_bundles, 1);
    }
}
<commit_msg>Rust breaking change: Deque has been removed<commit_after>use std::collections::RingBuf;

use message::IrcMessage;
use event::{
    IrcEvent,
    IrcEventMessage,
};

pub trait MessageResponder {
    fn on_message(&mut self, message: &IrcMessage) -> Vec<IrcMessage>;

    fn finished(&self) -> bool { false }
}

pub trait MessageWatcher {
    fn on_message(&mut self, message: &IrcMessage);

    /// If true, the `MessageWatcher` should be removed from the watcher set
    fn finished(&self) -> bool;
}

pub trait EventWatcher{
    fn on_event(&mut self, message: &IrcEvent);

    /// If true, the `EventWatcher` should be removed from the watcher set
    fn is_finished(&self) -> bool;

    fn get_name(&self) -> &'static str;
}

pub trait Bundler {
    fn on_message(&mut self, message: &IrcMessage) -> Vec<IrcEvent>;

    /// If true, the `Bundler` should be removed from the bundler set
    fn is_finished(&mut self) -> bool;

    fn get_name(&self) -> &'static str;
}

pub trait BundlerTrigger {
	fn on_message(&mut self, message: &IrcMessage) -> Vec<Box<Bundler+Send>>;
}

pub struct BundlerManager {
    // Unfinished watchers currently attached to the stream
    event_watchers: RingBuf<Box<EventWatcher+Send>>,

    // Active event bundlers.
    event_bundlers: RingBuf<Box<Bundler+Send>>,

    // Bundler triggers.  They create Bundlers.
    bundler_triggers: Vec<Box<BundlerTrigger+Send>>,
}

impl BundlerManager {
    pub fn new() -> BundlerManager {
        BundlerManager {
            event_watchers: RingBuf::new(),
            event_bundlers: RingBuf::new(),
            bundler_triggers: Vec::new(),
        }
    }

    // Do we really need +Send here?
    pub fn add_watcher(&mut self, watcher: Box<EventWatcher+Send>) {
        self.event_watchers.push(watcher);
    }

    pub fn add_bundler(&mut self, bundler: Box<Bundler+Send>) {
        self.event_bundlers.push(bundler);
    }

    pub fn add_bundler_trigger(&mut self, bundler: Box<BundlerTrigger+Send>) {
        self.bundler_triggers.push(bundler);
    }

    pub fn on_message(&mut self, message: &IrcMessage) -> Vec<IrcEvent> {
        let mut outgoing_events: Vec<IrcEvent> = Vec::new();

        for new_bundler in bundler_trigger_impl(&mut self.bundler_triggers, message).into_iter() {
            self.event_bundlers.push(new_bundler);
        }

        for event in bundler_accept_impl(&mut self.event_bundlers, message).into_iter() {
            outgoing_events.push(event);
        }

        outgoing_events.push(IrcEventMessage(message.clone()));

        for event in outgoing_events.iter() {
            for watcher in watcher_accept_impl(&mut self.event_watchers, event).into_iter() {
                drop(watcher);
            }
        }

        outgoing_events
    }
}

fn bundler_trigger_impl(triggers: &mut Vec<Box<BundlerTrigger+Send>>,
                       message: &IrcMessage
                      ) -> Vec<Box<Bundler+Send>> {

    let mut activating: Vec<Box<Bundler+Send>> = Vec::new();
    for trigger in triggers.iter_mut() {
        let new_bundlers = trigger.on_message(message);
        activating.reserve_additional(new_bundlers.len());
        for bundler in new_bundlers.into_iter() {
            activating.push(bundler);
        }
    }
    activating
}

fn watcher_accept_impl(buf: &mut RingBuf<Box<EventWatcher+Send>>,
                       event: &IrcEvent
                      ) -> Vec<Box<EventWatcher+Send>> {
    let mut keep_watchers: RingBuf<Box<EventWatcher+Send>> = RingBuf::new();
    let mut finished_watchers: Vec<Box<EventWatcher+Send>> = Vec::new();

    loop {
        match buf.pop_front() {
            Some(mut watcher) => {
                watcher.on_event(event);
                if watcher.is_finished() {
                    finished_watchers.push(watcher);
                } else {
                    keep_watchers.push(watcher);
                }
            },
            None => break
        }
    }
    loop {
        match keep_watchers.pop_front() {
            Some(watcher) => buf.push(watcher),
            None => break
        }
    }
    finished_watchers
}

fn bundler_accept_impl(buf: &mut RingBuf<Box<Bundler+Send>>,
                       message: &IrcMessage
                      ) -> Vec<IrcEvent> {

    let mut keep_bundlers: RingBuf<Box<Bundler+Send>> = RingBuf::new();
    let mut emit_events: Vec<IrcEvent> = Vec::new();

    loop {
        match buf.pop_front() {
            Some(mut bundler) => {
                for event in bundler.on_message(message).into_iter() {
                    emit_events.push(event);
                }
                if !bundler.is_finished() {
                    keep_bundlers.push(bundler);
                }
            },
            None => break
        }
    }
    loop {
        match keep_bundlers.pop_front() {
            Some(watcher) => buf.push(watcher),
            None => break
        }
    }
    emit_events
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::{IoResult, BufReader};

    use message::IrcMessage;
    use watchers::{
        WhoBundlerTrigger,
        JoinBundlerTrigger,
    };
    use event::{
        IrcEventJoinBundle,
        IrcEventWhoBundle,
    };

    const TEST_DATA: &'static [u8] = include_bin!("../../testdata/watcher.txt");

    fn unsafe_to_irc_message(line_res: IoResult<String>) -> IrcMessage {
        let line = match line_res {
            Ok(line) => line,
            Err(err) => panic!("err: {}", err)
        };
        let totrim: &[_] = &['\n', '\r'];
        match IrcMessage::from_str(line.as_slice().trim_right_chars(totrim)) {
            Ok(message) => message,
            Err(err) => panic!("err: {}", err)
        }
    }

    #[test]
    fn test_bundle_watcher() {
        let mut reader = BufReader::new(TEST_DATA);
        let mut bunman = BundlerManager::new();
        bunman.add_bundler_trigger(box JoinBundlerTrigger::new());
        bunman.add_bundler_trigger(box WhoBundlerTrigger::new());
        let mut events = Vec::new();

        for msg in reader.lines().map(unsafe_to_irc_message) {
            events.extend(bunman.on_message(&msg).into_iter());
        }

        let mut join_bundles = 0u;
        let mut who_bundles = 0u;

        for event in events.into_iter() {
            if let IrcEventJoinBundle(_) = event {
                join_bundles += 1;
            }
            if let IrcEventWhoBundle(_) = event {
                who_bundles += 1
            }
        }
        assert_eq!(join_bundles, 1);
        assert_eq!(who_bundles, 1);
    }
}
<|endoftext|>


<commit_before>mod get;

pub use self::get::*;

use r2d2;
use r2d2_postgres::PostgresConnectionManager;

use super::errors::*;
use super::board::NewBoard;
use super::moderator::Mod;
use super::post::Post;

pub type Pool = r2d2::Pool<PostgresConnectionManager>;

pub fn create_tables(pool: Pool) -> Result<()> {
    let conn = pool.get().unwrap();
    // Create the boards table
    conn.execute("CREATE TABLE IF NOT EXISTS boards (
                    short_name VARCHAR PRIMARY KEY,
                    long_name VARCHAR NOT NULL,
                    description TEXT NOT NULL,
                    post_number BIGINT NOT NULL,
                    active_threads INT NOT NULL
                  )",
                 &[])?;

    // Create the admins table
    conn.execute("CREATE TABLE IF NOT EXISTS admins (
                    username VARCHAR PRIMARY KEY,
                    password VARCHAR NOT NULL,
                    boards VARCHAR[] NOT NULL,
                    admin boolean NOT NULL
                  )",
                 &[])?;

    // Create the posts table
    conn.execute("CREATE TABLE IF NOT EXISTS posts (
                    post_number BIGINT NOT NULL,
                    parent BIGINT NOT NULL,
                    board VARCHAR NOT NULL,
                    subject VARCHAR,
                    name VARCHAR NOT NULL,
                    email VARCHAR NOT NULL,
                    content TEXT NOT NULL,
                    thread boolean NOT NULL,
                    pinned boolean NOT NULL,
                    active boolean NOT NULL,
                    last_modified TIMESTAMP,
                    PRIMARY KEY (board, post_number)
                  )",
                 &[])?;

    Ok(())
}

pub fn create_admin(pool: Pool, admin: Mod) -> Result<()> {
    let conn = pool.get().unwrap();

    conn.execute("INSERT INTO admins (username, password, boards, admin) VALUES ($1, $2, $3, $4)",
                 &[&admin.username, &admin.password, &admin.boards, &admin.admin])?;
    Ok(())
}

pub fn create_boards(pool: Pool, boards: &[NewBoard]) -> Result<()> {
    let conn = pool.get().unwrap();

    for b in boards {
        conn.execute("INSERT INTO boards (short_name, long_name, description, post_number, \
                      active_threads) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (short_name) DO \
                      NOTHING",
                     &[&b.short_name, &b.long_name, &b.description, &0i64, &b.active_threads])?;
    }

    Ok(())
}

pub fn board_exists(pool: Pool, board_name: &str) -> Result<bool> {
    let conn = pool.get().unwrap();

    // TODO(nokaa): Look at the EXISTS keyword for this check:
    // https://stackoverflow.com/questions/7471625/fastest-check-if-row-exists-in-postgresql
    let rows = conn.query("SELECT short_name FROM boards WHERE short_name = $1",
               &[&board_name])?;
    Ok(!rows.is_empty())

}

pub fn thread_exists(pool: Pool, board_name: &str, thread_number: i64) -> Result<bool> {
    let conn = pool.get().unwrap();

    // TODO(nokaa): Look at the EXISTS keyword for this check:
    // https://stackoverflow.com/questions/7471625/fastest-check-if-row-exists-in-postgresql
    let rows =
        conn.query("SELECT post_number, board, thread FROM posts WHERE board = $1 AND \
                    post_number = $2 AND thread = $3",
                   &[&board_name, &thread_number, &true])?;
    Ok(!rows.is_empty())
}

pub fn create_thread(pool: Pool, thread: Post) -> Result<i64> {
    let conn = pool.get().unwrap();
    // let trans = conn.transaction()?;

    // Get the post number for this post
    let rows =
        conn.query("UPDATE boards SET post_number = post_number + 1 WHERE short_name = $1 \
                    RETURNING post_number",
                   &[&thread.board])?;
    let post_number: i64 = rows.get(0).get(0);

    let time = ::chrono::UTC::now().naive_utc();
    conn.execute("INSERT INTO posts (post_number, parent, board, subject, name, email, content, \
                  thread, pinned, active, last_modified) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, \
                  $9, $10, $11);",
                 &[&post_number,
                   &post_number,
                   &thread.board,
                   &thread.subject,
                   &thread.name,
                   &thread.email,
                   &thread.content,
                   &true,
                   &false,
                   &true,
                   &Some(time)])?;
    // trans.commit()?;
    Ok(post_number)
}

pub fn create_post(pool: Pool, post: Post) -> Result<()> {
    let conn = pool.get().unwrap();

    // Get the post number for this post
    let rows =
        conn.query("UPDATE boards SET post_number = post_number + 1 WHERE short_name = $1 \
                    RETURNING post_number",
                   &[&post.board])?;
    let post_number: i64 = rows.get(0).get(0);

    conn.execute("INSERT INTO posts (post_number, parent, board, subject, name, email, content, \
                  thread, pinned, active, last_modified) VALUES ($1, $2, $3, \
                  $4, $5, $6, $7, $8, $9, $10, $11);",
                 &[&post_number,
                   &post.parent,
                   &post.board,
                   &post.subject,
                   &post.name,
                   &post.email,
                   &post.content,
                   &false,
                   &false,
                   &false,
                   &None::<::chrono::NaiveDateTime>])?;
    Ok(())
}
<commit_msg>Update thread last_modified field on reply<commit_after>mod get;

pub use self::get::*;

use r2d2;
use r2d2_postgres::PostgresConnectionManager;

use super::errors::*;
use super::board::NewBoard;
use super::moderator::Mod;
use super::post::Post;

pub type Pool = r2d2::Pool<PostgresConnectionManager>;

pub fn create_tables(pool: Pool) -> Result<()> {
    let conn = pool.get().unwrap();
    // Create the boards table
    conn.execute("CREATE TABLE IF NOT EXISTS boards (
                    short_name VARCHAR PRIMARY KEY,
                    long_name VARCHAR NOT NULL,
                    description TEXT NOT NULL,
                    post_number BIGINT NOT NULL,
                    active_threads INT NOT NULL
                  )",
                 &[])?;

    // Create the admins table
    conn.execute("CREATE TABLE IF NOT EXISTS admins (
                    username VARCHAR PRIMARY KEY,
                    password VARCHAR NOT NULL,
                    boards VARCHAR[] NOT NULL,
                    admin boolean NOT NULL
                  )",
                 &[])?;

    // Create the posts table
    conn.execute("CREATE TABLE IF NOT EXISTS posts (
                    post_number BIGINT NOT NULL,
                    parent BIGINT NOT NULL,
                    board VARCHAR NOT NULL,
                    subject VARCHAR,
                    name VARCHAR NOT NULL,
                    email VARCHAR NOT NULL,
                    content TEXT NOT NULL,
                    thread boolean NOT NULL,
                    pinned boolean NOT NULL,
                    active boolean NOT NULL,
                    last_modified TIMESTAMP,
                    PRIMARY KEY (board, post_number)
                  )",
                 &[])?;

    Ok(())
}

pub fn create_admin(pool: Pool, admin: Mod) -> Result<()> {
    let conn = pool.get().unwrap();

    conn.execute("INSERT INTO admins (username, password, boards, admin) VALUES ($1, $2, $3, $4)",
                 &[&admin.username, &admin.password, &admin.boards, &admin.admin])?;
    Ok(())
}

pub fn create_boards(pool: Pool, boards: &[NewBoard]) -> Result<()> {
    let conn = pool.get().unwrap();

    for b in boards {
        conn.execute("INSERT INTO boards (short_name, long_name, description, post_number, \
                      active_threads) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (short_name) DO \
                      NOTHING",
                     &[&b.short_name, &b.long_name, &b.description, &0i64, &b.active_threads])?;
    }

    Ok(())
}

pub fn board_exists(pool: Pool, board_name: &str) -> Result<bool> {
    let conn = pool.get().unwrap();

    // TODO(nokaa): Look at the EXISTS keyword for this check:
    // https://stackoverflow.com/questions/7471625/fastest-check-if-row-exists-in-postgresql
    let rows = conn.query("SELECT short_name FROM boards WHERE short_name = $1",
               &[&board_name])?;
    Ok(!rows.is_empty())

}

pub fn thread_exists(pool: Pool, board_name: &str, thread_number: i64) -> Result<bool> {
    let conn = pool.get().unwrap();

    // TODO(nokaa): Look at the EXISTS keyword for this check:
    // https://stackoverflow.com/questions/7471625/fastest-check-if-row-exists-in-postgresql
    let rows =
        conn.query("SELECT post_number, board, thread FROM posts WHERE board = $1 AND \
                    post_number = $2 AND thread = $3",
                   &[&board_name, &thread_number, &true])?;
    Ok(!rows.is_empty())
}

pub fn create_thread(pool: Pool, thread: Post) -> Result<i64> {
    let conn = pool.get().unwrap();

    // Get the post number for this post
    let rows =
        conn.query("UPDATE boards SET post_number = post_number + 1 WHERE short_name = $1 \
                    RETURNING post_number",
                   &[&thread.board])?;
    let post_number: i64 = rows.get(0).get(0);
    let time = ::chrono::UTC::now().naive_utc();

    let trans = conn.transaction()?;
    trans.execute("INSERT INTO posts (post_number, parent, board, subject, name, email, content, \
                  thread, pinned, active, last_modified) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, \
                  $9, $10, $11);",
                 &[&post_number,
                   &post_number,
                   &thread.board,
                   &thread.subject,
                   &thread.name,
                   &thread.email,
                   &thread.content,
                   &true,
                   &false,
                   &true,
                   &Some(time)])?;
    trans.commit()?;
    Ok(post_number)
}

pub fn create_post(pool: Pool, post: Post) -> Result<()> {
    let conn = pool.get().unwrap();
    let trans = conn.transaction()?;

    let time = ::chrono::UTC::now().naive_utc();
    trans.execute("UPDATE posts SET last_modified = $1 WHERE board = $2 AND post_number = $3 AND \
                  thread = true",
                 &[&time, &post.board, &post.parent])?;
    trans.execute("with rows as (UPDATE boards SET post_number = post_number + 1 where short_name \
                  = $1 returning post_number) INSERT INTO posts (post_number, parent, board, \
                  subject, name, email, content, thread, pinned, active, last_modified) VALUES \
                  ((SELECT post_number FROM rows), $2, $1, $3, $4, $5, $6, $7, $8, $9, $10);",
                 &[&post.board,
                   &post.parent,
                   &post.subject,
                   &post.name,
                   &post.email,
                   &post.content,
                   &false,
                   &false,
                   &false,
                   &None::<::chrono::NaiveDateTime>])?;
    trans.commit()?;
    Ok(())
}
<|endoftext|>


<commit_before>use std::io::fs::PathExtensions;
use std::io;
use toml;

use app_result::{AppResult, app_err};

#[deriving(Show)]
pub enum Dependency
{
   /// the depedency is based on a git repository
   Git {
      lib_name: String,
      commit_hash: String,
      dependencies: Vec<Dependency>
   },

   /// the depedency is from crates.io
   CratesIo {
      lib_name: String,
      version: String,
      dependencies: Vec<Dependency>
   }
}

pub type Dependencies = Vec<Dependency>;

/// Reads the dependecies from the `Cargo.toml` located in `cargo_toml_dir`
/// and for git depedencies the commit hash is read from the `Cargo.lock`.
pub fn read_dependencies(cargo_toml_dir: &Path) -> AppResult<Dependencies>
{
   let mut cargo_toml = cargo_toml_dir.clone();
   cargo_toml.push("Cargo.toml");

   let toml_string = try!(io::File::open(&cargo_toml).read_to_string());
   let mut toml_parser = toml::Parser::new(&*toml_string);

   let toml_table = try!(
      toml_parser.parse()
         .ok_or_else(|| app_err(format!("Couldn't parse '{}': {}", cargo_toml.display(), toml_parser.errors)))
   );

   let mut deps: Dependencies = Vec::new();
   let deps_string = "dependencies".to_string();
   if ! toml_table.contains_key(&deps_string) {
      return Ok(deps);
   }

   let deps_table = try!(
      toml_table.get(&deps_string)
         .and_then(|value| value.as_table())
         .ok_or(app_err(format!("Couldn't get toml::Table entry for 'dependency'!")))
   );

   let mut cargo_lock = cargo_toml_dir.clone();
   cargo_lock.push("Cargo.lock");

   let lock_string = try!(io::File::open(&cargo_lock).read_to_string());
   let mut lock_parser = toml::Parser::new(&*lock_string);

   let lock_table = try!(
      lock_parser.parse()
         .ok_or_else(|| app_err(format!("Couldn't parse '{}': {}", cargo_lock.display(), lock_parser.errors)))
   );

   let packages = try!(
      lock_table.get(&"package".to_string())
         .and_then(|value| value.as_slice())
         .and_then(|slice| {
            let tables = slice.iter()
               .filter_map(|value| value.as_table())
               .collect::<Vec<&toml::TomlTable>>();

            if ! tables.is_empty() { Some(tables) } else { None }
         })
         .ok_or(app_err(format!("Couldn't get Array of Tables entry for 'package'!")))
   );

   let name_str = &String::from_str("name");
   let version_str = &String::from_str("version");
   let source_str = &String::from_str("source");

   for (lib_name, value) in deps_table.iter() {
      match *value {
         toml::String(_) | toml::Table(_) => {
            deps.push(try!(get_dependency(&packages, lib_name)));
         }

         _ => {
            return Err(app_err(format!(
               "Expected a String or a Table for the dependency with the name '{}', but got: '{}'", lib_name, value
            )));
         }
      }
   }

   Ok(deps)
}

fn get_dependency(packages: &Vec<&toml::TomlTable>, lib_name: &String) -> AppResult<Dependency>
{
   let name_str = &String::from_str("name");

   let package = try!(
      packages.iter()
         .find(|t| t.get(name_str).and_then(|n| n.as_str()) == Some(lib_name[]))
         .ok_or_else(|| app_err(format!("Couldn't find package with name = '{}'!", lib_name)))
   );

   let version_str = &String::from_str("version");
   let source_str = &String::from_str("source");
   let deps_str = &String::from_str("dependencies");

   let source = try!(
      package.get(source_str)
         .and_then(|src| src.as_str())
         .ok_or_else(|| app_err(format!("Couldn't find source string in package: '{}'!", package)))
   );

   let src_type = try!(
      source.split('+')
         .nth(0)
         .ok_or_else(|| app_err(format!("Couldn't find source type in: '{}'!", source)))
   );

   let version = try!(
      package.get(version_str)
         .and_then(|vers| vers.as_str())
         .ok_or_else(|| app_err(format!("Couldn't find version string in package: '{}'!", package)))
   );

   let dep_strs = try!(
      package.get(deps_str)
         .and_then(|ds| ds.as_slice())
         .and_then(|slice| {
            let ds = slice.iter()
               .filter_map(|v| v.as_str())
               .collect::<Vec<&str>>();

            if ! ds.is_empty() { Some(ds) } else { None }
         })
         .ok_or_else(|| app_err(format!("Couldn't get Array of Strings for 'dependencies' entry: '{}'!", package)))
   );

   let mut dep_names: Vec<&str> = Vec::new();
   for dep_str in dep_strs.iter() {
      let dep_name = try!(
         dep_str.split(' ')
            .nth(0)
            .ok_or_else(|| app_err(format!("Couldn't get name from dependency: '{}'!", dep_str)))
      );

      dep_names.push(dep_name);
   }

   let mut deps: Vec<Dependency> = Vec::new();
   for dep_name in dep_names.iter() {
      deps.push(try!(get_dependency(packages, &dep_name.to_string())));
   }

   match src_type {
      "git" => {
         let commit_hash = try!(
            source.split('#')
               .last()
               .ok_or_else(|| app_err(format!("Couldn't find commit hash in source entry: '{}'!", source)))
         );

         Ok(Dependency::Git { lib_name: lib_name.clone(), commit_hash: commit_hash.to_string(), dependencies: deps })
      },

      "registry" => {
         Ok(Dependency::CratesIo { lib_name: lib_name.clone(), version: version.to_string(), dependencies: deps })
      }

      _ => {
         Err(app_err(format!("Unexpected source type '{}' in package: '{}'!", src_type, package)))
      }
   }
}
<commit_msg>Don't expect a 'dependency' entry in 'package', it's optional<commit_after>use std::io::fs::PathExtensions;
use std::io;
use toml;

use app_result::{AppResult, app_err};

#[deriving(Show)]
pub enum Dependency
{
   /// the depedency is based on a git repository
   Git {
      lib_name: String,
      commit_hash: String,
      dependencies: Vec<Dependency>
   },

   /// the depedency is from crates.io
   CratesIo {
      lib_name: String,
      version: String,
      dependencies: Vec<Dependency>
   }
}

pub type Dependencies = Vec<Dependency>;

/// Reads the dependecies from the `Cargo.toml` located in `cargo_toml_dir`
/// and for git depedencies the commit hash is read from the `Cargo.lock`.
pub fn read_dependencies(cargo_toml_dir: &Path) -> AppResult<Dependencies>
{
   let mut cargo_toml = cargo_toml_dir.clone();
   cargo_toml.push("Cargo.toml");

   let toml_string = try!(io::File::open(&cargo_toml).read_to_string());
   let mut toml_parser = toml::Parser::new(&*toml_string);

   let toml_table = try!(
      toml_parser.parse()
         .ok_or_else(|| app_err(format!("Couldn't parse '{}': {}", cargo_toml.display(), toml_parser.errors)))
   );

   let mut deps: Dependencies = Vec::new();
   let deps_string = "dependencies".to_string();
   if ! toml_table.contains_key(&deps_string) {
      return Ok(deps);
   }

   let deps_table = try!(
      toml_table.get(&deps_string)
         .and_then(|value| value.as_table())
         .ok_or(app_err(format!("Couldn't get toml::Table entry for 'dependency'!")))
   );

   let mut cargo_lock = cargo_toml_dir.clone();
   cargo_lock.push("Cargo.lock");

   let lock_string = try!(io::File::open(&cargo_lock).read_to_string());
   let mut lock_parser = toml::Parser::new(&*lock_string);

   let lock_table = try!(
      lock_parser.parse()
         .ok_or_else(|| app_err(format!("Couldn't parse '{}': {}", cargo_lock.display(), lock_parser.errors)))
   );

   let packages = try!(
      lock_table.get(&"package".to_string())
         .and_then(|value| value.as_slice())
         .and_then(|slice| {
            let tables = slice.iter()
               .filter_map(|value| value.as_table())
               .collect::<Vec<&toml::TomlTable>>();

            if ! tables.is_empty() { Some(tables) } else { None }
         })
         .ok_or(app_err(format!("Couldn't get Array of Tables entry for 'package'!")))
   );

   let name_str = &String::from_str("name");
   let version_str = &String::from_str("version");
   let source_str = &String::from_str("source");

   for (lib_name, value) in deps_table.iter() {
      match *value {
         toml::String(_) | toml::Table(_) => {
            deps.push(try!(get_dependency(&packages, lib_name)));
         }

         _ => {
            return Err(app_err(format!(
               "Expected a String or a Table for the dependency with the name '{}', but got: '{}'", lib_name, value
            )));
         }
      }
   }

   Ok(deps)
}

fn get_dependency(packages: &Vec<&toml::TomlTable>, lib_name: &String) -> AppResult<Dependency>
{
   let name_str = &String::from_str("name");

   let package = try!(
      packages.iter()
         .find(|t| t.get(name_str).and_then(|n| n.as_str()) == Some(lib_name[]))
         .ok_or_else(|| app_err(format!("Couldn't find package with name = '{}'!", lib_name)))
   );

   let version_str = &String::from_str("version");
   let source_str = &String::from_str("source");
   let deps_str = &String::from_str("dependencies");

   let source = try!(
      package.get(source_str)
         .and_then(|src| src.as_str())
         .ok_or_else(|| app_err(format!("Couldn't find source string in package: '{}'!", package)))
   );

   let src_type = try!(
      source.split('+')
         .nth(0)
         .ok_or_else(|| app_err(format!("Couldn't find source type in: '{}'!", source)))
   );

   let version = try!(
      package.get(version_str)
         .and_then(|vers| vers.as_str())
         .ok_or_else(|| app_err(format!("Couldn't find version string in package: '{}'!", package)))
   );

   let dep_strs = match package.get(deps_str) {
      None => {
         Vec::<&str>::new()
      },

      Some(value) => {
         try!(
            value.as_slice()
               .and_then(|slice| {
                  let ds = slice.iter()
                     .filter_map(|v| v.as_str())
                     .collect::<Vec<&str>>();

                  if ! ds.is_empty() { Some(ds) } else { None }
               })
               .ok_or_else(|| app_err(format!("Couldn't get Array of Strings for 'dependencies' entry: '{}'!", package)))
         )
      }
   };

   let mut dep_names: Vec<&str> = Vec::new();
   for dep_str in dep_strs.iter() {
      let dep_name = try!(
         dep_str.split(' ')
            .nth(0)
            .ok_or_else(|| app_err(format!("Couldn't get name from dependency: '{}'!", dep_str)))
      );

      dep_names.push(dep_name);
   }

   let mut deps: Vec<Dependency> = Vec::new();
   for dep_name in dep_names.iter() {
      deps.push(try!(get_dependency(packages, &dep_name.to_string())));
   }

   match src_type {
      "git" => {
         let commit_hash = try!(
            source.split('#')
               .last()
               .ok_or_else(|| app_err(format!("Couldn't find commit hash in source entry: '{}'!", source)))
         );

         Ok(Dependency::Git { lib_name: lib_name.clone(), commit_hash: commit_hash.to_string(), dependencies: deps })
      },

      "registry" => {
         Ok(Dependency::CratesIo { lib_name: lib_name.clone(), version: version.to_string(), dependencies: deps })
      }

      _ => {
         Err(app_err(format!("Unexpected source type '{}' in package: '{}'!", src_type, package)))
      }
   }
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2020 Fabian Schuiki

use heck::CamelCase;
use proc_macro::TokenStream;
use quote::{format_ident, quote, ToTokens};
use std::{
    cell::RefCell,
    collections::{BTreeSet, HashSet},
};

// CAUTION: This is all wildly unstable and relies on the compiler maintaining
// a certain order between proc macro expansions. So this could break any
// minute. Better have a robust CI.
thread_local! {
    static QUERIES: RefCell<Vec<String>> = Default::default();
}

pub(crate) fn mark_query(_args: TokenStream, input: TokenStream) -> TokenStream {
    // Parse the input.
    let input = syn::parse_macro_input!(input as syn::ItemFn);

    // Map everything to a string here. Compiler panics horribly if we hand out
    // the actual idents and generics.
    QUERIES.with(|c| c.borrow_mut().push(input.to_token_stream().to_string()));

    // Produce some output.
    let output = quote! { #input };
    output.into()
}

pub(crate) fn derive_query_db(input: TokenStream) -> TokenStream {
    let input = proc_macro2::TokenStream::from(input);
    let mut output = proc_macro2::TokenStream::new();

    // Flush the accumulated queries.
    let queries = QUERIES.with(|c| std::mem::replace(&mut *c.borrow_mut(), Default::default()));

    // Make a set of conflicting argument names.
    let reserved: HashSet<_> = ["query_key", "query_storage", "query_tag"]
        .iter()
        .copied()
        .collect();

    // Collect all query lifetimes.
    let mut ltset = BTreeSet::new();

    // Process the queries.
    let mut funcs = vec![];
    let mut caches = vec![];
    let mut query_tags = vec![];

    for raw_query in &queries {
        // Parse the fn.
        let query: syn::ItemFn = syn::parse_str(raw_query).unwrap();

        // Disect a few things.
        let name = query.sig.ident.clone();
        let mut generics = query.sig.generics.clone();
        let args = query.sig.inputs.iter().skip(1);
        let result = match query.sig.output {
            syn::ReturnType::Type(_, ty) => ty.as_ref().clone(),
            _ => panic!("query {} has no return type", name),
        };

        // Filter out the doc comments such that we can apply them to the trait
        // fn as well.
        let doc_attrs = query.attrs.iter().filter(|a| a.path.is_ident("doc"));

        // Determine the argument list of the query.
        let arg_pats = args.clone().map(|arg| match arg {
            syn::FnArg::Typed(pat) => pat,
            _ => unreachable!("there should be no self args left at this point"),
        });
        let arg_names: Vec<_> = arg_pats
            .clone()
            .enumerate()
            .map(|(i, pat)| match pat.pat.as_ref() {
                syn::Pat::Ident(id) if !reserved.contains(id.ident.to_string().as_str()) => {
                    id.ident.clone()
                }
                _ => format_ident!("arg{}", i),
            })
            .collect();
        let arg_types: Vec<_> = arg_pats
            .clone()
            .map(|pat| pat.ty.as_ref().clone())
            .collect();

        // Collect lifetimes and strip them from the generics.
        for param in std::mem::replace(&mut generics.params, Default::default()) {
            match param {
                syn::GenericParam::Lifetime(ltdef) => {
                    ltset.insert(ltdef.lifetime);
                }
                _ => generics.params.push(param),
            }
        }

        // Render the key type that we use to look up things in the database.
        let key_type = quote! {
            ( #(#arg_types),* )
        };
        let key = quote! {
            ( #(#arg_names),* )
        };

        // Determine the cache field name.
        let cache_name = format_ident!("cached_{}", name);

        // Render a query tag that can be pushed onto the query stack to break
        // cycles.
        let tag_name = format_ident!("{}", name.to_string().to_camel_case());
        let doc = format!("The `{}` query.", name);
        query_tags.push(quote! {
            #[doc = #doc]
            #tag_name (#key_type),
        });

        // Render the query for the database trait.
        funcs.push(quote! {
            #(#doc_attrs)*
            fn #name #generics (&self, #(#arg_names: #arg_types),*) -> #result {
                let query_storage = self.storage();
                let query_key = #key;
                let query_tag = QueryTag::#tag_name(query_key.clone());

                // Check if we already have a result for this query.
                if let Some(result) = query_storage.#cache_name.borrow().get(&query_key) {
                    trace!("Serving {} {:?} from cache", stringify!(#name), query_key);
                    return result.clone();
                }
                trace!("Executing {} {:?}", stringify!(#name), query_key);

                // Push the query onto the stack, checking if it is already in
                // flight.
                query_storage.stack.borrow_mut().push(query_tag.clone());
                if !query_storage.inflight.borrow_mut().insert(query_tag.clone()) {
                    self.handle_cycle();
                    // The above never returns.
                }

                // Execute the query.
                let result: #result = #name(self.context(), #(#arg_names),*);
                query_storage.#cache_name.borrow_mut().insert(query_key, result.clone());

                // Pop the query from the stack.
                query_storage.inflight.borrow_mut().remove(&query_tag);
                query_storage.stack.borrow_mut().pop();
                result
            }
        });

        // Render the cache field for the storage struct.
        let doc = format!("Cached results of the `{}` query.", name);
        caches.push(quote! {
            #[doc = #doc]
            pub #cache_name: RefCell<HashMap<#key_type, #result>>,
        });
    }

    // Extract query lifetimes.
    let lts = if ltset.is_empty() {
        quote! {}
    } else {
        let lts = ltset.iter();
        quote! { <#(#lts),*> }
    };

    // Generate the query database trait.
    output.extend(quote! {
        #input
        pub trait QueryDatabase #lts {
            /// The type passed as first argument to compiler query
            /// implementations.
            type Context: crate::Context #lts;

            /// Get the context that is passed to compiler query
            /// implementations.
            fn context(&self) -> &Self::Context;

            /// Get the query caches and runtime data.
            fn storage(&self) -> &QueryStorage #lts;

            /// Called when a query cycle is detected.
            fn handle_cycle(&self) -> ! {
                panic!("query cycle detected");
            }

            #(#funcs)*
        }
    });

    // Generate the query storage struct.
    output.extend(quote! {
        /// A collection of query caches and runtime data for a `QueryDatabase`.
        #[derive(Default)]
        pub struct QueryStorage #lts {
            /// A stack of the currently-executing queries.
            pub stack: RefCell<Vec<QueryTag #lts>>,
            /// A set of the currently-executing queries.
            pub inflight: RefCell<HashSet<QueryTag #lts>>,

            #(#caches)*
        }
    });

    // Generate the query tag enum.
    output.extend(quote! {
        /// A tag identifying any of the queries in `QueryDatabase`.
        #[derive(Clone, Hash, Eq, PartialEq, Debug)]
        pub enum QueryTag #lts {
            #(#query_tags)*
        }
    });

    // Produce some output.
    // println!("{}", output);
    output.into()
}
<commit_msg>derive: Generate proper query keys<commit_after>// Copyright (c) 2016-2020 Fabian Schuiki

use heck::CamelCase;
use proc_macro::TokenStream;
use quote::{format_ident, quote, ToTokens};
use std::{
    cell::RefCell,
    collections::{BTreeSet, HashSet},
};

// CAUTION: This is all wildly unstable and relies on the compiler maintaining
// a certain order between proc macro expansions. So this could break any
// minute. Better have a robust CI.
thread_local! {
    static QUERIES: RefCell<Vec<String>> = Default::default();
}

pub(crate) fn mark_query(_args: TokenStream, input: TokenStream) -> TokenStream {
    // Parse the input.
    let input = syn::parse_macro_input!(input as syn::ItemFn);

    // Map everything to a string here. Compiler panics horribly if we hand out
    // the actual idents and generics.
    QUERIES.with(|c| c.borrow_mut().push(input.to_token_stream().to_string()));

    // Produce some output.
    let output = quote! { #input };
    output.into()
}

pub(crate) fn derive_query_db(input: TokenStream) -> TokenStream {
    let input = proc_macro2::TokenStream::from(input);
    let mut output = proc_macro2::TokenStream::new();

    // Flush the accumulated queries.
    let queries = QUERIES.with(|c| std::mem::replace(&mut *c.borrow_mut(), Default::default()));

    // Make a set of conflicting argument names.
    let reserved: HashSet<_> = ["query_key", "query_storage", "query_tag", "query_result"]
        .iter()
        .copied()
        .collect();

    // Collect all query lifetimes.
    let mut ltset = BTreeSet::new();

    // Process the queries.
    let mut funcs = vec![];
    let mut caches = vec![];
    let mut tags = vec![];
    let mut tag_debugs = vec![];
    let mut keys = vec![];

    for raw_query in &queries {
        // Parse the fn.
        let query: syn::ItemFn = syn::parse_str(raw_query).unwrap();

        // Disect a few things.
        let name = query.sig.ident.clone();
        let mut generics = query.sig.generics.clone();
        let args = query.sig.inputs.iter().skip(1);
        let result = match query.sig.output {
            syn::ReturnType::Type(_, ty) => ty.as_ref().clone(),
            _ => panic!("query {} has no return type", name),
        };

        // Filter out the doc comments such that we can apply them to the trait
        // fn as well.
        let doc_attrs = query.attrs.iter().filter(|a| a.path.is_ident("doc"));

        // Determine the argument list of the query.
        let arg_pats = args.clone().map(|arg| match arg {
            syn::FnArg::Typed(pat) => pat,
            _ => unreachable!("there should be no self args left at this point"),
        });
        let arg_names: Vec<_> = arg_pats
            .clone()
            .enumerate()
            .map(|(i, pat)| match pat.pat.as_ref() {
                syn::Pat::Ident(id) if !reserved.contains(id.ident.to_string().as_str()) => {
                    id.ident.clone()
                }
                _ => format_ident!("arg{}", i),
            })
            .collect();
        let arg_types: Vec<_> = arg_pats
            .clone()
            .map(|pat| pat.ty.as_ref().clone())
            .collect();

        // Collect lifetimes and strip them from the generics.
        let original_generics = generics.clone();
        for param in std::mem::replace(&mut generics.params, Default::default()) {
            match param {
                syn::GenericParam::Lifetime(ltdef) => {
                    ltset.insert(ltdef.lifetime);
                }
                _ => generics.params.push(param),
            }
        }

        // Render the key type that we use to look up things in the database.
        let key_name = format_ident!("{}QueryKey", name.to_string().to_camel_case());
        let key_indices = (0..arg_types.len()).map(syn::Index::from);
        let doc = format!("The arguments passed to the `{}` query.", name);
        keys.push(quote! {
            #[doc = #doc]
            #[derive(Clone, PartialEq, Eq, Hash)]
            pub struct #key_name #original_generics (#(#arg_types),*);

            impl #original_generics std::fmt::Debug for #key_name #original_generics {
                fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
                    f.debug_tuple("")#(.field(&self.#key_indices))*.finish()
                }
            }
        });

        // Determine the cache field name.
        let cache_name = format_ident!("cached_{}", name);

        // Render a query tag that can be pushed onto the query stack to break
        // cycles.
        let tag_name = format_ident!("{}", name.to_string().to_camel_case());
        let doc = format!("The `{}` query.", name);
        tags.push(quote! {
            #[doc = #doc]
            #tag_name (#key_name #original_generics),
        });
        tag_debugs.push(quote! {
            QueryTag::#tag_name (x) => write!(f, "{}{:?}", stringify!(#name), x),
        });

        // Render the query for the database trait.
        funcs.push(quote! {
            #(#doc_attrs)*
            fn #name #generics (&self, #(#arg_names: #arg_types),*) -> #result {
                let query_storage = self.storage();
                let query_key = #key_name(#(#arg_names),*);
                let query_tag = QueryTag::#tag_name(query_key.clone());

                // Check if we already have a result for this query.
                if let Some(result) = query_storage.#cache_name.borrow().get(&query_key) {
                    trace!("Serving {}{:?} from cache", stringify!(#name), query_key);
                    return result.clone();
                }
                trace!("Executing {}{:?}", stringify!(#name), query_key);

                // Push the query onto the stack, checking if it is already in
                // flight.
                query_storage.stack.borrow_mut().push(query_tag.clone());
                if !query_storage.inflight.borrow_mut().insert(query_tag.clone()) {
                    self.handle_cycle();
                    // The above never returns.
                }

                // Execute the query.
                let query_result = #name(self.context(), #(#arg_names),*);
                query_storage.#cache_name.borrow_mut().insert(query_key, query_result.clone());

                // Pop the query from the stack.
                query_storage.inflight.borrow_mut().remove(&query_tag);
                query_storage.stack.borrow_mut().pop();
                query_result
            }
        });

        // Render the cache field for the storage struct.
        let doc = format!("Cached results of the `{}` query.", name);
        caches.push(quote! {
            #[doc = #doc]
            pub #cache_name: RefCell<HashMap<#key_name #original_generics, #result>>,
        });
    }

    // Extract query lifetimes.
    let lts = if ltset.is_empty() {
        quote! {}
    } else {
        let lts = ltset.iter();
        quote! { <#(#lts),*> }
    };

    // Generate the query database trait.
    output.extend(quote! {
        #input
        pub trait QueryDatabase #lts {
            /// The type passed as first argument to compiler query
            /// implementations.
            type Context: crate::Context #lts;

            /// Get the context that is passed to compiler query
            /// implementations.
            fn context(&self) -> &Self::Context;

            /// Get the query caches and runtime data.
            fn storage(&self) -> &QueryStorage #lts;

            /// Called when a query cycle is detected.
            fn handle_cycle(&self) -> ! {
                error!("Query cycle detected:");
                for x in self.storage().stack.borrow().iter() {
                    error!(" - {:?}", x);
                }
                panic!("query cycle detected");
            }

            #(#funcs)*
        }
    });

    // Generate the query storage struct.
    output.extend(quote! {
        /// A collection of query caches and runtime data for a `QueryDatabase`.
        #[derive(Default)]
        pub struct QueryStorage #lts {
            /// A stack of the currently-executing queries.
            pub stack: RefCell<Vec<QueryTag #lts>>,
            /// A set of the currently-executing queries.
            pub inflight: RefCell<HashSet<QueryTag #lts>>,

            #(#caches)*
        }
    });

    // Generate the query tag enum.
    output.extend(quote! {
        /// A tag identifying any of the queries in `QueryDatabase`.
        #[derive(Clone, Hash, Eq, PartialEq)]
        pub enum QueryTag #lts {
            #(#tags)*
        }

        impl #lts std::fmt::Debug for QueryTag #lts {
            fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
                match self {
                    #(#tag_debugs)*
                }
            }
        }
    });

    // Generate the query keys.
    output.extend(quote! {
        #(#keys)*
    });

    // Produce some output.
    // println!("{}", output);
    output.into()
}
<|endoftext|>


<commit_before>use app_units::Au;
use display_item::{DisplayItem, SpecificDisplayItem, ImageDisplayItem, WebGLDisplayItem};
use display_item::{RectangleDisplayItem, TextDisplayItem, GradientDisplayItem};
use display_item::{BorderDisplayItem, BoxShadowDisplayItem};
use euclid::{Point2D, Rect, Size2D};
use std::mem;
use types::{ClipRegion, ColorF, FontKey, ImageKey, PipelineId, StackingLevel};
use types::{BorderRadius, BorderSide, BoxShadowClipMode, GlyphInstance};
use types::{DisplayListMode, GradientStop, StackingContextId, ImageRendering};
use webgl::{WebGLContextId};

#[derive(Serialize, Deserialize)]
pub struct DrawListInfo {
    pub items: Vec<DisplayItem>,
}

#[derive(Serialize, Deserialize)]
pub struct StackingContextInfo {
    pub id: StackingContextId,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IframeInfo {
    pub id: PipelineId,
    pub offset: Point2D<f32>,
    pub clip: Rect<f32>,
}

#[derive(Serialize, Deserialize)]
pub enum SpecificDisplayListItem {
    DrawList(DrawListInfo),
    StackingContext(StackingContextInfo),
    Iframe(Box<IframeInfo>),
}

#[derive(Serialize, Deserialize)]
pub struct DisplayListItem {
    pub stacking_level: StackingLevel,
    pub specific: SpecificDisplayListItem,
}

#[derive(Serialize, Deserialize)]
pub struct DisplayListBuilder {
    pub mode: DisplayListMode,
    pub has_stacking_contexts: bool,

    pub work_background_and_borders: Vec<DisplayItem>,
    pub work_block_backgrounds_and_borders: Vec<DisplayItem>,
    pub work_floats: Vec<DisplayItem>,
    pub work_content: Vec<DisplayItem>,
    pub work_positioned_content: Vec<DisplayItem>,
    pub work_outlines: Vec<DisplayItem>,

    pub items: Vec<DisplayListItem>,
}

impl DisplayListBuilder {
    pub fn new() -> DisplayListBuilder {
        DisplayListBuilder {
            mode: DisplayListMode::Default,
            has_stacking_contexts: false,

            work_background_and_borders: Vec::new(),
            work_block_backgrounds_and_borders: Vec::new(),
            work_floats: Vec::new(),
            work_content: Vec::new(),
            work_positioned_content: Vec::new(),
            work_outlines: Vec::new(),

            items: Vec::new(),
        }
    }

    pub fn push_rect(&mut self,
                     level: StackingLevel,
                     rect: Rect<f32>,
                     clip: ClipRegion,
                     color: ColorF) {
        let item = RectangleDisplayItem {
            color: color,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::Rectangle(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_image(&mut self,
                      level: StackingLevel,
                      rect: Rect<f32>,
                      clip: ClipRegion,
                      stretch_size: Size2D<f32>,
                      image_rendering: ImageRendering,
                      key: ImageKey) {
        let item = ImageDisplayItem {
            image_key: key,
            stretch_size: stretch_size,
            image_rendering: image_rendering,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::Image(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_webgl_canvas(&mut self,
                             level: StackingLevel,
                             rect: Rect<f32>,
                             clip: ClipRegion,
                             context_id: WebGLContextId) {
        let item = WebGLDisplayItem {
            context_id: context_id,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::WebGL(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_text(&mut self,
                     level: StackingLevel,
                     rect: Rect<f32>,
                     clip: ClipRegion,
                     glyphs: Vec<GlyphInstance>,
                     font_key: FontKey,
                     color: ColorF,
                     size: Au,
                     blur_radius: Au) {
        // Sanity check - anything with glyphs bigger than this
        // is probably going to consume too much memory to render
        // efficiently anyway. This is specifically to work around
        // the font_advance.html reftest, which creates a very large
        // font as a crash test - the rendering is also ignored
        // by the azure renderer.
        if size < Au::from_px(4096) {
            let item = TextDisplayItem {
                color: color,
                glyphs: glyphs,
                font_key: font_key,
                size: size,
                blur_radius: blur_radius,
            };

            let display_item = DisplayItem {
                item: SpecificDisplayItem::Text(item),
                rect: rect,
                clip: clip,
            };

            self.push_item(level, display_item);
        }
    }

    pub fn push_border(&mut self,
                       level: StackingLevel,
                       rect: Rect<f32>,
                       clip: ClipRegion,
                       left: BorderSide,
                       top: BorderSide,
                       right: BorderSide,
                       bottom: BorderSide,
                       radius: BorderRadius) {
        let item = BorderDisplayItem {
            left: left,
            top: top,
            right: right,
            bottom: bottom,
            radius: radius,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::Border(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_box_shadow(&mut self,
                           level: StackingLevel,
                           rect: Rect<f32>,
                           clip: ClipRegion,
                           box_bounds: Rect<f32>,
                           offset: Point2D<f32>,
                           color: ColorF,
                           blur_radius: f32,
                           spread_radius: f32,
                           border_radius: f32,
                           clip_mode: BoxShadowClipMode) {
        let item = BoxShadowDisplayItem {
            box_bounds: box_bounds,
            offset: offset,
            color: color,
            blur_radius: blur_radius,
            spread_radius: spread_radius,
            border_radius: border_radius,
            clip_mode: clip_mode,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::BoxShadow(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_stacking_context(&mut self,
                                 level: StackingLevel,
                                 stacking_context_id: StackingContextId) {
        self.has_stacking_contexts = true;
        self.flush_list(level);
        let info = StackingContextInfo {
            id: stacking_context_id,
        };
        let item = DisplayListItem {
            stacking_level: level,
            specific: SpecificDisplayListItem::StackingContext(info),
        };
        self.items.push(item);
    }

    pub fn push_gradient(&mut self,
                         level: StackingLevel,
                         rect: Rect<f32>,
                         clip: ClipRegion,
                         start_point: Point2D<f32>,
                         end_point: Point2D<f32>,
                         stops: Vec<GradientStop>) {
        let item = GradientDisplayItem {
            start_point: start_point,
            end_point: end_point,
            stops: stops,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::Gradient(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_iframe(&mut self,
                       level: StackingLevel,
                       rect: Rect<f32>,
                       _clip: ClipRegion,
                       iframe: PipelineId) {
        self.flush_list(level);
        let info = Box::new(IframeInfo {
            id: iframe,
            offset: rect.origin,
            clip: rect,
        });
        let item = DisplayListItem {
            stacking_level: level,
            specific: SpecificDisplayListItem::Iframe(info),
        };
        self.items.push(item);
    }

    fn push_item(&mut self, level: StackingLevel, item: DisplayItem) {
        match level {
            StackingLevel::BackgroundAndBorders => {
                self.work_background_and_borders.push(item);
            }
            StackingLevel::BlockBackgroundAndBorders => {
                self.work_block_backgrounds_and_borders.push(item);
            }
            StackingLevel::Floats => {
                self.work_floats.push(item);
            }
            StackingLevel::Content => {
                self.work_content.push(item);
            }
            StackingLevel::PositionedContent => {
                self.work_positioned_content.push(item);
            }
            StackingLevel::Outlines => {
                self.work_outlines.push(item);
            }
        }
    }

    fn flush_list(&mut self, level: StackingLevel) {
        let list = match level {
            StackingLevel::BackgroundAndBorders => {
                &mut self.work_background_and_borders
            }
            StackingLevel::BlockBackgroundAndBorders => {
                &mut self.work_block_backgrounds_and_borders
            }
            StackingLevel::Floats => {
                &mut self.work_floats
            }
            StackingLevel::Content => {
                &mut self.work_content
            }
            StackingLevel::PositionedContent => {
                &mut self.work_positioned_content
            }
            StackingLevel::Outlines => {
                &mut self.work_outlines
            }
        };

        let items = mem::replace(list, Vec::new());
        if items.len() > 0 {
            let draw_list = DrawListInfo {
                items: items,
            };
            self.items.push(DisplayListItem {
                stacking_level: level,
                specific: SpecificDisplayListItem::DrawList(draw_list),
            });
        }
    }

    fn flush(&mut self) {
        self.flush_list(StackingLevel::BackgroundAndBorders);
        self.flush_list(StackingLevel::BlockBackgroundAndBorders);
        self.flush_list(StackingLevel::Floats);
        self.flush_list(StackingLevel::Content);
        self.flush_list(StackingLevel::PositionedContent);
        self.flush_list(StackingLevel::Outlines);
    }

    pub fn finalize(&mut self) {
        self.flush();
    }
}
<commit_msg>Update api for WR changes<commit_after>use app_units::Au;
use display_item::{DisplayItem, SpecificDisplayItem, ImageDisplayItem, WebGLDisplayItem};
use display_item::{RectangleDisplayItem, TextDisplayItem, GradientDisplayItem};
use display_item::{BorderDisplayItem, BoxShadowDisplayItem};
use euclid::{Point2D, Rect, Size2D};
use std::mem;
use types::{ClipRegion, ColorF, FontKey, ImageKey, PipelineId, StackingLevel};
use types::{BorderRadius, BorderSide, BoxShadowClipMode, GlyphInstance};
use types::{DisplayListMode, GradientStop, StackingContextId, ImageRendering};
use webgl::{WebGLContextId};

#[derive(Serialize, Deserialize)]
pub struct DrawListInfo {
    pub items: Vec<DisplayItem>,
}

#[derive(Serialize, Deserialize)]
pub struct StackingContextInfo {
    pub id: StackingContextId,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IframeInfo {
    pub id: PipelineId,
    pub bounds: Rect<f32>,
}

#[derive(Serialize, Deserialize)]
pub enum SpecificDisplayListItem {
    DrawList(DrawListInfo),
    StackingContext(StackingContextInfo),
    Iframe(Box<IframeInfo>),
}

#[derive(Serialize, Deserialize)]
pub struct DisplayListItem {
    pub stacking_level: StackingLevel,
    pub specific: SpecificDisplayListItem,
}

#[derive(Serialize, Deserialize)]
pub struct DisplayListBuilder {
    pub mode: DisplayListMode,
    pub has_stacking_contexts: bool,

    pub work_background_and_borders: Vec<DisplayItem>,
    pub work_block_backgrounds_and_borders: Vec<DisplayItem>,
    pub work_floats: Vec<DisplayItem>,
    pub work_content: Vec<DisplayItem>,
    pub work_positioned_content: Vec<DisplayItem>,
    pub work_outlines: Vec<DisplayItem>,

    pub items: Vec<DisplayListItem>,
}

impl DisplayListBuilder {
    pub fn new() -> DisplayListBuilder {
        DisplayListBuilder {
            mode: DisplayListMode::Default,
            has_stacking_contexts: false,

            work_background_and_borders: Vec::new(),
            work_block_backgrounds_and_borders: Vec::new(),
            work_floats: Vec::new(),
            work_content: Vec::new(),
            work_positioned_content: Vec::new(),
            work_outlines: Vec::new(),

            items: Vec::new(),
        }
    }

    pub fn push_rect(&mut self,
                     level: StackingLevel,
                     rect: Rect<f32>,
                     clip: ClipRegion,
                     color: ColorF) {
        let item = RectangleDisplayItem {
            color: color,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::Rectangle(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_image(&mut self,
                      level: StackingLevel,
                      rect: Rect<f32>,
                      clip: ClipRegion,
                      stretch_size: Size2D<f32>,
                      image_rendering: ImageRendering,
                      key: ImageKey) {
        let item = ImageDisplayItem {
            image_key: key,
            stretch_size: stretch_size,
            image_rendering: image_rendering,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::Image(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_webgl_canvas(&mut self,
                             level: StackingLevel,
                             rect: Rect<f32>,
                             clip: ClipRegion,
                             context_id: WebGLContextId) {
        let item = WebGLDisplayItem {
            context_id: context_id,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::WebGL(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_text(&mut self,
                     level: StackingLevel,
                     rect: Rect<f32>,
                     clip: ClipRegion,
                     glyphs: Vec<GlyphInstance>,
                     font_key: FontKey,
                     color: ColorF,
                     size: Au,
                     blur_radius: Au) {
        // Sanity check - anything with glyphs bigger than this
        // is probably going to consume too much memory to render
        // efficiently anyway. This is specifically to work around
        // the font_advance.html reftest, which creates a very large
        // font as a crash test - the rendering is also ignored
        // by the azure renderer.
        if size < Au::from_px(4096) {
            let item = TextDisplayItem {
                color: color,
                glyphs: glyphs,
                font_key: font_key,
                size: size,
                blur_radius: blur_radius,
            };

            let display_item = DisplayItem {
                item: SpecificDisplayItem::Text(item),
                rect: rect,
                clip: clip,
            };

            self.push_item(level, display_item);
        }
    }

    pub fn push_border(&mut self,
                       level: StackingLevel,
                       rect: Rect<f32>,
                       clip: ClipRegion,
                       left: BorderSide,
                       top: BorderSide,
                       right: BorderSide,
                       bottom: BorderSide,
                       radius: BorderRadius) {
        let item = BorderDisplayItem {
            left: left,
            top: top,
            right: right,
            bottom: bottom,
            radius: radius,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::Border(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_box_shadow(&mut self,
                           level: StackingLevel,
                           rect: Rect<f32>,
                           clip: ClipRegion,
                           box_bounds: Rect<f32>,
                           offset: Point2D<f32>,
                           color: ColorF,
                           blur_radius: f32,
                           spread_radius: f32,
                           border_radius: f32,
                           clip_mode: BoxShadowClipMode) {
        let item = BoxShadowDisplayItem {
            box_bounds: box_bounds,
            offset: offset,
            color: color,
            blur_radius: blur_radius,
            spread_radius: spread_radius,
            border_radius: border_radius,
            clip_mode: clip_mode,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::BoxShadow(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_stacking_context(&mut self,
                                 level: StackingLevel,
                                 stacking_context_id: StackingContextId) {
        self.has_stacking_contexts = true;
        self.flush_list(level);
        let info = StackingContextInfo {
            id: stacking_context_id,
        };
        let item = DisplayListItem {
            stacking_level: level,
            specific: SpecificDisplayListItem::StackingContext(info),
        };
        self.items.push(item);
    }

    pub fn push_gradient(&mut self,
                         level: StackingLevel,
                         rect: Rect<f32>,
                         clip: ClipRegion,
                         start_point: Point2D<f32>,
                         end_point: Point2D<f32>,
                         stops: Vec<GradientStop>) {
        let item = GradientDisplayItem {
            start_point: start_point,
            end_point: end_point,
            stops: stops,
        };

        let display_item = DisplayItem {
            item: SpecificDisplayItem::Gradient(item),
            rect: rect,
            clip: clip,
        };

        self.push_item(level, display_item);
    }

    pub fn push_iframe(&mut self,
                       level: StackingLevel,
                       rect: Rect<f32>,
                       _clip: ClipRegion,
                       iframe: PipelineId) {
        self.flush_list(level);
        let info = Box::new(IframeInfo {
            id: iframe,
            bounds: rect,
        });
        let item = DisplayListItem {
            stacking_level: level,
            specific: SpecificDisplayListItem::Iframe(info),
        };
        self.items.push(item);
    }

    fn push_item(&mut self, level: StackingLevel, item: DisplayItem) {
        match level {
            StackingLevel::BackgroundAndBorders => {
                self.work_background_and_borders.push(item);
            }
            StackingLevel::BlockBackgroundAndBorders => {
                self.work_block_backgrounds_and_borders.push(item);
            }
            StackingLevel::Floats => {
                self.work_floats.push(item);
            }
            StackingLevel::Content => {
                self.work_content.push(item);
            }
            StackingLevel::PositionedContent => {
                self.work_positioned_content.push(item);
            }
            StackingLevel::Outlines => {
                self.work_outlines.push(item);
            }
        }
    }

    fn flush_list(&mut self, level: StackingLevel) {
        let list = match level {
            StackingLevel::BackgroundAndBorders => {
                &mut self.work_background_and_borders
            }
            StackingLevel::BlockBackgroundAndBorders => {
                &mut self.work_block_backgrounds_and_borders
            }
            StackingLevel::Floats => {
                &mut self.work_floats
            }
            StackingLevel::Content => {
                &mut self.work_content
            }
            StackingLevel::PositionedContent => {
                &mut self.work_positioned_content
            }
            StackingLevel::Outlines => {
                &mut self.work_outlines
            }
        };

        let items = mem::replace(list, Vec::new());
        if items.len() > 0 {
            let draw_list = DrawListInfo {
                items: items,
            };
            self.items.push(DisplayListItem {
                stacking_level: level,
                specific: SpecificDisplayListItem::DrawList(draw_list),
            });
        }
    }

    fn flush(&mut self) {
        self.flush_list(StackingLevel::BackgroundAndBorders);
        self.flush_list(StackingLevel::BlockBackgroundAndBorders);
        self.flush_list(StackingLevel::Floats);
        self.flush_list(StackingLevel::Content);
        self.flush_list(StackingLevel::PositionedContent);
        self.flush_list(StackingLevel::Outlines);
    }

    pub fn finalize(&mut self) {
        self.flush();
    }
}
<|endoftext|>


<commit_before>/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0.
 */

// snippet-start:[rust.example_code.s3.large_files.scenario]

use std::convert::TryInto;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

use aws_config::meta::region::RegionProviderChain;
use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart};
use aws_sdk_s3::output::{CreateMultipartUploadOutput, GetObjectOutput};
use aws_sdk_s3::{Client as S3Client, Error, Region};
use aws_smithy_http::byte_stream::{ByteStream, Length};
use rand::distributions::Alphanumeric;
use rand::{thread_rng, Rng};
use uuid::Uuid;

//In bytes, minimum chunk size of 5MB. Increase CHUNK_SIZE to send larger chunks.
const CHUNK_SIZE: u64 = 1024 * 1024 * 5;
const MAX_CHUNKS: u64 = 10000;

#[tokio::main]
pub async fn main() -> Result<(), Error> {
    let shared_config = aws_config::load_from_env().await;
    let client = S3Client::new(&shared_config);

    let bucket_name = format!("doc-example-bucket-{}", Uuid::new_v4());
    let region_provider = RegionProviderChain::first_try(Region::new("us-west-2"));
    let region = region_provider.region().await.unwrap();
    s3_service::create_bucket(&client, &bucket_name, region.as_ref()).await?;

    let key = "sample.txt".to_string();
    // snippet-start:[rust.example_code.s3.create_multipart_upload]
    let multipart_upload_res: CreateMultipartUploadOutput = client
        .create_multipart_upload()
        .bucket(&bucket_name)
        .key(&key)
        .send()
        .await
        .unwrap();
    // snippet-end:[rust.example_code.s3.create_multipart_upload]
    let upload_id = multipart_upload_res.upload_id().unwrap();

    //Create a file of random characters for the upload.
    let mut file = File::create(&key).expect("Could not create sample file.");
    loop {
        let rand_string: String = thread_rng()
            .sample_iter(&Alphanumeric)
            .take(256)
            .map(char::from)
            .collect();
        let return_string: String = "\n".to_string();
        file.write_all(rand_string.as_ref())
            .expect("Error writing to file.");
        file.write_all(return_string.as_ref())
            .expect("Error writing to file.");
        if file.metadata().unwrap().len() > CHUNK_SIZE * 4 {
            // Create a 5 chunk file.
            break;
        }
    }

    let path = Path::new(&key);
    let file_size = tokio::fs::metadata(path)
        .await
        .expect("it exists I swear")
        .len();

    let mut chunk_count = (file_size / CHUNK_SIZE) + 1;
    let mut size_of_last_chunk = file_size % CHUNK_SIZE;
    if size_of_last_chunk == 0 {
        size_of_last_chunk = CHUNK_SIZE;
        chunk_count -= 1;
    }

    if file_size == 0 {
        panic!("Bad file size.");
    }
    if chunk_count > MAX_CHUNKS {
        panic!("Too many chunks! Try increasing your chunk size.")
    }

    let mut upload_parts: Vec<CompletedPart> = Vec::new();

    for chunk_index in 0..chunk_count {
        let this_chunk = if chunk_count - 1 == chunk_index {
            size_of_last_chunk
        } else {
            CHUNK_SIZE
        };
        let stream = ByteStream::read_from()
            .path(path)
            .offset(chunk_index * CHUNK_SIZE)
            .length(Length::Exact(this_chunk))
            .build()
            .await
            .unwrap();
        //Chunk index needs to start at 0, but part numbers start at 1.
        let part_number = (chunk_index as i32) + 1;
        // snippet-start:[rust.example_code.s3.upload_part]
        let upload_part_res = client
            .upload_part()
            .key(&key)
            .bucket(&bucket_name)
            .upload_id(upload_id)
            .body(stream)
            .part_number(part_number)
            .send()
            .await?;
        upload_parts.push(
            CompletedPart::builder()
                .e_tag(upload_part_res.e_tag.unwrap_or_default())
                .part_number(part_number)
                .build(),
        );
        // snippet-end:[rust.example_code.s3.upload_part]
    }
    // snippet-start:[rust.example_code.s3.upload_part.CompletedMultipartUpload]
    let completed_multipart_upload: CompletedMultipartUpload = CompletedMultipartUpload::builder()
        .set_parts(Some(upload_parts))
        .build();
    // snippet-end:[rust.example_code.s3.upload_part.CompletedMultipartUpload]

    // snippet-start:[rust.example_code.s3.complete_multipart_upload]
    let _complete_multipart_upload_res = match client
        .complete_multipart_upload()
        .bucket(&bucket_name)
        .key(&key)
        .multipart_upload(completed_multipart_upload)
        .upload_id(upload_id)
        .send()
        .await
    {
        Ok(res) => res,
        Err(e) => {
            println!("{:?}", e);
            panic!();
        }
    };
    // snippet-end:[rust.example_code.s3.complete_multipart_upload]

    let data: GetObjectOutput = s3_service::download_object(&client, &bucket_name, &key).await;
    let data_length: u64 = data.content_length().try_into().unwrap();
    if file.metadata().unwrap().len() == data_length {
        println!("Data lengths match.");
    } else {
        println!("The data was not the same size!");
    }

    s3_service::delete_objects(&client, &bucket_name)
        .await
        .expect("Error emptying bucket.");
    s3_service::delete_bucket(&client, &bucket_name)
        .await
        .expect("Error deleting bucket.");

    Ok(())
}
// snippet-end:[rust.example_code.s3.large_files.scenario]
<commit_msg>Review suggestions.<commit_after>/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0.
 */

// snippet-start:[rust.example_code.s3.large_files.scenario]

use std::convert::TryInto;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

use aws_config::meta::region::RegionProviderChain;
use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart};
use aws_sdk_s3::output::{CreateMultipartUploadOutput, GetObjectOutput};
use aws_sdk_s3::{Client as S3Client, Error, Region};
use aws_smithy_http::byte_stream::{ByteStream, Length};
use rand::distributions::Alphanumeric;
use rand::{thread_rng, Rng};
use uuid::Uuid;

//In bytes, minimum chunk size of 5MB. Increase CHUNK_SIZE to send larger chunks.
const CHUNK_SIZE: u64 = 1024 * 1024 * 5;
const MAX_CHUNKS: u64 = 10000;

#[tokio::main]
pub async fn main() -> Result<(), Error> {
    let shared_config = aws_config::load_from_env().await;
    let client = S3Client::new(&shared_config);

    let bucket_name = format!("doc-example-bucket-{}", Uuid::new_v4());
    let region_provider = RegionProviderChain::first_try(Region::new("us-west-2"));
    let region = region_provider.region().await.unwrap();
    s3_service::create_bucket(&client, &bucket_name, region.as_ref()).await?;

    let key = "sample.txt".to_string();
    // snippet-start:[rust.example_code.s3.create_multipart_upload]
    let multipart_upload_res: CreateMultipartUploadOutput = client
        .create_multipart_upload()
        .bucket(&bucket_name)
        .key(&key)
        .send()
        .await
        .unwrap();
    // snippet-end:[rust.example_code.s3.create_multipart_upload]
    let upload_id = multipart_upload_res.upload_id().unwrap();

    //Create a file of random characters for the upload.
    let mut file = File::create(&key).expect("Could not create sample file.");
    // Loop until the file is 5 chunks.
    while file.metadata().unwrap().len() <= CHUNK_SIZE * 4 {
        let rand_string: String = thread_rng()
            .sample_iter(&Alphanumeric)
            .take(256)
            .map(char::from)
            .collect();
        let return_string: String = "\n".to_string();
        file.write_all(rand_string.as_ref())
            .expect("Error writing to file.");
        file.write_all(return_string.as_ref())
            .expect("Error writing to file.");
    }

    let path = Path::new(&key);
    let file_size = tokio::fs::metadata(path)
        .await
        .expect("it exists I swear")
        .len();

    let mut chunk_count = (file_size / CHUNK_SIZE) + 1;
    let mut size_of_last_chunk = file_size % CHUNK_SIZE;
    if size_of_last_chunk == 0 {
        size_of_last_chunk = CHUNK_SIZE;
        chunk_count -= 1;
    }

    if file_size == 0 {
        panic!("Bad file size.");
    }
    if chunk_count > MAX_CHUNKS {
        panic!("Too many chunks! Try increasing your chunk size.")
    }

    let mut upload_parts: Vec<CompletedPart> = Vec::new();

    for chunk_index in 0..chunk_count {
        let this_chunk = if chunk_count - 1 == chunk_index {
            size_of_last_chunk
        } else {
            CHUNK_SIZE
        };
        let stream = ByteStream::read_from()
            .path(path)
            .offset(chunk_index * CHUNK_SIZE)
            .length(Length::Exact(this_chunk))
            .build()
            .await
            .unwrap();
        //Chunk index needs to start at 0, but part numbers start at 1.
        let part_number = (chunk_index as i32) + 1;
        // snippet-start:[rust.example_code.s3.upload_part]
        let upload_part_res = client
            .upload_part()
            .key(&key)
            .bucket(&bucket_name)
            .upload_id(upload_id)
            .body(stream)
            .part_number(part_number)
            .send()
            .await?;
        upload_parts.push(
            CompletedPart::builder()
                .e_tag(upload_part_res.e_tag.unwrap_or_default())
                .part_number(part_number)
                .build(),
        );
        // snippet-end:[rust.example_code.s3.upload_part]
    }
    // snippet-start:[rust.example_code.s3.upload_part.CompletedMultipartUpload]
    let completed_multipart_upload: CompletedMultipartUpload = CompletedMultipartUpload::builder()
        .set_parts(Some(upload_parts))
        .build();
    // snippet-end:[rust.example_code.s3.upload_part.CompletedMultipartUpload]

    // snippet-start:[rust.example_code.s3.complete_multipart_upload]
    let _complete_multipart_upload_res = client
        .complete_multipart_upload()
        .bucket(&bucket_name)
        .key(&key)
        .multipart_upload(completed_multipart_upload)
        .upload_id(upload_id)
        .send()
        .await.unwrap();
    // snippet-end:[rust.example_code.s3.complete_multipart_upload]

    let data: GetObjectOutput = s3_service::download_object(&client, &bucket_name, &key).await;
    let data_length: u64 = data.content_length().try_into().unwrap();
    if file.metadata().unwrap().len() == data_length {
        println!("Data lengths match.");
    } else {
        println!("The data was not the same size!");
    }

    s3_service::delete_objects(&client, &bucket_name)
        .await
        .expect("Error emptying bucket.");
    s3_service::delete_bucket(&client, &bucket_name)
        .await
        .expect("Error deleting bucket.");

    Ok(())
}
// snippet-end:[rust.example_code.s3.large_files.scenario]
<|endoftext|>