<commit_before>//! Types in this module are mostly internal and automatically generated. You
//! shouldn't need to interact with these types during normal usage, other than
//! the methods on [`Table`](/diesel/query_source/trait.Table.html)
#[doc(hidden)]
pub mod joins;
mod peano_numbers;

use std::error::Error;

use backend::Backend;
use expression::{Expression, NonAggregate, SelectableExpression};
use query_builder::*;
use row::NamedRow;
use types::{FromSqlRow, HasSqlType};

pub use self::joins::JoinTo;
pub use self::peano_numbers::*;

/// Trait indicating that a record can be queried from the database. This trait
/// can be derived automatically using `#[derive(Queryable)]`. This trait can
/// only be derived for structs, not enums.
pub trait Queryable<ST, DB>
where
    DB: Backend + HasSqlType<ST>,
{
    type Row: FromSqlRow<ST, DB>;

    fn build(row: Self::Row) -> Self;
}

// Reasons we can't write this:
//
// impl<T, ST, DB> Queryable<ST, DB> for T
// where
//     DB: Backend + HasSqlType<ST>,
//     T: FromSqlRow<ST, DB>,
// {
//     type Row = Self;
//
//     fn build(row: Self::Row) -> Self {
//         row
//     }
// }
//
// (this is mostly a reference for @sgrif so he has a better reference every
// time he thinks he has a breakthrough on this problem).
//
// See the comment under `FromSqlRow`. All of the same impls conflict there that
// would here. If we had `#[derive(FromSqlRow)]`, we would also have that
// implement `Queryable`. I know it doesn't look like the `Option` impl
// conflicts, but it definitely does -- It covers types which implement
// `Queryable` but not `FromSqlRow`, while this impl wouldn't.
//
// The same "we could remove one of these traits" applies. Really `FromSqlRow`
// is the only trait that *needs* to exist. At the end of the day, `FromSql` is
// meant to be "I only want to deal with deserializing a single field" case, and
// `Queryable` is meant to be easier to implement by hand than `FromSqlRow`
// would be.

/// Deserializes the result of a query constructed with [`sql_query`].
///
/// # Deriving
///
/// To derive this trait, Diesel needs to know the SQL type of each field. You
/// can do this by either annotating your struct with `#[table_name =
/// "some_table"]` (in which case the SQL type will be
/// `diesel::dsl::SqlTypeOf<table_name::column_name>`), or by annotating each
/// field with `#[sql_type = "SomeType"]`.
///
/// If you are using `#[table_name]`, the module for that table must be in
/// scope. For example, to derive this for a struct called `User`, you will
/// likely need a line such as `use schema::users;`
///
/// If the name of a field on your struct is different than the column in your
/// `table!` declaration, or if you are deriving this trait on a tuple struct,
/// you can annotate the field with `#[column_name = "some_column"]`. For tuple
/// structs, all fields must have this annotation.
///
/// If a field is another struct which implements `QueryableByName`, instead of
/// a column, you can annotate that struct with `#[diesel(embed)]`
///
/// [`sql_query`]: ../fn.sql_query.html
pub trait QueryableByName<DB>
where
    Self: Sized,
    DB: Backend,
{
    fn build<R: NamedRow<DB>>(row: &R) -> Result<Self, Box<Error + Send + Sync>>;
}

#[doc(hidden)]
pub trait QuerySource {
    type FromClause;
    type DefaultSelection: SelectableExpression<Self>;

    fn from_clause(&self) -> Self::FromClause;
    fn default_selection(&self) -> Self::DefaultSelection;
}

/// A column on a database table. Types which implement this trait should have
/// been generated by the [`table!` macro](../macro.table.html).
pub trait Column: Expression {
    type Table: Table;

    const NAME: &'static str;
}

/// A SQL database table. Types which implement this trait should have been
/// generated by the [`table!` macro](../macro.table.html).
pub trait Table: QuerySource + AsQuery + Sized {
    type PrimaryKey: SelectableExpression<Self> + NonAggregate;
    type AllColumns: SelectableExpression<Self> + NonAggregate;

    fn primary_key(&self) -> Self::PrimaryKey;
    fn all_columns() -> Self::AllColumns;
}

/// Determines how many times `Self` appears in `QS`
///
/// This trait is primarily used to determine whether or not a column is
/// selectable from a given from clause. A column can be selected if its table
/// appears in the from clause *exactly once*.
///
/// We do not allow the same table to appear in a query multiple times in any
/// context where referencing that table would be ambiguous (depending on the
/// context and backend being used, this may or may not be something that would
/// otherwise result in a runtime error).
pub trait AppearsInFromClause<QS> {
    /// How many times does `Self` appear in `QS`?
    type Count;
}
<commit_msg>Document `Queryable`<commit_after>//! Types in this module are mostly internal and automatically generated. You
//! shouldn't need to interact with these types during normal usage, other than
//! the methods on [`Table`](/diesel/query_source/trait.Table.html)
#[doc(hidden)]
pub mod joins;
mod peano_numbers;

use std::error::Error;

use backend::Backend;
use expression::{Expression, NonAggregate, SelectableExpression};
use query_builder::*;
use row::NamedRow;
use types::{FromSqlRow, HasSqlType};

pub use self::joins::JoinTo;
pub use self::peano_numbers::*;

/// Trait indicating that a record can be queried from the database.
///
/// Types which implement `Queryable` represent the result of a SQL query. This
/// does not necessarily mean they represent a single database table.
///
/// This trait can be derived automatically using `#[derive(Queryable)]`. This
/// trait can only be derived for structs, not enums.
///
/// Diesel represents the return type of a query as a tuple. The purpose of this
/// trait is to convert from a tuple of Rust values that have been deserialized
/// into your struct.
///
/// When this trait is derived, it will assume that the order of fields on your
/// struct match the order of the fields in the query. This means that field
/// order is significant if you are using `#[derive(Queryable)]`. Field name has
/// no affect.
///
/// # Examples
///
/// If we just want to map a query to our struct, we can use `derive`.
///
/// ```rust
/// # #[macro_use] extern crate diesel;
/// # include!("../doctest_setup.rs");
/// #
/// #[derive(Queryable, PartialEq, Debug)]
/// struct User {
///     id: i32,
///     name: String,
/// }
///
/// # fn main() {
/// #     run_test();
/// # }
/// #
/// # fn run_test() -> QueryResult<()> {
/// #     use schema::users::dsl::*;
/// #     let connection = establish_connection();
/// let first_user = users.first(&connection)?;
/// let expected = User { id: 1, name: "Sean".into() };
/// assert_eq!(expected, first_user);
/// #     Ok(())
/// # }
/// ```
///
/// If we want to do additional work during deserializaiton, we can implement
/// the trait ourselves.
///
/// ```rust
/// # #[macro_use] extern crate diesel;
/// # include!("../doctest_setup.rs");
/// #
/// use schema::users;
/// use diesel::query_source::Queryable;
///
/// # /*
/// type DB = diesel::sqlite::Sqlite;
/// # */
///
/// #[derive(PartialEq, Debug)]
/// struct User {
///     id: i32,
///     name: String,
/// }
///
/// impl Queryable<users::SqlType, DB> for User {
///     type Row = (i32, String);
///
///     fn build(row: Self::Row) -> Self {
///         User {
///             id: row.0,
///             name: row.1.to_lowercase(),
///         }
///     }
/// }
///
/// # fn main() {
/// #     run_test();
/// # }
/// #
/// # fn run_test() -> QueryResult<()> {
/// #     use schema::users::dsl::*;
/// #     let connection = establish_connection();
/// let first_user = users.first(&connection)?;
/// let expected = User { id: 1, name: "sean".into() };
/// assert_eq!(expected, first_user);
/// #     Ok(())
/// # }
pub trait Queryable<ST, DB>
where
    DB: Backend + HasSqlType<ST>,
{
    /// The Rust type you'd like to map from.
    ///
    /// This is typically a tuple of all of your struct's fields.
    type Row: FromSqlRow<ST, DB>;

    /// Construct an instance of this type
    fn build(row: Self::Row) -> Self;
}

// Reasons we can't write this:
//
// impl<T, ST, DB> Queryable<ST, DB> for T
// where
//     DB: Backend + HasSqlType<ST>,
//     T: FromSqlRow<ST, DB>,
// {
//     type Row = Self;
//
//     fn build(row: Self::Row) -> Self {
//         row
//     }
// }
//
// (this is mostly a reference for @sgrif so he has a better reference every
// time he thinks he has a breakthrough on this problem).
//
// See the comment under `FromSqlRow`. All of the same impls conflict there that
// would here. If we had `#[derive(FromSqlRow)]`, we would also have that
// implement `Queryable`. I know it doesn't look like the `Option` impl
// conflicts, but it definitely does -- It covers types which implement
// `Queryable` but not `FromSqlRow`, while this impl wouldn't.
//
// The same "we could remove one of these traits" applies. Really `FromSqlRow`
// is the only trait that *needs* to exist. At the end of the day, `FromSql` is
// meant to be "I only want to deal with deserializing a single field" case, and
// `Queryable` is meant to be easier to implement by hand than `FromSqlRow`
// would be.

/// Deserializes the result of a query constructed with [`sql_query`].
///
/// # Deriving
///
/// To derive this trait, Diesel needs to know the SQL type of each field. You
/// can do this by either annotating your struct with `#[table_name =
/// "some_table"]` (in which case the SQL type will be
/// `diesel::dsl::SqlTypeOf<table_name::column_name>`), or by annotating each
/// field with `#[sql_type = "SomeType"]`.
///
/// If you are using `#[table_name]`, the module for that table must be in
/// scope. For example, to derive this for a struct called `User`, you will
/// likely need a line such as `use schema::users;`
///
/// If the name of a field on your struct is different than the column in your
/// `table!` declaration, or if you are deriving this trait on a tuple struct,
/// you can annotate the field with `#[column_name = "some_column"]`. For tuple
/// structs, all fields must have this annotation.
///
/// If a field is another struct which implements `QueryableByName`, instead of
/// a column, you can annotate that struct with `#[diesel(embed)]`
///
/// [`sql_query`]: ../fn.sql_query.html
pub trait QueryableByName<DB>
where
    Self: Sized,
    DB: Backend,
{
    fn build<R: NamedRow<DB>>(row: &R) -> Result<Self, Box<Error + Send + Sync>>;
}

#[doc(hidden)]
pub trait QuerySource {
    type FromClause;
    type DefaultSelection: SelectableExpression<Self>;

    fn from_clause(&self) -> Self::FromClause;
    fn default_selection(&self) -> Self::DefaultSelection;
}

/// A column on a database table. Types which implement this trait should have
/// been generated by the [`table!` macro](../macro.table.html).
pub trait Column: Expression {
    type Table: Table;

    const NAME: &'static str;
}

/// A SQL database table. Types which implement this trait should have been
/// generated by the [`table!` macro](../macro.table.html).
pub trait Table: QuerySource + AsQuery + Sized {
    type PrimaryKey: SelectableExpression<Self> + NonAggregate;
    type AllColumns: SelectableExpression<Self> + NonAggregate;

    fn primary_key(&self) -> Self::PrimaryKey;
    fn all_columns() -> Self::AllColumns;
}

/// Determines how many times `Self` appears in `QS`
///
/// This trait is primarily used to determine whether or not a column is
/// selectable from a given from clause. A column can be selected if its table
/// appears in the from clause *exactly once*.
///
/// We do not allow the same table to appear in a query multiple times in any
/// context where referencing that table would be ambiguous (depending on the
/// context and backend being used, this may or may not be something that would
/// otherwise result in a runtime error).
pub trait AppearsInFromClause<QS> {
    /// How many times does `Self` appear in `QS`?
    type Count;
}
<|endoftext|>


<commit_before>use std::collections::VecDeque;
use std::fmt;
use term;
use time::precise_time_s;
use rustup::Notification;
use rustup_dist::Notification as In;
use rustup_utils::Notification as Un;
use tty;

/// Keep track of this many past download amounts
const DOWNLOAD_TRACK_COUNT: usize = 5;

/// Tracks download progress and displays information about it to a terminal.
pub struct DownloadTracker {
    /// Content-Length of the to-be downloaded object.
    content_len: Option<u64>,
    /// Total data downloaded in bytes.
    total_downloaded: usize,
    /// Data downloaded this second.
    downloaded_this_sec: usize,
    /// Keeps track of amount of data downloaded every last few secs.
    /// Used for averaging the download speed.
    downloaded_last_few_secs: VecDeque<usize>,
    /// Time stamp of the last second
    last_sec: Option<f64>,
    /// How many seconds have elapsed since the download started
    seconds_elapsed: u32,
    /// The terminal we write the information to.
    term: Option<Box<term::StdoutTerminal>>,
    /// Whether we displayed progress for the download or not.
    ///
    /// If the download is quick enough, we don't have time to
    /// display the progress info.
    /// In that case, we do not want to do some cleanup stuff we normally do.
    displayed_progress: bool,
}

impl DownloadTracker {
    /// Creates a new DownloadTracker.
    pub fn new() -> Self {
        DownloadTracker {
            content_len: None,
            total_downloaded: 0,
            downloaded_this_sec: 0,
            downloaded_last_few_secs: VecDeque::with_capacity(DOWNLOAD_TRACK_COUNT),
            seconds_elapsed: 0,
            last_sec: None,
            term: term::stdout(),
            displayed_progress: false,
        }
    }

    pub fn handle_notification(&mut self, n: &Notification) -> bool {
        match n {
            &Notification::Install(In::Utils(Un::DownloadContentLengthReceived(content_len))) => {
                self.content_length_received(content_len);

                true
            }
            &Notification::Install(In::Utils(Un::DownloadDataReceived(len))) => {
                if tty::stdout_isatty() && self.term.is_some() {
                    self.data_received(len);
                }
                true
            }
            &Notification::Install(In::Utils(Un::DownloadFinished)) => {
                self.download_finished();
                true
            }
            _ => false
        }
    }

    /// Notifies self that Content-Length information has been received.
    pub fn content_length_received(&mut self, content_len: u64) {
        self.content_len = Some(content_len);
    }
    /// Notifies self that data of size `len` has been received.
    pub fn data_received(&mut self, len: usize) {
        self.total_downloaded += len;
        self.downloaded_this_sec += len;

        let current_time = precise_time_s();

        match self.last_sec {
            None => self.last_sec = Some(current_time),
            Some(start) => {
                let elapsed = current_time - start;
                if elapsed >= 1.0 {
                    self.seconds_elapsed += 1;

                    self.display();
                    self.last_sec = Some(current_time);
                    if self.downloaded_last_few_secs.len() == DOWNLOAD_TRACK_COUNT {
                        self.downloaded_last_few_secs.pop_back();
                    }
                    self.downloaded_last_few_secs.push_front(self.downloaded_this_sec);
                    self.downloaded_this_sec = 0;
                }
            }
        }
    }
    /// Notifies self that the download has finished.
    pub fn download_finished(&mut self) {
        if self.displayed_progress {
            // Display the finished state
            self.display();
            let _ = writeln!(self.term.as_mut().unwrap(), "");
        }
        self.prepare_for_new_download();
    }
    /// Resets the state to be ready for a new download.
    fn prepare_for_new_download(&mut self) {
        self.content_len = None;
        self.total_downloaded = 0;
        self.downloaded_this_sec = 0;
        self.downloaded_last_few_secs.clear();
        self.seconds_elapsed = 0;
        self.last_sec = None;
        self.displayed_progress = false;
    }
    /// Display the tracked download information to the terminal.
    fn display(&mut self) {
        let total_h = HumanReadable(self.total_downloaded as u64);
        let sum = self.downloaded_last_few_secs
                      .iter()
                      .fold(0usize, |a, &v| a + v);
        let len = self.downloaded_last_few_secs.len();
        let speed = if len > 0 {
            (sum / len) as u64
        } else {
            0
        };
        let speed_h = HumanReadable(speed);

        match self.content_len {
            Some(content_len) => {
                use std::borrow::Cow;

                let percent = (self.total_downloaded as f64 / content_len as f64) * 100.;
                let content_len_h = HumanReadable(content_len);
                let remaining = content_len - self.total_downloaded as u64;
                let eta = if speed > 0 {
                    Cow::Owned(format!("{}s", remaining / speed))
                } else {
                    Cow::Borrowed("Unknown")
                };
                let _ = write!(self.term.as_mut().unwrap(),
                               "{} / {} ({:.2}%) {}/s ETA: {}",
                               total_h,
                               content_len_h,
                               percent,
                               speed_h,
                               eta);
            }
            None => {
                let _ = write!(self.term.as_mut().unwrap(),
                               "Total: {} Speed: {}/s", total_h, speed_h);
            }
        }
        // delete_line() doesn't seem to clear the line properly.
        // Instead, let's just print some whitespace to clear it.
        let _ = write!(self.term.as_mut().unwrap(), "                ");
        let _ = self.term.as_mut().unwrap().flush();
        let _ = self.term.as_mut().unwrap().carriage_return();
        self.displayed_progress = true;
    }
}

/// Human readable representation of data size in bytes
struct HumanReadable(u64);

impl fmt::Display for HumanReadable {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        const KIB: f64 = 1024.0;
        const MIB: f64 = 1048576.0;
        let size = self.0 as f64;

        if size >= MIB {
            write!(f, "{:.2} MiB", size / MIB)
        } else if size >= KIB {
            write!(f, "{:.2} KiB", size / KIB)
        } else {
            write!(f, "{} B", size)
        }
    }
}
<commit_msg>Improve download progress<commit_after>use std::collections::VecDeque;
use std::fmt;
use term;
use time::precise_time_s;
use rustup::Notification;
use rustup_dist::Notification as In;
use rustup_utils::Notification as Un;
use tty;

/// Keep track of this many past download amounts
const DOWNLOAD_TRACK_COUNT: usize = 5;

/// Tracks download progress and displays information about it to a terminal.
pub struct DownloadTracker {
    /// Content-Length of the to-be downloaded object.
    content_len: Option<u64>,
    /// Total data downloaded in bytes.
    total_downloaded: usize,
    /// Data downloaded this second.
    downloaded_this_sec: usize,
    /// Keeps track of amount of data downloaded every last few secs.
    /// Used for averaging the download speed.
    downloaded_last_few_secs: VecDeque<usize>,
    /// Time stamp of the last second
    last_sec: Option<f64>,
    /// How many seconds have elapsed since the download started
    seconds_elapsed: u32,
    /// The terminal we write the information to.
    term: Option<Box<term::StdoutTerminal>>,
    /// Whether we displayed progress for the download or not.
    ///
    /// If the download is quick enough, we don't have time to
    /// display the progress info.
    /// In that case, we do not want to do some cleanup stuff we normally do.
    displayed_progress: bool,
}

impl DownloadTracker {
    /// Creates a new DownloadTracker.
    pub fn new() -> Self {
        DownloadTracker {
            content_len: None,
            total_downloaded: 0,
            downloaded_this_sec: 0,
            downloaded_last_few_secs: VecDeque::with_capacity(DOWNLOAD_TRACK_COUNT),
            seconds_elapsed: 0,
            last_sec: None,
            term: term::stdout(),
            displayed_progress: false,
        }
    }

    pub fn handle_notification(&mut self, n: &Notification) -> bool {
        match n {
            &Notification::Install(In::Utils(Un::DownloadContentLengthReceived(content_len))) => {
                self.content_length_received(content_len);

                true
            }
            &Notification::Install(In::Utils(Un::DownloadDataReceived(len))) => {
                if tty::stdout_isatty() && self.term.is_some() {
                    self.data_received(len);
                }
                true
            }
            &Notification::Install(In::Utils(Un::DownloadFinished)) => {
                self.download_finished();
                true
            }
            _ => false
        }
    }

    /// Notifies self that Content-Length information has been received.
    pub fn content_length_received(&mut self, content_len: u64) {
        self.content_len = Some(content_len);
    }
    /// Notifies self that data of size `len` has been received.
    pub fn data_received(&mut self, len: usize) {
        self.total_downloaded += len;
        self.downloaded_this_sec += len;

        let current_time = precise_time_s();

        match self.last_sec {
            None => self.last_sec = Some(current_time),
            Some(start) => {
                let elapsed = current_time - start;
                if elapsed >= 1.0 {
                    self.seconds_elapsed += 1;

                    self.display();
                    self.last_sec = Some(current_time);
                    if self.downloaded_last_few_secs.len() == DOWNLOAD_TRACK_COUNT {
                        self.downloaded_last_few_secs.pop_back();
                    }
                    self.downloaded_last_few_secs.push_front(self.downloaded_this_sec);
                    self.downloaded_this_sec = 0;
                }
            }
        }
    }
    /// Notifies self that the download has finished.
    pub fn download_finished(&mut self) {
        if self.displayed_progress {
            // Display the finished state
            self.display();
            let _ = writeln!(self.term.as_mut().unwrap(), "");
        }
        self.prepare_for_new_download();
    }
    /// Resets the state to be ready for a new download.
    fn prepare_for_new_download(&mut self) {
        self.content_len = None;
        self.total_downloaded = 0;
        self.downloaded_this_sec = 0;
        self.downloaded_last_few_secs.clear();
        self.seconds_elapsed = 0;
        self.last_sec = None;
        self.displayed_progress = false;
    }
    /// Display the tracked download information to the terminal.
    fn display(&mut self) {
        let total_h = HumanReadable(self.total_downloaded as f64);
        let sum = self.downloaded_last_few_secs
                      .iter()
                      .fold(0., |a, &v| a + v as f64);
        let len = self.downloaded_last_few_secs.len();
        let speed = if len > 0 {
            sum / len as f64
        } else {
            0.
        };
        let speed_h = HumanReadable(speed);

        match self.content_len {
            Some(content_len) => {
                let percent = (self.total_downloaded as f64 / content_len as f64) * 100.;
                let content_len_h = HumanReadable(content_len as f64);
                let remaining = content_len - self.total_downloaded as u64;
                let eta_h = HumanReadable(remaining as f64 / speed);
                let _ = write!(self.term.as_mut().unwrap(),
                               "{} / {} ({:3.0} %) {}/s ETA: {:#}",
                               total_h,
                               content_len_h,
                               percent,
                               speed_h,
                               eta_h);
            }
            None => {
                let _ = write!(self.term.as_mut().unwrap(),
                               "Total: {} Speed: {}/s", total_h, speed_h);
            }
        }
        // delete_line() doesn't seem to clear the line properly.
        // Instead, let's just print some whitespace to clear it.
        let _ = write!(self.term.as_mut().unwrap(), "                ");
        let _ = self.term.as_mut().unwrap().flush();
        let _ = self.term.as_mut().unwrap().carriage_return();
        self.displayed_progress = true;
    }
}

/// Human readable representation of data size in bytes
struct HumanReadable(f64);

impl fmt::Display for HumanReadable {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        if f.alternate() {  // repurposing the alternate mode for ETA
            let sec = self.0;

            if sec <= 0. {
                write!(f, "Unknown")
            } else if sec > 1e3 {
                let sec = self.0 as u64;
                let min = sec / 60;
                let sec = sec % 60;

                write!(f, "{:3} min {:2} s", min, sec)  // XYZ min PQ s
            } else {
                write!(f, "{:3.0} s", self.0)  // XYZ s
            }
        } else {
            const KIB: f64 = 1024.0;
            const MIB: f64 = KIB * KIB;
            let size = self.0;

            if size >= MIB {
                write!(f, "{:5.1} MiB", size / MIB)  // XYZ.P MiB
            } else if size >= KIB {
                write!(f, "{:5.1} KiB", size / KIB)
            } else {
                write!(f, "{:3.0} B", size)
            }
        }
    }
}
<|endoftext|>


<commit_before>use std::io::Write;

use inflector::Inflector;

use botocore::{Operation, Shape, ShapeType};
use ::Service;
use super::{GenerateProtocol, error_type_name, FileWriter, IoResult, rest_response_parser,
            rest_request_generator, generate_field_name};

pub struct RestJsonGenerator;

impl GenerateProtocol for RestJsonGenerator {
    fn generate_method_signatures(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        for (operation_name, operation) in service.operations().iter() {
            let input_type = operation.input_shape();
            let output_type = operation.output_shape_or("()");

            // Retrieve the `Shape` for the input for this operation.
            let input_shape = &service.get_shape(input_type).unwrap();

            writeln!(writer,
                     "
                {documentation}
                {method_signature} -> \
                      RusotoFuture<{output_type}, {error_type}>;
                ",
                     documentation = generate_documentation(operation).unwrap_or_else(|| "".to_owned()),
                     method_signature = generate_method_signature(operation, input_shape),
                     error_type = error_type_name(service, operation_name),
                     output_type = output_type)?
        }
        Ok(())
    }

    fn generate_method_impls(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        for (operation_name, operation) in service.operations().iter() {
            let input_type = operation.input_shape();
            let output_type = operation.output_shape_or("()");

            // Retrieve the `Shape` for the input for this operation.
            let input_shape = &service.get_shape(input_type).unwrap();

            let (request_uri, _) = rest_request_generator::parse_query_string(&operation.http
                .request_uri);

            writeln!(writer,"
                {documentation}
                {method_signature} -> RusotoFuture<{output_type}, {error_type}> {{
                    {request_uri_formatter}

                    let mut request = SignedRequest::new(\"{http_method}\", \"{endpoint_prefix}\", &self.region, &request_uri);
                    request.set_content_type(\"application/x-amz-json-1.1\".to_owned());
                    {set_headers}
                    {modify_endpoint_prefix}
                    {load_payload}
                    {load_headers}
                    {load_params}

                    self.client.sign_and_dispatch(request, |response| {{
                        if {status_check} {{
                            Box::new(response.buffer().from_err().map(|response| {{
                                {parse_body}
                                {parse_headers}
                                {parse_status_code}
                                result
                            }}))
                        }} else {{
                            Box::new(response.buffer().from_err().and_then(|response| {{
                                Err({error_type}::from_response(response))
                            }}))
                        }}
                    }})
                }}
                ",
                documentation = generate_documentation(operation).unwrap_or_else(|| "".to_owned()),
                method_signature = generate_method_signature(operation, input_shape),
                endpoint_prefix = service.signing_name(),
                modify_endpoint_prefix = generate_endpoint_modification(service).unwrap_or_else(|| "".to_owned()),
                http_method = operation.http.method,
                error_type = error_type_name(service, operation_name),
                status_check = http_code_expected(operation.http.response_code),
                parse_body = generate_body_parser(operation, service),
                parse_status_code = generate_status_code_parser(operation, service),
                output_type = output_type,
                load_headers = rest_request_generator::generate_headers(service, operation).unwrap_or_else(|| "".to_string()),
                parse_headers = rest_response_parser::generate_response_headers_parser(service, operation)
                    .unwrap_or_else(|| "".to_owned()),
                request_uri_formatter = rest_request_generator::generate_uri_formatter(
                    &request_uri,
                    service,
                    operation
                ).unwrap_or_else(|| "".to_string()),
                load_payload = generate_payload(service, input_shape).unwrap_or_else(|| "".to_string()),
                load_params = rest_request_generator::generate_params_loading_string(service, operation).unwrap_or_else(|| "".to_string()),
                set_headers = generate_headers(service).unwrap_or_else(|| "".to_string()),
            )?
        }
        Ok(())
    }

    fn generate_prelude(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        writeln!(writer, "use serde_json;")?;

        // avoid unused imports when building services that don't use params
        if service_has_query_parameters(service) {
            writeln!(writer, "use rusoto_core::param::{{Params, ServiceParams}};")?;
        }

        writeln!(writer,
                 "use rusoto_core::signature::SignedRequest;
                  use serde_json::from_slice;
                  use serde_json::Value as SerdeJsonValue;")

    }

    fn serialize_trait(&self) -> Option<&'static str> {
        Some("Serialize")
    }

    fn deserialize_trait(&self) -> Option<&'static str> {
        Some("Deserialize")
    }

    fn timestamp_type(&self) -> &'static str {
        "f64"
    }
}

fn http_code_expected(code: Option<i32>) -> String {
    match code {
        Some(actual_code) => format!("response.status.as_u16() == {}", actual_code),
        // Some service definitions such as elastictranscoder don't specify
        // the response code, we'll assume 2xx is okay:
        None => "response.status.is_success()".to_string(),
    }
}

// Glacier needs a special header added, not included in botocore definition.  See
// http://docs.aws.amazon.com/amazonglacier/latest/dev/api-common-request-headers.html
fn generate_headers(service: &Service) -> Option<String> {
    if service.full_name() == "Amazon Glacier" {
        return Some("request.add_header(\"x-amz-glacier-version\", \"2012-06-01\");".to_string());
    }
    None
}

// IoT has an endpoint_prefix and a signing_name that differ
fn generate_endpoint_modification(service: &Service) -> Option<String> {
    if service.signing_name() == service.endpoint_prefix() {
        None
    } else {
        Some(format!("request.set_endpoint_prefix(\"{}\".to_string());",
                     service.endpoint_prefix()))
    }
}

// IoT defines a lot of empty (and therefore unnecessary) request shapes
// don't clutter method signatures with them
fn generate_method_signature(operation: &Operation, shape: &Shape) -> String {
    if shape.members.is_some() && !shape.members.as_ref().unwrap().is_empty() {
        format!("fn {method_name}(&self, input: {input_type})",
                method_name = operation.name.to_snake_case(),
                input_type = operation.input_shape())
    } else {
        format!("fn {method_name}(&self)",
                method_name = operation.name.to_snake_case())
    }
}

// Figure out what, if anything, should be sent as the body of the http request
fn generate_payload(service: &Service, input_shape: &Shape) -> Option<String> {
    let declare_payload = match input_shape.payload {

        // if the input shape explicitly specifies a payload field, use that
        Some(ref payload_member_name) => {
            Some(declared_payload(input_shape, payload_member_name, service))
        }

        // otherwise see if the input_shape itself should be serialized as the payload
        None => {
            // only use the input_shape if it has non-query, non-header members
            // (i.e., location unspecified)
            if input_shape.members
                .as_ref()
                .unwrap()
                .iter()
                .any(|(_, member)| member.location.is_none()) {
                Some("let encoded = Some(serde_json::to_vec(&input).unwrap());".to_owned())
            } else {
                None
            }
        }
    };

    if declare_payload.is_some() {
        Some(declare_payload.unwrap() + "request.set_payload(encoded);")
    } else {
        None
    }
}

fn declared_payload(input_shape: &Shape, payload_member_name: &str, service: &Service) -> String {
    let payload_member_shape = &input_shape.members.as_ref().unwrap()[payload_member_name].shape;
    let payload_shape = &service.get_shape(payload_member_shape)
        .expect("Shape missing from service definition");

    let field_name = generate_field_name(payload_member_name);

    match payload_shape.shape_type {
        // if it's a String or a Blob, send it as the raw payload
        payload_type if payload_type == ShapeType::Blob || payload_type == ShapeType::String => {
            if input_shape.required(payload_member_name) {
                format!("let encoded = Some(input.{}.to_owned());", field_name)
            } else {
                format!("let encoded = if let Some(ref payload) = input.{} {{
                            Some(payload.to_owned())
                        }} else {{
                            None
                        }};",
                        field_name)
            }
        }

        // otherwise serialize the payload member as json and use that
        _ => {
            format!("let encoded = Some(serde_json::to_vec(&input.{}).unwrap());",
                    field_name)
        }
    }
}

// Do any input operations in the entire service use query parameters?
fn service_has_query_parameters(service: &Service) -> bool {
    service.operations()
        .iter()
        .map(|(_, operation)| operation.input_shape())
        .map(|input_type| service.get_shape(input_type).unwrap())
        .any(|input_shape| input_shape.has_query_parameters())
}

fn generate_documentation(operation: &Operation) -> Option<String> {
    operation.documentation
        .as_ref()
        .map(|docs| ::doco::Item(docs).to_string())
}

/// Generate code to plumb the response status code into any fields
/// in the output shape that specify it
fn generate_status_code_parser(operation: &Operation, service: &Service) -> String {
    if operation.output.is_none() {
        return "".to_owned();
    }

    let shape_name = &operation.output.as_ref().unwrap().shape;
    let output_shape = &service.get_shape(shape_name)
        .expect("Shape missing from service definition");

    let mut status_code_parser = "".to_string();

    for (member_name, member) in output_shape.members.as_ref().unwrap() {
        if let Some(ref location) = member.location {
            if location == "statusCode" {
                if output_shape.required(member_name) {
                    status_code_parser += &format!("result.{} = Some(response.status.as_u16());",
                                                   member_name.to_snake_case());
                } else {
                    status_code_parser += &format!("result.{} = Some(response.status.as_u16() as i64);",
                                                   member_name.to_snake_case());
                }
            }
        }
    }

    status_code_parser
}

/// Generate code to parse the http response body, either as a JSON object
/// deserialized with serde, or as a raw payload that's assigned to one of
/// the fields in the result object.
///
/// Needs to determine whether or not other fields in the result object
/// will be set later (e.g. from headers), so the compiler won't spit out
/// warnings about unnecessary mutability
fn generate_body_parser(operation: &Operation, service: &Service) -> String {

    if operation.output.is_none() {
        return "let result = ::std::mem::drop(response);".to_string();
    }

    let shape_name = &operation.output.as_ref().unwrap().shape;
    let output_shape = &service.get_shape(shape_name)
        .expect("Shape missing from service definition");

    let mutable_result = output_shape.members
        .as_ref()
        .unwrap()
        .iter()
        .any(|(_, member)| member.location.is_some()) || output_shape.payload.is_some();

    match output_shape.payload {
        None => json_body_parser(shape_name, mutable_result),
        Some(ref payload_member_name) => {
            let payload_member_shape = &output_shape.members.as_ref().unwrap()[payload_member_name]
                .shape;
            let payload_shape = &service.get_shape(payload_member_shape)
                .expect("Shape missing from service definition");
            // is the shape required?
            let payload_shape_required = match output_shape.required {
                Some(ref s) => {
                    // if there's any required shape present the body payload parser will handle it
                    s.len() > 0
                },
                None => false,
            };
            match payload_shape.shape_type {
                payload_type if payload_type == ShapeType::Blob ||
                                payload_type == ShapeType::String => {
                    payload_body_parser(payload_type,
                                        shape_name,
                                        payload_member_name,
                                        mutable_result,
                                        payload_shape_required)
                }
                _ => json_body_parser(shape_name, false),
            }
        }
    }
}

/// Take the raw http response body and assign it to the payload field
/// on the result object
fn payload_body_parser(payload_type: ShapeType,
                       output_shape: &str,
                       payload_member: &str,
                       mutable_result: bool,
                       payload_required: bool)
                       -> String {

    let response_body = match payload_required {
        true => {
            match payload_type {
                ShapeType::Blob => "response.body",
                _ => "String::from_utf8_lossy(response.body.as_ref())",
            }
        },
        false => {
            match payload_type {
                ShapeType::Blob => "Some(response.body)",
                _ => "Some(String::from_utf8_lossy(response.body.as_ref()).into_owned())",
            }
        }
    };

    format!("
        let {mutable} result = {output_shape}::default();
        result.{payload_member} = {response_body};
        ",
            output_shape = output_shape,
            payload_member = payload_member.to_snake_case(),
            response_body = response_body,
            mutable = if mutable_result { "mut" } else { "" })
}

/// Parse the http response body as a JSON object with serde, and use that
/// as the result object
fn json_body_parser(output_shape: &str, mutable_result: bool) -> String {
    // `serde-json` serializes field-less structs as "null", but AWS returns
    // "{{}}" for a field-less response, so we must check for this result
    // and convert it if necessary.
    format!("
            let mut body = response.body;

            if body == b\"null\" || body.is_empty() {{
                body = b\"{{}}\".to_vec();
            }}

            debug!(\"Response body: {{:?}}\", body);
            debug!(\"Response status: {{}}\", response.status);
            let {mutable} result = serde_json::from_slice::<{output_shape}>(&body).unwrap();
            ",
            output_shape = output_shape,
            mutable = if mutable_result { "mut" } else { "" })
}
<commit_msg>Fix Content-Type header in SageMaker Runtime<commit_after>use std::io::Write;

use inflector::Inflector;

use botocore::{Operation, Shape, ShapeType};
use ::Service;
use super::{GenerateProtocol, error_type_name, FileWriter, IoResult, rest_response_parser,
            rest_request_generator, generate_field_name};

pub struct RestJsonGenerator;

impl GenerateProtocol for RestJsonGenerator {
    fn generate_method_signatures(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        for (operation_name, operation) in service.operations().iter() {
            let input_type = operation.input_shape();
            let output_type = operation.output_shape_or("()");

            // Retrieve the `Shape` for the input for this operation.
            let input_shape = &service.get_shape(input_type).unwrap();

            writeln!(writer,
                     "
                {documentation}
                {method_signature} -> \
                      RusotoFuture<{output_type}, {error_type}>;
                ",
                     documentation = generate_documentation(operation).unwrap_or_else(|| "".to_owned()),
                     method_signature = generate_method_signature(operation, input_shape),
                     error_type = error_type_name(service, operation_name),
                     output_type = output_type)?
        }
        Ok(())
    }

    fn generate_method_impls(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        for (operation_name, operation) in service.operations().iter() {
            let input_type = operation.input_shape();
            let output_type = operation.output_shape_or("()");

            // Retrieve the `Shape` for the input for this operation.
            let input_shape = &service.get_shape(input_type).unwrap();

            let (request_uri, _) = rest_request_generator::parse_query_string(&operation.http
                .request_uri);

            writeln!(writer,"
                {documentation}
                {method_signature} -> RusotoFuture<{output_type}, {error_type}> {{
                    {request_uri_formatter}

                    let mut request = SignedRequest::new(\"{http_method}\", \"{endpoint_prefix}\", &self.region, &request_uri);
                    {default_headers}
                    {set_headers}
                    {modify_endpoint_prefix}
                    {load_payload}
                    {load_headers}
                    {load_params}

                    self.client.sign_and_dispatch(request, |response| {{
                        if {status_check} {{
                            Box::new(response.buffer().from_err().map(|response| {{
                                {parse_body}
                                {parse_headers}
                                {parse_status_code}
                                result
                            }}))
                        }} else {{
                            Box::new(response.buffer().from_err().and_then(|response| {{
                                Err({error_type}::from_response(response))
                            }}))
                        }}
                    }})
                }}
                ",
                documentation = generate_documentation(operation).unwrap_or_else(|| "".to_owned()),
                method_signature = generate_method_signature(operation, input_shape),
                endpoint_prefix = service.signing_name(),
                modify_endpoint_prefix = generate_endpoint_modification(service).unwrap_or_else(|| "".to_owned()),
                http_method = operation.http.method,
                error_type = error_type_name(service, operation_name),
                status_check = http_code_expected(operation.http.response_code),
                parse_body = generate_body_parser(operation, service),
                parse_status_code = generate_status_code_parser(operation, service),
                output_type = output_type,
                load_headers = rest_request_generator::generate_headers(service, operation).unwrap_or_else(|| "".to_string()),
                parse_headers = rest_response_parser::generate_response_headers_parser(service, operation)
                    .unwrap_or_else(|| "".to_owned()),
                request_uri_formatter = rest_request_generator::generate_uri_formatter(
                    &request_uri,
                    service,
                    operation
                ).unwrap_or_else(|| "".to_string()),
                load_payload = generate_payload(service, input_shape).unwrap_or_else(|| "".to_string()),
                load_params = rest_request_generator::generate_params_loading_string(service, operation).unwrap_or_else(|| "".to_string()),
                default_headers = generate_default_headers(service).unwrap_or_else(|| "".to_string()),
                set_headers = generate_headers(service).unwrap_or_else(|| "".to_string()),
            )?
        }
        Ok(())
    }

    fn generate_prelude(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        writeln!(writer, "use serde_json;")?;

        // avoid unused imports when building services that don't use params
        if service_has_query_parameters(service) {
            writeln!(writer, "use rusoto_core::param::{{Params, ServiceParams}};")?;
        }

        writeln!(writer,
                 "use rusoto_core::signature::SignedRequest;
                  use serde_json::from_slice;
                  use serde_json::Value as SerdeJsonValue;")

    }

    fn serialize_trait(&self) -> Option<&'static str> {
        Some("Serialize")
    }

    fn deserialize_trait(&self) -> Option<&'static str> {
        Some("Deserialize")
    }

    fn timestamp_type(&self) -> &'static str {
        "f64"
    }
}

fn http_code_expected(code: Option<i32>) -> String {
    match code {
        Some(actual_code) => format!("response.status.as_u16() == {}", actual_code),
        // Some service definitions such as elastictranscoder don't specify
        // the response code, we'll assume 2xx is okay:
        None => "response.status.is_success()".to_string(),
    }
}

// Glacier needs a special header added, not included in botocore definition.  See
// http://docs.aws.amazon.com/amazonglacier/latest/dev/api-common-request-headers.html
fn generate_headers(service: &Service) -> Option<String> {
    if service.full_name() == "Amazon Glacier" {
        return Some("request.add_header(\"x-amz-glacier-version\", \"2012-06-01\");".to_string());
    }
    None
}

// SageMaker Runtime allows to overwrite content-type
fn generate_default_headers(service: &Service) -> Option<String> {
    if service.full_name() == "Amazon SageMaker Runtime" {
        return Some("if input.content_type.is_none() {
                         request.set_content_type(\"application/x-amz-json-1.1\".to_owned());
                     }".to_string());
    }
    Some("request.set_content_type(\"application/x-amz-json-1.1\".to_owned());".to_string())
}

// IoT has an endpoint_prefix and a signing_name that differ
fn generate_endpoint_modification(service: &Service) -> Option<String> {
    if service.signing_name() == service.endpoint_prefix() {
        None
    } else {
        Some(format!("request.set_endpoint_prefix(\"{}\".to_string());",
                     service.endpoint_prefix()))
    }
}

// IoT defines a lot of empty (and therefore unnecessary) request shapes
// don't clutter method signatures with them
fn generate_method_signature(operation: &Operation, shape: &Shape) -> String {
    if shape.members.is_some() && !shape.members.as_ref().unwrap().is_empty() {
        format!("fn {method_name}(&self, input: {input_type})",
                method_name = operation.name.to_snake_case(),
                input_type = operation.input_shape())
    } else {
        format!("fn {method_name}(&self)",
                method_name = operation.name.to_snake_case())
    }
}

// Figure out what, if anything, should be sent as the body of the http request
fn generate_payload(service: &Service, input_shape: &Shape) -> Option<String> {
    let declare_payload = match input_shape.payload {

        // if the input shape explicitly specifies a payload field, use that
        Some(ref payload_member_name) => {
            Some(declared_payload(input_shape, payload_member_name, service))
        }

        // otherwise see if the input_shape itself should be serialized as the payload
        None => {
            // only use the input_shape if it has non-query, non-header members
            // (i.e., location unspecified)
            if input_shape.members
                .as_ref()
                .unwrap()
                .iter()
                .any(|(_, member)| member.location.is_none()) {
                Some("let encoded = Some(serde_json::to_vec(&input).unwrap());".to_owned())
            } else {
                None
            }
        }
    };

    if declare_payload.is_some() {
        Some(declare_payload.unwrap() + "request.set_payload(encoded);")
    } else {
        None
    }
}

fn declared_payload(input_shape: &Shape, payload_member_name: &str, service: &Service) -> String {
    let payload_member_shape = &input_shape.members.as_ref().unwrap()[payload_member_name].shape;
    let payload_shape = &service.get_shape(payload_member_shape)
        .expect("Shape missing from service definition");

    let field_name = generate_field_name(payload_member_name);

    match payload_shape.shape_type {
        // if it's a String or a Blob, send it as the raw payload
        payload_type if payload_type == ShapeType::Blob || payload_type == ShapeType::String => {
            if input_shape.required(payload_member_name) {
                format!("let encoded = Some(input.{}.to_owned());", field_name)
            } else {
                format!("let encoded = if let Some(ref payload) = input.{} {{
                            Some(payload.to_owned())
                        }} else {{
                            None
                        }};",
                        field_name)
            }
        }

        // otherwise serialize the payload member as json and use that
        _ => {
            format!("let encoded = Some(serde_json::to_vec(&input.{}).unwrap());",
                    field_name)
        }
    }
}

// Do any input operations in the entire service use query parameters?
fn service_has_query_parameters(service: &Service) -> bool {
    service.operations()
        .iter()
        .map(|(_, operation)| operation.input_shape())
        .map(|input_type| service.get_shape(input_type).unwrap())
        .any(|input_shape| input_shape.has_query_parameters())
}

fn generate_documentation(operation: &Operation) -> Option<String> {
    operation.documentation
        .as_ref()
        .map(|docs| ::doco::Item(docs).to_string())
}

/// Generate code to plumb the response status code into any fields
/// in the output shape that specify it
fn generate_status_code_parser(operation: &Operation, service: &Service) -> String {
    if operation.output.is_none() {
        return "".to_owned();
    }

    let shape_name = &operation.output.as_ref().unwrap().shape;
    let output_shape = &service.get_shape(shape_name)
        .expect("Shape missing from service definition");

    let mut status_code_parser = "".to_string();

    for (member_name, member) in output_shape.members.as_ref().unwrap() {
        if let Some(ref location) = member.location {
            if location == "statusCode" {
                if output_shape.required(member_name) {
                    status_code_parser += &format!("result.{} = Some(response.status.as_u16());",
                                                   member_name.to_snake_case());
                } else {
                    status_code_parser += &format!("result.{} = Some(response.status.as_u16() as i64);",
                                                   member_name.to_snake_case());
                }
            }
        }
    }

    status_code_parser
}

/// Generate code to parse the http response body, either as a JSON object
/// deserialized with serde, or as a raw payload that's assigned to one of
/// the fields in the result object.
///
/// Needs to determine whether or not other fields in the result object
/// will be set later (e.g. from headers), so the compiler won't spit out
/// warnings about unnecessary mutability
fn generate_body_parser(operation: &Operation, service: &Service) -> String {

    if operation.output.is_none() {
        return "let result = ::std::mem::drop(response);".to_string();
    }

    let shape_name = &operation.output.as_ref().unwrap().shape;
    let output_shape = &service.get_shape(shape_name)
        .expect("Shape missing from service definition");

    let mutable_result = output_shape.members
        .as_ref()
        .unwrap()
        .iter()
        .any(|(_, member)| member.location.is_some()) || output_shape.payload.is_some();

    match output_shape.payload {
        None => json_body_parser(shape_name, mutable_result),
        Some(ref payload_member_name) => {
            let payload_member_shape = &output_shape.members.as_ref().unwrap()[payload_member_name]
                .shape;
            let payload_shape = &service.get_shape(payload_member_shape)
                .expect("Shape missing from service definition");
            // is the shape required?
            let payload_shape_required = match output_shape.required {
                Some(ref s) => {
                    // if there's any required shape present the body payload parser will handle it
                    s.len() > 0
                },
                None => false,
            };
            match payload_shape.shape_type {
                payload_type if payload_type == ShapeType::Blob ||
                                payload_type == ShapeType::String => {
                    payload_body_parser(payload_type,
                                        shape_name,
                                        payload_member_name,
                                        mutable_result,
                                        payload_shape_required)
                }
                _ => json_body_parser(shape_name, false),
            }
        }
    }
}

/// Take the raw http response body and assign it to the payload field
/// on the result object
fn payload_body_parser(payload_type: ShapeType,
                       output_shape: &str,
                       payload_member: &str,
                       mutable_result: bool,
                       payload_required: bool)
                       -> String {

    let response_body = match payload_required {
        true => {
            match payload_type {
                ShapeType::Blob => "response.body",
                _ => "String::from_utf8_lossy(response.body.as_ref())",
            }
        },
        false => {
            match payload_type {
                ShapeType::Blob => "Some(response.body)",
                _ => "Some(String::from_utf8_lossy(response.body.as_ref()).into_owned())",
            }
        }
    };

    format!("
        let {mutable} result = {output_shape}::default();
        result.{payload_member} = {response_body};
        ",
            output_shape = output_shape,
            payload_member = payload_member.to_snake_case(),
            response_body = response_body,
            mutable = if mutable_result { "mut" } else { "" })
}

/// Parse the http response body as a JSON object with serde, and use that
/// as the result object
fn json_body_parser(output_shape: &str, mutable_result: bool) -> String {
    // `serde-json` serializes field-less structs as "null", but AWS returns
    // "{{}}" for a field-less response, so we must check for this result
    // and convert it if necessary.
    format!("
            let mut body = response.body;

            if body == b\"null\" || body.is_empty() {{
                body = b\"{{}}\".to_vec();
            }}

            debug!(\"Response body: {{:?}}\", body);
            debug!(\"Response status: {{}}\", response.status);
            let {mutable} result = serde_json::from_slice::<{output_shape}>(&body).unwrap();
            ",
            output_shape = output_shape,
            mutable = if mutable_result { "mut" } else { "" })
}
<|endoftext|>


<commit_before>use crate::test_utils::TestRandom;
use crate::*;
use bls::Signature;
use rand::RngCore;
use serde_derive::{Deserialize, Serialize};
use ssz_derive::{Decode, Encode};
use test_random_derive::TestRandom;
use tree_hash::TreeHash;
use tree_hash_derive::{SignedRoot, TreeHash};

/// A header of a `BeaconBlock`.
///
/// Spec v0.5.1
#[derive(
    Debug,
    PartialEq,
    Clone,
    Serialize,
    Deserialize,
    Encode,
    Decode,
    TreeHash,
    TestRandom,
    SignedRoot,
)]
pub struct BeaconBlockHeader {
    pub slot: Slot,
    pub previous_block_root: Hash256,
    pub state_root: Hash256,
    pub block_body_root: Hash256,
    #[signed_root(skip_hashing)]
    pub signature: Signature,
}

impl BeaconBlockHeader {
    /// Returns the `tree_hash_root` of the header.
    ///
    /// Spec v0.5.1
    pub fn canonical_root(&self) -> Hash256 {
        Hash256::from_slice(&self.tree_hash_root()[..])
    }

    /// Given a `body`, consumes `self` and returns a complete `BeaconBlock`.
    ///
    /// Spec v0.5.1
    pub fn into_block(self, body: BeaconBlockBody) -> BeaconBlock {
        BeaconBlock {
            slot: self.slot,
            previous_block_root: self.previous_block_root,
            state_root: self.state_root,
            body,
            signature: self.signature,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    ssz_tests!(BeaconBlockHeader);
}
<commit_msg>Use signed_root for canonical header ID<commit_after>use crate::test_utils::TestRandom;
use crate::*;
use bls::Signature;
use rand::RngCore;
use serde_derive::{Deserialize, Serialize};
use ssz_derive::{Decode, Encode};
use test_random_derive::TestRandom;
use tree_hash::{SignedRoot, TreeHash};
use tree_hash_derive::{SignedRoot, TreeHash};

/// A header of a `BeaconBlock`.
///
/// Spec v0.5.1
#[derive(
    Debug,
    PartialEq,
    Clone,
    Serialize,
    Deserialize,
    Encode,
    Decode,
    TreeHash,
    TestRandom,
    SignedRoot,
)]
pub struct BeaconBlockHeader {
    pub slot: Slot,
    pub previous_block_root: Hash256,
    pub state_root: Hash256,
    pub block_body_root: Hash256,
    #[signed_root(skip_hashing)]
    pub signature: Signature,
}

impl BeaconBlockHeader {
    /// Returns the `tree_hash_root` of the header.
    ///
    /// Spec v0.5.1
    pub fn canonical_root(&self) -> Hash256 {
        Hash256::from_slice(&self.signed_root()[..])
    }

    /// Given a `body`, consumes `self` and returns a complete `BeaconBlock`.
    ///
    /// Spec v0.5.1
    pub fn into_block(self, body: BeaconBlockBody) -> BeaconBlock {
        BeaconBlock {
            slot: self.slot,
            previous_block_root: self.previous_block_root,
            state_root: self.state_root,
            body,
            signature: self.signature,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    ssz_tests!(BeaconBlockHeader);
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#![feature(box_syntax)]
#![feature(custom_derive)]
#![feature(iter_cmp)]
#![feature(plugin)]
#![feature(slice_bytes)]
#![feature(mpsc_select)]
#![feature(plugin)]
#![plugin(plugins)]

#![deny(unsafe_code)]
#![plugin(serde_macros)]

extern crate app_units;

extern crate azure;
extern crate canvas;
extern crate canvas_traits;
extern crate clipboard;
#[cfg(target_os = "macos")]
extern crate core_graphics;
#[cfg(target_os = "macos")]
extern crate core_text;
extern crate devtools_traits;
extern crate euclid;
extern crate gaol;
extern crate gfx;
extern crate gfx_traits;
extern crate gleam;
extern crate image;
extern crate ipc_channel;
extern crate layers;
extern crate layout_traits;
extern crate libc;
#[macro_use]
extern crate log;
extern crate msg;
extern crate net_traits;
extern crate num;
extern crate offscreen_gl_context;
#[macro_use]
extern crate profile_traits;
extern crate script_traits;
extern crate serde;
extern crate style_traits;
extern crate time;
extern crate url;
#[macro_use]
extern crate util;

pub use compositor_task::{CompositorEventListener, CompositorProxy, CompositorTask};
pub use constellation::Constellation;
use euclid::size::{Size2D};
use ipc_channel::ipc::{IpcSender};
use msg::compositor_msg::Epoch;
use msg::constellation_msg::{FrameId, Key, KeyState, KeyModifiers, LoadData};
use msg::constellation_msg::{NavigationDirection, PipelineId, SubpageId};
use msg::constellation_msg::{WebDriverCommandMsg, WindowSizeData};
use std::collections::HashMap;
use url::Url;

mod compositor;
mod compositor_layer;
pub mod compositor_task;
pub mod constellation;
mod headless;
pub mod pipeline;
pub mod sandboxing;
mod scrolling;
mod surface_map;
mod timer_scheduler;
pub mod windowing;

/// Messages from the compositor to the constellation.
#[derive(Deserialize, Serialize)]
pub enum CompositorMsg {
    Exit,
    FrameSize(PipelineId, Size2D<f32>),
    /// Request that the constellation send the FrameId corresponding to the document
    /// with the provided pipeline id
    GetFrame(PipelineId, IpcSender<Option<FrameId>>),
    /// Request that the constellation send the current pipeline id for the provided frame
    /// id, or for the root frame if this is None, over a provided channel
    GetPipeline(Option<FrameId>, IpcSender<Option<PipelineId>>),
    /// Requests that the constellation inform the compositor of the title of the pipeline
    /// immediately.
    GetPipelineTitle(PipelineId),
    InitLoadUrl(Url),
    /// Query the constellation to see if the current compositor output is stable
    IsReadyToSaveImage(HashMap<PipelineId, Epoch>),
    KeyEvent(Key, KeyState, KeyModifiers),
    LoadUrl(PipelineId, LoadData),
    Navigate(Option<(PipelineId, SubpageId)>, NavigationDirection),
    ResizedWindow(WindowSizeData),
    /// Requests that the constellation instruct layout to begin a new tick of the animation.
    TickAnimation(PipelineId),
    /// Dispatch a webdriver command
    WebDriverCommand(WebDriverCommandMsg),
}
<commit_msg>Fix warning: needless 'iter_cmp' flags<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#![feature(box_syntax)]
#![feature(custom_derive)]
#![feature(plugin)]
#![feature(slice_bytes)]
#![feature(mpsc_select)]
#![feature(plugin)]
#![plugin(plugins)]

#![deny(unsafe_code)]
#![plugin(serde_macros)]

extern crate app_units;

extern crate azure;
extern crate canvas;
extern crate canvas_traits;
extern crate clipboard;
#[cfg(target_os = "macos")]
extern crate core_graphics;
#[cfg(target_os = "macos")]
extern crate core_text;
extern crate devtools_traits;
extern crate euclid;
extern crate gaol;
extern crate gfx;
extern crate gfx_traits;
extern crate gleam;
extern crate image;
extern crate ipc_channel;
extern crate layers;
extern crate layout_traits;
extern crate libc;
#[macro_use]
extern crate log;
extern crate msg;
extern crate net_traits;
extern crate num;
extern crate offscreen_gl_context;
#[macro_use]
extern crate profile_traits;
extern crate script_traits;
extern crate serde;
extern crate style_traits;
extern crate time;
extern crate url;
#[macro_use]
extern crate util;

pub use compositor_task::{CompositorEventListener, CompositorProxy, CompositorTask};
pub use constellation::Constellation;
use euclid::size::{Size2D};
use ipc_channel::ipc::{IpcSender};
use msg::compositor_msg::Epoch;
use msg::constellation_msg::{FrameId, Key, KeyState, KeyModifiers, LoadData};
use msg::constellation_msg::{NavigationDirection, PipelineId, SubpageId};
use msg::constellation_msg::{WebDriverCommandMsg, WindowSizeData};
use std::collections::HashMap;
use url::Url;

mod compositor;
mod compositor_layer;
pub mod compositor_task;
pub mod constellation;
mod headless;
pub mod pipeline;
pub mod sandboxing;
mod scrolling;
mod surface_map;
mod timer_scheduler;
pub mod windowing;

/// Messages from the compositor to the constellation.
#[derive(Deserialize, Serialize)]
pub enum CompositorMsg {
    Exit,
    FrameSize(PipelineId, Size2D<f32>),
    /// Request that the constellation send the FrameId corresponding to the document
    /// with the provided pipeline id
    GetFrame(PipelineId, IpcSender<Option<FrameId>>),
    /// Request that the constellation send the current pipeline id for the provided frame
    /// id, or for the root frame if this is None, over a provided channel
    GetPipeline(Option<FrameId>, IpcSender<Option<PipelineId>>),
    /// Requests that the constellation inform the compositor of the title of the pipeline
    /// immediately.
    GetPipelineTitle(PipelineId),
    InitLoadUrl(Url),
    /// Query the constellation to see if the current compositor output is stable
    IsReadyToSaveImage(HashMap<PipelineId, Epoch>),
    KeyEvent(Key, KeyState, KeyModifiers),
    LoadUrl(PipelineId, LoadData),
    Navigate(Option<(PipelineId, SubpageId)>, NavigationDirection),
    ResizedWindow(WindowSizeData),
    /// Requests that the constellation instruct layout to begin a new tick of the animation.
    TickAnimation(PipelineId),
    /// Dispatch a webdriver command
    WebDriverCommand(WebDriverCommandMsg),
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use dom::bindings::cell::DOMRefCell;
use dom::bindings::codegen::Bindings::BlobBinding;
use dom::bindings::codegen::Bindings::BlobBinding::BlobMethods;
use dom::bindings::codegen::UnionTypes::BlobOrString;
use dom::bindings::error::{Error, Fallible};
use dom::bindings::global::GlobalRef;
use dom::bindings::js::{JS, Root};
use dom::bindings::reflector::{Reflectable, Reflector, reflect_dom_object};
use dom::bindings::str::DOMString;
use dom::globalscope::GlobalScope;
use encoding::all::UTF_8;
use encoding::types::{EncoderTrap, Encoding};
use ipc_channel::ipc;
use net_traits::{CoreResourceMsg, IpcSend};
use net_traits::blob_url_store::{BlobBuf, get_blob_origin};
use net_traits::filemanager_thread::{FileManagerThreadMsg, ReadFileProgress, RelativePos};
use std::cell::Cell;
use std::mem;
use std::ops::Index;
use std::path::PathBuf;
use uuid::Uuid;

/// File-based blob
#[derive(JSTraceable)]
pub struct FileBlob {
    id: Uuid,
    name: Option<PathBuf>,
    cache: DOMRefCell<Option<Vec<u8>>>,
    size: u64,
}

/// Different backends of Blob
#[must_root]
#[derive(JSTraceable)]
pub enum BlobImpl {
    /// File-based blob, whose content lives in the net process
    File(FileBlob),
    /// Memory-based blob, whose content lives in the script process
    Memory(Vec<u8>),
    /// Sliced blob, including parent blob reference and
    /// relative positions of current slicing range,
    /// IMPORTANT: The depth of tree is only two, i.e. the parent Blob must be
    /// either File-based or Memory-based
    Sliced(JS<Blob>, RelativePos),
}

impl BlobImpl {
    /// Construct memory-backed BlobImpl
    #[allow(unrooted_must_root)]
    pub fn new_from_bytes(bytes: Vec<u8>) -> BlobImpl {
        BlobImpl::Memory(bytes)
    }

    /// Construct file-backed BlobImpl from File ID
    pub fn new_from_file(file_id: Uuid, name: PathBuf, size: u64) -> BlobImpl {
        BlobImpl::File(FileBlob {
            id: file_id,
            name: Some(name),
            cache: DOMRefCell::new(None),
            size: size,
        })
    }
}

// https://w3c.github.io/FileAPI/#blob
#[dom_struct]
pub struct Blob {
    reflector_: Reflector,
    #[ignore_heap_size_of = "No clear owner"]
    blob_impl: DOMRefCell<BlobImpl>,
    /// content-type string
    type_string: String,
    is_closed: Cell<bool>,
}

impl Blob {
    #[allow(unrooted_must_root)]
    pub fn new(
            global: &GlobalScope, blob_impl: BlobImpl, typeString: String)
            -> Root<Blob> {
        let boxed_blob = box Blob::new_inherited(blob_impl, typeString);
        reflect_dom_object(boxed_blob, global, BlobBinding::Wrap)
    }

    #[allow(unrooted_must_root)]
    pub fn new_inherited(blob_impl: BlobImpl, type_string: String) -> Blob {
        Blob {
            reflector_: Reflector::new(),
            blob_impl: DOMRefCell::new(blob_impl),
            // NOTE: Guarding the format correctness here,
            // https://w3c.github.io/FileAPI/#dfn-type
            type_string: normalize_type_string(&type_string),
            is_closed: Cell::new(false),
        }
    }

    #[allow(unrooted_must_root)]
    fn new_sliced(parent: &Blob, rel_pos: RelativePos,
                  relative_content_type: DOMString) -> Root<Blob> {
        let blob_impl = match *parent.blob_impl.borrow() {
            BlobImpl::File(_) => {
                // Create new parent node
                BlobImpl::Sliced(JS::from_ref(parent), rel_pos)
            }
            BlobImpl::Memory(_) => {
                // Create new parent node
                BlobImpl::Sliced(JS::from_ref(parent), rel_pos)
            }
            BlobImpl::Sliced(ref grandparent, ref old_rel_pos) => {
                // Adjust the slicing position, using same parent
                BlobImpl::Sliced(grandparent.clone(), old_rel_pos.slice_inner(&rel_pos))
            }
        };

        Blob::new(&parent.global_scope(), blob_impl, relative_content_type.into())
    }

    // https://w3c.github.io/FileAPI/#constructorBlob
    pub fn Constructor(global: GlobalRef,
                       blobParts: Option<Vec<BlobOrString>>,
                       blobPropertyBag: &BlobBinding::BlobPropertyBag)
                       -> Fallible<Root<Blob>> {
        // TODO: accept other blobParts types - ArrayBuffer or ArrayBufferView
        let bytes: Vec<u8> = match blobParts {
            None => Vec::new(),
            Some(blobparts) => match blob_parts_to_bytes(blobparts) {
                Ok(bytes) => bytes,
                Err(_) => return Err(Error::InvalidCharacter),
            }
        };

        Ok(Blob::new(global.as_global_scope(), BlobImpl::new_from_bytes(bytes), blobPropertyBag.type_.to_string()))
    }

    /// Get a slice to inner data, this might incur synchronous read and caching
    pub fn get_bytes(&self) -> Result<Vec<u8>, ()> {
        match *self.blob_impl.borrow() {
            BlobImpl::File(ref f) => {
                let (buffer, is_new_buffer) = match *f.cache.borrow() {
                    Some(ref bytes) => (bytes.clone(), false),
                    None => {
                        let global = self.global();
                        let bytes = read_file(global.r(), f.id.clone())?;
                        (bytes, true)
                    }
                };

                // Cache
                if is_new_buffer {
                    *f.cache.borrow_mut() = Some(buffer.clone());
                }

                Ok(buffer)
            }
            BlobImpl::Memory(ref s) => Ok(s.clone()),
            BlobImpl::Sliced(ref parent, ref rel_pos) => {
                parent.get_bytes().map(|v| {
                    let range = rel_pos.to_abs_range(v.len());
                    v.index(range).to_vec()
                })
            }
        }
    }

    /// Get a FileID representing the Blob content,
    /// used by URL.createObjectURL
    pub fn get_blob_url_id(&self) -> Uuid {
        let opt_sliced_parent = match *self.blob_impl.borrow() {
            BlobImpl::Sliced(ref parent, ref rel_pos) => {
                Some((parent.promote(/* set_valid is */ false), rel_pos.clone(), parent.Size()))
            }
            _ => None
        };

        match opt_sliced_parent {
            Some((parent_id, rel_pos, size)) => self.create_sliced_url_id(&parent_id, &rel_pos, size),
            None => self.promote(/* set_valid is */ true),
        }
    }

    /// Promote non-Slice blob:
    /// 1. Memory-based: The bytes in data slice will be transferred to file manager thread.
    /// 2. File-based: If set_valid, then activate the FileID so it can serve as URL
    /// Depending on set_valid, the returned FileID can be part of
    /// valid or invalid Blob URL.
    fn promote(&self, set_valid: bool) -> Uuid {
        let mut bytes = vec![];
        let global_url = self.global_scope().get_url();

        match *self.blob_impl.borrow_mut() {
            BlobImpl::Sliced(_, _) => {
                debug!("Sliced can't have a sliced parent");
                // Return dummy id
                return Uuid::new_v4();
            }
            BlobImpl::File(ref f) => {
                if set_valid {
                    let origin = get_blob_origin(&global_url);
                    let (tx, rx) = ipc::channel().unwrap();

                    let msg = FileManagerThreadMsg::ActivateBlobURL(f.id.clone(), tx, origin.clone());
                    self.send_to_file_manager(msg);

                    match rx.recv().unwrap() {
                        Ok(_) => return f.id.clone(),
                        // Return a dummy id on error
                        Err(_) => return Uuid::new_v4(),
                    }
                } else {
                    // no need to activate
                    return f.id.clone();
                }
            }
            BlobImpl::Memory(ref mut bytes_in) => mem::swap(bytes_in, &mut bytes),
        };

        let origin = get_blob_origin(&global_url);

        let blob_buf = BlobBuf {
            filename: None,
            type_string: self.type_string.clone(),
            size: bytes.len() as u64,
            bytes: bytes.to_vec(),
        };

        let (tx, rx) = ipc::channel().unwrap();
        let msg = FileManagerThreadMsg::PromoteMemory(blob_buf, set_valid, tx, origin.clone());
        self.send_to_file_manager(msg);

        match rx.recv().unwrap() {
            Ok(id) => {
                *self.blob_impl.borrow_mut() = BlobImpl::File(FileBlob {
                    id: id.clone(),
                    name: None,
                    cache: DOMRefCell::new(Some(bytes.to_vec())),
                    size: bytes.len() as u64,
                });
                id
            }
            // Dummy id
            Err(_) => Uuid::new_v4(),
        }
    }

    /// Get a FileID representing sliced parent-blob content
    fn create_sliced_url_id(&self, parent_id: &Uuid,
                            rel_pos: &RelativePos, parent_len: u64) -> Uuid {
        let origin = get_blob_origin(&self.global_scope().get_url());

        let (tx, rx) = ipc::channel().unwrap();
        let msg = FileManagerThreadMsg::AddSlicedURLEntry(parent_id.clone(),
                                                          rel_pos.clone(),
                                                          tx, origin.clone());
        self.send_to_file_manager(msg);
        match rx.recv().expect("File manager thread is down") {
            Ok(new_id) => {
                *self.blob_impl.borrow_mut() = BlobImpl::File(FileBlob {
                    id: new_id.clone(),
                    name: None,
                    cache: DOMRefCell::new(None),
                    size: rel_pos.to_abs_range(parent_len as usize).len() as u64,
                });

                // Return the indirect id reference
                new_id
            }
            Err(_) => {
                // Return dummy id
                Uuid::new_v4()
            }
        }
    }

    /// Cleanups at the time of destruction/closing
    fn clean_up_file_resource(&self) {
        if let BlobImpl::File(ref f) = *self.blob_impl.borrow() {
            let origin = get_blob_origin(&self.global_scope().get_url());

            let (tx, rx) = ipc::channel().unwrap();

            let msg = FileManagerThreadMsg::DecRef(f.id.clone(), origin, tx);
            self.send_to_file_manager(msg);
            let _ = rx.recv().unwrap();
        }
    }

    fn send_to_file_manager(&self, msg: FileManagerThreadMsg) {
        let global = self.global_scope();
        let resource_threads = global.resource_threads();
        let _ = resource_threads.send(CoreResourceMsg::ToFileManager(msg));
    }
}

impl Drop for Blob {
    fn drop(&mut self) {
        if !self.IsClosed() {
            self.clean_up_file_resource();
        }
    }
}

fn read_file(global: GlobalRef, id: Uuid) -> Result<Vec<u8>, ()> {
    let resource_threads = global.as_global_scope().resource_threads();
    let (chan, recv) = ipc::channel().map_err(|_|())?;
    let origin = get_blob_origin(&global.as_global_scope().get_url());
    let check_url_validity = false;
    let msg = FileManagerThreadMsg::ReadFile(chan, id, check_url_validity, origin);
    let _ = resource_threads.send(CoreResourceMsg::ToFileManager(msg));

    let mut bytes = vec![];

    loop {
        match recv.recv().unwrap() {
            Ok(ReadFileProgress::Meta(mut blob_buf)) => {
                bytes.append(&mut blob_buf.bytes);
            }
            Ok(ReadFileProgress::Partial(mut bytes_in)) => {
                bytes.append(&mut bytes_in);
            }
            Ok(ReadFileProgress::EOF) => {
                return Ok(bytes);
            }
            Err(_) => return Err(()),
        }
    }
}

/// Extract bytes from BlobParts, used by Blob and File constructor
/// https://w3c.github.io/FileAPI/#constructorBlob
pub fn blob_parts_to_bytes(blobparts: Vec<BlobOrString>) -> Result<Vec<u8>, ()> {
    let mut ret = vec![];

    for blobpart in &blobparts {
        match blobpart {
            &BlobOrString::String(ref s) => {
                let mut bytes = UTF_8.encode(s, EncoderTrap::Replace).map_err(|_|())?;
                ret.append(&mut bytes);
            },
            &BlobOrString::Blob(ref b) => {
                let mut bytes = b.get_bytes().unwrap_or(vec![]);
                ret.append(&mut bytes);
            },
        }
    }

    Ok(ret)
}

impl BlobMethods for Blob {
    // https://w3c.github.io/FileAPI/#dfn-size
    fn Size(&self) -> u64 {
         match *self.blob_impl.borrow() {
            BlobImpl::File(ref f) => f.size,
            BlobImpl::Memory(ref v) => v.len() as u64,
            BlobImpl::Sliced(ref parent, ref rel_pos) =>
                rel_pos.to_abs_range(parent.Size() as usize).len() as u64,
         }
    }

    // https://w3c.github.io/FileAPI/#dfn-type
    fn Type(&self) -> DOMString {
        DOMString::from(self.type_string.clone())
    }

    // https://w3c.github.io/FileAPI/#slice-method-algo
    fn Slice(&self,
             start: Option<i64>,
             end: Option<i64>,
             content_type: Option<DOMString>)
             -> Root<Blob> {
        let rel_pos = RelativePos::from_opts(start, end);
        Blob::new_sliced(self, rel_pos, content_type.unwrap_or(DOMString::from("")))
    }

    // https://w3c.github.io/FileAPI/#dfn-isClosed
    fn IsClosed(&self) -> bool {
        self.is_closed.get()
    }

    // https://w3c.github.io/FileAPI/#dfn-close
    fn Close(&self) {
        // Step 1
        if self.is_closed.get() {
            return;
        }

        // Step 2
        self.is_closed.set(true);

        // Step 3
        self.clean_up_file_resource();
    }
}

/// Get the normalized, MIME-parsable type string
/// https://w3c.github.io/FileAPI/#dfn-type
/// XXX: We will relax the restriction here,
/// since the spec has some problem over this part.
/// see https://github.com/w3c/FileAPI/issues/43
fn normalize_type_string(s: &str) -> String {
    if is_ascii_printable(s) {
        let s_lower = s.to_lowercase();
        // match s_lower.parse() as Result<Mime, ()> {
            // Ok(_) => s_lower,
            // Err(_) => "".to_string()
        s_lower
    } else {
        "".to_string()
    }
}

fn is_ascii_printable(string: &str) -> bool {
    // Step 5.1 in Sec 5.1 of File API spec
    // https://w3c.github.io/FileAPI/#constructorBlob
    string.chars().all(|c| c >= '\x20' && c <= '\x7E')
}
<commit_msg>Make dom::blob::read_file take a &GlobalScope<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use dom::bindings::cell::DOMRefCell;
use dom::bindings::codegen::Bindings::BlobBinding;
use dom::bindings::codegen::Bindings::BlobBinding::BlobMethods;
use dom::bindings::codegen::UnionTypes::BlobOrString;
use dom::bindings::error::{Error, Fallible};
use dom::bindings::global::GlobalRef;
use dom::bindings::js::{JS, Root};
use dom::bindings::reflector::{Reflectable, Reflector, reflect_dom_object};
use dom::bindings::str::DOMString;
use dom::globalscope::GlobalScope;
use encoding::all::UTF_8;
use encoding::types::{EncoderTrap, Encoding};
use ipc_channel::ipc;
use net_traits::{CoreResourceMsg, IpcSend};
use net_traits::blob_url_store::{BlobBuf, get_blob_origin};
use net_traits::filemanager_thread::{FileManagerThreadMsg, ReadFileProgress, RelativePos};
use std::cell::Cell;
use std::mem;
use std::ops::Index;
use std::path::PathBuf;
use uuid::Uuid;

/// File-based blob
#[derive(JSTraceable)]
pub struct FileBlob {
    id: Uuid,
    name: Option<PathBuf>,
    cache: DOMRefCell<Option<Vec<u8>>>,
    size: u64,
}

/// Different backends of Blob
#[must_root]
#[derive(JSTraceable)]
pub enum BlobImpl {
    /// File-based blob, whose content lives in the net process
    File(FileBlob),
    /// Memory-based blob, whose content lives in the script process
    Memory(Vec<u8>),
    /// Sliced blob, including parent blob reference and
    /// relative positions of current slicing range,
    /// IMPORTANT: The depth of tree is only two, i.e. the parent Blob must be
    /// either File-based or Memory-based
    Sliced(JS<Blob>, RelativePos),
}

impl BlobImpl {
    /// Construct memory-backed BlobImpl
    #[allow(unrooted_must_root)]
    pub fn new_from_bytes(bytes: Vec<u8>) -> BlobImpl {
        BlobImpl::Memory(bytes)
    }

    /// Construct file-backed BlobImpl from File ID
    pub fn new_from_file(file_id: Uuid, name: PathBuf, size: u64) -> BlobImpl {
        BlobImpl::File(FileBlob {
            id: file_id,
            name: Some(name),
            cache: DOMRefCell::new(None),
            size: size,
        })
    }
}

// https://w3c.github.io/FileAPI/#blob
#[dom_struct]
pub struct Blob {
    reflector_: Reflector,
    #[ignore_heap_size_of = "No clear owner"]
    blob_impl: DOMRefCell<BlobImpl>,
    /// content-type string
    type_string: String,
    is_closed: Cell<bool>,
}

impl Blob {
    #[allow(unrooted_must_root)]
    pub fn new(
            global: &GlobalScope, blob_impl: BlobImpl, typeString: String)
            -> Root<Blob> {
        let boxed_blob = box Blob::new_inherited(blob_impl, typeString);
        reflect_dom_object(boxed_blob, global, BlobBinding::Wrap)
    }

    #[allow(unrooted_must_root)]
    pub fn new_inherited(blob_impl: BlobImpl, type_string: String) -> Blob {
        Blob {
            reflector_: Reflector::new(),
            blob_impl: DOMRefCell::new(blob_impl),
            // NOTE: Guarding the format correctness here,
            // https://w3c.github.io/FileAPI/#dfn-type
            type_string: normalize_type_string(&type_string),
            is_closed: Cell::new(false),
        }
    }

    #[allow(unrooted_must_root)]
    fn new_sliced(parent: &Blob, rel_pos: RelativePos,
                  relative_content_type: DOMString) -> Root<Blob> {
        let blob_impl = match *parent.blob_impl.borrow() {
            BlobImpl::File(_) => {
                // Create new parent node
                BlobImpl::Sliced(JS::from_ref(parent), rel_pos)
            }
            BlobImpl::Memory(_) => {
                // Create new parent node
                BlobImpl::Sliced(JS::from_ref(parent), rel_pos)
            }
            BlobImpl::Sliced(ref grandparent, ref old_rel_pos) => {
                // Adjust the slicing position, using same parent
                BlobImpl::Sliced(grandparent.clone(), old_rel_pos.slice_inner(&rel_pos))
            }
        };

        Blob::new(&parent.global_scope(), blob_impl, relative_content_type.into())
    }

    // https://w3c.github.io/FileAPI/#constructorBlob
    pub fn Constructor(global: GlobalRef,
                       blobParts: Option<Vec<BlobOrString>>,
                       blobPropertyBag: &BlobBinding::BlobPropertyBag)
                       -> Fallible<Root<Blob>> {
        // TODO: accept other blobParts types - ArrayBuffer or ArrayBufferView
        let bytes: Vec<u8> = match blobParts {
            None => Vec::new(),
            Some(blobparts) => match blob_parts_to_bytes(blobparts) {
                Ok(bytes) => bytes,
                Err(_) => return Err(Error::InvalidCharacter),
            }
        };

        Ok(Blob::new(global.as_global_scope(), BlobImpl::new_from_bytes(bytes), blobPropertyBag.type_.to_string()))
    }

    /// Get a slice to inner data, this might incur synchronous read and caching
    pub fn get_bytes(&self) -> Result<Vec<u8>, ()> {
        match *self.blob_impl.borrow() {
            BlobImpl::File(ref f) => {
                let (buffer, is_new_buffer) = match *f.cache.borrow() {
                    Some(ref bytes) => (bytes.clone(), false),
                    None => {
                        let bytes = read_file(&self.global_scope(), f.id.clone())?;
                        (bytes, true)
                    }
                };

                // Cache
                if is_new_buffer {
                    *f.cache.borrow_mut() = Some(buffer.clone());
                }

                Ok(buffer)
            }
            BlobImpl::Memory(ref s) => Ok(s.clone()),
            BlobImpl::Sliced(ref parent, ref rel_pos) => {
                parent.get_bytes().map(|v| {
                    let range = rel_pos.to_abs_range(v.len());
                    v.index(range).to_vec()
                })
            }
        }
    }

    /// Get a FileID representing the Blob content,
    /// used by URL.createObjectURL
    pub fn get_blob_url_id(&self) -> Uuid {
        let opt_sliced_parent = match *self.blob_impl.borrow() {
            BlobImpl::Sliced(ref parent, ref rel_pos) => {
                Some((parent.promote(/* set_valid is */ false), rel_pos.clone(), parent.Size()))
            }
            _ => None
        };

        match opt_sliced_parent {
            Some((parent_id, rel_pos, size)) => self.create_sliced_url_id(&parent_id, &rel_pos, size),
            None => self.promote(/* set_valid is */ true),
        }
    }

    /// Promote non-Slice blob:
    /// 1. Memory-based: The bytes in data slice will be transferred to file manager thread.
    /// 2. File-based: If set_valid, then activate the FileID so it can serve as URL
    /// Depending on set_valid, the returned FileID can be part of
    /// valid or invalid Blob URL.
    fn promote(&self, set_valid: bool) -> Uuid {
        let mut bytes = vec![];
        let global_url = self.global_scope().get_url();

        match *self.blob_impl.borrow_mut() {
            BlobImpl::Sliced(_, _) => {
                debug!("Sliced can't have a sliced parent");
                // Return dummy id
                return Uuid::new_v4();
            }
            BlobImpl::File(ref f) => {
                if set_valid {
                    let origin = get_blob_origin(&global_url);
                    let (tx, rx) = ipc::channel().unwrap();

                    let msg = FileManagerThreadMsg::ActivateBlobURL(f.id.clone(), tx, origin.clone());
                    self.send_to_file_manager(msg);

                    match rx.recv().unwrap() {
                        Ok(_) => return f.id.clone(),
                        // Return a dummy id on error
                        Err(_) => return Uuid::new_v4(),
                    }
                } else {
                    // no need to activate
                    return f.id.clone();
                }
            }
            BlobImpl::Memory(ref mut bytes_in) => mem::swap(bytes_in, &mut bytes),
        };

        let origin = get_blob_origin(&global_url);

        let blob_buf = BlobBuf {
            filename: None,
            type_string: self.type_string.clone(),
            size: bytes.len() as u64,
            bytes: bytes.to_vec(),
        };

        let (tx, rx) = ipc::channel().unwrap();
        let msg = FileManagerThreadMsg::PromoteMemory(blob_buf, set_valid, tx, origin.clone());
        self.send_to_file_manager(msg);

        match rx.recv().unwrap() {
            Ok(id) => {
                *self.blob_impl.borrow_mut() = BlobImpl::File(FileBlob {
                    id: id.clone(),
                    name: None,
                    cache: DOMRefCell::new(Some(bytes.to_vec())),
                    size: bytes.len() as u64,
                });
                id
            }
            // Dummy id
            Err(_) => Uuid::new_v4(),
        }
    }

    /// Get a FileID representing sliced parent-blob content
    fn create_sliced_url_id(&self, parent_id: &Uuid,
                            rel_pos: &RelativePos, parent_len: u64) -> Uuid {
        let origin = get_blob_origin(&self.global_scope().get_url());

        let (tx, rx) = ipc::channel().unwrap();
        let msg = FileManagerThreadMsg::AddSlicedURLEntry(parent_id.clone(),
                                                          rel_pos.clone(),
                                                          tx, origin.clone());
        self.send_to_file_manager(msg);
        match rx.recv().expect("File manager thread is down") {
            Ok(new_id) => {
                *self.blob_impl.borrow_mut() = BlobImpl::File(FileBlob {
                    id: new_id.clone(),
                    name: None,
                    cache: DOMRefCell::new(None),
                    size: rel_pos.to_abs_range(parent_len as usize).len() as u64,
                });

                // Return the indirect id reference
                new_id
            }
            Err(_) => {
                // Return dummy id
                Uuid::new_v4()
            }
        }
    }

    /// Cleanups at the time of destruction/closing
    fn clean_up_file_resource(&self) {
        if let BlobImpl::File(ref f) = *self.blob_impl.borrow() {
            let origin = get_blob_origin(&self.global_scope().get_url());

            let (tx, rx) = ipc::channel().unwrap();

            let msg = FileManagerThreadMsg::DecRef(f.id.clone(), origin, tx);
            self.send_to_file_manager(msg);
            let _ = rx.recv().unwrap();
        }
    }

    fn send_to_file_manager(&self, msg: FileManagerThreadMsg) {
        let global = self.global_scope();
        let resource_threads = global.resource_threads();
        let _ = resource_threads.send(CoreResourceMsg::ToFileManager(msg));
    }
}

impl Drop for Blob {
    fn drop(&mut self) {
        if !self.IsClosed() {
            self.clean_up_file_resource();
        }
    }
}

fn read_file(global: &GlobalScope, id: Uuid) -> Result<Vec<u8>, ()> {
    let resource_threads = global.resource_threads();
    let (chan, recv) = ipc::channel().map_err(|_|())?;
    let origin = get_blob_origin(&global.get_url());
    let check_url_validity = false;
    let msg = FileManagerThreadMsg::ReadFile(chan, id, check_url_validity, origin);
    let _ = resource_threads.send(CoreResourceMsg::ToFileManager(msg));

    let mut bytes = vec![];

    loop {
        match recv.recv().unwrap() {
            Ok(ReadFileProgress::Meta(mut blob_buf)) => {
                bytes.append(&mut blob_buf.bytes);
            }
            Ok(ReadFileProgress::Partial(mut bytes_in)) => {
                bytes.append(&mut bytes_in);
            }
            Ok(ReadFileProgress::EOF) => {
                return Ok(bytes);
            }
            Err(_) => return Err(()),
        }
    }
}

/// Extract bytes from BlobParts, used by Blob and File constructor
/// https://w3c.github.io/FileAPI/#constructorBlob
pub fn blob_parts_to_bytes(blobparts: Vec<BlobOrString>) -> Result<Vec<u8>, ()> {
    let mut ret = vec![];

    for blobpart in &blobparts {
        match blobpart {
            &BlobOrString::String(ref s) => {
                let mut bytes = UTF_8.encode(s, EncoderTrap::Replace).map_err(|_|())?;
                ret.append(&mut bytes);
            },
            &BlobOrString::Blob(ref b) => {
                let mut bytes = b.get_bytes().unwrap_or(vec![]);
                ret.append(&mut bytes);
            },
        }
    }

    Ok(ret)
}

impl BlobMethods for Blob {
    // https://w3c.github.io/FileAPI/#dfn-size
    fn Size(&self) -> u64 {
         match *self.blob_impl.borrow() {
            BlobImpl::File(ref f) => f.size,
            BlobImpl::Memory(ref v) => v.len() as u64,
            BlobImpl::Sliced(ref parent, ref rel_pos) =>
                rel_pos.to_abs_range(parent.Size() as usize).len() as u64,
         }
    }

    // https://w3c.github.io/FileAPI/#dfn-type
    fn Type(&self) -> DOMString {
        DOMString::from(self.type_string.clone())
    }

    // https://w3c.github.io/FileAPI/#slice-method-algo
    fn Slice(&self,
             start: Option<i64>,
             end: Option<i64>,
             content_type: Option<DOMString>)
             -> Root<Blob> {
        let rel_pos = RelativePos::from_opts(start, end);
        Blob::new_sliced(self, rel_pos, content_type.unwrap_or(DOMString::from("")))
    }

    // https://w3c.github.io/FileAPI/#dfn-isClosed
    fn IsClosed(&self) -> bool {
        self.is_closed.get()
    }

    // https://w3c.github.io/FileAPI/#dfn-close
    fn Close(&self) {
        // Step 1
        if self.is_closed.get() {
            return;
        }

        // Step 2
        self.is_closed.set(true);

        // Step 3
        self.clean_up_file_resource();
    }
}

/// Get the normalized, MIME-parsable type string
/// https://w3c.github.io/FileAPI/#dfn-type
/// XXX: We will relax the restriction here,
/// since the spec has some problem over this part.
/// see https://github.com/w3c/FileAPI/issues/43
fn normalize_type_string(s: &str) -> String {
    if is_ascii_printable(s) {
        let s_lower = s.to_lowercase();
        // match s_lower.parse() as Result<Mime, ()> {
            // Ok(_) => s_lower,
            // Err(_) => "".to_string()
        s_lower
    } else {
        "".to_string()
    }
}

fn is_ascii_printable(string: &str) -> bool {
    // Step 5.1 in Sec 5.1 of File API spec
    // https://w3c.github.io/FileAPI/#constructorBlob
    string.chars().all(|c| c >= '\x20' && c <= '\x7E')
}
<|endoftext|>