<commit_before>//! Types for extensible file message events ([MSC3551]).
//!
//! [MSC3551]: https://github.com/matrix-org/matrix-spec-proposals/pull/3551

use std::collections::BTreeMap;

use js_int::UInt;
use ruma_macros::EventContent;
use serde::{Deserialize, Serialize};

use super::{
    message::MessageContent,
    room::{message::Relation, JsonWebKey},
};
use crate::{serde::Base64, MxcUri};

/// The encryption info of a file sent to a room with end-to-end encryption enabled.
///
/// To create an instance of this type, first create a `EncryptedContentInit` and convert it via
/// `EncryptedContent::from` / `.into()`.
#[derive(Clone, Debug, Deserialize, Serialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct EncryptedContent {
    /// A [JSON Web Key](https://tools.ietf.org/html/rfc7517#appendix-A.3) object.
    pub key: JsonWebKey,

    /// The 128-bit unique counter block used by AES-CTR, encoded as unpadded base64.
    pub iv: Base64,

    /// A map from an algorithm name to a hash of the ciphertext, encoded as unpadded base64.
    ///
    /// Clients should support the SHA-256 hash, which uses the key sha256.
    pub hashes: BTreeMap<String, Base64>,

    /// Version of the encrypted attachments protocol.
    ///
    /// Must be `v2`.
    pub v: String,
}

/// Initial set of fields of `EncryptedContent`.
///
/// This struct will not be updated even if additional fields are added to `EncryptedContent` in a
/// new (non-breaking) release of the Matrix specification.
#[derive(Debug)]
#[allow(clippy::exhaustive_structs)]
pub struct EncryptedContentInit {
    /// A [JSON Web Key](https://tools.ietf.org/html/rfc7517#appendix-A.3) object.
    pub key: JsonWebKey,

    /// The 128-bit unique counter block used by AES-CTR, encoded as unpadded base64.
    pub iv: Base64,

    /// A map from an algorithm name to a hash of the ciphertext, encoded as unpadded base64.
    ///
    /// Clients should support the SHA-256 hash, which uses the key sha256.
    pub hashes: BTreeMap<String, Base64>,

    /// Version of the encrypted attachments protocol.
    ///
    /// Must be `v2`.
    pub v: String,
}

impl From<EncryptedContentInit> for EncryptedContent {
    fn from(init: EncryptedContentInit) -> Self {
        let EncryptedContentInit { key, iv, hashes, v } = init;
        Self { key, iv, hashes, v }
    }
}

/// Information about a file content.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct FileContentInfo {
    /// The original filename of the uploaded file.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub name: Option<String>,

    /// The mimetype of the file, e.g. “application/msword”.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub mimetype: Option<String>,

    /// The size of the file in bytes.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub size: Option<UInt>,
}

impl FileContentInfo {
    /// Creates an empty `FileContentInfo`.
    pub fn new() -> Self {
        Self::default()
    }
}

/// File content.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct FileContent {
    /// The URL to the file.
    pub url: Box<MxcUri>,

    /// Information about the uploaded file.
    #[serde(flatten, skip_serializing_if = "Option::is_none")]
    pub info: Option<Box<FileContentInfo>>,

    /// Information on the encrypted file.
    ///
    /// Required if file is encrypted.
    #[serde(flatten, skip_serializing_if = "Option::is_none")]
    pub encryption_info: Option<Box<EncryptedContent>>,
}

impl FileContent {
    /// Creates a new non-encrypted `FileContent` with the given url and file info.
    pub fn plain(url: Box<MxcUri>, info: Option<Box<FileContentInfo>>) -> Self {
        Self { url, info, encryption_info: None }
    }

    /// Creates a new encrypted `FileContent` with the given url, encryption info and file info.
    pub fn encrypted(
        url: Box<MxcUri>,
        encryption_info: EncryptedContent,
        info: Option<Box<FileContentInfo>>,
    ) -> Self {
        Self { url, info, encryption_info: Some(Box::new(encryption_info)) }
    }

    /// Whether the file is encrypted.
    pub fn is_encrypted(&self) -> bool {
        self.encryption_info.is_some()
    }
}

/// The payload for an extensible text message.
#[derive(Clone, Debug, Serialize, Deserialize, EventContent)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
#[ruma_event(type = "m.file", kind = MessageLike)]
pub struct FileEventContent {
    /// The text representation of the message.
    #[serde(flatten)]
    pub message: MessageContent,

    /// The file content of the message.
    #[serde(rename = "m.file")]
    pub file: FileContent,

    /// Information about related messages for [rich replies].
    ///
    /// [rich replies]: https://spec.matrix.org/v1.2/client-server-api/#rich-replies
    #[serde(flatten, skip_serializing_if = "Option::is_none")]
    pub relates_to: Option<Relation>,
}

impl FileEventContent {
    /// Creates a new non-encrypted `FileEventContent` with the given plain text message, url and
    /// file info.
    pub fn plain(
        message: impl Into<String>,
        url: Box<MxcUri>,
        info: Option<Box<FileContentInfo>>,
    ) -> Self {
        Self {
            message: MessageContent::plain(message),
            file: FileContent::plain(url, info),
            relates_to: None,
        }
    }

    /// Creates a new non-encrypted `FileEventContent` with the given message, url and
    /// file info.
    pub fn plain_message(
        message: MessageContent,
        url: Box<MxcUri>,
        info: Option<Box<FileContentInfo>>,
    ) -> Self {
        Self { message, file: FileContent::plain(url, info), relates_to: None }
    }

    /// Creates a new encrypted `FileEventContent` with the given plain text message, url,
    /// encryption info and file info.
    pub fn encrypted(
        message: impl Into<String>,
        url: Box<MxcUri>,
        encryption_info: EncryptedContent,
        info: Option<Box<FileContentInfo>>,
    ) -> Self {
        Self {
            message: MessageContent::plain(message),
            file: FileContent::encrypted(url, encryption_info, info),
            relates_to: None,
        }
    }

    /// Creates a new encrypted `FileEventContent` with the given message, url,
    /// encryption info and file info.
    pub fn encrypted_message(
        message: MessageContent,
        url: Box<MxcUri>,
        encryption_info: EncryptedContent,
        info: Option<Box<FileContentInfo>>,
    ) -> Self {
        Self { message, file: FileContent::encrypted(url, encryption_info, info), relates_to: None }
    }
}
<commit_msg>events: reorder types in file module<commit_after>//! Types for extensible file message events ([MSC3551]).
//!
//! [MSC3551]: https://github.com/matrix-org/matrix-spec-proposals/pull/3551

use std::collections::BTreeMap;

use js_int::UInt;
use ruma_macros::EventContent;
use serde::{Deserialize, Serialize};

use super::{
    message::MessageContent,
    room::{message::Relation, JsonWebKey},
};
use crate::{serde::Base64, MxcUri};

/// The payload for an extensible text message.
#[derive(Clone, Debug, Serialize, Deserialize, EventContent)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
#[ruma_event(type = "m.file", kind = MessageLike)]
pub struct FileEventContent {
    /// The text representation of the message.
    #[serde(flatten)]
    pub message: MessageContent,

    /// The file content of the message.
    #[serde(rename = "m.file")]
    pub file: FileContent,

    /// Information about related messages for [rich replies].
    ///
    /// [rich replies]: https://spec.matrix.org/v1.2/client-server-api/#rich-replies
    #[serde(flatten, skip_serializing_if = "Option::is_none")]
    pub relates_to: Option<Relation>,
}

impl FileEventContent {
    /// Creates a new non-encrypted `FileEventContent` with the given plain text message, url and
    /// file info.
    pub fn plain(
        message: impl Into<String>,
        url: Box<MxcUri>,
        info: Option<Box<FileContentInfo>>,
    ) -> Self {
        Self {
            message: MessageContent::plain(message),
            file: FileContent::plain(url, info),
            relates_to: None,
        }
    }

    /// Creates a new non-encrypted `FileEventContent` with the given message, url and
    /// file info.
    pub fn plain_message(
        message: MessageContent,
        url: Box<MxcUri>,
        info: Option<Box<FileContentInfo>>,
    ) -> Self {
        Self { message, file: FileContent::plain(url, info), relates_to: None }
    }

    /// Creates a new encrypted `FileEventContent` with the given plain text message, url,
    /// encryption info and file info.
    pub fn encrypted(
        message: impl Into<String>,
        url: Box<MxcUri>,
        encryption_info: EncryptedContent,
        info: Option<Box<FileContentInfo>>,
    ) -> Self {
        Self {
            message: MessageContent::plain(message),
            file: FileContent::encrypted(url, encryption_info, info),
            relates_to: None,
        }
    }

    /// Creates a new encrypted `FileEventContent` with the given message, url,
    /// encryption info and file info.
    pub fn encrypted_message(
        message: MessageContent,
        url: Box<MxcUri>,
        encryption_info: EncryptedContent,
        info: Option<Box<FileContentInfo>>,
    ) -> Self {
        Self { message, file: FileContent::encrypted(url, encryption_info, info), relates_to: None }
    }
}

/// File content.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct FileContent {
    /// The URL to the file.
    pub url: Box<MxcUri>,

    /// Information about the uploaded file.
    #[serde(flatten, skip_serializing_if = "Option::is_none")]
    pub info: Option<Box<FileContentInfo>>,

    /// Information on the encrypted file.
    ///
    /// Required if file is encrypted.
    #[serde(flatten, skip_serializing_if = "Option::is_none")]
    pub encryption_info: Option<Box<EncryptedContent>>,
}

impl FileContent {
    /// Creates a new non-encrypted `FileContent` with the given url and file info.
    pub fn plain(url: Box<MxcUri>, info: Option<Box<FileContentInfo>>) -> Self {
        Self { url, info, encryption_info: None }
    }

    /// Creates a new encrypted `FileContent` with the given url, encryption info and file info.
    pub fn encrypted(
        url: Box<MxcUri>,
        encryption_info: EncryptedContent,
        info: Option<Box<FileContentInfo>>,
    ) -> Self {
        Self { url, info, encryption_info: Some(Box::new(encryption_info)) }
    }

    /// Whether the file is encrypted.
    pub fn is_encrypted(&self) -> bool {
        self.encryption_info.is_some()
    }
}

/// Information about a file content.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct FileContentInfo {
    /// The original filename of the uploaded file.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub name: Option<String>,

    /// The mimetype of the file, e.g. “application/msword”.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub mimetype: Option<String>,

    /// The size of the file in bytes.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub size: Option<UInt>,
}

impl FileContentInfo {
    /// Creates an empty `FileContentInfo`.
    pub fn new() -> Self {
        Self::default()
    }
}

/// The encryption info of a file sent to a room with end-to-end encryption enabled.
///
/// To create an instance of this type, first create a `EncryptedContentInit` and convert it via
/// `EncryptedContent::from` / `.into()`.
#[derive(Clone, Debug, Deserialize, Serialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct EncryptedContent {
    /// A [JSON Web Key](https://tools.ietf.org/html/rfc7517#appendix-A.3) object.
    pub key: JsonWebKey,

    /// The 128-bit unique counter block used by AES-CTR, encoded as unpadded base64.
    pub iv: Base64,

    /// A map from an algorithm name to a hash of the ciphertext, encoded as unpadded base64.
    ///
    /// Clients should support the SHA-256 hash, which uses the key sha256.
    pub hashes: BTreeMap<String, Base64>,

    /// Version of the encrypted attachments protocol.
    ///
    /// Must be `v2`.
    pub v: String,
}

/// Initial set of fields of `EncryptedContent`.
///
/// This struct will not be updated even if additional fields are added to `EncryptedContent` in a
/// new (non-breaking) release of the Matrix specification.
#[derive(Debug)]
#[allow(clippy::exhaustive_structs)]
pub struct EncryptedContentInit {
    /// A [JSON Web Key](https://tools.ietf.org/html/rfc7517#appendix-A.3) object.
    pub key: JsonWebKey,

    /// The 128-bit unique counter block used by AES-CTR, encoded as unpadded base64.
    pub iv: Base64,

    /// A map from an algorithm name to a hash of the ciphertext, encoded as unpadded base64.
    ///
    /// Clients should support the SHA-256 hash, which uses the key sha256.
    pub hashes: BTreeMap<String, Base64>,

    /// Version of the encrypted attachments protocol.
    ///
    /// Must be `v2`.
    pub v: String,
}

impl From<EncryptedContentInit> for EncryptedContent {
    fn from(init: EncryptedContentInit) -> Self {
        let EncryptedContentInit { key, iv, hashes, v } = init;
        Self { key, iv, hashes, v }
    }
}
<|endoftext|>


<commit_before>use std::fmt::{self, Formatter, Debug};

use backend::Backend;
use super::js_var::{JsPtrEnum, JsVar};

#[derive(Clone)]
pub struct NativeFn(fn(Box<Backend>, Vec<(JsVar, Option<JsPtrEnum>)>) -> JsVar);

impl Debug for NativeFn {
    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
        write!(fmt, "[native_code]")
    }
}
<commit_msg>fix return type of native functions and add constructor<commit_after>use std::fmt::{self, Formatter, Debug};

use backend::Backend;
use super::js_var::{JsPtrEnum, JsVar};

#[derive(Clone)]
pub struct NativeFn(fn(Box<Backend>, Vec<(JsVar, Option<JsPtrEnum>)>) -> (JsVar, Option<JsPtrEnum>));

impl NativeFn {
  fn new(func: fn(Box<Backend>, Vec<(JsVar, Option<JsPtrEnum>)>) -> (JsVar, Option<JsPtrEnum>)) -> NativeFn {
    NativeFn(func)
  }
}

impl Debug for NativeFn {
    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
        write!(fmt, "[native_code]")
    }
}
<|endoftext|>


<commit_before>//! Tools for the user interface.

use error::UIErrorKind;

use error::Result;
use std::io;
use std::io::Write;
use std::result;

type CommandArg<'a, T> = (&'a str, T, &'a str);
pub type CommandList<'a, T> = Vec<CommandArg<'a, T>>;

/// Read and trim a string from stdin.
pub fn get_input(query: &'static str) -> Result<String> {
    let stdin = io::stdin();
    let mut stdout = io::stdout();
    let mut selection = String::new();

    print!("{}: ", query);
    stdout.flush()?;
    stdin.read_line(&mut selection)?;

    Ok(selection.trim().to_string())
}

/// Parse a string for a command from the input list.
pub fn get_selection<'a, 'b, T: Copy>(input: &'a str, commands: &CommandList<'b, T>) -> Option<T> {
    if let Some(needle) = input.trim().to_lowercase().split_whitespace().next() {
        for &(ident, cmd, _) in commands.iter() {
            if needle == ident {
                return Some(cmd);
            }
        }
    }

    None
}

/// Parse a string for a command from the input list and return along with the remaining string.
pub fn get_selection_and_tail<'a, 'b, T: Copy>(input: &'a str, commands: &CommandList<'b, T>)
        -> Option<(T, String)> {
    let mut iter = input.split_whitespace();

    iter.next()
        .and_then(|needle| get_selection(needle, &commands))
        .and_then(|cmd| {
            let tail = iter.collect::<Vec<&str>>().join(" ");
            Some((cmd, tail))
        })
}

/// Parse an input tail for values. The input tail is a string with white space
/// separated values. Return an Error if any value of the string could not be parsed.
pub fn parse_tail<'a>(tail: &'a str) -> result::Result<Vec<usize>, UIErrorKind> {
    // Note to self: This uses `FromIterator` to turn a Vec<Result> into Result<Vec>. Neat!
    tail.split_whitespace()
        .map(|s| s.parse::<usize>().map_err(|_| {
            UIErrorKind::BadNumber(format!("'{}' is not a valid index", s))
        }))
        .collect()
}

/// Print a menu for the input commands.
pub fn print_menu<'a, T>(commands: &CommandList<'a, T>) {
    println!("Commands:");

    for &(ref c, _, ref info) in commands.iter() {
        println!("{:>4}{:4}{}", c, ' ', info);
    }

    println!("");
}

#[cfg(test)]
mod tests {
    use super::*;

    #[derive(Clone, Copy, Debug, PartialEq)]
    enum TestCommands {
        One,
        Two,
    }

    #[test]
    fn select_a_command() {
        let command_list = vec![
            ("a", TestCommands::One, "one"),
            ("b", TestCommands::Two, "two")
        ];

        assert_eq!(Some(TestCommands::One), get_selection("a", &command_list));
        assert_eq!(Some(TestCommands::Two), get_selection("b", &command_list));
        assert_eq!(None, get_selection("nope", &command_list));
    }

    #[test]
    fn select_a_command_trims_and_lowercases() {
        let command_list = vec![
            ("a", TestCommands::One, "one"),
            ("b", TestCommands::Two, "two")
        ];

        assert_eq!(Some(TestCommands::One), get_selection("\n\ta\n", &command_list));
        assert_eq!(Some(TestCommands::Two), get_selection("  B  ", &command_list));
    }

    #[test]
    fn select_a_command_checks_first_word() {
        let command_list = vec![
            ("a", TestCommands::One, "one"),
            ("b", TestCommands::Two, "two")
        ];

        assert_eq!(Some(TestCommands::One), get_selection("a ignore", &command_list));
        assert_eq!(None, get_selection("aignore", &command_list));
    }

    #[test]
    fn select_a_command_differentiates_between_similar() {
        let command_list = vec![
            ("at", TestCommands::One, "one"),
            ("ar", TestCommands::Two, "two")
        ];

        assert_eq!(None, get_selection("a", &command_list));
        assert_eq!(Some(TestCommands::One), get_selection("at", &command_list));
        assert_eq!(Some(TestCommands::Two), get_selection("ar", &command_list));
    }

    #[test]
    fn select_a_command_and_tail() {
        let command_list = vec![
            ("a", TestCommands::One, "one"),
            ("b", TestCommands::Two, "two")
        ];

        let (cmd, tail) = get_selection_and_tail("a and a tail", &command_list).unwrap();
        assert_eq!(TestCommands::One, cmd);
        assert_eq!("and a tail", &tail);

        let (cmd, tail) = get_selection_and_tail("b\twith trimming\n", &command_list).unwrap();
        assert_eq!(TestCommands::Two, cmd);
        assert_eq!("with trimming", &tail);

        assert_eq!(None, get_selection_and_tail("", &command_list));
        assert_eq!(None, get_selection_and_tail("tail", &command_list));
    }

    #[test]
    fn parse_tail_for_usize() {
        assert_eq!(vec![3, 2, 1, 100], parse_tail("3 2 1 100").unwrap());
        assert_eq!(vec![3, 2, 1, 0], parse_tail("3\t2\t1 0").unwrap());
        assert!(parse_tail("").unwrap().is_empty());
    }

    #[test]
    fn parse_tail_for_usize_bad_numbers() {
        assert!(parse_tail("3a").is_err());
        assert!(parse_tail("3 1 2 a 3").is_err());
    }
}
<commit_msg>Document some UI tools<commit_after>//! Tools for the user interface.

use error::UIErrorKind;

use error::Result;
use std::io;
use std::io::Write;
use std::result;

/// Define commands with a selection string, enum and information string.
type CommandArg<'a, T> = (&'a str, T, &'a str);

/// These are put into a list. Unfortunately it is difficult to use
/// a const array for this since we need to send a Sized type to functions.
///
/// # Examples:
/// ```
/// # use define::uitools::CommandList;
/// // enum Command { First, Second }
/// let commands: CommandList<'static, Command> = vec![
///     ("a", Command::First, "Select option `First` by inputting "a")
///     ("bad", Command::Second, "and `Second` by inputting "bad")
/// ];
///
/// print_menu(&commands);
/// if let Some(cmd) = get_selection("bad", &commands) {
///     // Do something
/// }
/// ```
pub type CommandList<'a, T> = Vec<CommandArg<'a, T>>;

/// Read and trim a string from stdin.
pub fn get_input(query: &'static str) -> Result<String> {
    let stdin = io::stdin();
    let mut stdout = io::stdout();
    let mut selection = String::new();

    print!("{}: ", query);
    stdout.flush()?;
    stdin.read_line(&mut selection)?;

    Ok(selection.trim().to_string())
}

/// Parse a string for a command from the input list.
pub fn get_selection<'a, 'b, T: Copy>(input: &'a str, commands: &CommandList<'b, T>) -> Option<T> {
    if let Some(needle) = input.trim().to_lowercase().split_whitespace().next() {
        for &(ident, cmd, _) in commands.iter() {
            if needle == ident {
                return Some(cmd);
            }
        }
    }

    None
}

/// Parse a string for a command from the input list and return along with the remaining string.
pub fn get_selection_and_tail<'a, 'b, T: Copy>(input: &'a str, commands: &CommandList<'b, T>)
        -> Option<(T, String)> {
    let mut iter = input.split_whitespace();

    iter.next()
        .and_then(|needle| get_selection(needle, &commands))
        .and_then(|cmd| {
            let tail = iter.collect::<Vec<&str>>().join(" ");
            Some((cmd, tail))
        })
}

/// Parse an input tail for values. The input tail is a string with white space
/// separated values. Return an Error if any value of the string could not be parsed.
pub fn parse_tail<'a>(tail: &'a str) -> result::Result<Vec<usize>, UIErrorKind> {
    // Note to self: This uses `FromIterator` to turn a Vec<Result> into Result<Vec>. Neat!
    tail.split_whitespace()
        .map(|s| s.parse::<usize>().map_err(|_| {
            UIErrorKind::BadNumber(format!("'{}' is not a valid index", s))
        }))
        .collect()
}

/// Print a menu for the input commands.
pub fn print_menu<'a, T>(commands: &CommandList<'a, T>) {
    println!("Commands:");

    for &(ref c, _, ref info) in commands.iter() {
        println!("{:>4}{:4}{}", c, ' ', info);
    }

    println!("");
}

#[cfg(test)]
mod tests {
    use super::*;

    #[derive(Clone, Copy, Debug, PartialEq)]
    enum TestCommands {
        One,
        Two,
    }

    #[test]
    fn select_a_command() {
        let command_list = vec![
            ("a", TestCommands::One, "one"),
            ("b", TestCommands::Two, "two")
        ];

        assert_eq!(Some(TestCommands::One), get_selection("a", &command_list));
        assert_eq!(Some(TestCommands::Two), get_selection("b", &command_list));
        assert_eq!(None, get_selection("nope", &command_list));
    }

    #[test]
    fn select_a_command_trims_and_lowercases() {
        let command_list = vec![
            ("a", TestCommands::One, "one"),
            ("b", TestCommands::Two, "two")
        ];

        assert_eq!(Some(TestCommands::One), get_selection("\n\ta\n", &command_list));
        assert_eq!(Some(TestCommands::Two), get_selection("  B  ", &command_list));
    }

    #[test]
    fn select_a_command_checks_first_word() {
        let command_list = vec![
            ("a", TestCommands::One, "one"),
            ("b", TestCommands::Two, "two")
        ];

        assert_eq!(Some(TestCommands::One), get_selection("a ignore", &command_list));
        assert_eq!(None, get_selection("aignore", &command_list));
    }

    #[test]
    fn select_a_command_differentiates_between_similar() {
        let command_list = vec![
            ("at", TestCommands::One, "one"),
            ("ar", TestCommands::Two, "two")
        ];

        assert_eq!(None, get_selection("a", &command_list));
        assert_eq!(Some(TestCommands::One), get_selection("at", &command_list));
        assert_eq!(Some(TestCommands::Two), get_selection("ar", &command_list));
    }

    #[test]
    fn select_a_command_and_tail() {
        let command_list = vec![
            ("a", TestCommands::One, "one"),
            ("b", TestCommands::Two, "two")
        ];

        let (cmd, tail) = get_selection_and_tail("a and a tail", &command_list).unwrap();
        assert_eq!(TestCommands::One, cmd);
        assert_eq!("and a tail", &tail);

        let (cmd, tail) = get_selection_and_tail("b\twith trimming\n", &command_list).unwrap();
        assert_eq!(TestCommands::Two, cmd);
        assert_eq!("with trimming", &tail);

        assert_eq!(None, get_selection_and_tail("", &command_list));
        assert_eq!(None, get_selection_and_tail("tail", &command_list));
    }

    #[test]
    fn parse_tail_for_usize() {
        assert_eq!(vec![3, 2, 1, 100], parse_tail("3 2 1 100").unwrap());
        assert_eq!(vec![3, 2, 1, 0], parse_tail("3\t2\t1 0").unwrap());
        assert!(parse_tail("").unwrap().is_empty());
    }

    #[test]
    fn parse_tail_for_usize_bad_numbers() {
        assert!(parse_tail("3a").is_err());
        assert!(parse_tail("3 1 2 a 3").is_err());
    }
}
<|endoftext|>


<commit_before>// Iterates over every crate versions ever uploaded and (re-)renders their
// readme using the Markdown renderer from the cargo_registry crate.
//
// Warning: this can take a lot of time.

#![deny(warnings)]

#[macro_use]
extern crate serde_derive;

extern crate cargo_registry;
extern crate chrono;
extern crate curl;
extern crate diesel;
extern crate docopt;
extern crate flate2;
extern crate itertools;
extern crate s3;
extern crate tar;
extern crate time;
extern crate toml;
extern crate url;

use curl::easy::{Easy, List};
use chrono::{Utc, TimeZone};
use diesel::prelude::*;
use diesel::expression::any;
use docopt::Docopt;
use flate2::read::GzDecoder;
use itertools::Itertools;
use std::io::{Cursor, Read};
use std::path::Path;
use std::thread;
use tar::Archive;
use url::Url;

use cargo_registry::{Config, Version};
use cargo_registry::schema::*;
use cargo_registry::render::markdown_to_html;

const DEFAULT_PAGE_SIZE: usize = 25;
const USAGE: &'static str = "
Usage: render-readmes [options]
       render-readmes --help

Options:
    -h, --help         Show this message.
    --page-size NUM    How many versions should be queried and processed at a time.
    --older-than DATE  Only rerender readmes that are older than this date.
    --crate NAME       Only rerender readmes for the specified crate.
";

#[derive(Deserialize)]
struct Args {
    flag_page_size: Option<usize>,
    flag_older_than: Option<String>,
    flag_crate: Option<String>,
}

fn main() {
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.deserialize())
        .unwrap_or_else(|e| e.exit());
    let config: Config = Default::default();
    let conn = cargo_registry::db::connect_now().unwrap();

    let start_time = Utc::now();

    let older_than = if let Some(ref time) = args.flag_older_than {
        Utc.datetime_from_str(&time, "%Y-%m-%d %H:%M:%S").expect(
            "Could not parse --older-than argument as a time",
        )
    } else {
        start_time
    };
    let older_than = older_than.naive_utc();

    println!("Start time:                   {}", start_time);
    println!("Rendering readmes older than: {}", older_than);

    let mut query = versions::table
        .inner_join(readme_rendering::table)
        .inner_join(crates::table)
        .filter(readme_rendering::rendered_at.lt(older_than))
        .select(versions::id)
        .into_boxed();

    if let Some(crate_name) = args.flag_crate {
        println!("Rendering readmes for {}", crate_name);
        query = query.filter(crates::name.eq(crate_name));
    }

    let version_ids = query.load::<(i32)>(&conn).expect(
        "error loading version ids",
    );

    let total_versions = version_ids.len();
    println!("Rendering {} versions", total_versions);

    let page_size = args.flag_page_size.unwrap_or(DEFAULT_PAGE_SIZE);

    let total_pages = total_versions / page_size;
    let total_pages = if total_versions % page_size == 0 {
        total_pages
    } else {
        total_pages + 1
    };

    for (page_num, version_ids_chunk) in
        version_ids
            .into_iter()
            .chunks(page_size)
            .into_iter()
            .enumerate()
    {
        println!(
            "= Page {} of {} ==================================",
            page_num + 1,
            total_pages
        );

        let ids: Vec<_> = version_ids_chunk.collect();

        let versions = versions::table
            .inner_join(crates::table)
            .filter(versions::id.eq(any(ids)))
            .select((versions::all_columns, crates::name))
            .load::<(Version, String)>(&conn)
            .expect("error loading versions");

        let mut tasks = Vec::with_capacity(page_size as usize);
        for (version, krate_name) in versions {
            let config = config.clone();
            version.record_readme_rendering(&conn).expect(&format!(
                "[{}-{}] Couldn't record rendering time",
                krate_name,
                version.num
            ));
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", krate_name, version.num);
                let readme = get_readme(&config, &version, &krate_name);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
                let readme_len = readme.len();
                let mut body = Cursor::new(readme.into_bytes());
                config
                    .uploader
                    .upload(
                        Easy::new(),
                        &readme_path,
                        &mut body,
                        "text/html",
                        readme_len as u64,
                    )
                    .expect(&format!(
                        "[{}-{}] Couldn't upload file to S3",
                        krate_name,
                        version.num
                    ));
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thread panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(config: &Config, version: &Version, krate_name: &str) -> Option<String> {
    let mut handle = Easy::new();
    let location = match config.uploader.crate_location(
        &krate_name,
        &version.num.to_string(),
    ) {
        Some(l) => l,
        None => return None,
    };
    let date = time::now().rfc822z().to_string();
    let url = Url::parse(&location).expect(&format!(
        "[{}-{}] Couldn't parse crate URL",
        krate_name,
        version.num
    ));

    let mut headers = List::new();
    headers
        .append(&format!("Host: {}", url.host().unwrap()))
        .unwrap();
    headers.append(&format!("Date: {}", date)).unwrap();

    handle.url(url.as_str()).unwrap();
    handle.get(true).unwrap();
    handle.http_headers(headers).unwrap();

    let mut response = Vec::new();
    {
        let mut req = handle.transfer();
        req.write_function(|data| {
            response.extend(data);
            Ok(data.len())
        }).unwrap();
        if let Err(err) = req.perform() {
            println!(
                "[{}-{}] Unable to fetch crate: {}",
                krate_name,
                version.num,
                err
            );
            return None;
        }
    }
    if handle.response_code().unwrap() != 200 {
        let response = String::from_utf8_lossy(&response);
        println!(
            "[{}-{}] Failed to get a 200 response: {}",
            krate_name,
            version.num,
            response
        );
        return None;
    }
    let reader = Cursor::new(response);
    let reader = GzDecoder::new(reader).expect(&format!(
        "[{}-{}] Invalid gzip header",
        krate_name,
        version.num
    ));
    let mut archive = Archive::new(reader);
    let mut entries = archive.entries().expect(&format!(
        "[{}-{}] Invalid tar archive entries",
        krate_name,
        version.num
    ));
    let manifest: Manifest = {
        let path = format!("{}-{}/Cargo.toml", krate_name, version.num);
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version, &krate_name);
        toml::from_str(&contents).expect(&format!(
            "[{}-{}] Syntax error in manifest file",
            krate_name,
            version.num
        ))
    };
    if manifest.package.readme.is_none() {
        return None;
    }
    let rendered = {
        let path = format!(
            "{}-{}/{}",
            krate_name,
            version.num,
            manifest.package.readme.unwrap()
        );
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version, &krate_name);
        markdown_to_html(&contents).expect(&format!(
            "[{}-{}] Couldn't render README",
            krate_name,
            version.num
        ))
    };
    return Some(rendered);
    #[derive(Deserialize)]
    struct Package {
        readme: Option<String>,
    }
    #[derive(Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    entries: &mut tar::Entries<R>,
    path: &Path,
    version: &Version,
    krate_name: &str,
) -> String {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                return filepath == path;
            }
        })
        .expect(&format!(
            "[{}-{}] couldn't open file: {}",
            krate_name,
            version.num,
            path.display()
        ))
        .expect(&format!(
            "[{}-{}] file is not present: {}",
            krate_name,
            version.num,
            path.display()
        ));
    let mut contents = String::new();
    file.read_to_string(&mut contents).expect(&format!(
        "[{}-{}] Couldn't read file contents",
        krate_name,
        version.num
    ));
    contents
}
<commit_msg>Need to rerender readmes that haven't been recorded<commit_after>// Iterates over every crate versions ever uploaded and (re-)renders their
// readme using the Markdown renderer from the cargo_registry crate.
//
// Warning: this can take a lot of time.

#![deny(warnings)]

#[macro_use]
extern crate serde_derive;

extern crate cargo_registry;
extern crate chrono;
extern crate curl;
extern crate diesel;
extern crate docopt;
extern crate flate2;
extern crate itertools;
extern crate s3;
extern crate tar;
extern crate time;
extern crate toml;
extern crate url;

use curl::easy::{Easy, List};
use chrono::{Utc, TimeZone};
use diesel::prelude::*;
use diesel::expression::any;
use docopt::Docopt;
use flate2::read::GzDecoder;
use itertools::Itertools;
use std::io::{Cursor, Read};
use std::path::Path;
use std::thread;
use tar::Archive;
use url::Url;

use cargo_registry::{Config, Version};
use cargo_registry::schema::*;
use cargo_registry::render::markdown_to_html;

const DEFAULT_PAGE_SIZE: usize = 25;
const USAGE: &'static str = "
Usage: render-readmes [options]
       render-readmes --help

Options:
    -h, --help         Show this message.
    --page-size NUM    How many versions should be queried and processed at a time.
    --older-than DATE  Only rerender readmes that are older than this date.
    --crate NAME       Only rerender readmes for the specified crate.
";

#[derive(Deserialize)]
struct Args {
    flag_page_size: Option<usize>,
    flag_older_than: Option<String>,
    flag_crate: Option<String>,
}

fn main() {
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.deserialize())
        .unwrap_or_else(|e| e.exit());
    let config: Config = Default::default();
    let conn = cargo_registry::db::connect_now().unwrap();

    let start_time = Utc::now();

    let older_than = if let Some(ref time) = args.flag_older_than {
        Utc.datetime_from_str(&time, "%Y-%m-%d %H:%M:%S").expect(
            "Could not parse --older-than argument as a time",
        )
    } else {
        start_time
    };
    let older_than = older_than.naive_utc();

    println!("Start time:                   {}", start_time);
    println!("Rendering readmes older than: {}", older_than);

    let mut query = versions::table
        .inner_join(crates::table)
        .left_join(readme_rendering::table)
        .filter(readme_rendering::rendered_at.lt(older_than).or(
            readme_rendering::rendered_at.is_null(),
        ))
        .select(versions::id)
        .into_boxed();

    if let Some(crate_name) = args.flag_crate {
        println!("Rendering readmes for {}", crate_name);
        query = query.filter(crates::name.eq(crate_name));
    }

    let version_ids = query.load::<(i32)>(&conn).expect(
        "error loading version ids",
    );

    let total_versions = version_ids.len();
    println!("Rendering {} versions", total_versions);

    let page_size = args.flag_page_size.unwrap_or(DEFAULT_PAGE_SIZE);

    let total_pages = total_versions / page_size;
    let total_pages = if total_versions % page_size == 0 {
        total_pages
    } else {
        total_pages + 1
    };

    for (page_num, version_ids_chunk) in
        version_ids
            .into_iter()
            .chunks(page_size)
            .into_iter()
            .enumerate()
    {
        println!(
            "= Page {} of {} ==================================",
            page_num + 1,
            total_pages
        );

        let ids: Vec<_> = version_ids_chunk.collect();

        let versions = versions::table
            .inner_join(crates::table)
            .filter(versions::id.eq(any(ids)))
            .select((versions::all_columns, crates::name))
            .load::<(Version, String)>(&conn)
            .expect("error loading versions");

        let mut tasks = Vec::with_capacity(page_size as usize);
        for (version, krate_name) in versions {
            let config = config.clone();
            version.record_readme_rendering(&conn).expect(&format!(
                "[{}-{}] Couldn't record rendering time",
                krate_name,
                version.num
            ));
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", krate_name, version.num);
                let readme = get_readme(&config, &version, &krate_name);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
                let readme_len = readme.len();
                let mut body = Cursor::new(readme.into_bytes());
                config
                    .uploader
                    .upload(
                        Easy::new(),
                        &readme_path,
                        &mut body,
                        "text/html",
                        readme_len as u64,
                    )
                    .expect(&format!(
                        "[{}-{}] Couldn't upload file to S3",
                        krate_name,
                        version.num
                    ));
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thread panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(config: &Config, version: &Version, krate_name: &str) -> Option<String> {
    let mut handle = Easy::new();
    let location = match config.uploader.crate_location(
        &krate_name,
        &version.num.to_string(),
    ) {
        Some(l) => l,
        None => return None,
    };
    let date = time::now().rfc822z().to_string();
    let url = Url::parse(&location).expect(&format!(
        "[{}-{}] Couldn't parse crate URL",
        krate_name,
        version.num
    ));

    let mut headers = List::new();
    headers
        .append(&format!("Host: {}", url.host().unwrap()))
        .unwrap();
    headers.append(&format!("Date: {}", date)).unwrap();

    handle.url(url.as_str()).unwrap();
    handle.get(true).unwrap();
    handle.http_headers(headers).unwrap();

    let mut response = Vec::new();
    {
        let mut req = handle.transfer();
        req.write_function(|data| {
            response.extend(data);
            Ok(data.len())
        }).unwrap();
        if let Err(err) = req.perform() {
            println!(
                "[{}-{}] Unable to fetch crate: {}",
                krate_name,
                version.num,
                err
            );
            return None;
        }
    }
    if handle.response_code().unwrap() != 200 {
        let response = String::from_utf8_lossy(&response);
        println!(
            "[{}-{}] Failed to get a 200 response: {}",
            krate_name,
            version.num,
            response
        );
        return None;
    }
    let reader = Cursor::new(response);
    let reader = GzDecoder::new(reader).expect(&format!(
        "[{}-{}] Invalid gzip header",
        krate_name,
        version.num
    ));
    let mut archive = Archive::new(reader);
    let mut entries = archive.entries().expect(&format!(
        "[{}-{}] Invalid tar archive entries",
        krate_name,
        version.num
    ));
    let manifest: Manifest = {
        let path = format!("{}-{}/Cargo.toml", krate_name, version.num);
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version, &krate_name);
        toml::from_str(&contents).expect(&format!(
            "[{}-{}] Syntax error in manifest file",
            krate_name,
            version.num
        ))
    };
    if manifest.package.readme.is_none() {
        return None;
    }
    let rendered = {
        let path = format!(
            "{}-{}/{}",
            krate_name,
            version.num,
            manifest.package.readme.unwrap()
        );
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version, &krate_name);
        markdown_to_html(&contents).expect(&format!(
            "[{}-{}] Couldn't render README",
            krate_name,
            version.num
        ))
    };
    return Some(rendered);
    #[derive(Deserialize)]
    struct Package {
        readme: Option<String>,
    }
    #[derive(Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    entries: &mut tar::Entries<R>,
    path: &Path,
    version: &Version,
    krate_name: &str,
) -> String {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                return filepath == path;
            }
        })
        .expect(&format!(
            "[{}-{}] couldn't open file: {}",
            krate_name,
            version.num,
            path.display()
        ))
        .expect(&format!(
            "[{}-{}] file is not present: {}",
            krate_name,
            version.num,
            path.display()
        ));
    let mut contents = String::new();
    file.read_to_string(&mut contents).expect(&format!(
        "[{}-{}] Couldn't read file contents",
        krate_name,
        version.num
    ));
    contents
}
<|endoftext|>


<commit_before>// Iterates over every crate versions ever uploaded and (re-)renders their
// readme using the Markdown renderer from the cargo_registry crate.
//
// Warning: this can take a lot of time.
//
// Usage:
//     cargo run --bin render-readmes [page-size: optional = 25]
// The page-size argument dictate how much versions should be queried and processed at once.

#![deny(warnings)]

#[macro_use]
extern crate serde_derive;

extern crate cargo_registry;
extern crate curl;
extern crate diesel;
extern crate flate2;
extern crate postgres;
extern crate s3;
extern crate tar;
extern crate time;
extern crate toml;
extern crate url;

use curl::easy::{Easy, List};
use diesel::prelude::*;
use flate2::read::GzDecoder;
use std::env;
use std::io::{Cursor, Read};
use std::path::Path;
use std::sync::Arc;
use std::thread;
use tar::Archive;
use url::Url;

use cargo_registry::{App, Config, Version};
use cargo_registry::version::EncodableVersion;
use cargo_registry::schema::*;
use cargo_registry::render::markdown_to_html;

const DEFAULT_PAGE_SIZE: i64 = 25;

fn main() {
    let config: Config = Default::default();
    let app = Arc::new(App::new(&config));
    let conn = app.diesel_database.get().unwrap();
    let versions_count = versions::table
        .select(versions::all_columns)
        .count()
        .get_result::<i64>(&*conn)
        .expect("error counting versions");
    let page_size = match env::args().nth(1) {
        None => DEFAULT_PAGE_SIZE,
        Some(s) => s.parse::<i64>().unwrap_or(DEFAULT_PAGE_SIZE),
    };
    let pages = if versions_count % page_size == 0 {
        versions_count / page_size
    } else {
        versions_count / page_size + 1
    };
    for current_page in 0..pages {
        let versions: Vec<EncodableVersion> = versions::table
            .inner_join(crates::table)
            .select((versions::all_columns, crates::name))
            .limit(page_size)
            .offset(current_page * page_size)
            .load::<(Version, String)>(&*conn)
            .expect("error loading versions")
            .into_iter()
            .map(|(version, crate_name)| version.encodable(&crate_name))
            .collect();
        let mut tasks = Vec::with_capacity(page_size as usize);
        for version in versions {
            let app = app.clone();
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", version.krate, version.num);
                let readme = get_readme(app.clone(), &version);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let readme_path = format!(
                    "readmes/{}/{}-{}.html",
                    version.krate,
                    version.krate,
                    version.num
                );
                let readme_len = readme.len();
                let mut body = Cursor::new(readme.into_bytes());
                app.config
                    .uploader
                    .upload(
                        Easy::new(),
                        &readme_path,
                        &mut body,
                        "text/html",
                        readme_len as u64,
                    )
                    .expect(&format!(
                        "[{}-{}] Couldn't upload file to S3",
                        version.krate,
                        version.num
                    ));
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thead panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(app: Arc<App>, version: &EncodableVersion) -> Option<String> {
    let mut handle = Easy::new();
    let location = match app.config.uploader.crate_location(
        &version.krate,
        &version.num,
    ) {
        Some(l) => l,
        None => return None,
    };
    let date = time::now().rfc822z().to_string();
    let url = Url::parse(&location).expect(&format!(
        "[{}-{}] Couldn't parse crate URL",
        version.krate,
        version.num
    ));

    let mut headers = List::new();
    headers
        .append(&format!("Host: {}", url.host().unwrap()))
        .unwrap();
    headers.append(&format!("Date: {}", date)).unwrap();

    handle.url(url.as_str()).unwrap();
    handle.get(true).unwrap();
    handle.http_headers(headers).unwrap();

    let mut response = Vec::new();
    {
        let mut req = handle.transfer();
        req.write_function(|data| {
            response.extend(data);
            Ok(data.len())
        }).unwrap();
        if let Err(err) = req.perform() {
            println!(
                "[{}-{}] Unable to fetch crate: {}",
                version.krate,
                version.num,
                err
            );
            return None;
        }
    }
    if handle.response_code().unwrap() != 200 {
        let response = String::from_utf8_lossy(&response);
        println!(
            "[{}-{}] Failed to get a 200 response: {}",
            version.krate,
            version.num,
            response
        );
        return None;
    }
    let reader = Cursor::new(response);
    let reader = GzDecoder::new(reader).expect(&format!(
        "[{}-{}] Invalid gzip header",
        version.krate,
        version.num
    ));
    let mut archive = Archive::new(reader);
    let mut entries = archive.entries().expect(&format!(
        "[{}-{}] Invalid tar archive entries",
        version.krate,
        version.num
    ));
    let manifest: Manifest = {
        let path = format!("{}-{}/Cargo.toml", version.krate, version.num);
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version);
        toml::from_str(&contents).expect(&format!(
            "[{}-{}] Syntax error in manifest file",
            version.krate,
            version.num
        ))
    };
    if manifest.package.readme.is_none() {
        return None;
    }
    let rendered = {
        let path = format!(
            "{}-{}/{}",
            version.krate,
            version.num,
            manifest.package.readme.unwrap()
        );
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version);
        markdown_to_html(&contents).expect(&format!(
            "[{}-{}] Couldn't render README",
            version.krate,
            version.num
        ))
    };
    return Some(rendered);
    #[derive(Deserialize)]
    struct Package {
        readme: Option<String>,
    }
    #[derive(Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    mut entries: &mut tar::Entries<R>,
    path: &Path,
    version: &EncodableVersion,
) -> String {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                return filepath == path;
            }
        })
        .expect(&format!(
            "[{}-{}] couldn't open file: {}",
            version.krate,
            version.num,
            path.display()
        ))
        .expect(&format!(
            "[{}-{}] file is not present: {}",
            version.krate,
            version.num,
            path.display()
        ));
    let mut contents = String::new();
    file.read_to_string(&mut contents).expect(&format!(
        "[{}-{}] Couldn't read file contents",
        version.krate,
        version.num
    ));
    contents
}
<commit_msg>Get a diesel connection from connect_now instead of app<commit_after>// Iterates over every crate versions ever uploaded and (re-)renders their
// readme using the Markdown renderer from the cargo_registry crate.
//
// Warning: this can take a lot of time.
//
// Usage:
//     cargo run --bin render-readmes [page-size: optional = 25]
// The page-size argument dictate how much versions should be queried and processed at once.

#![deny(warnings)]

#[macro_use]
extern crate serde_derive;

extern crate cargo_registry;
extern crate curl;
extern crate diesel;
extern crate flate2;
extern crate postgres;
extern crate s3;
extern crate tar;
extern crate time;
extern crate toml;
extern crate url;

use curl::easy::{Easy, List};
use diesel::prelude::*;
use flate2::read::GzDecoder;
use std::env;
use std::io::{Cursor, Read};
use std::path::Path;
use std::sync::Arc;
use std::thread;
use tar::Archive;
use url::Url;

use cargo_registry::{App, Config, Version};
use cargo_registry::version::EncodableVersion;
use cargo_registry::schema::*;
use cargo_registry::render::markdown_to_html;

const DEFAULT_PAGE_SIZE: i64 = 25;

fn main() {
    let config: Config = Default::default();
    let app = Arc::new(App::new(&config));
    let conn = cargo_registry::db::connect_now().unwrap();
    let versions_count = versions::table
        .select(versions::all_columns)
        .count()
        .get_result::<i64>(&conn)
        .expect("error counting versions");
    let page_size = match env::args().nth(1) {
        None => DEFAULT_PAGE_SIZE,
        Some(s) => s.parse::<i64>().unwrap_or(DEFAULT_PAGE_SIZE),
    };
    let pages = if versions_count % page_size == 0 {
        versions_count / page_size
    } else {
        versions_count / page_size + 1
    };
    for current_page in 0..pages {
        let versions: Vec<EncodableVersion> = versions::table
            .inner_join(crates::table)
            .select((versions::all_columns, crates::name))
            .limit(page_size)
            .offset(current_page * page_size)
            .load::<(Version, String)>(&conn)
            .expect("error loading versions")
            .into_iter()
            .map(|(version, crate_name)| version.encodable(&crate_name))
            .collect();
        let mut tasks = Vec::with_capacity(page_size as usize);
        for version in versions {
            let app = app.clone();
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", version.krate, version.num);
                let readme = get_readme(app.clone(), &version);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let readme_path = format!(
                    "readmes/{}/{}-{}.html",
                    version.krate,
                    version.krate,
                    version.num
                );
                let readme_len = readme.len();
                let mut body = Cursor::new(readme.into_bytes());
                app.config
                    .uploader
                    .upload(
                        Easy::new(),
                        &readme_path,
                        &mut body,
                        "text/html",
                        readme_len as u64,
                    )
                    .expect(&format!(
                        "[{}-{}] Couldn't upload file to S3",
                        version.krate,
                        version.num
                    ));
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thead panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(app: Arc<App>, version: &EncodableVersion) -> Option<String> {
    let mut handle = Easy::new();
    let location = match app.config.uploader.crate_location(
        &version.krate,
        &version.num,
    ) {
        Some(l) => l,
        None => return None,
    };
    let date = time::now().rfc822z().to_string();
    let url = Url::parse(&location).expect(&format!(
        "[{}-{}] Couldn't parse crate URL",
        version.krate,
        version.num
    ));

    let mut headers = List::new();
    headers
        .append(&format!("Host: {}", url.host().unwrap()))
        .unwrap();
    headers.append(&format!("Date: {}", date)).unwrap();

    handle.url(url.as_str()).unwrap();
    handle.get(true).unwrap();
    handle.http_headers(headers).unwrap();

    let mut response = Vec::new();
    {
        let mut req = handle.transfer();
        req.write_function(|data| {
            response.extend(data);
            Ok(data.len())
        }).unwrap();
        if let Err(err) = req.perform() {
            println!(
                "[{}-{}] Unable to fetch crate: {}",
                version.krate,
                version.num,
                err
            );
            return None;
        }
    }
    if handle.response_code().unwrap() != 200 {
        let response = String::from_utf8_lossy(&response);
        println!(
            "[{}-{}] Failed to get a 200 response: {}",
            version.krate,
            version.num,
            response
        );
        return None;
    }
    let reader = Cursor::new(response);
    let reader = GzDecoder::new(reader).expect(&format!(
        "[{}-{}] Invalid gzip header",
        version.krate,
        version.num
    ));
    let mut archive = Archive::new(reader);
    let mut entries = archive.entries().expect(&format!(
        "[{}-{}] Invalid tar archive entries",
        version.krate,
        version.num
    ));
    let manifest: Manifest = {
        let path = format!("{}-{}/Cargo.toml", version.krate, version.num);
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version);
        toml::from_str(&contents).expect(&format!(
            "[{}-{}] Syntax error in manifest file",
            version.krate,
            version.num
        ))
    };
    if manifest.package.readme.is_none() {
        return None;
    }
    let rendered = {
        let path = format!(
            "{}-{}/{}",
            version.krate,
            version.num,
            manifest.package.readme.unwrap()
        );
        let contents = find_file_by_path(&mut entries, Path::new(&path), &version);
        markdown_to_html(&contents).expect(&format!(
            "[{}-{}] Couldn't render README",
            version.krate,
            version.num
        ))
    };
    return Some(rendered);
    #[derive(Deserialize)]
    struct Package {
        readme: Option<String>,
    }
    #[derive(Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    mut entries: &mut tar::Entries<R>,
    path: &Path,
    version: &EncodableVersion,
) -> String {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                return filepath == path;
            }
        })
        .expect(&format!(
            "[{}-{}] couldn't open file: {}",
            version.krate,
            version.num,
            path.display()
        ))
        .expect(&format!(
            "[{}-{}] file is not present: {}",
            version.krate,
            version.num,
            path.display()
        ));
    let mut contents = String::new();
    file.read_to_string(&mut contents).expect(&format!(
        "[{}-{}] Couldn't read file contents",
        version.krate,
        version.num
    ));
    contents
}
<|endoftext|>


<commit_before>//! rustbuild, the Rust build system
//!
//! This is the entry point for the build system used to compile the `rustc`
//! compiler. Lots of documentation can be found in the `README.md` file in the
//! parent directory, and otherwise documentation can be found throughout the `build`
//! directory in each respective module.

use std::env;

use bootstrap::{Build, Config, Subcommand, VERSION};

fn main() {
    let args = env::args().skip(1).collect::<Vec<_>>();
    let config = Config::parse(&args);

    // check_version warnings are not printed during setup
    let changelog_suggestion =
        if matches!(config.cmd, Subcommand::Setup { .. }) { None } else { check_version(&config) };

    // NOTE: Since `./configure` generates a `config.toml`, distro maintainers will see the
    // changelog warning, not the `x.py setup` message.
    let suggest_setup = !config.config.exists() && !matches!(config.cmd, Subcommand::Setup { .. });
    if suggest_setup {
        println!("warning: you have not made a `config.toml`");
        println!("help: consider running `x.py setup` or copying `config.toml.example`");
    } else if let Some(suggestion) = &changelog_suggestion {
        println!("{}", suggestion);
    }

    Build::new(config).build();

    if suggest_setup {
        println!("warning: you have not made a `config.toml`");
        println!("help: consider running `x.py setup` or copying `config.toml.example`");
    } else if let Some(suggestion) = &changelog_suggestion {
        println!("{}", suggestion);
    }

    if suggest_setup || changelog_suggestion.is_some() {
        println!("note: this message was printed twice to make it more likely to be seen");
    }
}

fn check_version(config: &Config) -> Option<String> {
    let mut msg = String::new();

    let suggestion = if let Some(seen) = config.changelog_seen {
        if seen != VERSION {
            msg.push_str("warning: there have been changes to x.py since you last updated.\n");
            format!("update `config.toml` to use `changelog-seen = {}` instead", VERSION)
        } else {
            return None;
        }
    } else {
        msg.push_str("warning: x.py has made several changes recently you may want to look at\n");
        format!("add `changelog-seen = {}` at the top of `config.toml`", VERSION)
    };

    msg.push_str("help: consider looking at the changes in `src/bootstrap/CHANGELOG.md`\n");
    msg.push_str("note: to silence this warning, ");
    msg.push_str(&suggestion);

    Some(msg)
}
<commit_msg>Rollup merge of #82211 - henryboisdequin:make-setup-msgs-better, r=jyn514<commit_after>//! rustbuild, the Rust build system
//!
//! This is the entry point for the build system used to compile the `rustc`
//! compiler. Lots of documentation can be found in the `README.md` file in the
//! parent directory, and otherwise documentation can be found throughout the `build`
//! directory in each respective module.

use std::env;

use bootstrap::{Build, Config, Subcommand, VERSION};

fn main() {
    let args = env::args().skip(1).collect::<Vec<_>>();
    let config = Config::parse(&args);

    // check_version warnings are not printed during setup
    let changelog_suggestion =
        if matches!(config.cmd, Subcommand::Setup { .. }) { None } else { check_version(&config) };

    // NOTE: Since `./configure` generates a `config.toml`, distro maintainers will see the
    // changelog warning, not the `x.py setup` message.
    let suggest_setup = !config.config.exists() && !matches!(config.cmd, Subcommand::Setup { .. });
    if suggest_setup {
        println!("warning: you have not made a `config.toml`");
        println!(
            "help: consider running `./x.py setup` or copying `config.toml.example` by running \
            `cp config.toml.example config.toml`"
        );
    } else if let Some(suggestion) = &changelog_suggestion {
        println!("{}", suggestion);
    }

    Build::new(config).build();

    if suggest_setup {
        println!("warning: you have not made a `config.toml`");
        println!(
            "help: consider running `./x.py setup` or copying `config.toml.example` by running \
            `cp config.toml.example config.toml`"
        );
    } else if let Some(suggestion) = &changelog_suggestion {
        println!("{}", suggestion);
    }

    if suggest_setup || changelog_suggestion.is_some() {
        println!("note: this message was printed twice to make it more likely to be seen");
    }
}

fn check_version(config: &Config) -> Option<String> {
    let mut msg = String::new();

    let suggestion = if let Some(seen) = config.changelog_seen {
        if seen != VERSION {
            msg.push_str("warning: there have been changes to x.py since you last updated.\n");
            format!("update `config.toml` to use `changelog-seen = {}` instead", VERSION)
        } else {
            return None;
        }
    } else {
        msg.push_str("warning: x.py has made several changes recently you may want to look at\n");
        format!("add `changelog-seen = {}` at the top of `config.toml`", VERSION)
    };

    msg.push_str("help: consider looking at the changes in `src/bootstrap/CHANGELOG.md`\n");
    msg.push_str("note: to silence this warning, ");
    msg.push_str(&suggestion);

    Some(msg)
}
<|endoftext|>


<commit_before>// Copyright lowRISC contributors.
// Licensed under the Apache License, Version 2.0, see LICENSE for details.
// SPDX-License-Identifier: Apache-2.0

use anyhow::{anyhow, bail, Result};
use directories::{BaseDirs, ProjectDirs};
use erased_serde::Serialize;
use log::LevelFilter;
use nix::sys::signal::{self, Signal};
use nix::unistd::{dup2, setsid, Pid};
use std::env::{self, args_os, ArgsOs};
use std::ffi::OsString;
use std::fs::{self, read_to_string, File};
use std::io::{self, ErrorKind, Write};
use std::iter::{IntoIterator, Iterator};
use std::os::unix::io::AsRawFd;
use std::path::PathBuf;
use std::process::{self, ChildStdout, Command, Stdio};
use std::str::FromStr;
use std::time::Duration;
use structopt::StructOpt;

use opentitanlib::backend;
use opentitanlib::proxy::SessionHandler;

#[derive(Debug, StructOpt)]
#[structopt(
    name = "opentitansession",
    about = "A tool for interacting with OpenTitan chips."
)]
struct Opts {
    #[structopt(
        long,
        default_value = "config",
        help = "Filename of a default flagsfile.  Relative to $XDG_CONFIG_HOME/opentitantool."
    )]
    rcfile: PathBuf,

    #[structopt(long, default_value = "off")]
    logging: LevelFilter,

    #[structopt(flatten)]
    backend_opts: backend::BackendOpts,

    #[structopt(
        long,
        help = "Stop a running session, optionally combine with --listen_port for disambiguation"
    )]
    stop: bool,

    #[structopt(
        long,
        help = "Optional, defaults to 9900 or nearest higher available port."
    )]
    listen_port: Option<u16>,

    #[structopt(long, help = "Start session, staying in foreground (do not daemonize)")]
    debug: bool,

    #[structopt(
        long,
        help = "Internal, used to tell the child process run as a daemon."
    )]
    child: bool,
}

// Given some existing option configuration, maybe re-evaluate command
// line options by reading an `rcfile`.
fn parse_command_line(opts: Opts, mut args: ArgsOs) -> Result<Opts> {
    // Initialize the logger if the user requested the non-defualt option.
    let logging = opts.logging;
    if logging != LevelFilter::Off {
        env_logger::Builder::from_default_env()
            .filter(None, opts.logging)
            .init();
    }
    if opts.rcfile.as_os_str().is_empty() {
        // No rcfile to parse.
        return Ok(opts);
    }

    // Construct the rcfile path based on the user's config directory
    // (ie: $HOME/.config/opentitantool/<filename>).
    let rcfile = if let Some(base) = ProjectDirs::from("org", "opentitan", "opentitantool") {
        base.config_dir().join(&opts.rcfile)
    } else {
        opts.rcfile
    };

    // argument[0] is the executable name.
    let mut arguments = vec![args.next().unwrap()];

    // Read in the rcfile and extend the argument list.
    match read_to_string(&rcfile) {
        Ok(content) => {
            for line in content.split('\n') {
                // Strip basic comments as shellwords won't handle comments.
                let (line, _) = line.split_once('#').unwrap_or((line, ""));
                arguments.extend(shellwords::split(line)?.iter().map(OsString::from));
            }
            Ok(())
        }
        Err(e) if e.kind() == ErrorKind::NotFound => {
            log::warn!("Could not read {:?}. Ignoring.", rcfile);
            Ok(())
        }
        Err(e) => Err(anyhow::Error::new(e).context(format!("Reading file {:?}", rcfile))),
    }?;

    // Extend the argument list with all remaining command line arguments.
    arguments.extend(args.into_iter());
    let opts = Opts::from_iter(&arguments);
    if opts.logging != logging {
        // Try re-initializing the logger.  Ignore errors.
        let _ = env_logger::Builder::from_default_env()
            .filter(None, opts.logging)
            .try_init();
    }
    Ok(opts)
}

#[derive(serde::Serialize, serde::Deserialize)]
pub struct SessionStartResult {
    port: u16,
}

/// Spawn a child process, passing all the same arguments to the child, letting it instantiate a
/// Transport based on the command line arguments, listen on a TCP socket, and run as a daemon
/// process serving network requests.  Success of the child is verified by means of a
/// `SessionStartResult` JSON message sent through the standard output pipe.
fn start_session(run_file_fn: impl FnOnce(u16) -> PathBuf) -> Result<Box<dyn Serialize>> {
    let mut child = Command::new(env::current_exe()?) // Same executable
        .arg("--child") // Add argument to let the new process know it is the daemon child
        .args(args_os().skip(1)) // Propagate all existing arguments: --interface, etc.
        .stdin(Stdio::null()) // Not used by child, disconnect from terminal
        .stdout(Stdio::piped()) // Used for signalling completion of daemon startup
        .stderr(Stdio::inherit()) // May be used for error messages during daemon startup
        .current_dir("/")
        .spawn()?;

    match serde_json::from_reader::<&mut ChildStdout, Result<SessionStartResult, String>>(
        child.stdout.as_mut().unwrap(),
    ) {
        Ok(Ok(result)) => {
            // Create a pid file corresponding to the requested TCP port.
            let path = run_file_fn(result.port);
            File::create(path)?.write_all(format!("{}\n", child.id()).as_bytes())?;
            Ok(Box::new(result))
        }
        Ok(Err(e)) => bail!(e),
        Err(e) => bail!("Child process failed to start: {}", e),
    }
}

// This method runs in the daemon child.  It will instantiate SessionHandler to bind to a
// socket, then report the chosen port number to the parent process by means of a serialized
// `SessionStartResult` sent through the stdout anonymous pipe, and finally enter an infnite
// loop, processing connections on that socket
fn session_child(listen_port: Option<u16>, backend_opts: &backend::BackendOpts) -> Result<()> {
    let transport = backend::create(backend_opts)?;
    let mut session = SessionHandler::init(&transport, listen_port)?;
    // Instantiation of Transport backend, and binding to a socket was successful, now go
    // through the process of making this process a daemon, disconnected from the
    // terminal that was used to start it.

    // Close stderr, which remained open in order to allow any errors from the above code to
    // surface, but needs to be severed in order for the daemon to avoid being killed by SIGHUP
    // if the user closes the terminal window.
    dup2(File::open("/dev/null")?.as_raw_fd(), 2)?;

    // After severing the only connection to the controlling terminal inherited from the parent,
    // we can now establish a new Unix "session" for this process, which will not be
    // "controlled" by any terminal.  This means that this daemon will not be killed by SIGHUP,
    // in case the terminal that was used for running `session start` is later closed.
    setsid()?;

    // Report startup success to parent process.
    serde_json::to_writer::<io::Stdout, Result<SessionStartResult, String>>(
        io::stdout(),
        &Ok(SessionStartResult {
            port: session.get_port(),
        }),
    )?;
    io::stdout().flush()?;

    // Closing the standard output pipe is the signal to the parent process that this child has
    // started up successfully.  We close the pipe indirectly, by replacing file descriptor 1
    // with one pointing to /dev/null.  This will ensure that any subsequent accidentally
    // executed println!() will be a no-op, rather than trigger termination via SIGPIPE.
    dup2(2, 1)?;

    // Indefinitely run command processing loop in this daemon process.
    session.run_loop()
}

#[derive(serde::Serialize, serde::Deserialize)]
pub struct SessionStopResult {}

/// Load .pid file based on given port number, and send SIGTERM to the process identified in the
/// file, to request the daemon gracefully shut down.
fn stop_session(run_file_fn: impl FnOnce(u16) -> PathBuf, port: u16) -> Result<Box<dyn Serialize>> {
    // Read the pid file corresponding to the requested TCP port.
    let path = run_file_fn(port);
    let pid: i32 = FromStr::from_str(&fs::read_to_string(&path)?.trim())?;
    // Send signal to daemon process, asking it to terminate.
    signal::kill(Pid::from_raw(pid), Signal::SIGTERM)?;
    // Wait for daemon process to stop.
    loop {
        std::thread::sleep(Duration::from_millis(100));
        // Send "signal 0", meaning that the kernel performs error checks (among those, checking
        // that the target process exists), without actually sending any signal.
        match signal::kill(Pid::from_raw(pid), None) {
            Ok(()) => (), // Process still running, repeat.
            Err(nix::Error::Sys(nix::errno::Errno::ESRCH)) => {
                // Process could not be found, meaning that it has terminated, as expected.
                fs::remove_file(&path)?;
                return Ok(Box::new(SessionStopResult {}));
            }
            Err(e) => bail!("Unexpected error querying process presence: {}", e),
        }
    }
}

fn main() -> Result<()> {
    let opts = parse_command_line(Opts::from_args(), args_os())?;

    if opts.debug {
        // Start session process in foreground (do not daemonize)
        let transport = backend::create(&opts.backend_opts)?;
        let mut session = SessionHandler::init(&transport, opts.listen_port)?;
        println!("Listening on port {}", session.get_port());
        session.run_loop()?;
        return Ok(());
    }

    if opts.child {
        // This process is a child, which is supposed to stay running as a daemon.
        match session_child(opts.listen_port, &opts.backend_opts) {
            Ok(()) => process::exit(0),
            Err(e) => {
                // Report any error to parent process though stdout pipe.
                serde_json::to_writer::<io::Stdout, Result<SessionStartResult, String>>(
                    io::stdout(),
                    &Err(format!("{}", e).to_string()),
                )?;
                process::exit(1)
            }
        }
    }

    // Locate directory to use for .pid files
    let base_dirs = BaseDirs::new().unwrap();
    let run_user_dir = base_dirs
        .runtime_dir()
        .ok_or(anyhow!("No /run/user directory"))?;
    let run_file_fn = |port: u16| {
        let mut p = PathBuf::from(run_user_dir);
        p.push(format!("opentitansession.{}.pid", port));
        p
    };

    let value = if opts.stop {
        // Send signal to daemon process to stop
        stop_session(run_file_fn, opts.listen_port.unwrap_or(9900))?
    } else {
        // Fork a daemon process
        start_session(run_file_fn)?
    };
    println!("{}", serde_json::to_string_pretty(&value)?);
    Ok(())
}
<commit_msg>[opentitantool] Conf files relative to current directory<commit_after>// Copyright lowRISC contributors.
// Licensed under the Apache License, Version 2.0, see LICENSE for details.
// SPDX-License-Identifier: Apache-2.0

use anyhow::{anyhow, bail, Result};
use directories::{BaseDirs, ProjectDirs};
use erased_serde::Serialize;
use log::LevelFilter;
use nix::sys::signal::{self, Signal};
use nix::unistd::{dup2, setsid, Pid};
use std::env::{self, args_os, ArgsOs};
use std::ffi::OsString;
use std::fs::{self, read_to_string, File};
use std::io::{self, ErrorKind, Write};
use std::iter::{IntoIterator, Iterator};
use std::os::unix::io::AsRawFd;
use std::path::PathBuf;
use std::process::{self, ChildStdout, Command, Stdio};
use std::str::FromStr;
use std::time::Duration;
use structopt::StructOpt;

use opentitanlib::backend;
use opentitanlib::proxy::SessionHandler;

#[derive(Debug, StructOpt)]
#[structopt(
    name = "opentitansession",
    about = "A tool for interacting with OpenTitan chips."
)]
struct Opts {
    #[structopt(
        long,
        default_value = "config",
        help = "Filename of a default flagsfile.  Relative to $XDG_CONFIG_HOME/opentitantool."
    )]
    rcfile: PathBuf,

    #[structopt(long, default_value = "off")]
    logging: LevelFilter,

    #[structopt(flatten)]
    backend_opts: backend::BackendOpts,

    #[structopt(
        long,
        help = "Stop a running session, optionally combine with --listen_port for disambiguation"
    )]
    stop: bool,

    #[structopt(
        long,
        help = "Optional, defaults to 9900 or nearest higher available port."
    )]
    listen_port: Option<u16>,

    #[structopt(long, help = "Start session, staying in foreground (do not daemonize)")]
    debug: bool,

    #[structopt(
        long,
        help = "Internal, used to tell the child process run as a daemon."
    )]
    child: bool,
}

// Given some existing option configuration, maybe re-evaluate command
// line options by reading an `rcfile`.
fn parse_command_line(opts: Opts, mut args: ArgsOs) -> Result<Opts> {
    // Initialize the logger if the user requested the non-defualt option.
    let logging = opts.logging;
    if logging != LevelFilter::Off {
        env_logger::Builder::from_default_env()
            .filter(None, opts.logging)
            .init();
    }
    if opts.rcfile.as_os_str().is_empty() {
        // No rcfile to parse.
        return Ok(opts);
    }

    // Construct the rcfile path based on the user's config directory
    // (ie: $HOME/.config/opentitantool/<filename>).
    let rcfile = if let Some(base) = ProjectDirs::from("org", "opentitan", "opentitantool") {
        base.config_dir().join(&opts.rcfile)
    } else {
        opts.rcfile
    };

    // argument[0] is the executable name.
    let mut arguments = vec![args.next().unwrap()];

    // Read in the rcfile and extend the argument list.
    match read_to_string(&rcfile) {
        Ok(content) => {
            for line in content.split('\n') {
                // Strip basic comments as shellwords won't handle comments.
                let (line, _) = line.split_once('#').unwrap_or((line, ""));
                arguments.extend(shellwords::split(line)?.iter().map(OsString::from));
            }
            Ok(())
        }
        Err(e) if e.kind() == ErrorKind::NotFound => {
            log::warn!("Could not read {:?}. Ignoring.", rcfile);
            Ok(())
        }
        Err(e) => Err(anyhow::Error::new(e).context(format!("Reading file {:?}", rcfile))),
    }?;

    // Extend the argument list with all remaining command line arguments.
    arguments.extend(args.into_iter());
    let opts = Opts::from_iter(&arguments);
    if opts.logging != logging {
        // Try re-initializing the logger.  Ignore errors.
        let _ = env_logger::Builder::from_default_env()
            .filter(None, opts.logging)
            .try_init();
    }
    Ok(opts)
}

#[derive(serde::Serialize, serde::Deserialize)]
pub struct SessionStartResult {
    port: u16,
}

/// Spawn a child process, passing all the same arguments to the child, letting it instantiate a
/// Transport based on the command line arguments, listen on a TCP socket, and run as a daemon
/// process serving network requests.  Success of the child is verified by means of a
/// `SessionStartResult` JSON message sent through the standard output pipe.
fn start_session(run_file_fn: impl FnOnce(u16) -> PathBuf) -> Result<Box<dyn Serialize>> {
    let mut child = Command::new(env::current_exe()?) // Same executable
        .arg("--child") // Add argument to let the new process know it is the daemon child
        .args(args_os().skip(1)) // Propagate all existing arguments: --interface, etc.
        .stdin(Stdio::null()) // Not used by child, disconnect from terminal
        .stdout(Stdio::piped()) // Used for signalling completion of daemon startup
        .stderr(Stdio::inherit()) // May be used for error messages during daemon startup
        .spawn()?;

    match serde_json::from_reader::<&mut ChildStdout, Result<SessionStartResult, String>>(
        child.stdout.as_mut().unwrap(),
    ) {
        Ok(Ok(result)) => {
            // Create a pid file corresponding to the requested TCP port.
            let path = run_file_fn(result.port);
            File::create(path)?.write_all(format!("{}\n", child.id()).as_bytes())?;
            Ok(Box::new(result))
        }
        Ok(Err(e)) => bail!(e),
        Err(e) => bail!("Child process failed to start: {}", e),
    }
}

// This method runs in the daemon child.  It will instantiate SessionHandler to bind to a
// socket, then report the chosen port number to the parent process by means of a serialized
// `SessionStartResult` sent through the stdout anonymous pipe, and finally enter an infnite
// loop, processing connections on that socket
fn session_child(listen_port: Option<u16>, backend_opts: &backend::BackendOpts) -> Result<()> {
    let transport = backend::create(backend_opts)?;
    let mut session = SessionHandler::init(&transport, listen_port)?;
    // Instantiation of Transport backend, and binding to a socket was successful, now go
    // through the process of making this process a daemon, disconnected from the
    // terminal that was used to start it.

    // All configuration files have been processed (relative to current direction), we can now
    // drop the reference to the file system, (in case the admin wants to unmount while this
    // daemon is still running.)
    env::set_current_dir("/")?;

    // Close stderr, which remained open in order to allow any errors from the above code to
    // surface, but needs to be severed in order for the daemon to avoid being killed by SIGHUP
    // if the user closes the terminal window.
    dup2(File::open("/dev/null")?.as_raw_fd(), 2)?;

    // After severing the only connection to the controlling terminal inherited from the parent,
    // we can now establish a new Unix "session" for this process, which will not be
    // "controlled" by any terminal.  This means that this daemon will not be killed by SIGHUP,
    // in case the terminal that was used for running `session start` is later closed.
    setsid()?;

    // Report startup success to parent process.
    serde_json::to_writer::<io::Stdout, Result<SessionStartResult, String>>(
        io::stdout(),
        &Ok(SessionStartResult {
            port: session.get_port(),
        }),
    )?;
    io::stdout().flush()?;

    // Closing the standard output pipe is the signal to the parent process that this child has
    // started up successfully.  We close the pipe indirectly, by replacing file descriptor 1
    // with one pointing to /dev/null.  This will ensure that any subsequent accidentally
    // executed println!() will be a no-op, rather than trigger termination via SIGPIPE.
    dup2(2, 1)?;

    // Indefinitely run command processing loop in this daemon process.
    session.run_loop()
}

#[derive(serde::Serialize, serde::Deserialize)]
pub struct SessionStopResult {}

/// Load .pid file based on given port number, and send SIGTERM to the process identified in the
/// file, to request the daemon gracefully shut down.
fn stop_session(run_file_fn: impl FnOnce(u16) -> PathBuf, port: u16) -> Result<Box<dyn Serialize>> {
    // Read the pid file corresponding to the requested TCP port.
    let path = run_file_fn(port);
    let pid: i32 = FromStr::from_str(&fs::read_to_string(&path)?.trim())?;
    // Send signal to daemon process, asking it to terminate.
    signal::kill(Pid::from_raw(pid), Signal::SIGTERM)?;
    // Wait for daemon process to stop.
    loop {
        std::thread::sleep(Duration::from_millis(100));
        // Send "signal 0", meaning that the kernel performs error checks (among those, checking
        // that the target process exists), without actually sending any signal.
        match signal::kill(Pid::from_raw(pid), None) {
            Ok(()) => (), // Process still running, repeat.
            Err(nix::Error::Sys(nix::errno::Errno::ESRCH)) => {
                // Process could not be found, meaning that it has terminated, as expected.
                fs::remove_file(&path)?;
                return Ok(Box::new(SessionStopResult {}));
            }
            Err(e) => bail!("Unexpected error querying process presence: {}", e),
        }
    }
}

fn main() -> Result<()> {
    let opts = parse_command_line(Opts::from_args(), args_os())?;

    if opts.debug {
        // Start session process in foreground (do not daemonize)
        let transport = backend::create(&opts.backend_opts)?;
        let mut session = SessionHandler::init(&transport, opts.listen_port)?;
        println!("Listening on port {}", session.get_port());
        session.run_loop()?;
        return Ok(());
    }

    if opts.child {
        // This process is a child, which is supposed to stay running as a daemon.
        match session_child(opts.listen_port, &opts.backend_opts) {
            Ok(()) => process::exit(0),
            Err(e) => {
                // Report any error to parent process though stdout pipe.
                serde_json::to_writer::<io::Stdout, Result<SessionStartResult, String>>(
                    io::stdout(),
                    &Err(format!("{}", e).to_string()),
                )?;
                process::exit(1)
            }
        }
    }

    // Locate directory to use for .pid files
    let base_dirs = BaseDirs::new().unwrap();
    let run_user_dir = base_dirs
        .runtime_dir()
        .ok_or(anyhow!("No /run/user directory"))?;
    let run_file_fn = |port: u16| {
        let mut p = PathBuf::from(run_user_dir);
        p.push(format!("opentitansession.{}.pid", port));
        p
    };

    let value = if opts.stop {
        // Send signal to daemon process to stop
        stop_session(run_file_fn, opts.listen_port.unwrap_or(9900))?
    } else {
        // Fork a daemon process
        start_session(run_file_fn)?
    };
    println!("{}", serde_json::to_string_pretty(&value)?);
    Ok(())
}
<|endoftext|>


<commit_before>use regex::Regex;

use support::project;

#[test]
fn migration_generate_creates_a_migration_with_the_proper_name() {
    let p = project("migration_name")
        .folder("migrations")
        .build();
    let result = p.command("migration")
        .arg("generate")
        .arg("hello")
        .run();

    let expected_stdout = Regex::new("\
Creating migrations.\\d{14}_hello.up.sql
Creating migrations.\\d{14}_hello.down.sql\
        ").unwrap();
    assert!(result.is_success(), "Command failed: {:?}", result);
    assert!(expected_stdout.is_match(result.stdout()));

    let migrations = p.migrations();
    assert_eq!(1, migrations.len());

    let migration = &migrations[0];
    assert_eq!("hello", migration.name());
    assert!(migration.path().join("up.sql").exists());
    assert!(migration.path().join("down.sql").exists());
}

#[test]
fn migration_generate_creates_a_migration_with_initital_contents() {
    let p = project("migration_name")
        .folder("migrations")
        .build();
    let result = p.command("migration")
        .arg("generate")
        .arg("hello")
        .run();
    assert!(result.is_success(), "Command failed: {:?}", result);

    let migrations = p.migrations();
    let migration = &migrations[0];

    let up = file_content(&migration.path().join("up.sql"));
    let down = file_content(&migration.path().join("down.sql"));

    assert_eq!(up.trim(), "-- Your SQL goes here");
    assert_eq!(down.trim(), "-- This file should undo anything in `up.sql`");

    fn file_content<P: AsRef<::std::path::Path>>(path: P) -> String {
        use std::io::Read;

        let mut file = ::std::fs::File::open(path).unwrap();
        let mut contents = String::new();
        file.read_to_string(&mut contents).unwrap();

        contents
    }
}

#[test]
fn migration_generate_doesnt_require_database_url_to_be_set() {
    let p = project("migration_name")
        .folder("migrations")
        .build();
    let result = p.command_without_database_url("migration")
        .arg("generate")
        .arg("hello")
        .run();

    assert!(result.is_success(), "Command failed: {:?}", result);
}

#[test]
fn migration_version_can_be_specified_on_creation() {
    let p = project("migration_name")
        .folder("migrations")
        .build();
    let result = p.command("migration")
        .arg("generate")
        .arg("hello")
        .arg("--version=1234")
        .run();

    let expected_stdout = Regex::new("\
Creating migrations.1234_hello.up.sql
Creating migrations.1234_hello.down.sql
").unwrap();
    assert!(result.is_success(), "Command failed: {:?}", result);
    assert!(expected_stdout.is_match(result.stdout()));

    assert!(p.has_file("migrations/1234_hello/up.sql"));
    assert!(p.has_file("migrations/1234_hello/down.sql"));
}

#[test]
fn migration_directory_can_be_specified_for_generate_by_command_line_arg() {
    let p = project("migration_name")
        .folder("foo")
        .build();
    let result = p.command("migration")
        .arg("generate")
        .arg("stuff")
        .arg("--version=12345")
        .arg("--migration-dir=foo")
        .run();

    let expected_stdout = Regex::new("\
Creating foo.12345_stuff.up.sql
Creating foo.12345_stuff.down.sql
").unwrap();
    assert!(result.is_success(), "Command failed: {:?}", result);
    assert!(expected_stdout.is_match(result.stdout()));

    assert!(p.has_file("foo/12345_stuff/up.sql"));
    assert!(p.has_file("foo/12345_stuff/down.sql"));
}

#[test]
fn migration_directory_can_be_specified_for_generate_by_env_var() {
    let p = project("migration_name")
        .folder("bar")
        .build();
    let result = p.command("migration")
        .arg("generate")
        .arg("stuff")
        .arg("--version=12345")
        .env("MIGRATION_DIRECTORY", "bar")
        .run();

    let expected_stdout = Regex::new("\
Creating bar.12345_stuff.up.sql
Creating bar.12345_stuff.down.sql
").unwrap();
    assert!(result.is_success(), "Command failed: {:?}", result);
    assert!(expected_stdout.is_match(result.stdout()));

    assert!(p.has_file("bar/12345_stuff/up.sql"));
    assert!(p.has_file("bar/12345_stuff/down.sql"));
}
<commit_msg>update `migration generate` tests<commit_after>use regex::Regex;
use chrono::prelude::*;

use support::project;
use diesel::migrations::TIMESTAMP_FORMAT;

#[test]
fn migration_generate_creates_a_migration_with_the_proper_name() {
    let p = project("migration_name")
        .folder("migrations")
        .build();
    let result = p.command("migration")
        .arg("generate")
        .arg("hello")
        .run();

    // check overall output
    let expected_stdout = Regex::new("\
Creating migrations.\\d*_hello.up.sql
Creating migrations.\\d*_hello.down.sql\
        ").unwrap();
    assert!(result.is_success(), "Command failed: {:?}", result);
    assert!(expected_stdout.is_match(result.stdout()));

    // check timestamps are properly formatted
    let captured_timestamps = Regex::new(r"(?P<stamp>\d*)_hello").unwrap();
    let mut stamps_found = 0;
    for caps in captured_timestamps.captures_iter(result.stdout()) {
        let timestamp = UTC.datetime_from_str(&caps["stamp"], TIMESTAMP_FORMAT);
        assert!(timestamp.is_ok(), "Found invalid timestamp format: {:?}", &caps["stamp"]);
        stamps_found += 1;
    }
    assert_eq!(stamps_found, 2);

    let migrations = p.migrations();
    assert_eq!(1, migrations.len());

    let migration = &migrations[0];
    assert_eq!("hello", migration.name());
    assert!(migration.path().join("up.sql").exists());
    assert!(migration.path().join("down.sql").exists());
}

#[test]
fn migration_generate_creates_a_migration_with_initital_contents() {
    let p = project("migration_name")
        .folder("migrations")
        .build();
    let result = p.command("migration")
        .arg("generate")
        .arg("hello")
        .run();
    assert!(result.is_success(), "Command failed: {:?}", result);

    let migrations = p.migrations();
    let migration = &migrations[0];

    let up = file_content(&migration.path().join("up.sql"));
    let down = file_content(&migration.path().join("down.sql"));

    assert_eq!(up.trim(), "-- Your SQL goes here");
    assert_eq!(down.trim(), "-- This file should undo anything in `up.sql`");

    fn file_content<P: AsRef<::std::path::Path>>(path: P) -> String {
        use std::io::Read;

        let mut file = ::std::fs::File::open(path).unwrap();
        let mut contents = String::new();
        file.read_to_string(&mut contents).unwrap();

        contents
    }
}

#[test]
fn migration_generate_doesnt_require_database_url_to_be_set() {
    let p = project("migration_name")
        .folder("migrations")
        .build();
    let result = p.command_without_database_url("migration")
        .arg("generate")
        .arg("hello")
        .run();

    assert!(result.is_success(), "Command failed: {:?}", result);
}

#[test]
fn migration_version_can_be_specified_on_creation() {
    let p = project("migration_name")
        .folder("migrations")
        .build();
    let result = p.command("migration")
        .arg("generate")
        .arg("hello")
        .arg("--version=1234")
        .run();

    let expected_stdout = Regex::new("\
Creating migrations.1234_hello.up.sql
Creating migrations.1234_hello.down.sql
").unwrap();
    assert!(result.is_success(), "Command failed: {:?}", result);
    assert!(expected_stdout.is_match(result.stdout()));

    assert!(p.has_file("migrations/1234_hello/up.sql"));
    assert!(p.has_file("migrations/1234_hello/down.sql"));
}

#[test]
fn migration_directory_can_be_specified_for_generate_by_command_line_arg() {
    let p = project("migration_name")
        .folder("foo")
        .build();
    let result = p.command("migration")
        .arg("generate")
        .arg("stuff")
        .arg("--version=12345")
        .arg("--migration-dir=foo")
        .run();

    let expected_stdout = Regex::new("\
Creating foo.12345_stuff.up.sql
Creating foo.12345_stuff.down.sql
").unwrap();
    assert!(result.is_success(), "Command failed: {:?}", result);
    assert!(expected_stdout.is_match(result.stdout()));

    assert!(p.has_file("foo/12345_stuff/up.sql"));
    assert!(p.has_file("foo/12345_stuff/down.sql"));
}

#[test]
fn migration_directory_can_be_specified_for_generate_by_env_var() {
    let p = project("migration_name")
        .folder("bar")
        .build();
    let result = p.command("migration")
        .arg("generate")
        .arg("stuff")
        .arg("--version=12345")
        .env("MIGRATION_DIRECTORY", "bar")
        .run();

    let expected_stdout = Regex::new("\
Creating bar.12345_stuff.up.sql
Creating bar.12345_stuff.down.sql
").unwrap();
    assert!(result.is_success(), "Command failed: {:?}", result);
    assert!(expected_stdout.is_match(result.stdout()));

    assert!(p.has_file("bar/12345_stuff/up.sql"));
    assert!(p.has_file("bar/12345_stuff/down.sql"));
}
<|endoftext|>