<commit_before>#[macro_use]
extern crate askama;

use askama::Template;

use std::collections::HashMap;

#[derive(Template)]
#[template(path = "simple.html")]
struct VariablesTemplate<'a> {
    strvar: &'a str,
    num: i64,
    i18n: String,
}

#[test]
fn test_variables() {
    let s = VariablesTemplate {
        strvar: "foo",
        num: 42,
        i18n: "Iñtërnâtiônàlizætiøn".to_string(),
    };
    assert_eq!(
        s.render().unwrap(),
        "\nhello world, foo\n\
         with number: 42\n\
         Iñtërnâtiônàlizætiøn is important\n\
         in vars too: Iñtërnâtiônàlizætiøn"
    );
    assert_eq!(VariablesTemplate::extension(), Some("html"));
}

#[derive(Template)]
#[template(path = "simple-no-escape.txt")]
struct VariablesTemplateNoEscape<'a> {
    strvar: &'a str,
    num: i64,
    i18n: String,
}

#[test]
fn test_variables_no_escape() {
    let s = VariablesTemplateNoEscape {
        strvar: "foo",
        num: 42,
        i18n: "Iñtërnâtiônàlizætiøn".to_string(),
    };
    assert_eq!(
        s.render().unwrap(),
        "\nhello world, foo\n\
         with number: 42\n\
         Iñtërnâtiônàlizætiøn is important\n\
         in vars too: Iñtërnâtiônàlizætiøn"
    );
}

#[derive(Template)]
#[template(path = "if.html")]
struct IfTemplate {
    cond: bool,
}

#[test]
fn test_if() {
    let s = IfTemplate { cond: true };
    assert_eq!(s.render().unwrap(), "true");
}

#[derive(Template)]
#[template(path = "else.html")]
struct ElseTemplate {
    cond: bool,
}

#[test]
fn test_else() {
    let s = ElseTemplate { cond: false };
    assert_eq!(s.render().unwrap(), "false");
}

#[derive(Template)]
#[template(path = "else-if.html")]
struct ElseIfTemplate {
    cond: bool,
    check: bool,
}

#[test]
fn test_else_if() {
    let s = ElseIfTemplate {
        cond: false,
        check: true,
    };
    assert_eq!(s.render().unwrap(), "checked");
}

#[derive(Template)]
#[template(path = "literals.html")]
struct LiteralsTemplate {}

#[test]
fn test_literals() {
    let s = LiteralsTemplate {};
    assert_eq!(s.render().unwrap(), "a");
}

struct Holder {
    a: usize,
}

#[derive(Template)]
#[template(path = "attr.html")]
struct AttrTemplate {
    inner: Holder,
}

#[test]
fn test_attr() {
    let t = AttrTemplate {
        inner: Holder { a: 5 },
    };
    assert_eq!(t.render().unwrap(), "5");
}

#[derive(Template)]
#[template(path = "tuple-attr.html")]
struct TupleAttrTemplate<'a> {
    tuple: (&'a str, &'a str),
}

#[test]
fn test_tuple_attr() {
    let t = TupleAttrTemplate {
        tuple: ("foo", "bar"),
    };
    assert_eq!(t.render().unwrap(), "foobar");
}

struct NestedHolder {
    holder: Holder,
}

#[derive(Template)]
#[template(path = "nested-attr.html")]
struct NestedAttrTemplate {
    inner: NestedHolder,
}

#[test]
fn test_nested_attr() {
    let t = NestedAttrTemplate {
        inner: NestedHolder {
            holder: Holder { a: 5 },
        },
    };
    assert_eq!(t.render().unwrap(), "5");
}

#[derive(Template)]
#[template(path = "option.html")]
struct OptionTemplate<'a> {
    var: Option<&'a str>,
}

#[test]
fn test_option() {
    let some = OptionTemplate { var: Some("foo") };
    assert_eq!(some.render().unwrap(), "some: foo");
    let none = OptionTemplate { var: None };
    assert_eq!(none.render().unwrap(), "none");
}

#[derive(Template)]
#[template(path = "generics.html")]
struct GenericsTemplate<T: std::fmt::Display, U = u8>
where
    U: std::fmt::Display,
{
    t: T,
    u: U,
}

#[test]
fn test_generics() {
    let t = GenericsTemplate { t: "a", u: 42 };
    assert_eq!(t.render().unwrap(), "a42");
}

#[derive(Template)]
#[template(path = "composition.html")]
struct CompositionTemplate {
    foo: IfTemplate,
}

#[test]
fn test_composition() {
    let t = CompositionTemplate {
        foo: IfTemplate { cond: true },
    };
    assert_eq!(t.render().unwrap(), "composed: true");
}

#[derive(PartialEq, Eq)]
enum Alphabet {
    Alpha,
}

#[derive(Template)]
#[template(
    source = "{% if x == Alphabet::Alpha %}true{% endif %}",
    ext = "txt"
)]
struct PathCompareTemplate {
    x: Alphabet,
}

#[test]
fn test_path_compare() {
    let t = PathCompareTemplate { x: Alphabet::Alpha };
    assert_eq!(t.render().unwrap(), "true");
}

#[derive(Template)]
#[template(
    source = "{% for i in [\"a\", \"\"] %}{{ i }}{% endfor %}",
    ext = "txt"
)]
struct ArrayTemplate {}

#[test]
fn test_slice_literal() {
    let t = ArrayTemplate {};
    assert_eq!(t.render().unwrap(), "a");
}

#[derive(Template)]
#[template(source = "  {# foo -#} ", ext = "txt")]
struct CommentTemplate {}

#[test]
fn test_comment() {
    let t = CommentTemplate {};
    assert_eq!(t.render().unwrap(), "  ");
}

#[derive(Template)]
#[template(source = "{% if !foo %}Hello{% endif %}", ext = "txt")]
struct NegationTemplate {
    foo: bool,
}

#[test]
fn test_negation() {
    let t = NegationTemplate { foo: false };
    assert_eq!(t.render().unwrap(), "Hello");
}

#[derive(Template)]
#[template(source = "{% if foo > -2 %}Hello{% endif %}", ext = "txt")]
struct MinusTemplate {
    foo: i8,
}

#[test]
fn test_minus() {
    let t = MinusTemplate { foo: 1 };
    assert_eq!(t.render().unwrap(), "Hello");
}

#[derive(Template)]
#[template(source = "{{ foo[\"bar\"] }}", ext = "txt")]
struct IndexTemplate {
    foo: HashMap<String, String>,
}

#[test]
fn test_index() {
    let mut foo = HashMap::new();
    foo.insert("bar".into(), "baz".into());
    let t = IndexTemplate { foo };
    assert_eq!(t.render().unwrap(), "baz");
}

#[derive(Template)]
#[template(source = "foo", ext = "txt")]
struct Empty;

#[test]
fn test_empty() {
    assert_eq!(Empty.render().unwrap(), "foo");
}

mod without_import_on_derive {
    #[derive(askama::Template)]
    #[template(source = "foo", ext = "txt")]
    struct WithoutImport;

    #[test]
    fn test_without_import() {
        use askama::Template;
        assert_eq!(WithoutImport.render().unwrap(), "foo");
    }
}
<commit_msg>Add e2e test for escape<commit_after>#[macro_use]
extern crate askama;

use askama::Template;

use std::collections::HashMap;

#[derive(Template)]
#[template(path = "simple.html")]
struct VariablesTemplate<'a> {
    strvar: &'a str,
    num: i64,
    i18n: String,
}

#[test]
fn test_variables() {
    let s = VariablesTemplate {
        strvar: "foo",
        num: 42,
        i18n: "Iñtërnâtiônàlizætiøn".to_string(),
    };
    assert_eq!(
        s.render().unwrap(),
        "\nhello world, foo\n\
         with number: 42\n\
         Iñtërnâtiônàlizætiøn is important\n\
         in vars too: Iñtërnâtiônàlizætiøn"
    );
    assert_eq!(VariablesTemplate::extension(), Some("html"));
}

#[derive(Template)]
#[template(path = "hello.html")]
struct EscapeTemplate<'a> {
    name: &'a str,
}

#[test]
fn test_escape() {
    let s = EscapeTemplate { name: "<>&\"'/" };

    assert_eq!(
        s.render().unwrap(),
        "Hello, <>&"'/!"
    );
}

#[derive(Template)]
#[template(path = "simple-no-escape.txt")]
struct VariablesTemplateNoEscape<'a> {
    strvar: &'a str,
    num: i64,
    i18n: String,
}

#[test]
fn test_variables_no_escape() {
    let s = VariablesTemplateNoEscape {
        strvar: "foo",
        num: 42,
        i18n: "Iñtërnâtiônàlizætiøn".to_string(),
    };
    assert_eq!(
        s.render().unwrap(),
        "\nhello world, foo\n\
         with number: 42\n\
         Iñtërnâtiônàlizætiøn is important\n\
         in vars too: Iñtërnâtiônàlizætiøn"
    );
}

#[derive(Template)]
#[template(path = "if.html")]
struct IfTemplate {
    cond: bool,
}

#[test]
fn test_if() {
    let s = IfTemplate { cond: true };
    assert_eq!(s.render().unwrap(), "true");
}

#[derive(Template)]
#[template(path = "else.html")]
struct ElseTemplate {
    cond: bool,
}

#[test]
fn test_else() {
    let s = ElseTemplate { cond: false };
    assert_eq!(s.render().unwrap(), "false");
}

#[derive(Template)]
#[template(path = "else-if.html")]
struct ElseIfTemplate {
    cond: bool,
    check: bool,
}

#[test]
fn test_else_if() {
    let s = ElseIfTemplate {
        cond: false,
        check: true,
    };
    assert_eq!(s.render().unwrap(), "checked");
}

#[derive(Template)]
#[template(path = "literals.html")]
struct LiteralsTemplate {}

#[test]
fn test_literals() {
    let s = LiteralsTemplate {};
    assert_eq!(s.render().unwrap(), "a");
}

struct Holder {
    a: usize,
}

#[derive(Template)]
#[template(path = "attr.html")]
struct AttrTemplate {
    inner: Holder,
}

#[test]
fn test_attr() {
    let t = AttrTemplate {
        inner: Holder { a: 5 },
    };
    assert_eq!(t.render().unwrap(), "5");
}

#[derive(Template)]
#[template(path = "tuple-attr.html")]
struct TupleAttrTemplate<'a> {
    tuple: (&'a str, &'a str),
}

#[test]
fn test_tuple_attr() {
    let t = TupleAttrTemplate {
        tuple: ("foo", "bar"),
    };
    assert_eq!(t.render().unwrap(), "foobar");
}

struct NestedHolder {
    holder: Holder,
}

#[derive(Template)]
#[template(path = "nested-attr.html")]
struct NestedAttrTemplate {
    inner: NestedHolder,
}

#[test]
fn test_nested_attr() {
    let t = NestedAttrTemplate {
        inner: NestedHolder {
            holder: Holder { a: 5 },
        },
    };
    assert_eq!(t.render().unwrap(), "5");
}

#[derive(Template)]
#[template(path = "option.html")]
struct OptionTemplate<'a> {
    var: Option<&'a str>,
}

#[test]
fn test_option() {
    let some = OptionTemplate { var: Some("foo") };
    assert_eq!(some.render().unwrap(), "some: foo");
    let none = OptionTemplate { var: None };
    assert_eq!(none.render().unwrap(), "none");
}

#[derive(Template)]
#[template(path = "generics.html")]
struct GenericsTemplate<T: std::fmt::Display, U = u8>
where
    U: std::fmt::Display,
{
    t: T,
    u: U,
}

#[test]
fn test_generics() {
    let t = GenericsTemplate { t: "a", u: 42 };
    assert_eq!(t.render().unwrap(), "a42");
}

#[derive(Template)]
#[template(path = "composition.html")]
struct CompositionTemplate {
    foo: IfTemplate,
}

#[test]
fn test_composition() {
    let t = CompositionTemplate {
        foo: IfTemplate { cond: true },
    };
    assert_eq!(t.render().unwrap(), "composed: true");
}

#[derive(PartialEq, Eq)]
enum Alphabet {
    Alpha,
}

#[derive(Template)]
#[template(
    source = "{% if x == Alphabet::Alpha %}true{% endif %}",
    ext = "txt"
)]
struct PathCompareTemplate {
    x: Alphabet,
}

#[test]
fn test_path_compare() {
    let t = PathCompareTemplate { x: Alphabet::Alpha };
    assert_eq!(t.render().unwrap(), "true");
}

#[derive(Template)]
#[template(
    source = "{% for i in [\"a\", \"\"] %}{{ i }}{% endfor %}",
    ext = "txt"
)]
struct ArrayTemplate {}

#[test]
fn test_slice_literal() {
    let t = ArrayTemplate {};
    assert_eq!(t.render().unwrap(), "a");
}

#[derive(Template)]
#[template(source = "  {# foo -#} ", ext = "txt")]
struct CommentTemplate {}

#[test]
fn test_comment() {
    let t = CommentTemplate {};
    assert_eq!(t.render().unwrap(), "  ");
}

#[derive(Template)]
#[template(source = "{% if !foo %}Hello{% endif %}", ext = "txt")]
struct NegationTemplate {
    foo: bool,
}

#[test]
fn test_negation() {
    let t = NegationTemplate { foo: false };
    assert_eq!(t.render().unwrap(), "Hello");
}

#[derive(Template)]
#[template(source = "{% if foo > -2 %}Hello{% endif %}", ext = "txt")]
struct MinusTemplate {
    foo: i8,
}

#[test]
fn test_minus() {
    let t = MinusTemplate { foo: 1 };
    assert_eq!(t.render().unwrap(), "Hello");
}

#[derive(Template)]
#[template(source = "{{ foo[\"bar\"] }}", ext = "txt")]
struct IndexTemplate {
    foo: HashMap<String, String>,
}

#[test]
fn test_index() {
    let mut foo = HashMap::new();
    foo.insert("bar".into(), "baz".into());
    let t = IndexTemplate { foo };
    assert_eq!(t.render().unwrap(), "baz");
}

#[derive(Template)]
#[template(source = "foo", ext = "txt")]
struct Empty;

#[test]
fn test_empty() {
    assert_eq!(Empty.render().unwrap(), "foo");
}

mod without_import_on_derive {
    #[derive(askama::Template)]
    #[template(source = "foo", ext = "txt")]
    struct WithoutImport;

    #[test]
    fn test_without_import() {
        use askama::Template;
        assert_eq!(WithoutImport.render().unwrap(), "foo");
    }
}
<|endoftext|>


<commit_before>use js_int::Int;
use serde::{Deserialize, Serialize};
use serde_json::{from_str as from_json_str, value::RawValue as RawJsonValue};

#[cfg(feature = "unstable-msc2675")]
use super::relation::Relations;
use super::{room::redaction::SyncRoomRedactionEvent, EventContent, StateEventType};
use crate::{serde::Raw, TransactionId};

/// Extra information about a message event that is not incorporated into the event's hash.
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct MessageLikeUnsigned {
    /// The time in milliseconds that has elapsed since the event was sent.
    ///
    /// This field is generated by the local homeserver, and may be incorrect if the local time on
    /// at least one of the two servers is out of sync, which can cause the age to either be
    /// negative or greater than it actually is.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub age: Option<Int>,

    /// The client-supplied transaction ID, if the client being given the event is the same one
    /// which sent it.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub transaction_id: Option<Box<TransactionId>>,

    /// Server-compiled information from other events relating to this event.
    #[cfg(feature = "unstable-msc2675")]
    #[serde(rename = "m.relations", skip_serializing_if = "Option::is_none")]
    pub relations: Option<Relations>,
}

impl MessageLikeUnsigned {
    /// Create a new `Unsigned` with fields set to `None`.
    pub fn new() -> Self {
        Self::default()
    }

    /// Whether this unsigned data is empty (all fields are `None`).
    ///
    /// This method is used to determine whether to skip serializing the `unsigned` field in room
    /// events. Do not use it to determine whether an incoming `unsigned` field was present - it
    /// could still have been present but contained none of the known fields.
    pub fn is_empty(&self) -> bool {
        #[cfg(not(feature = "unstable-msc2675"))]
        {
            self.age.is_none() && self.transaction_id.is_none()
        }

        #[cfg(feature = "unstable-msc2675")]
        {
            self.age.is_none() && self.transaction_id.is_none() && self.relations.is_none()
        }
    }
}

/// Extra information about a state event that is not incorporated into the event's hash.
#[derive(Clone, Debug, Deserialize, Serialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct StateUnsigned<C: EventContent<EventType = StateEventType>> {
    /// The time in milliseconds that has elapsed since the event was sent.
    ///
    /// This field is generated by the local homeserver, and may be incorrect if the local time on
    /// at least one of the two servers is out of sync, which can cause the age to either be
    /// negative or greater than it actually is.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub age: Option<Int>,

    /// The client-supplied transaction ID, if the client being given the event is the same one
    /// which sent it.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub transaction_id: Option<Box<TransactionId>>,

    /// Optional previous content of the event.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub prev_content: Option<C>,

    /// Server-compiled information from other events relating to this event.
    #[cfg(feature = "unstable-pre-spec")]
    #[serde(rename = "m.relations", skip_serializing_if = "Option::is_none")]
    pub relations: Option<Relations>,
}

impl<C: EventContent<EventType = StateEventType>> StateUnsigned<C> {
    /// Create a new `Unsigned` with fields set to `None`.
    pub fn new() -> Self {
        Self {
            age: None,
            transaction_id: None,
            prev_content: None,
            #[cfg(feature = "unstable-pre-spec")]
            relations: None,
        }
    }

    /// Whether this unsigned data is empty (all fields are `None`).
    ///
    /// This method is used to determine whether to skip serializing the `unsigned` field in room
    /// events. Do not use it to determine whether an incoming `unsigned` field was present - it
    /// could still have been present but contained none of the known fields.
    pub fn is_empty(&self) -> bool {
        #[cfg(not(feature = "unstable-msc2675"))]
        {
            self.age.is_none() && self.transaction_id.is_none() && self.prev_content.is_none()
        }

        #[cfg(feature = "unstable-msc2675")]
        {
            self.age.is_none()
                && self.transaction_id.is_none()
                && self.prev_content.is_none()
                && self.relations.is_none()
        }
    }
}

/// Helper functions for proc-macro code.
///
/// Needs to be public for UI tests.
#[doc(hidden)]
impl<C: EventContent<EventType = StateEventType>> StateUnsigned<C> {
    pub fn _from_parts(event_type: &str, object: &RawJsonValue) -> serde_json::Result<Self> {
        #[derive(Deserialize)]
        #[serde(bound = "")] // Disable default C: Deserialize bound
        struct WithRawPrevContent<C> {
            #[serde(skip_serializing_if = "Option::is_none")]
            age: Option<Int>,
            #[serde(skip_serializing_if = "Option::is_none")]
            transaction_id: Option<Box<TransactionId>>,
            prev_content: Option<Raw<C>>,
            #[cfg(feature = "unstable-pre-spec")]
            #[serde(rename = "m.relations", skip_serializing_if = "Option::is_none")]
            relations: Option<Relations>,
        }

        let raw: WithRawPrevContent<C> = from_json_str(object.get())?;
        let prev_content =
            raw.prev_content.map(|r| r.deserialize_content(event_type.into())).transpose()?;

        Ok(Self {
            age: raw.age,
            transaction_id: raw.transaction_id,
            #[cfg(feature = "unstable-pre-spec")]
            relations: raw.relations,
            prev_content,
        })
    }

    pub fn _map_prev_unsigned<T>(&self, f: impl FnOnce(&C) -> T) -> StateUnsigned<T>
    where
        T: EventContent<EventType = StateEventType>,
    {
        StateUnsigned {
            age: self.age,
            transaction_id: self.transaction_id.clone(),
            prev_content: self.prev_content.as_ref().map(f),
            #[cfg(feature = "unstable-pre-spec")]
            relations: self.relations.clone(),
        }
    }
}

impl<C: EventContent<EventType = StateEventType>> Default for StateUnsigned<C> {
    fn default() -> Self {
        Self::new()
    }
}

/// Extra information about a redacted event that is not incorporated into the event's hash.
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct RedactedUnsigned {
    /// The event that redacted this event, if any.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub redacted_because: Option<Box<SyncRoomRedactionEvent>>,
}

impl RedactedUnsigned {
    /// Create a new `RedactedUnsigned` with field set to `None`.
    pub fn new() -> Self {
        Self::default()
    }

    /// Create a new `RedactedUnsigned` with the given redacted because.
    pub fn new_because(redacted_because: Box<SyncRoomRedactionEvent>) -> Self {
        Self { redacted_because: Some(redacted_because) }
    }

    /// Whether this unsigned data is empty (`redacted_because` is `None`).
    ///
    /// This method is used to determine whether to skip serializing the `unsigned` field in
    /// redacted room events. Do not use it to determine whether an incoming `unsigned` field
    /// was present - it could still have been present but contained none of the known fields.
    pub fn is_empty(&self) -> bool {
        self.redacted_because.is_none()
    }
}
<commit_msg>events: Fix feature gate for aggregated relations<commit_after>use js_int::Int;
use serde::{Deserialize, Serialize};
use serde_json::{from_str as from_json_str, value::RawValue as RawJsonValue};

#[cfg(feature = "unstable-msc2675")]
use super::relation::Relations;
use super::{room::redaction::SyncRoomRedactionEvent, EventContent, StateEventType};
use crate::{serde::Raw, TransactionId};

/// Extra information about a message event that is not incorporated into the event's hash.
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct MessageLikeUnsigned {
    /// The time in milliseconds that has elapsed since the event was sent.
    ///
    /// This field is generated by the local homeserver, and may be incorrect if the local time on
    /// at least one of the two servers is out of sync, which can cause the age to either be
    /// negative or greater than it actually is.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub age: Option<Int>,

    /// The client-supplied transaction ID, if the client being given the event is the same one
    /// which sent it.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub transaction_id: Option<Box<TransactionId>>,

    /// Server-compiled information from other events relating to this event.
    #[cfg(feature = "unstable-msc2675")]
    #[serde(rename = "m.relations", skip_serializing_if = "Option::is_none")]
    pub relations: Option<Relations>,
}

impl MessageLikeUnsigned {
    /// Create a new `Unsigned` with fields set to `None`.
    pub fn new() -> Self {
        Self::default()
    }

    /// Whether this unsigned data is empty (all fields are `None`).
    ///
    /// This method is used to determine whether to skip serializing the `unsigned` field in room
    /// events. Do not use it to determine whether an incoming `unsigned` field was present - it
    /// could still have been present but contained none of the known fields.
    pub fn is_empty(&self) -> bool {
        #[cfg(not(feature = "unstable-msc2675"))]
        {
            self.age.is_none() && self.transaction_id.is_none()
        }

        #[cfg(feature = "unstable-msc2675")]
        {
            self.age.is_none() && self.transaction_id.is_none() && self.relations.is_none()
        }
    }
}

/// Extra information about a state event that is not incorporated into the event's hash.
#[derive(Clone, Debug, Deserialize, Serialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct StateUnsigned<C: EventContent<EventType = StateEventType>> {
    /// The time in milliseconds that has elapsed since the event was sent.
    ///
    /// This field is generated by the local homeserver, and may be incorrect if the local time on
    /// at least one of the two servers is out of sync, which can cause the age to either be
    /// negative or greater than it actually is.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub age: Option<Int>,

    /// The client-supplied transaction ID, if the client being given the event is the same one
    /// which sent it.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub transaction_id: Option<Box<TransactionId>>,

    /// Optional previous content of the event.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub prev_content: Option<C>,

    /// Server-compiled information from other events relating to this event.
    #[cfg(feature = "unstable-msc2675")]
    #[serde(rename = "m.relations", skip_serializing_if = "Option::is_none")]
    pub relations: Option<Relations>,
}

impl<C: EventContent<EventType = StateEventType>> StateUnsigned<C> {
    /// Create a new `Unsigned` with fields set to `None`.
    pub fn new() -> Self {
        Self {
            age: None,
            transaction_id: None,
            prev_content: None,
            #[cfg(feature = "unstable-msc2675")]
            relations: None,
        }
    }

    /// Whether this unsigned data is empty (all fields are `None`).
    ///
    /// This method is used to determine whether to skip serializing the `unsigned` field in room
    /// events. Do not use it to determine whether an incoming `unsigned` field was present - it
    /// could still have been present but contained none of the known fields.
    pub fn is_empty(&self) -> bool {
        #[cfg(not(feature = "unstable-msc2675"))]
        {
            self.age.is_none() && self.transaction_id.is_none() && self.prev_content.is_none()
        }

        #[cfg(feature = "unstable-msc2675")]
        {
            self.age.is_none()
                && self.transaction_id.is_none()
                && self.prev_content.is_none()
                && self.relations.is_none()
        }
    }
}

/// Helper functions for proc-macro code.
///
/// Needs to be public for UI tests.
#[doc(hidden)]
impl<C: EventContent<EventType = StateEventType>> StateUnsigned<C> {
    pub fn _from_parts(event_type: &str, object: &RawJsonValue) -> serde_json::Result<Self> {
        #[derive(Deserialize)]
        #[serde(bound = "")] // Disable default C: Deserialize bound
        struct WithRawPrevContent<C> {
            #[serde(skip_serializing_if = "Option::is_none")]
            age: Option<Int>,
            #[serde(skip_serializing_if = "Option::is_none")]
            transaction_id: Option<Box<TransactionId>>,
            prev_content: Option<Raw<C>>,
            #[cfg(feature = "unstable-msc2675")]
            #[serde(rename = "m.relations", skip_serializing_if = "Option::is_none")]
            relations: Option<Relations>,
        }

        let raw: WithRawPrevContent<C> = from_json_str(object.get())?;
        let prev_content =
            raw.prev_content.map(|r| r.deserialize_content(event_type.into())).transpose()?;

        Ok(Self {
            age: raw.age,
            transaction_id: raw.transaction_id,
            #[cfg(feature = "unstable-msc2675")]
            relations: raw.relations,
            prev_content,
        })
    }

    pub fn _map_prev_unsigned<T>(&self, f: impl FnOnce(&C) -> T) -> StateUnsigned<T>
    where
        T: EventContent<EventType = StateEventType>,
    {
        StateUnsigned {
            age: self.age,
            transaction_id: self.transaction_id.clone(),
            prev_content: self.prev_content.as_ref().map(f),
            #[cfg(feature = "unstable-msc2675")]
            relations: self.relations.clone(),
        }
    }
}

impl<C: EventContent<EventType = StateEventType>> Default for StateUnsigned<C> {
    fn default() -> Self {
        Self::new()
    }
}

/// Extra information about a redacted event that is not incorporated into the event's hash.
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)]
pub struct RedactedUnsigned {
    /// The event that redacted this event, if any.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub redacted_because: Option<Box<SyncRoomRedactionEvent>>,
}

impl RedactedUnsigned {
    /// Create a new `RedactedUnsigned` with field set to `None`.
    pub fn new() -> Self {
        Self::default()
    }

    /// Create a new `RedactedUnsigned` with the given redacted because.
    pub fn new_because(redacted_because: Box<SyncRoomRedactionEvent>) -> Self {
        Self { redacted_because: Some(redacted_because) }
    }

    /// Whether this unsigned data is empty (`redacted_because` is `None`).
    ///
    /// This method is used to determine whether to skip serializing the `unsigned` field in
    /// redacted room events. Do not use it to determine whether an incoming `unsigned` field
    /// was present - it could still have been present but contained none of the known fields.
    pub fn is_empty(&self) -> bool {
        self.redacted_because.is_none()
    }
}
<|endoftext|>


<commit_before>use std::io::Read;
use error::{Result, Error, ErrorKind};
use reader::{FbxEvent, PropertyValue};
use super::CommonState;

#[derive(Debug, Clone)]
pub struct BinaryParser {
    version: u32,
    end_offset_stack: Vec<u32>,
}

impl BinaryParser {
    pub fn new(version: u32) -> Self {
        BinaryParser {
            version: version,
            end_offset_stack: vec![],
        }
    }

    pub fn next<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<FbxEvent> {
        // Check if the previously read node ends here.
        if let Some(&end_pos_top) = self.end_offset_stack.last() {
            if end_pos_top as u64 == common.pos {
                // Reached the end of previously read node.
                self.end_offset_stack.pop();
                return Ok(FbxEvent::EndNode);
            }
        }

        // Read a node record header.
        let node_record_header = try!(NodeRecordHeader::read(reader, &mut common.pos));
        if node_record_header.is_null_record() {
            // End of a node.
            return if let Some(expected_pos) = self.end_offset_stack.pop() {
                if common.pos == expected_pos as u64 {
                    Ok(FbxEvent::EndNode)
                } else {
                    // Data is collapsed (the node doesn't end at expected position).
                    Err(Error::new(
                            common.pos,
                            ErrorKind::DataError(format!("Node does not end at expected position (expected {}, now at {})", expected_pos, common.pos))))
                }
            } else {
                // Data is collapsed (an extra node end marker found).
                Err(Error::new(common.pos, ErrorKind::DataError("An extra node end marker found".to_string())))
            };
        } else {
            // Start of a node.
            self.end_offset_stack.push(node_record_header.end_offset);
        }

        // Read a node name.
        let name = try_read_fixstr!(common.pos, reader, node_record_header.name_len);

        // Read properties.
        let mut properties = Vec::<PropertyValue>::with_capacity(node_record_header.num_properties as usize);
        for _ in 0..node_record_header.num_properties {
            let prop = try!(self.read_property(reader, common));
            properties.push(prop);
        }

        Ok(FbxEvent::StartNode {
            name: name,
            properties: properties,
        })
    }

    fn read_property<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<PropertyValue> {
        let type_code = try_read_le_u8!(common.pos, reader);
        // type code must be ASCII.
        let type_code = if type_code > 0x80 {
            return Err(Error::new(common.pos-1, ErrorKind::DataError(format!("Expected property type code (ASCII) but got {:#x}", type_code))));
        } else {
            type_code as char
        };
        let value = match type_code {
            // 1 bit boolean (1: true, 0: false) encoded as the LSB of a 1 byte value.
            'C' => {
                let val = try_read_le_u8!(common.pos, reader);
                // It seems 'T' (0x54) is used as `false`, 'T' (0x59) is used as `true`.
                if (val != 'T' as u8) && (val != 'Y' as u8) {
                    // Should this treated as error?
                    // (I don't know whether other characters than 'T' and 'Y' are allowed...)
                    warn!("Expected 'T' or 'Y' for representaton of boolean property value, but got {:#x}", val);
                }
                // Check LSB.
                PropertyValue::Bool(val & 1 == 1)
            },
            // 2 byte signed integer.
            'Y' => {
                PropertyValue::I16(try_read_le_i16!(common.pos, reader))
            },
            // 4 byte signed integer.
            'I' => {
                PropertyValue::I32(try_read_le_i32!(common.pos, reader))
            },
            // 4 byte single-precision IEEE 754 floating-point number.
            'F' => {
                PropertyValue::F32(try_read_le_f32!(common.pos, reader))
            },
            // 8 byte double-precision IEEE 754 floating-point number.
            'D' => {
                PropertyValue::F64(try_read_le_f64!(common.pos, reader))
            },
            // 8 byte signed integer.
            'L' => {
                PropertyValue::I64(try_read_le_i64!(common.pos, reader))
            },
            // Array types
            'f'|'d'|'l'|'i'|'b' => {
                return Err(Error::new(
                        common.pos,
                        ErrorKind::Unimplemented("Parser for array type of property value is not implemented yet".to_string())));
            },
            // String
            'S' => {
                let length = try_read_le_u32!(common.pos, reader);
                PropertyValue::String(try_read_fixstr!(common.pos, reader, length))
            },
            // Raw binary data
            'R' => {
                let length = try_read_le_u32!(common.pos, reader);
                PropertyValue::Binary(try_read_exact!(common.pos, reader, length))
            },
            _ => {
                return Err(Error::new(
                        common.pos,
                        ErrorKind::DataError(format!(
                                "Unsupported type code appears in node property: type_code={}({:#x})",
                                type_code, type_code as u8))));
            }
        };
        Ok(value)
    }
}

#[derive(Debug, Copy, Clone)]
struct NodeRecordHeader {
    end_offset: u32,
    num_properties: u32,
    property_list_len: u32,
    name_len: u8,
}

impl NodeRecordHeader {
    pub fn read<R: Read>(reader: &mut R, pos: &mut u64) -> Result<Self> {
        let end_offset = try_read_le_u32!(*pos, reader);
        let num_properties = try_read_le_u32!(*pos, reader);
        let property_list_len = try_read_le_u32!(*pos, reader);
        let name_len = try_read_le_u8!(*pos, reader);
        Ok(NodeRecordHeader {
            end_offset: end_offset,
            num_properties: num_properties,
            property_list_len: property_list_len,
            name_len: name_len,
        })
    }

    /// Check whether the header indicates there are no more children.
    pub fn is_null_record(&self) -> bool {
        self.end_offset == 0
            && self.num_properties == 0
            && self.property_list_len == 0
            && self.name_len == 0
    }
}
<commit_msg>Add `PropertyArrayHeader` type and its member `read()`<commit_after>use std::io::Read;
use error::{Result, Error, ErrorKind};
use reader::{FbxEvent, PropertyValue};
use super::CommonState;

#[derive(Debug, Clone)]
pub struct BinaryParser {
    version: u32,
    end_offset_stack: Vec<u32>,
}

impl BinaryParser {
    pub fn new(version: u32) -> Self {
        BinaryParser {
            version: version,
            end_offset_stack: vec![],
        }
    }

    pub fn next<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<FbxEvent> {
        // Check if the previously read node ends here.
        if let Some(&end_pos_top) = self.end_offset_stack.last() {
            if end_pos_top as u64 == common.pos {
                // Reached the end of previously read node.
                self.end_offset_stack.pop();
                return Ok(FbxEvent::EndNode);
            }
        }

        // Read a node record header.
        let node_record_header = try!(NodeRecordHeader::read(reader, &mut common.pos));
        if node_record_header.is_null_record() {
            // End of a node.
            return if let Some(expected_pos) = self.end_offset_stack.pop() {
                if common.pos == expected_pos as u64 {
                    Ok(FbxEvent::EndNode)
                } else {
                    // Data is collapsed (the node doesn't end at expected position).
                    Err(Error::new(
                            common.pos,
                            ErrorKind::DataError(format!("Node does not end at expected position (expected {}, now at {})", expected_pos, common.pos))))
                }
            } else {
                // Data is collapsed (an extra node end marker found).
                Err(Error::new(common.pos, ErrorKind::DataError("An extra node end marker found".to_string())))
            };
        } else {
            // Start of a node.
            self.end_offset_stack.push(node_record_header.end_offset);
        }

        // Read a node name.
        let name = try_read_fixstr!(common.pos, reader, node_record_header.name_len);

        // Read properties.
        let mut properties = Vec::<PropertyValue>::with_capacity(node_record_header.num_properties as usize);
        for _ in 0..node_record_header.num_properties {
            let prop = try!(self.read_property(reader, common));
            properties.push(prop);
        }

        Ok(FbxEvent::StartNode {
            name: name,
            properties: properties,
        })
    }

    fn read_property<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<PropertyValue> {
        let type_code = try_read_le_u8!(common.pos, reader);
        // type code must be ASCII.
        let type_code = if type_code > 0x80 {
            return Err(Error::new(common.pos-1, ErrorKind::DataError(format!("Expected property type code (ASCII) but got {:#x}", type_code))));
        } else {
            type_code as char
        };
        let value = match type_code {
            // 1 bit boolean (1: true, 0: false) encoded as the LSB of a 1 byte value.
            'C' => {
                let val = try_read_le_u8!(common.pos, reader);
                // It seems 'T' (0x54) is used as `false`, 'T' (0x59) is used as `true`.
                if (val != 'T' as u8) && (val != 'Y' as u8) {
                    // Should this treated as error?
                    // (I don't know whether other characters than 'T' and 'Y' are allowed...)
                    warn!("Expected 'T' or 'Y' for representaton of boolean property value, but got {:#x}", val);
                }
                // Check LSB.
                PropertyValue::Bool(val & 1 == 1)
            },
            // 2 byte signed integer.
            'Y' => {
                PropertyValue::I16(try_read_le_i16!(common.pos, reader))
            },
            // 4 byte signed integer.
            'I' => {
                PropertyValue::I32(try_read_le_i32!(common.pos, reader))
            },
            // 4 byte single-precision IEEE 754 floating-point number.
            'F' => {
                PropertyValue::F32(try_read_le_f32!(common.pos, reader))
            },
            // 8 byte double-precision IEEE 754 floating-point number.
            'D' => {
                PropertyValue::F64(try_read_le_f64!(common.pos, reader))
            },
            // 8 byte signed integer.
            'L' => {
                PropertyValue::I64(try_read_le_i64!(common.pos, reader))
            },
            // Array types
            'f'|'d'|'l'|'i'|'b' => {
                let array_header = try!(PropertyArrayHeader::read(reader, &mut common.pos));
                return Err(Error::new(
                        common.pos,
                        ErrorKind::Unimplemented("Parser for array type of property value is not implemented yet".to_string())));
            },
            // String
            'S' => {
                let length = try_read_le_u32!(common.pos, reader);
                PropertyValue::String(try_read_fixstr!(common.pos, reader, length))
            },
            // Raw binary data
            'R' => {
                let length = try_read_le_u32!(common.pos, reader);
                PropertyValue::Binary(try_read_exact!(common.pos, reader, length))
            },
            _ => {
                return Err(Error::new(
                        common.pos,
                        ErrorKind::DataError(format!(
                                "Unsupported type code appears in node property: type_code={}({:#x})",
                                type_code, type_code as u8))));
            }
        };
        Ok(value)
    }
}

#[derive(Debug, Copy, Clone)]
struct NodeRecordHeader {
    end_offset: u32,
    num_properties: u32,
    property_list_len: u32,
    name_len: u8,
}

impl NodeRecordHeader {
    pub fn read<R: Read>(reader: &mut R, pos: &mut u64) -> Result<Self> {
        let end_offset = try_read_le_u32!(*pos, reader);
        let num_properties = try_read_le_u32!(*pos, reader);
        let property_list_len = try_read_le_u32!(*pos, reader);
        let name_len = try_read_le_u8!(*pos, reader);
        Ok(NodeRecordHeader {
            end_offset: end_offset,
            num_properties: num_properties,
            property_list_len: property_list_len,
            name_len: name_len,
        })
    }

    /// Check whether the header indicates there are no more children.
    pub fn is_null_record(&self) -> bool {
        self.end_offset == 0
            && self.num_properties == 0
            && self.property_list_len == 0
            && self.name_len == 0
    }
}

#[derive(Debug, Copy, Clone)]
pub struct PropertyArrayHeader {
    array_length: u32,
    encoding: u32,
    compressed_length: u32,
}

impl PropertyArrayHeader {
    pub fn read<R: Read>(reader: &mut R, pos: &mut u64) -> Result<Self> {
        let array_length = try_read_le_u32!(*pos, reader);
        let encoding = try_read_le_u32!(*pos, reader);
        let compressed_length = try_read_le_u32!(*pos, reader);
        Ok(PropertyArrayHeader {
            array_length: array_length,
            encoding: encoding,
            compressed_length: compressed_length,
        })
    }
}
<|endoftext|>


<commit_before>// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::sync::Arc;
use std::thread;
use std::time::Duration;

use fail;
use kvproto::metapb::{Peer, Region};

use test_raftstore::*;
use tikv::pd::PdClient;
use tikv::raftstore::store::Callback;

fn stale_read_during_splitting(right_derive: bool) {
    let _guard = crate::setup();

    let count = 3;
    let mut cluster = new_node_cluster(0, count);
    cluster.cfg.raft_store.right_derive_when_split = right_derive;
    let election_timeout = configure_for_lease_read(&mut cluster, None, None);
    cluster.run();

    // Write the initial values.
    let key1 = b"k1";
    let v1 = b"v1";
    cluster.must_put(key1, v1);
    let key2 = b"k2";
    let v2 = b"v2";
    cluster.must_put(key2, v2);

    // Get the first region.
    let region_left = cluster.get_region(key1);
    let region_right = cluster.get_region(key2);
    assert_eq!(region_left, region_right);
    let region1 = region_left;
    assert_eq!(region1.get_id(), 1);
    let peer3 = region1
        .get_peers()
        .iter()
        .find(|p| p.get_id() == 3)
        .unwrap()
        .clone();
    cluster.must_transfer_leader(region1.get_id(), peer3.clone());

    // Get the current leader.
    let leader1 = peer3;

    // Pause the apply worker of peer 3.
    let apply_split = "apply_before_split_1_3";
    fail::cfg(apply_split, "pause").unwrap();

    // Split the frist region.
    cluster.split_region(&region1, key2, Callback::Write(Box::new(move |_| {})));

    // Sleep for a while.
    // The TiKVs that have followers of the old region will elected a leader
    // of the new region.
    //           TiKV A  TiKV B  TiKV C
    // Region 1    L       F       F
    // Region 2    X       L       F
    // Note: A has the peer 3,
    //       L: leader, F: follower, X: peer is not ready.
    thread::sleep(election_timeout);

    // A key that is covered by the old region and the new region.
    let stale_key = if right_derive { key1 } else { key2 };
    // Get the new region.
    let region2 = cluster.get_region_with(stale_key, |region| region != &region1);

    // Get the leader of the new region.
    let leader2 = cluster.leader_of_region(region2.get_id()).unwrap();
    assert_ne!(leader1.get_store_id(), leader2.get_store_id());

    must_not_stale_read(
        &mut cluster,
        stale_key,
        &region1,
        &leader1,
        &region2,
        &leader2,
        apply_split,
    );
}

fn must_not_stale_read(
    cluster: &mut Cluster<NodeCluster>,
    stale_key: &[u8],
    old_region: &Region,
    old_leader: &Peer,
    new_region: &Region,
    new_leader: &Peer,
    fp: &str,
) {
    // A new value for stale_key.
    let v3 = b"v3";
    let mut request = new_request(
        new_region.get_id(),
        new_region.get_region_epoch().clone(),
        vec![new_put_cf_cmd("default", stale_key, v3)],
        false,
    );
    request.mut_header().set_peer(new_leader.clone());
    cluster
        .call_command_on_node(new_leader.get_store_id(), request, Duration::from_secs(5))
        .unwrap();

    // LocalRead.
    let read_quorum = false;
    must_not_eq_on_key(
        cluster,
        stale_key,
        v3,
        read_quorum,
        old_region,
        old_leader,
        new_region,
        new_leader,
    );

    // ReadIndex.
    let read_quorum = true;
    must_not_eq_on_key(
        cluster,
        stale_key,
        v3,
        read_quorum,
        old_region,
        old_leader,
        new_region,
        new_leader,
    );

    // Leaders can always propose read index despite split/merge.
    let propose_readindex = "before_propose_readindex";
    fail::cfg(propose_readindex, "return(true)").unwrap();

    // Can not execute reads that are queued.
    let value1 = read_on_peer(
        cluster,
        old_leader.clone(),
        old_region.clone(),
        stale_key,
        read_quorum,
        Duration::from_secs(1),
    );
    debug!("stale_key: {:?}, {:?}", stale_key, value1);
    value1.unwrap_err(); // Error::Timeout

    // Remove the fp.
    fail::remove(fp);

    // It should read an error instead of timeout.
    let value1 = read_on_peer(
        cluster,
        old_leader.clone(),
        old_region.clone(),
        stale_key,
        read_quorum,
        Duration::from_secs(5),
    );
    debug!("stale_key: {:?}, {:?}", stale_key, value1);
    assert!(value1.unwrap().get_header().has_error());

    // Clean up.
    fail::remove(propose_readindex);
}

fn must_not_eq_on_key(
    cluster: &mut Cluster<NodeCluster>,
    key: &[u8],
    value: &[u8],
    read_quorum: bool,
    old_region: &Region,
    old_leader: &Peer,
    new_region: &Region,
    new_leader: &Peer,
) {
    let value1 = read_on_peer(
        cluster,
        old_leader.clone(),
        old_region.clone(),
        key,
        read_quorum,
        Duration::from_secs(1),
    );
    let value2 = read_on_peer(
        cluster,
        new_leader.clone(),
        new_region.clone(),
        key,
        read_quorum,
        Duration::from_secs(1),
    );
    debug!("stale_key: {:?}, {:?} vs {:?}", key, value1, value2);
    assert_eq!(must_get_value(value2.as_ref().unwrap()).as_slice(), value);
    // The old leader should return an error.
    assert!(
        value1.as_ref().unwrap().get_header().has_error(),
        "{:?}",
        value1
    );
}

#[test]
fn test_node_stale_read_during_splitting_left_derive() {
    stale_read_during_splitting(false);
}

#[test]
fn test_node_stale_read_during_splitting_right_derive() {
    stale_read_during_splitting(true);
}

#[test]
fn test_stale_read_during_merging() {
    let _guard = crate::setup();

    let count = 3;
    let mut cluster = new_node_cluster(0, count);
    configure_for_merge(&mut cluster);
    let lease = configure_for_lease_read(&mut cluster, None, None);
    cluster.cfg.raft_store.right_derive_when_split = false;
    cluster.cfg.raft_store.pd_heartbeat_tick_interval =
        cluster.cfg.raft_store.raft_base_tick_interval.clone();
    debug!("max leader lease: {:?}", lease);
    let pd_client = Arc::clone(&cluster.pd_client);
    pd_client.disable_default_operator();

    cluster.run_conf_change();

    // Write the initial values.
    let key1 = b"k1";
    let v1 = b"v1";
    cluster.must_put(key1, v1);
    let key2 = b"k2";
    let v2 = b"v2";
    cluster.must_put(key2, v2);
    let region = pd_client.get_region(b"k1").unwrap();
    pd_client.must_add_peer(region.get_id(), new_peer(2, 4));
    pd_client.must_add_peer(region.get_id(), new_peer(3, 5));

    cluster.must_split(&region, b"k2");

    let mut region1 = cluster.get_region(key1);
    let mut region1000 = cluster.get_region(key2);
    assert_ne!(region1, region1000);
    assert_eq!(region1.get_id(), 1); // requires disable right_derive.
    let leader1 = region1
        .get_peers()
        .iter()
        .find(|p| p.get_id() == 4)
        .unwrap()
        .clone();
    cluster.must_transfer_leader(region1.get_id(), leader1.clone());

    let leader1000 = region1000
        .get_peers()
        .iter()
        .find(|p| p.get_store_id() != leader1.get_store_id())
        .unwrap()
        .clone();
    cluster.must_transfer_leader(region1000.get_id(), leader1000.clone());
    assert_ne!(leader1.get_store_id(), leader1000.get_store_id());

    //             Merge into
    // region1000 ------------> region1,
    cluster.try_merge(region1000.get_id(), region1.get_id());

    // Pause the apply worker other than peer 4.
    let apply_commit_merge = "apply_before_commit_merge_except_1_4";
    fail::cfg(apply_commit_merge, "pause").unwrap();

    // Wait for commit merge.
    // The TiKVs that have followers of the old region will elected a leader
    // of the new region.
    //             TiKV A  TiKV B  TiKV C
    // Region    1   L       F       F
    // Region 1000   F       L       F
    //           after wait
    // Region    1   L       F       F
    // Region 1000   X       L       F
    // Note: L: leader, F: follower, X: peer is not exist.
    // TODO: what if cluster runs slow and lease is expired.
    // Epoch changed by prepare merge.
    // We can not use `get_region_with` to get the latest info of reigon 1000,
    // because leader1 is not paused, it executes commit merge very fast
    // and reports pd, its range covers region1000.
    //
    // region1000 does prepare merge, it increases ver and conf_ver by 1.
    debug!("before merge: {:?} | {:?}", region1000, region1);
    let region1000_version = region1000.get_region_epoch().get_version() + 1;
    region1000
        .mut_region_epoch()
        .set_version(region1000_version);
    let region1000_conf_version = region1000.get_region_epoch().get_conf_ver() + 1;
    region1000
        .mut_region_epoch()
        .set_conf_ver(region1000_conf_version);

    // Epoch changed by commit merge.
    region1 = cluster.get_region_with(key1, |region| region != &region1);
    debug!("after merge: {:?} | {:?}", region1000, region1);

    // A key that is covered by region 1000 and region 1.
    let stale_key = key2;

    must_not_stale_read(
        &mut cluster,
        stale_key,
        &region1000,
        &leader1000,
        &region1,
        &leader1,
        apply_commit_merge,
    );
}
<commit_msg>tests: stable test_stale_read_during_merging (#4253)<commit_after>// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::sync::Arc;
use std::thread;
use std::time::Duration;

use fail;
use kvproto::metapb::{Peer, Region};

use test_raftstore::*;
use tikv::pd::PdClient;
use tikv::raftstore::store::Callback;

fn stale_read_during_splitting(right_derive: bool) {
    let _guard = crate::setup();

    let count = 3;
    let mut cluster = new_node_cluster(0, count);
    cluster.cfg.raft_store.right_derive_when_split = right_derive;
    let election_timeout = configure_for_lease_read(&mut cluster, None, None);
    cluster.run();

    // Write the initial values.
    let key1 = b"k1";
    let v1 = b"v1";
    cluster.must_put(key1, v1);
    let key2 = b"k2";
    let v2 = b"v2";
    cluster.must_put(key2, v2);

    // Get the first region.
    let region_left = cluster.get_region(key1);
    let region_right = cluster.get_region(key2);
    assert_eq!(region_left, region_right);
    let region1 = region_left;
    assert_eq!(region1.get_id(), 1);
    let peer3 = region1
        .get_peers()
        .iter()
        .find(|p| p.get_id() == 3)
        .unwrap()
        .clone();
    cluster.must_transfer_leader(region1.get_id(), peer3.clone());

    // Get the current leader.
    let leader1 = peer3;

    // Pause the apply worker of peer 3.
    let apply_split = "apply_before_split_1_3";
    fail::cfg(apply_split, "pause").unwrap();

    // Split the frist region.
    cluster.split_region(&region1, key2, Callback::Write(Box::new(move |_| {})));

    // Sleep for a while.
    // The TiKVs that have followers of the old region will elected a leader
    // of the new region.
    //           TiKV A  TiKV B  TiKV C
    // Region 1    L       F       F
    // Region 2    X       L       F
    // Note: A has the peer 3,
    //       L: leader, F: follower, X: peer is not ready.
    thread::sleep(election_timeout);

    // A key that is covered by the old region and the new region.
    let stale_key = if right_derive { key1 } else { key2 };
    // Get the new region.
    let region2 = cluster.get_region_with(stale_key, |region| region != &region1);

    // Get the leader of the new region.
    let leader2 = cluster.leader_of_region(region2.get_id()).unwrap();
    assert_ne!(leader1.get_store_id(), leader2.get_store_id());

    must_not_stale_read(
        &mut cluster,
        stale_key,
        &region1,
        &leader1,
        &region2,
        &leader2,
        apply_split,
    );
}

fn must_not_stale_read(
    cluster: &mut Cluster<NodeCluster>,
    stale_key: &[u8],
    old_region: &Region,
    old_leader: &Peer,
    new_region: &Region,
    new_leader: &Peer,
    fp: &str,
) {
    // A new value for stale_key.
    let v3 = b"v3";
    let mut request = new_request(
        new_region.get_id(),
        new_region.get_region_epoch().clone(),
        vec![new_put_cf_cmd("default", stale_key, v3)],
        false,
    );
    request.mut_header().set_peer(new_leader.clone());
    cluster
        .call_command_on_node(new_leader.get_store_id(), request, Duration::from_secs(5))
        .unwrap();

    // LocalRead.
    let read_quorum = false;
    must_not_eq_on_key(
        cluster,
        stale_key,
        v3,
        read_quorum,
        old_region,
        old_leader,
        new_region,
        new_leader,
    );

    // ReadIndex.
    let read_quorum = true;
    must_not_eq_on_key(
        cluster,
        stale_key,
        v3,
        read_quorum,
        old_region,
        old_leader,
        new_region,
        new_leader,
    );

    // Leaders can always propose read index despite split/merge.
    let propose_readindex = "before_propose_readindex";
    fail::cfg(propose_readindex, "return(true)").unwrap();

    // Can not execute reads that are queued.
    let value1 = read_on_peer(
        cluster,
        old_leader.clone(),
        old_region.clone(),
        stale_key,
        read_quorum,
        Duration::from_secs(1),
    );
    debug!("stale_key: {:?}, {:?}", stale_key, value1);
    value1.unwrap_err(); // Error::Timeout

    // Remove the fp.
    fail::remove(fp);

    // It should read an error instead of timeout.
    let value1 = read_on_peer(
        cluster,
        old_leader.clone(),
        old_region.clone(),
        stale_key,
        read_quorum,
        Duration::from_secs(5),
    );
    debug!("stale_key: {:?}, {:?}", stale_key, value1);
    assert!(value1.unwrap().get_header().has_error());

    // Clean up.
    fail::remove(propose_readindex);
}

fn must_not_eq_on_key(
    cluster: &mut Cluster<NodeCluster>,
    key: &[u8],
    value: &[u8],
    read_quorum: bool,
    old_region: &Region,
    old_leader: &Peer,
    new_region: &Region,
    new_leader: &Peer,
) {
    let value1 = read_on_peer(
        cluster,
        old_leader.clone(),
        old_region.clone(),
        key,
        read_quorum,
        Duration::from_secs(1),
    );
    let value2 = read_on_peer(
        cluster,
        new_leader.clone(),
        new_region.clone(),
        key,
        read_quorum,
        Duration::from_secs(1),
    );
    debug!("stale_key: {:?}, {:?} vs {:?}", key, value1, value2);
    assert_eq!(must_get_value(value2.as_ref().unwrap()).as_slice(), value);
    // The old leader should return an error.
    assert!(
        value1.as_ref().unwrap().get_header().has_error(),
        "{:?}",
        value1
    );
}

#[test]
fn test_node_stale_read_during_splitting_left_derive() {
    stale_read_during_splitting(false);
}

#[test]
fn test_node_stale_read_during_splitting_right_derive() {
    stale_read_during_splitting(true);
}

#[test]
fn test_stale_read_during_merging() {
    let _guard = crate::setup();

    let count = 3;
    let mut cluster = new_node_cluster(0, count);
    configure_for_merge(&mut cluster);
    let election_timeout = configure_for_lease_read(&mut cluster, None, None);
    cluster.cfg.raft_store.right_derive_when_split = false;
    cluster.cfg.raft_store.pd_heartbeat_tick_interval =
        cluster.cfg.raft_store.raft_base_tick_interval.clone();
    debug!("max leader lease: {:?}", election_timeout);
    let pd_client = Arc::clone(&cluster.pd_client);
    pd_client.disable_default_operator();

    cluster.run_conf_change();

    // Write the initial values.
    let key1 = b"k1";
    let v1 = b"v1";
    cluster.must_put(key1, v1);
    let key2 = b"k2";
    let v2 = b"v2";
    cluster.must_put(key2, v2);
    let region = pd_client.get_region(b"k1").unwrap();
    pd_client.must_add_peer(region.get_id(), new_peer(2, 4));
    pd_client.must_add_peer(region.get_id(), new_peer(3, 5));

    cluster.must_split(&region, b"k2");

    let mut region1 = cluster.get_region(key1);
    let mut region1000 = cluster.get_region(key2);
    assert_ne!(region1, region1000);
    assert_eq!(region1.get_id(), 1); // requires disable right_derive.
    let leader1 = region1
        .get_peers()
        .iter()
        .find(|p| p.get_id() == 4)
        .unwrap()
        .clone();
    cluster.must_transfer_leader(region1.get_id(), leader1.clone());

    let leader1000 = region1000
        .get_peers()
        .iter()
        .find(|p| p.get_store_id() != leader1.get_store_id())
        .unwrap()
        .clone();
    cluster.must_transfer_leader(region1000.get_id(), leader1000.clone());
    assert_ne!(leader1.get_store_id(), leader1000.get_store_id());

    // Sleeps an election timeout. The new leader needs enough time to gather
    // all followers progress, in cause the merge request is reject by the
    // log gap too large (min_progress == 0).
    thread::sleep(election_timeout);

    //             merge into
    // region1000 ------------> region1
    cluster.try_merge(region1000.get_id(), region1.get_id());

    // Pause the apply workers except for the peer 4.
    let apply_commit_merge = "apply_before_commit_merge_except_1_4";
    fail::cfg(apply_commit_merge, "pause").unwrap();

    // Wait for commit merge.
    // The TiKVs that have followers of the old region will elected a leader
    // of the new region.
    //             TiKV A  TiKV B  TiKV C
    // Region    1   L       F       F
    // Region 1000   F       L       F
    //           after wait
    // Region    1   L       F       F
    // Region 1000   X       L       F
    // Note: L: leader, F: follower, X: peer is not exist.
    // TODO: what if cluster runs slow and lease is expired.
    // Epoch changed by prepare merge.
    // We can not use `get_region_with` to get the latest info of reigon 1000,
    // because leader1 is not paused, it executes commit merge very fast
    // and reports pd, its range covers region1000.
    //
    // region1000 does prepare merge, it increases ver and conf_ver by 1.
    debug!("before merge: {:?} | {:?}", region1000, region1);
    let region1000_version = region1000.get_region_epoch().get_version() + 1;
    region1000
        .mut_region_epoch()
        .set_version(region1000_version);
    let region1000_conf_version = region1000.get_region_epoch().get_conf_ver() + 1;
    region1000
        .mut_region_epoch()
        .set_conf_ver(region1000_conf_version);

    // Epoch changed by commit merge.
    region1 = cluster.get_region_with(key1, |region| region != &region1);
    debug!("after merge: {:?} | {:?}", region1000, region1);

    // A key that is covered by region 1000 and region 1.
    let stale_key = key2;

    must_not_stale_read(
        &mut cluster,
        stale_key,
        &region1000,
        &leader1000,
        &region1,
        &leader1,
        apply_commit_merge,
    );
}
<|endoftext|>


<commit_before>//  * This file is part of the uutils coreutils package.
//  *
//  * (c) Alex Lyon <arcterus@mail.com>
//  *
//  * For the full copyright and license information, please view the LICENSE
//  * file that was distributed with this source code.

// spell-checker:ignore (ToDOs) ncount routput

#[macro_use]
extern crate uucore;

use clap::{crate_version, App, Arg};
use std::fs::File;
use std::io::{stdin, BufRead, BufReader, Read};
use std::path::Path;
use uucore::InvalidEncodingHandling;

const TAB_WIDTH: usize = 8;

static NAME: &str = "fold";
static SYNTAX: &str = "[OPTION]... [FILE]...";
static SUMMARY: &str = "Writes each file (or standard input if no files are given)
 to standard output whilst breaking long lines";

mod options {
    pub const BYTES: &str = "bytes";
    pub const SPACES: &str = "spaces";
    pub const WIDTH: &str = "width";
    pub const FILE: &str = "file";
}

pub fn uumain(args: impl uucore::Args) -> i32 {
    let args = args
        .collect_str(InvalidEncodingHandling::ConvertLossy)
        .accept_any();

    let (args, obs_width) = handle_obsolete(&args[..]);
    let matches = App::new(executable!())
        .name(NAME)
        .version(crate_version!())
        .usage(SYNTAX)
        .about(SUMMARY)
        .arg(
            Arg::with_name(options::BYTES)
                .long(options::BYTES)
                .short("b")
                .help(
                    "count using bytes rather than columns (meaning control characters \
                     such as newline are not treated specially)",
                )
                .takes_value(false),
        )
        .arg(
            Arg::with_name(options::SPACES)
                .long(options::SPACES)
                .short("s")
                .help("break lines at word boundaries rather than a hard cut-off")
                .takes_value(false),
        )
        .arg(
            Arg::with_name(options::WIDTH)
                .long(options::WIDTH)
                .short("w")
                .help("set WIDTH as the maximum line width rather than 80")
                .value_name("WIDTH")
                .allow_hyphen_values(true)
                .takes_value(true),
        )
        .arg(Arg::with_name(options::FILE).hidden(true).multiple(true))
        .get_matches_from(args);

    let bytes = matches.is_present(options::BYTES);
    let spaces = matches.is_present(options::SPACES);
    let poss_width = match matches.value_of(options::WIDTH) {
        Some(v) => Some(v.to_owned()),
        None => obs_width,
    };

    let width = match poss_width {
        Some(inp_width) => match inp_width.parse::<usize>() {
            Ok(width) => width,
            Err(e) => crash!(1, "illegal width value (\"{}\"): {}", inp_width, e),
        },
        None => 80,
    };

    let files = match matches.values_of(options::FILE) {
        Some(v) => v.map(|v| v.to_owned()).collect(),
        None => vec!["-".to_owned()],
    };

    fold(files, bytes, spaces, width);

    0
}

fn handle_obsolete(args: &[String]) -> (Vec<String>, Option<String>) {
    for (i, arg) in args.iter().enumerate() {
        let slice = &arg;
        if slice.starts_with('-') && slice.len() > 1 && slice.chars().nth(1).unwrap().is_digit(10) {
            let mut v = args.to_vec();
            v.remove(i);
            return (v, Some(slice[1..].to_owned()));
        }
    }
    (args.to_vec(), None)
}

fn fold(filenames: Vec<String>, bytes: bool, spaces: bool, width: usize) {
    for filename in &filenames {
        let filename: &str = &filename;
        let mut stdin_buf;
        let mut file_buf;
        let buffer = BufReader::new(if filename == "-" {
            stdin_buf = stdin();
            &mut stdin_buf as &mut dyn Read
        } else {
            file_buf = safe_unwrap!(File::open(Path::new(filename)));
            &mut file_buf as &mut dyn Read
        });

        if bytes {
            fold_file_bytewise(buffer, spaces, width);
        } else {
            fold_file(buffer, spaces, width);
        }
    }
}

/// Fold `file` to fit `width` (number of columns), counting all characters as
/// one column.
///
/// This function handles folding for the `-b`/`--bytes` option, counting
/// tab, backspace, and carriage return as occupying one column, identically
/// to all other characters in the stream.
///
///  If `spaces` is `true`, attempt to break lines at whitespace boundaries.
fn fold_file_bytewise<T: Read>(mut file: BufReader<T>, spaces: bool, width: usize) {
    let mut line = String::new();

    loop {
        if let Ok(0) = file.read_line(&mut line) {
            break;
        }

        if line == "\n" {
            println!();
            line.truncate(0);
            continue;
        }

        let len = line.len();
        let mut i = 0;

        while i < len {
            let width = if len - i >= width { width } else { len - i };
            let slice = {
                let slice = &line[i..i + width];
                if spaces && i + width < len {
                    match slice.rfind(|c: char| c.is_whitespace() && c != '\r') {
                        Some(m) => &slice[..=m],
                        None => slice,
                    }
                } else {
                    slice
                }
            };

            // Don't duplicate trailing newlines: if the slice is "\n", the
            // previous iteration folded just before the end of the line and
            // has already printed this newline.
            if slice == "\n" {
                break;
            }

            i += slice.len();

            let at_eol = i >= len;

            if at_eol {
                print!("{}", slice);
            } else {
                println!("{}", slice);
            }
        }

        line.truncate(0);
    }
}

/// Fold `file` to fit `width` (number of columns).
///
/// By default `fold` treats tab, backspace, and carriage return specially:
/// tab characters count as 8 columns, backspace decreases the
/// column count, and carriage return resets the column count to 0.
///
/// If `spaces` is `true`, attempt to break lines at whitespace boundaries.
#[allow(unused_assignments)]
fn fold_file<T: Read>(mut file: BufReader<T>, spaces: bool, width: usize) {
    let mut line = String::new();
    let mut output = String::new();
    let mut col_count = 0;
    let mut last_space = None;

    /// Print the output line, resetting the column and character counts.
    ///
    /// If `spaces` is `true`, print the output line up to the last
    /// encountered whitespace character (inclusive) and set the remaining
    /// characters as the start of the next line.
    macro_rules! emit_output {
        () => {
            let consume = match last_space {
                Some(i) => i + 1,
                None => output.len(),
            };

            println!("{}", &output[..consume]);
            output.replace_range(..consume, "");

            // we know there are no tabs left in output, so each char counts
            // as 1 column
            col_count = output.len();

            last_space = None;
        };
    }

    loop {
        if let Ok(0) = file.read_line(&mut line) {
            break;
        }

        for ch in line.chars() {
            if ch == '\n' {
                // make sure to _not_ split output at whitespace, since we
                // know the entire output will fit
                last_space = None;
                emit_output!();
                break;
            }

            if col_count >= width {
                emit_output!();
            }

            match ch {
                '\r' => col_count = 0,
                '\t' => {
                    let next_tab_stop = col_count + TAB_WIDTH - col_count % TAB_WIDTH;

                    if next_tab_stop > width && !output.is_empty() {
                        emit_output!();
                    }

                    col_count = next_tab_stop;
                    last_space = if spaces { Some(output.len()) } else { None };
                }
                '\x08' => {
                    if col_count > 0 {
                        col_count -= 1;
                    }
                }
                _ if spaces && ch.is_whitespace() => {
                    last_space = Some(output.len());
                    col_count += 1;
                }
                _ => col_count += 1,
            };

            output.push(ch);
        }

        if !output.is_empty() {
            print!("{}", output);
            output.truncate(0);
        }

        line.truncate(0);
    }
}
<commit_msg>refactor/fold ~ fix `cargo clippy` complaint (clippy::needless_borrow)<commit_after>//  * This file is part of the uutils coreutils package.
//  *
//  * (c) Alex Lyon <arcterus@mail.com>
//  *
//  * For the full copyright and license information, please view the LICENSE
//  * file that was distributed with this source code.

// spell-checker:ignore (ToDOs) ncount routput

#[macro_use]
extern crate uucore;

use clap::{crate_version, App, Arg};
use std::fs::File;
use std::io::{stdin, BufRead, BufReader, Read};
use std::path::Path;
use uucore::InvalidEncodingHandling;

const TAB_WIDTH: usize = 8;

static NAME: &str = "fold";
static SYNTAX: &str = "[OPTION]... [FILE]...";
static SUMMARY: &str = "Writes each file (or standard input if no files are given)
 to standard output whilst breaking long lines";

mod options {
    pub const BYTES: &str = "bytes";
    pub const SPACES: &str = "spaces";
    pub const WIDTH: &str = "width";
    pub const FILE: &str = "file";
}

pub fn uumain(args: impl uucore::Args) -> i32 {
    let args = args
        .collect_str(InvalidEncodingHandling::ConvertLossy)
        .accept_any();

    let (args, obs_width) = handle_obsolete(&args[..]);
    let matches = App::new(executable!())
        .name(NAME)
        .version(crate_version!())
        .usage(SYNTAX)
        .about(SUMMARY)
        .arg(
            Arg::with_name(options::BYTES)
                .long(options::BYTES)
                .short("b")
                .help(
                    "count using bytes rather than columns (meaning control characters \
                     such as newline are not treated specially)",
                )
                .takes_value(false),
        )
        .arg(
            Arg::with_name(options::SPACES)
                .long(options::SPACES)
                .short("s")
                .help("break lines at word boundaries rather than a hard cut-off")
                .takes_value(false),
        )
        .arg(
            Arg::with_name(options::WIDTH)
                .long(options::WIDTH)
                .short("w")
                .help("set WIDTH as the maximum line width rather than 80")
                .value_name("WIDTH")
                .allow_hyphen_values(true)
                .takes_value(true),
        )
        .arg(Arg::with_name(options::FILE).hidden(true).multiple(true))
        .get_matches_from(args);

    let bytes = matches.is_present(options::BYTES);
    let spaces = matches.is_present(options::SPACES);
    let poss_width = match matches.value_of(options::WIDTH) {
        Some(v) => Some(v.to_owned()),
        None => obs_width,
    };

    let width = match poss_width {
        Some(inp_width) => match inp_width.parse::<usize>() {
            Ok(width) => width,
            Err(e) => crash!(1, "illegal width value (\"{}\"): {}", inp_width, e),
        },
        None => 80,
    };

    let files = match matches.values_of(options::FILE) {
        Some(v) => v.map(|v| v.to_owned()).collect(),
        None => vec!["-".to_owned()],
    };

    fold(files, bytes, spaces, width);

    0
}

fn handle_obsolete(args: &[String]) -> (Vec<String>, Option<String>) {
    for (i, arg) in args.iter().enumerate() {
        let slice = &arg;
        if slice.starts_with('-') && slice.len() > 1 && slice.chars().nth(1).unwrap().is_digit(10) {
            let mut v = args.to_vec();
            v.remove(i);
            return (v, Some(slice[1..].to_owned()));
        }
    }
    (args.to_vec(), None)
}

fn fold(filenames: Vec<String>, bytes: bool, spaces: bool, width: usize) {
    for filename in &filenames {
        let filename: &str = filename;
        let mut stdin_buf;
        let mut file_buf;
        let buffer = BufReader::new(if filename == "-" {
            stdin_buf = stdin();
            &mut stdin_buf as &mut dyn Read
        } else {
            file_buf = safe_unwrap!(File::open(Path::new(filename)));
            &mut file_buf as &mut dyn Read
        });

        if bytes {
            fold_file_bytewise(buffer, spaces, width);
        } else {
            fold_file(buffer, spaces, width);
        }
    }
}

/// Fold `file` to fit `width` (number of columns), counting all characters as
/// one column.
///
/// This function handles folding for the `-b`/`--bytes` option, counting
/// tab, backspace, and carriage return as occupying one column, identically
/// to all other characters in the stream.
///
///  If `spaces` is `true`, attempt to break lines at whitespace boundaries.
fn fold_file_bytewise<T: Read>(mut file: BufReader<T>, spaces: bool, width: usize) {
    let mut line = String::new();

    loop {
        if let Ok(0) = file.read_line(&mut line) {
            break;
        }

        if line == "\n" {
            println!();
            line.truncate(0);
            continue;
        }

        let len = line.len();
        let mut i = 0;

        while i < len {
            let width = if len - i >= width { width } else { len - i };
            let slice = {
                let slice = &line[i..i + width];
                if spaces && i + width < len {
                    match slice.rfind(|c: char| c.is_whitespace() && c != '\r') {
                        Some(m) => &slice[..=m],
                        None => slice,
                    }
                } else {
                    slice
                }
            };

            // Don't duplicate trailing newlines: if the slice is "\n", the
            // previous iteration folded just before the end of the line and
            // has already printed this newline.
            if slice == "\n" {
                break;
            }

            i += slice.len();

            let at_eol = i >= len;

            if at_eol {
                print!("{}", slice);
            } else {
                println!("{}", slice);
            }
        }

        line.truncate(0);
    }
}

/// Fold `file` to fit `width` (number of columns).
///
/// By default `fold` treats tab, backspace, and carriage return specially:
/// tab characters count as 8 columns, backspace decreases the
/// column count, and carriage return resets the column count to 0.
///
/// If `spaces` is `true`, attempt to break lines at whitespace boundaries.
#[allow(unused_assignments)]
fn fold_file<T: Read>(mut file: BufReader<T>, spaces: bool, width: usize) {
    let mut line = String::new();
    let mut output = String::new();
    let mut col_count = 0;
    let mut last_space = None;

    /// Print the output line, resetting the column and character counts.
    ///
    /// If `spaces` is `true`, print the output line up to the last
    /// encountered whitespace character (inclusive) and set the remaining
    /// characters as the start of the next line.
    macro_rules! emit_output {
        () => {
            let consume = match last_space {
                Some(i) => i + 1,
                None => output.len(),
            };

            println!("{}", &output[..consume]);
            output.replace_range(..consume, "");

            // we know there are no tabs left in output, so each char counts
            // as 1 column
            col_count = output.len();

            last_space = None;
        };
    }

    loop {
        if let Ok(0) = file.read_line(&mut line) {
            break;
        }

        for ch in line.chars() {
            if ch == '\n' {
                // make sure to _not_ split output at whitespace, since we
                // know the entire output will fit
                last_space = None;
                emit_output!();
                break;
            }

            if col_count >= width {
                emit_output!();
            }

            match ch {
                '\r' => col_count = 0,
                '\t' => {
                    let next_tab_stop = col_count + TAB_WIDTH - col_count % TAB_WIDTH;

                    if next_tab_stop > width && !output.is_empty() {
                        emit_output!();
                    }

                    col_count = next_tab_stop;
                    last_space = if spaces { Some(output.len()) } else { None };
                }
                '\x08' => {
                    if col_count > 0 {
                        col_count -= 1;
                    }
                }
                _ if spaces && ch.is_whitespace() => {
                    last_space = Some(output.len());
                    col_count += 1;
                }
                _ => col_count += 1,
            };

            output.push(ch);
        }

        if !output.is_empty() {
            print!("{}", output);
            output.truncate(0);
        }

        line.truncate(0);
    }
}
<|endoftext|>


<commit_before>extern crate bincode;

extern crate clap;
#[macro_use]
extern crate error_chain;
extern crate definitions;

mod vm;
mod error;

use clap::{App, Arg};
use definitions::program::{Interrupt, Program};
use definitions::typedef::*;
use error::*;
use std::fs::File;
use std::io::Read;
use std::sync::mpsc;
use std::thread;
use vm::VM;

fn main() {
    if let Err(ref e) = run() {
        println!("error: {}", e);

        for e in e.iter().skip(1) {
            println!("caused by: {}", e);
        }

        // The backtrace is not always generated. Try to run this example
        // with `RUST_BACKTRACE=1`.
        if let Some(backtrace) = e.backtrace() {
            println!("backtrace: {:?}", backtrace);
        }

        ::std::process::exit(1);
    }
}

fn run() -> VMResult<()> {
    let matches = App::new("bakerVM")
        .version(env!("CARGO_PKG_VERSION"))
        .author("Julian Laubstein <contact@julianlaubstein.de>")
        .about("A virtual machine for building and running retro games")
        .arg(
            Arg::with_name("input")
                .index(1)
                .help("Sets the image file to use. Uses a standard image if nothing is specified.")
        )
        .get_matches();

    let program: Program = if let Some(input) = matches.value_of("input") {
        let mut file = File::open(input).chain_err(|| "unable to open file")?;
        let mut buf: ImageData = ImageData::new();
        file.read_to_end(&mut buf).chain_err(|| "unable to read from file")?;

        bincode::deserialize(&buf[..]).chain_err(|| "unable to decode image file")?
    } else {
        Program::default()
    };

    let display_resolution = program.config.display_resolution.clone();
    let display_scale = program.config.display_scale.clone();

    let (vm_sender, outer_receiver) = mpsc::channel::<Frame>();
    let (outer_sender, vm_receiver) = mpsc::channel::<Interrupt>();

    thread::spawn(
        move || {
            VM::default().exec(program, vm_sender, vm_receiver).expect("unable to exec program");
        },
    );

    'main: loop {
        if let Ok(frame) = outer_receiver.try_recv() {
            for y in 0..display_resolution.height {
                for x in 0..display_resolution.width {
                    print!("{:?}", frame[(y * display_resolution.width) + x]);
                }
                println!();
            }
        } else {
            break 'main;
        }

        // TODO: Send interrupt based on real input
    }

    Ok(())
}
<commit_msg>Added error handling to vm thread<commit_after>extern crate bincode;

extern crate clap;
#[macro_use]
extern crate error_chain;
extern crate definitions;

mod vm;
mod error;

use clap::{App, Arg};
use definitions::program::{Interrupt, Program};
use definitions::typedef::*;
use error::*;
use std::fs::File;
use std::io::Read;
use std::sync::mpsc;
use std::thread;
use vm::VM;

fn main() {
    if let Err(ref e) = run() {
        println!("error: {}", e);

        for e in e.iter().skip(1) {
            println!("caused by: {}", e);
        }

        // The backtrace is not always generated. Try to run this example
        // with `RUST_BACKTRACE=1`.
        if let Some(backtrace) = e.backtrace() {
            println!("backtrace: {:?}", backtrace);
        }

        ::std::process::exit(1);
    }
}

fn run() -> VMResult<()> {
    let matches = App::new("bakerVM")
        .version(env!("CARGO_PKG_VERSION"))
        .author("Julian Laubstein <contact@julianlaubstein.de>")
        .about("A virtual machine for building and running retro games")
        .arg(
            Arg::with_name("input")
                .index(1)
                .help("Sets the image file to use. Uses a standard image if nothing is specified.")
        )
        .get_matches();

    let program: Program = if let Some(input) = matches.value_of("input") {
        let mut file = File::open(input).chain_err(|| "unable to open file")?;
        let mut buf: ImageData = ImageData::new();
        file.read_to_end(&mut buf).chain_err(|| "unable to read from file")?;

        bincode::deserialize(&buf[..]).chain_err(|| "unable to decode image file")?
    } else {
        Program::default()
    };

    let display_resolution = program.config.display_resolution.clone();
    let display_scale = program.config.display_scale.clone();

    let (vm_sender, outer_receiver) = mpsc::channel::<Frame>();
    let (outer_sender, vm_receiver) = mpsc::channel::<Interrupt>();

    thread::spawn(
        move || {
            if let Err(ref e) = VM::default().exec(program, vm_sender, vm_receiver) {
                println!("error: {}", e);

                for e in e.iter().skip(1) {
                    println!("caused by: {}", e);
                }

                // The backtrace is not always generated. Try to run this example
                // with `RUST_BACKTRACE=1`.
                if let Some(backtrace) = e.backtrace() {
                    println!("backtrace: {:?}", backtrace);
                }
            }
        },
    );

    'main: loop {
        if let Ok(frame) = outer_receiver.try_recv() {
            for y in 0..display_resolution.height {
                for x in 0..display_resolution.width {
                    print!("{:?}", frame[(y * display_resolution.width) + x]);
                }
                println!();
            }
        } else {
            break 'main;
        }

        // TODO: Send interrupt based on real input
    }

    Ok(())
}
<|endoftext|>


<commit_before>//! Reciprocal length (base unit reciprocal meter, m⁻¹).

quantity! {
    /// Reciprocal length (base unit reciprocal meter, m⁻¹).
    quantity: ReciprocalLength; "reciprocal length";
    /// Dimension of reciprocal length, L⁻¹ (base unit reciprocal meter, m⁻¹).
    dimension: ISQ<
        N1,     // length
        Z0,     // mass
        Z0,     // time
        Z0,     // electric current
        Z0,     // thermodynamic temperature
        Z0,     // amount of substance
        Z0>;    // luminous intensity
    units {
        @reciprocal_kilometer: prefix!(none) / prefix!(kilo); "km⁻¹", "reciprocal kilometer",
            "reciprocal kilometers";
        @reciprocal_meter: prefix!(none); "m⁻¹", "reciprocal meter", "reciprocal meters";
        @reciprocal_decimeter: prefix!(none) / prefix!(deci); "dm⁻¹", "reciprocal decimeter",
            "reciprocal decimeters";
        @reciprocal_centimeter: prefix!(none) / prefix!(centi); "cm⁻¹", "reciprocal centimeter",
            "reciprocal centimeters";
        @reciprocal_millimeter: prefix!(none) / prefix!(milli); "mm⁻¹", "reciprocal millimeter",
            "reciprocal millimeters";
        @reciprocal_micrometer: prefix!(none) / prefix!(micro); "µm⁻¹", "reciprocal micrometer",
            "reciprocal micrometers";
        @reciprocal_nanometer: prefix!(none) / prefix!(nano); "nm⁻¹", "reciprocal nanometer",
            "reciprocal nanometers";

        @reciprocal_angstrom: prefix!(none) / 1.0_E-10; "Å⁻¹", "reciprocal ångström",
            "reciprocal ångströms";
        @diopter: prefix!(none); "dpt", "diopter", "diopters";
    }
}

#[cfg(test)]
mod test {
    storage_types! {
        use crate::num::One;
        use crate::si::length as l;
        use crate::si::reciprocal_length as rl;
        use crate::si::quantities::*;
        use crate::tests::Test;

        #[test]
        fn check_dimension() {
            let _: ReciprocalLength<V> = V::one()
                / Length::new::<l::meter>(V::one());
        }

        #[test]
        fn check_units() {
            test::<l::kilometer, rl::reciprocal_kilometer>();
            test::<l::meter, rl::reciprocal_meter>();
            test::<l::decimeter, rl::reciprocal_decimeter>();
            test::<l::centimeter, rl::reciprocal_centimeter>();
            test::<l::millimeter, rl::reciprocal_millimeter>();
            test::<l::micrometer, rl::reciprocal_micrometer>();
            test::<l::nanometer, rl::reciprocal_nanometer>();
            test::<l::angstrom, rl::reciprocal_angstrom>();
            test::<l::meter, rl::diopter>();

            fn test<L: l::Conversion<V>, RL: rl::Conversion<V>>() {
                Test::assert_approx_eq(&ReciprocalLength::new::<RL>(V::one()),
                    &(V::one() / Length::new::<L>(V::one())));
            }
        }
    }
}
<commit_msg>Add Rydberg constant as unit of ReciprocalLength.<commit_after>//! Reciprocal length (base unit reciprocal meter, m⁻¹).

quantity! {
    /// Reciprocal length (base unit reciprocal meter, m⁻¹).
    quantity: ReciprocalLength; "reciprocal length";
    /// Dimension of reciprocal length, L⁻¹ (base unit reciprocal meter, m⁻¹).
    dimension: ISQ<
        N1,     // length
        Z0,     // mass
        Z0,     // time
        Z0,     // electric current
        Z0,     // thermodynamic temperature
        Z0,     // amount of substance
        Z0>;    // luminous intensity
    units {
        @reciprocal_kilometer: prefix!(none) / prefix!(kilo); "km⁻¹", "reciprocal kilometer",
            "reciprocal kilometers";
        @reciprocal_meter: prefix!(none); "m⁻¹", "reciprocal meter", "reciprocal meters";
        @reciprocal_decimeter: prefix!(none) / prefix!(deci); "dm⁻¹", "reciprocal decimeter",
            "reciprocal decimeters";
        @reciprocal_centimeter: prefix!(none) / prefix!(centi); "cm⁻¹", "reciprocal centimeter",
            "reciprocal centimeters";
        @reciprocal_millimeter: prefix!(none) / prefix!(milli); "mm⁻¹", "reciprocal millimeter",
            "reciprocal millimeters";
        @reciprocal_micrometer: prefix!(none) / prefix!(micro); "µm⁻¹", "reciprocal micrometer",
            "reciprocal micrometers";
        @reciprocal_nanometer: prefix!(none) / prefix!(nano); "nm⁻¹", "reciprocal nanometer",
            "reciprocal nanometers";

        @reciprocal_angstrom: prefix!(none) / 1.0_E-10; "Å⁻¹", "reciprocal ångström",
            "reciprocal ångströms";
        @diopter: prefix!(none); "dpt", "diopter", "diopters";
        @rydberg_constant: 10_973_731.568_160; "R∞", "Rydberg constant", "Rydberg constants";
    }
}

#[cfg(test)]
mod test {
    storage_types! {
        use crate::num::One;
        use crate::si::length as l;
        use crate::si::reciprocal_length as rl;
        use crate::si::quantities::*;
        use crate::tests::Test;

        #[test]
        fn check_dimension() {
            let _: ReciprocalLength<V> = V::one()
                / Length::new::<l::meter>(V::one());
        }

        #[test]
        fn check_units() {
            test::<l::kilometer, rl::reciprocal_kilometer>();
            test::<l::meter, rl::reciprocal_meter>();
            test::<l::decimeter, rl::reciprocal_decimeter>();
            test::<l::centimeter, rl::reciprocal_centimeter>();
            test::<l::millimeter, rl::reciprocal_millimeter>();
            test::<l::micrometer, rl::reciprocal_micrometer>();
            test::<l::nanometer, rl::reciprocal_nanometer>();
            test::<l::angstrom, rl::reciprocal_angstrom>();
            test::<l::meter, rl::diopter>();

            fn test<L: l::Conversion<V>, RL: rl::Conversion<V>>() {
                Test::assert_approx_eq(&ReciprocalLength::new::<RL>(V::one()),
                    &(V::one() / Length::new::<L>(V::one())));
            }
        }
    }
}
<|endoftext|>


<commit_before>#![feature(test)]

extern crate crossbeam;
extern crate ipc_channel;
extern crate test;

use ipc_channel::platform;

use std::sync::{mpsc, Mutex};

/// Allows doing multiple inner iterations per bench.iter() run.
///
/// This is mostly to amortise the overhead of spawning a thread in the benchmark
/// when sending larger messages (that might be fragmented).
///
/// Note that you need to compensate the displayed results
/// for the proportionally longer runs yourself,
/// as the benchmark framework doesn't know about the inner iterations...
const ITERATIONS: usize = 1;

fn bench_size(b: &mut test::Bencher, size: usize) {
    let data: Vec<u8> = (0..size).map(|i| (i % 251) as u8).collect();
    let (tx, rx) = platform::channel().unwrap();

    let (wait_tx, wait_rx) = mpsc::channel();
    let wait_rx = Mutex::new(wait_rx);

    if size > platform::OsIpcSender::get_max_fragment_size() {
        b.iter(|| {
            crossbeam::scope(|scope| {
                scope.spawn(|| {
                    let wait_rx = wait_rx.lock().unwrap();
                    for _ in 0..ITERATIONS {
                        tx.send(&data, vec![], vec![]).unwrap();
                        if ITERATIONS > 1 {
                            // Prevent beginning of the next send
                            // from overlapping with receive of last fragment,
                            // as otherwise results of runs with a large tail fragment
                            // are significantly skewed.
                            wait_rx.recv().unwrap();
                        }
                    }
                });
                for _ in 0..ITERATIONS {
                    rx.recv().unwrap();
                    if ITERATIONS > 1 {
                        wait_tx.send(()).unwrap();
                    }
                }
                // For reasons mysterious to me,
                // not returning a value *from every branch*
                // adds some 100 ns or so of overhead to all results --
                // which is quite significant for very short tests...
                0
            })
        });
    } else {
        b.iter(|| {
            for _ in 0..ITERATIONS {
                tx.send(&data, vec![], vec![]).unwrap();
                rx.recv().unwrap();
            }
            0
        });
    }
}

#[bench]
fn size_00_1(b: &mut test::Bencher) {
    bench_size(b, 1);
}
#[bench]
fn size_01_2(b: &mut test::Bencher) {
    bench_size(b, 2);
}
#[bench]
fn size_02_4(b: &mut test::Bencher) {
    bench_size(b, 4);
}
#[bench]
fn size_03_8(b: &mut test::Bencher) {
    bench_size(b, 8);
}
#[bench]
fn size_04_16(b: &mut test::Bencher) {
    bench_size(b, 16);
}
#[bench]
fn size_05_32(b: &mut test::Bencher) {
    bench_size(b, 32);
}
#[bench]
fn size_06_64(b: &mut test::Bencher) {
    bench_size(b, 64);
}
#[bench]
fn size_07_128(b: &mut test::Bencher) {
    bench_size(b, 128);
}
#[bench]
fn size_08_256(b: &mut test::Bencher) {
    bench_size(b, 256);
}
#[bench]
fn size_09_512(b: &mut test::Bencher) {
    bench_size(b, 512);
}
#[bench]
fn size_10_1k(b: &mut test::Bencher) {
    bench_size(b, 1 * 1024);
}
#[bench]
fn size_11_2k(b: &mut test::Bencher) {
    bench_size(b, 2 * 1024);
}
#[bench]
fn size_12_4k(b: &mut test::Bencher) {
    bench_size(b, 4 * 1024);
}
#[bench]
fn size_13_8k(b: &mut test::Bencher) {
    bench_size(b, 8 * 1024);
}
#[bench]
fn size_14_16k(b: &mut test::Bencher) {
    bench_size(b, 16 * 1024);
}
#[bench]
fn size_15_32k(b: &mut test::Bencher) {
    bench_size(b, 32 * 1024);
}
#[bench]
fn size_16_64k(b: &mut test::Bencher) {
    bench_size(b, 64 * 1024);
}
#[bench]
fn size_17_128k(b: &mut test::Bencher) {
    bench_size(b, 128 * 1024);
}
#[bench]
fn size_18_256k(b: &mut test::Bencher) {
    bench_size(b, 256 * 1024);
}
#[bench]
fn size_19_512k(b: &mut test::Bencher) {
    bench_size(b, 512 * 1024);
}
#[bench]
fn size_20_1m(b: &mut test::Bencher) {
    bench_size(b, 1 * 1024 * 1024);
}
#[bench]
fn size_21_2m(b: &mut test::Bencher) {
    bench_size(b, 2 * 1024 * 1024);
}
#[bench]
fn size_22_4m(b: &mut test::Bencher) {
    bench_size(b, 4 * 1024 * 1024);
}
#[bench]
fn size_23_8m(b: &mut test::Bencher) {
    bench_size(b, 8 * 1024 * 1024);
}
<commit_msg>Benchmarks: Add test for channel creation<commit_after>#![feature(test)]

extern crate crossbeam;
extern crate ipc_channel;
extern crate test;

use ipc_channel::platform;

use std::sync::{mpsc, Mutex};

/// Allows doing multiple inner iterations per bench.iter() run.
///
/// This is mostly to amortise the overhead of spawning a thread in the benchmark
/// when sending larger messages (that might be fragmented).
///
/// Note that you need to compensate the displayed results
/// for the proportionally longer runs yourself,
/// as the benchmark framework doesn't know about the inner iterations...
const ITERATIONS: usize = 1;

#[bench]
fn create_channel(b: &mut test::Bencher) {
    b.iter(|| {
        for _ in 0..ITERATIONS {
            platform::channel().unwrap();
        }
    });
}

fn bench_size(b: &mut test::Bencher, size: usize) {
    let data: Vec<u8> = (0..size).map(|i| (i % 251) as u8).collect();
    let (tx, rx) = platform::channel().unwrap();

    let (wait_tx, wait_rx) = mpsc::channel();
    let wait_rx = Mutex::new(wait_rx);

    if size > platform::OsIpcSender::get_max_fragment_size() {
        b.iter(|| {
            crossbeam::scope(|scope| {
                scope.spawn(|| {
                    let wait_rx = wait_rx.lock().unwrap();
                    for _ in 0..ITERATIONS {
                        tx.send(&data, vec![], vec![]).unwrap();
                        if ITERATIONS > 1 {
                            // Prevent beginning of the next send
                            // from overlapping with receive of last fragment,
                            // as otherwise results of runs with a large tail fragment
                            // are significantly skewed.
                            wait_rx.recv().unwrap();
                        }
                    }
                });
                for _ in 0..ITERATIONS {
                    rx.recv().unwrap();
                    if ITERATIONS > 1 {
                        wait_tx.send(()).unwrap();
                    }
                }
                // For reasons mysterious to me,
                // not returning a value *from every branch*
                // adds some 100 ns or so of overhead to all results --
                // which is quite significant for very short tests...
                0
            })
        });
    } else {
        b.iter(|| {
            for _ in 0..ITERATIONS {
                tx.send(&data, vec![], vec![]).unwrap();
                rx.recv().unwrap();
            }
            0
        });
    }
}

#[bench]
fn size_00_1(b: &mut test::Bencher) {
    bench_size(b, 1);
}
#[bench]
fn size_01_2(b: &mut test::Bencher) {
    bench_size(b, 2);
}
#[bench]
fn size_02_4(b: &mut test::Bencher) {
    bench_size(b, 4);
}
#[bench]
fn size_03_8(b: &mut test::Bencher) {
    bench_size(b, 8);
}
#[bench]
fn size_04_16(b: &mut test::Bencher) {
    bench_size(b, 16);
}
#[bench]
fn size_05_32(b: &mut test::Bencher) {
    bench_size(b, 32);
}
#[bench]
fn size_06_64(b: &mut test::Bencher) {
    bench_size(b, 64);
}
#[bench]
fn size_07_128(b: &mut test::Bencher) {
    bench_size(b, 128);
}
#[bench]
fn size_08_256(b: &mut test::Bencher) {
    bench_size(b, 256);
}
#[bench]
fn size_09_512(b: &mut test::Bencher) {
    bench_size(b, 512);
}
#[bench]
fn size_10_1k(b: &mut test::Bencher) {
    bench_size(b, 1 * 1024);
}
#[bench]
fn size_11_2k(b: &mut test::Bencher) {
    bench_size(b, 2 * 1024);
}
#[bench]
fn size_12_4k(b: &mut test::Bencher) {
    bench_size(b, 4 * 1024);
}
#[bench]
fn size_13_8k(b: &mut test::Bencher) {
    bench_size(b, 8 * 1024);
}
#[bench]
fn size_14_16k(b: &mut test::Bencher) {
    bench_size(b, 16 * 1024);
}
#[bench]
fn size_15_32k(b: &mut test::Bencher) {
    bench_size(b, 32 * 1024);
}
#[bench]
fn size_16_64k(b: &mut test::Bencher) {
    bench_size(b, 64 * 1024);
}
#[bench]
fn size_17_128k(b: &mut test::Bencher) {
    bench_size(b, 128 * 1024);
}
#[bench]
fn size_18_256k(b: &mut test::Bencher) {
    bench_size(b, 256 * 1024);
}
#[bench]
fn size_19_512k(b: &mut test::Bencher) {
    bench_size(b, 512 * 1024);
}
#[bench]
fn size_20_1m(b: &mut test::Bencher) {
    bench_size(b, 1 * 1024 * 1024);
}
#[bench]
fn size_21_2m(b: &mut test::Bencher) {
    bench_size(b, 2 * 1024 * 1024);
}
#[bench]
fn size_22_4m(b: &mut test::Bencher) {
    bench_size(b, 4 * 1024 * 1024);
}
#[bench]
fn size_23_8m(b: &mut test::Bencher) {
    bench_size(b, 8 * 1024 * 1024);
}
<|endoftext|>