<commit_before>use std::borrow;
use std::cmp::Ordering;
use std::fmt;

use chrono;

/// Liquid's native date/time type.
pub type Date = chrono::DateTime<chrono::FixedOffset>;

/// A Liquid scalar value
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Scalar(ScalarEnum);

/// An enum to represent different value types
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(untagged)]
enum ScalarEnum {
    Integer(i32),
    Float(f64),
    Bool(bool),
    #[serde(with = "friendly_date")]
    Date(Date),
    Str(borrow::Cow<'static, str>),
}

impl Scalar {
    /// Convert a value into a `Scalar`.
    pub fn new<T: Into<Self>>(value: T) -> Self {
        value.into()
    }

    /// Interpret as a string.
    pub fn to_str(&self) -> borrow::Cow<str> {
        match self.0 {
            ScalarEnum::Integer(ref x) => borrow::Cow::Owned(x.to_string()),
            ScalarEnum::Float(ref x) => borrow::Cow::Owned(x.to_string()),
            ScalarEnum::Bool(ref x) => borrow::Cow::Owned(x.to_string()),
            ScalarEnum::Date(ref x) => borrow::Cow::Owned(x.format(DATE_FORMAT).to_string()),
            ScalarEnum::Str(ref x) => borrow::Cow::Borrowed(x.as_ref()),
        }
    }

    /// Convert to a string.
    pub fn into_string(self) -> String {
        match self.0 {
            ScalarEnum::Integer(x) => x.to_string(),
            ScalarEnum::Float(x) => x.to_string(),
            ScalarEnum::Bool(x) => x.to_string(),
            ScalarEnum::Date(x) => x.to_string(),
            ScalarEnum::Str(x) => x.into_owned(),
        }
    }

    /// Interpret as an integer, if possible
    pub fn to_integer(&self) -> Option<i32> {
        match self.0 {
            ScalarEnum::Integer(ref x) => Some(*x),
            ScalarEnum::Str(ref x) => x.parse::<i32>().ok(),
            _ => None,
        }
    }

    /// Interpret as an float, if possible
    pub fn to_float(&self) -> Option<f64> {
        match self.0 {
            ScalarEnum::Integer(ref x) => Some(f64::from(*x)),
            ScalarEnum::Float(ref x) => Some(*x),
            ScalarEnum::Str(ref x) => x.parse::<f64>().ok(),
            _ => None,
        }
    }

    /// Interpret as an bool, if possible
    pub fn to_bool(&self) -> Option<bool> {
        match self.0 {
            ScalarEnum::Bool(ref x) => Some(*x),
            _ => None,
        }
    }

    /// Interpret as an bool, if possible
    pub fn to_date(&self) -> Option<Date> {
        match self.0 {
            ScalarEnum::Date(ref x) => Some(*x),
            ScalarEnum::Str(ref x) => parse_date(x.as_ref()),
            _ => None,
        }
    }

    /// Evaluate using Liquid "truthiness"
    pub fn is_truthy(&self) -> bool {
        // encode Ruby truthiness: all values except false and nil are true
        match self.0 {
            ScalarEnum::Bool(ref x) => *x,
            _ => true,
        }
    }

    /// Whether a default constructed value.
    pub fn is_default(&self) -> bool {
        // encode Ruby truthiness: all values except false and nil are true
        match self.0 {
            ScalarEnum::Bool(ref x) => !*x,
            ScalarEnum::Str(ref x) => x.is_empty(),
            _ => false,
        }
    }

    /// Report the data type (generally for error reporting).
    pub fn type_name(&self) -> &'static str {
        match self.0 {
            ScalarEnum::Integer(_) => "whole number",
            ScalarEnum::Float(_) => "fractional number",
            ScalarEnum::Bool(_) => "boolean",
            ScalarEnum::Date(_) => "date",
            ScalarEnum::Str(_) => "string",
        }
    }
}

impl From<i32> for Scalar {
    fn from(s: i32) -> Self {
        Scalar {
            0: ScalarEnum::Integer(s),
        }
    }
}

impl From<f64> for Scalar {
    fn from(s: f64) -> Self {
        Scalar {
            0: ScalarEnum::Float(s),
        }
    }
}

impl From<bool> for Scalar {
    fn from(s: bool) -> Self {
        Scalar {
            0: ScalarEnum::Bool(s),
        }
    }
}

impl From<Date> for Scalar {
    fn from(s: Date) -> Self {
        Scalar {
            0: ScalarEnum::Date(s),
        }
    }
}

impl From<String> for Scalar {
    fn from(s: String) -> Self {
        Scalar {
            0: ScalarEnum::Str(s.into()),
        }
    }
}

impl<'a> From<&'a String> for Scalar {
    fn from(s: &String) -> Self {
        Scalar {
            0: ScalarEnum::Str(s.to_owned().into()),
        }
    }
}

impl From<&'static str> for Scalar {
    fn from(s: &'static str) -> Self {
        Scalar {
            0: ScalarEnum::Str(s.into()),
        }
    }
}

impl PartialEq<Scalar> for Scalar {
    fn eq(&self, other: &Self) -> bool {
        match (&self.0, &other.0) {
            (&ScalarEnum::Integer(x), &ScalarEnum::Integer(y)) => x == y,
            (&ScalarEnum::Integer(x), &ScalarEnum::Float(y)) => (f64::from(x)) == y,
            (&ScalarEnum::Float(x), &ScalarEnum::Integer(y)) => x == (f64::from(y)),
            (&ScalarEnum::Float(x), &ScalarEnum::Float(y)) => x == y,
            (&ScalarEnum::Bool(x), &ScalarEnum::Bool(y)) => x == y,
            (&ScalarEnum::Date(x), &ScalarEnum::Date(y)) => x == y,
            (&ScalarEnum::Str(ref x), &ScalarEnum::Str(ref y)) => x == y,
            // encode Ruby truthiness: all values except false and nil are true
            (_, &ScalarEnum::Bool(b)) | (&ScalarEnum::Bool(b), _) => b,
            _ => false,
        }
    }
}

impl Eq for Scalar {}

impl PartialOrd<Scalar> for Scalar {
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        match (&self.0, &other.0) {
            (&ScalarEnum::Integer(x), &ScalarEnum::Integer(y)) => x.partial_cmp(&y),
            (&ScalarEnum::Integer(x), &ScalarEnum::Float(y)) => (f64::from(x)).partial_cmp(&y),
            (&ScalarEnum::Float(x), &ScalarEnum::Integer(y)) => x.partial_cmp(&(f64::from(y))),
            (&ScalarEnum::Float(x), &ScalarEnum::Float(y)) => x.partial_cmp(&y),
            (&ScalarEnum::Bool(x), &ScalarEnum::Bool(y)) => x.partial_cmp(&y),
            (&ScalarEnum::Date(x), &ScalarEnum::Date(y)) => x.partial_cmp(&y),
            (&ScalarEnum::Str(ref x), &ScalarEnum::Str(ref y)) => x.partial_cmp(y),
            _ => None,
        }
    }
}

impl fmt::Display for Scalar {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let data = self.to_str();
        write!(f, "{}", data)
    }
}

const DATE_FORMAT: &str = "%Y-%m-%d %H:%M:%S %z";

mod friendly_date {
    use super::*;
    use serde::{self, Deserialize, Deserializer, Serializer};

    pub(crate) fn serialize<S>(date: &Date, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let s = date.format(DATE_FORMAT).to_string();
        serializer.serialize_str(&s)
    }

    pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Date, D::Error>
    where
        D: Deserializer<'de>,
    {
        let s = String::deserialize(deserializer)?;
        Date::parse_from_str(&s, DATE_FORMAT).map_err(serde::de::Error::custom)
    }
}

fn parse_date(s: &str) -> Option<Date> {
    match s {
        "now" | "today" => {
            let now = chrono::offset::Utc::now();
            let now = now.naive_utc();
            let now = chrono::DateTime::from_utc(now, chrono::offset::FixedOffset::east(0));
            Some(now)
        }
        _ => {
            let formats = ["%d %B %Y %H:%M:%S %z", "%Y-%m-%d %H:%M:%S %z"];
            formats
                .iter()
                .filter_map(|f| Date::parse_from_str(s, f).ok())
                .next()
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    static TRUE: Scalar = Scalar(ScalarEnum::Bool(true));
    static FALSE: Scalar = Scalar(ScalarEnum::Bool(false));

    #[test]
    fn test_to_str_bool() {
        assert_eq!(TRUE.to_str(), "true");
    }

    #[test]
    fn test_to_str_integer() {
        let val: Scalar = 42i32.into();
        assert_eq!(val.to_str(), "42");
    }

    #[test]
    fn test_to_str_float() {
        let val: Scalar = 42f64.into();
        assert_eq!(val.to_str(), "42");

        let val: Scalar = 42.34.into();
        assert_eq!(val.to_str(), "42.34");
    }

    #[test]
    fn test_to_str_str() {
        let val: Scalar = "foobar".into();
        assert_eq!(val.to_str(), "foobar");
    }

    #[test]
    fn test_to_integer_bool() {
        assert_eq!(TRUE.to_integer(), None);
    }

    #[test]
    fn test_to_integer_integer() {
        let val: Scalar = 42i32.into();
        assert_eq!(val.to_integer(), Some(42i32));
    }

    #[test]
    fn test_to_integer_float() {
        let val: Scalar = 42f64.into();
        assert_eq!(val.to_integer(), None);

        let val: Scalar = 42.34.into();
        assert_eq!(val.to_integer(), None);
    }

    #[test]
    fn test_to_integer_str() {
        let val: Scalar = "foobar".into();
        assert_eq!(val.to_integer(), None);

        let val: Scalar = "42.34".into();
        assert_eq!(val.to_integer(), None);

        let val: Scalar = "42".into();
        assert_eq!(val.to_integer(), Some(42));
    }

    #[test]
    fn test_to_float_bool() {
        assert_eq!(TRUE.to_float(), None);
    }

    #[test]
    fn test_to_float_integer() {
        let val: Scalar = 42i32.into();
        assert_eq!(val.to_float(), Some(42f64));
    }

    #[test]
    fn test_to_float_float() {
        let val: Scalar = 42f64.into();
        assert_eq!(val.to_float(), Some(42f64));

        let val: Scalar = 42.34.into();
        assert_eq!(val.to_float(), Some(42.34));
    }

    #[test]
    fn test_to_float_str() {
        let val: Scalar = "foobar".into();
        assert_eq!(val.to_float(), None);

        let val: Scalar = "42.34".into();
        assert_eq!(val.to_float(), Some(42.34));

        let val: Scalar = "42".into();
        assert_eq!(val.to_float(), Some(42f64));
    }

    #[test]
    fn test_to_bool_bool() {
        assert_eq!(TRUE.to_bool(), Some(true));
    }

    #[test]
    fn test_to_bool_integer() {
        let val: Scalar = 42i32.into();
        assert_eq!(val.to_bool(), None);
    }

    #[test]
    fn test_to_bool_float() {
        let val: Scalar = 42f64.into();
        assert_eq!(val.to_bool(), None);

        let val: Scalar = 42.34.into();
        assert_eq!(val.to_bool(), None);
    }

    #[test]
    fn test_to_bool_str() {
        let val: Scalar = "foobar".into();
        assert_eq!(val.to_bool(), None);

        let val: Scalar = "true".into();
        assert_eq!(val.to_bool(), None);

        let val: Scalar = "false".into();
        assert_eq!(val.to_bool(), None);
    }

    #[test]
    fn integer_equality() {
        let val: Scalar = 42i32.into();
        let zero: Scalar = 0i32.into();
        assert_eq!(val, val);
        assert_eq!(zero, zero);
        assert!(val != zero);
        assert!(zero != val);
    }

    #[test]
    fn integers_have_ruby_truthiness() {
        let val: Scalar = 42i32.into();
        let zero: Scalar = 0i32.into();
        assert_eq!(TRUE, val);
        assert_eq!(val, TRUE);
        assert!(val.is_truthy());

        assert_eq!(TRUE, zero);
        assert_eq!(zero, TRUE);
        assert!(zero.is_truthy());
    }

    #[test]
    fn float_equality() {
        let val: Scalar = 42f64.into();
        let zero: Scalar = 0f64.into();
        assert_eq!(val, val);
        assert_eq!(zero, zero);
        assert!(val != zero);
        assert!(zero != val);
    }

    #[test]
    fn floats_have_ruby_truthiness() {
        let val: Scalar = 42f64.into();
        let zero: Scalar = 0f64.into();
        assert_eq!(TRUE, val);
        assert_eq!(val, TRUE);
        assert!(val.is_truthy());

        assert_eq!(TRUE, zero);
        assert_eq!(zero, TRUE);
        assert!(zero.is_truthy());
    }

    #[test]
    fn boolean_equality() {
        assert_eq!(TRUE, TRUE);
        assert_eq!(FALSE, FALSE);
        assert!(FALSE != TRUE);
        assert!(TRUE != FALSE);
    }

    #[test]
    fn booleans_have_ruby_truthiness() {
        assert!(TRUE.is_truthy());
        assert!(!FALSE.is_truthy());
    }

    #[test]
    fn string_equality() {
        let alpha: Scalar = "alpha".into();
        let beta: Scalar = "beta".into();
        let empty: Scalar = "".into();
        assert_eq!(alpha, alpha);
        assert_eq!(empty, empty);
        assert!(alpha != beta);
        assert!(beta != alpha);
    }

    #[test]
    fn strings_have_ruby_truthiness() {
        // all strings in ruby are true
        let alpha: Scalar = "alpha".into();
        let empty: Scalar = "".into();
        assert_eq!(TRUE, alpha);
        assert_eq!(alpha, TRUE);
        assert!(alpha.is_truthy());

        assert_eq!(TRUE, empty);
        assert_eq!(empty, TRUE);
        assert!(empty.is_truthy());
    }

    #[test]
    fn parse_date_empty_is_bad() {
        assert!(parse_date("").is_none());
    }

    #[test]
    fn parse_date_bad() {
        assert!(parse_date("aaaaa").is_none());
    }

    #[test]
    fn parse_date_now() {
        assert!(parse_date("now").is_some());
    }

    #[test]
    fn parse_date_today() {
        assert!(parse_date("today").is_some());
    }
}
<commit_msg>docs(scalar): fix minor mistakes in documentation<commit_after>use std::borrow;
use std::cmp::Ordering;
use std::fmt;

use chrono;

/// Liquid's native date/time type.
pub type Date = chrono::DateTime<chrono::FixedOffset>;

/// A Liquid scalar value
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Scalar(ScalarEnum);

/// An enum to represent different value types
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(untagged)]
enum ScalarEnum {
    Integer(i32),
    Float(f64),
    Bool(bool),
    #[serde(with = "friendly_date")]
    Date(Date),
    Str(borrow::Cow<'static, str>),
}

impl Scalar {
    /// Convert a value into a `Scalar`.
    pub fn new<T: Into<Self>>(value: T) -> Self {
        value.into()
    }

    /// Interpret as a string.
    pub fn to_str(&self) -> borrow::Cow<str> {
        match self.0 {
            ScalarEnum::Integer(ref x) => borrow::Cow::Owned(x.to_string()),
            ScalarEnum::Float(ref x) => borrow::Cow::Owned(x.to_string()),
            ScalarEnum::Bool(ref x) => borrow::Cow::Owned(x.to_string()),
            ScalarEnum::Date(ref x) => borrow::Cow::Owned(x.format(DATE_FORMAT).to_string()),
            ScalarEnum::Str(ref x) => borrow::Cow::Borrowed(x.as_ref()),
        }
    }

    /// Convert to a string.
    pub fn into_string(self) -> String {
        match self.0 {
            ScalarEnum::Integer(x) => x.to_string(),
            ScalarEnum::Float(x) => x.to_string(),
            ScalarEnum::Bool(x) => x.to_string(),
            ScalarEnum::Date(x) => x.to_string(),
            ScalarEnum::Str(x) => x.into_owned(),
        }
    }

    /// Interpret as an integer, if possible
    pub fn to_integer(&self) -> Option<i32> {
        match self.0 {
            ScalarEnum::Integer(ref x) => Some(*x),
            ScalarEnum::Str(ref x) => x.parse::<i32>().ok(),
            _ => None,
        }
    }

    /// Interpret as a float, if possible
    pub fn to_float(&self) -> Option<f64> {
        match self.0 {
            ScalarEnum::Integer(ref x) => Some(f64::from(*x)),
            ScalarEnum::Float(ref x) => Some(*x),
            ScalarEnum::Str(ref x) => x.parse::<f64>().ok(),
            _ => None,
        }
    }

    /// Interpret as a bool, if possible
    pub fn to_bool(&self) -> Option<bool> {
        match self.0 {
            ScalarEnum::Bool(ref x) => Some(*x),
            _ => None,
        }
    }

    /// Interpret as a date, if possible
    pub fn to_date(&self) -> Option<Date> {
        match self.0 {
            ScalarEnum::Date(ref x) => Some(*x),
            ScalarEnum::Str(ref x) => parse_date(x.as_ref()),
            _ => None,
        }
    }

    /// Evaluate using Liquid "truthiness"
    pub fn is_truthy(&self) -> bool {
        // encode Ruby truthiness: all values except false and nil are true
        match self.0 {
            ScalarEnum::Bool(ref x) => *x,
            _ => true,
        }
    }

    /// Whether a default constructed value.
    pub fn is_default(&self) -> bool {
        // encode Ruby truthiness: all values except false and nil are true
        match self.0 {
            ScalarEnum::Bool(ref x) => !*x,
            ScalarEnum::Str(ref x) => x.is_empty(),
            _ => false,
        }
    }

    /// Report the data type (generally for error reporting).
    pub fn type_name(&self) -> &'static str {
        match self.0 {
            ScalarEnum::Integer(_) => "whole number",
            ScalarEnum::Float(_) => "fractional number",
            ScalarEnum::Bool(_) => "boolean",
            ScalarEnum::Date(_) => "date",
            ScalarEnum::Str(_) => "string",
        }
    }
}

impl From<i32> for Scalar {
    fn from(s: i32) -> Self {
        Scalar {
            0: ScalarEnum::Integer(s),
        }
    }
}

impl From<f64> for Scalar {
    fn from(s: f64) -> Self {
        Scalar {
            0: ScalarEnum::Float(s),
        }
    }
}

impl From<bool> for Scalar {
    fn from(s: bool) -> Self {
        Scalar {
            0: ScalarEnum::Bool(s),
        }
    }
}

impl From<Date> for Scalar {
    fn from(s: Date) -> Self {
        Scalar {
            0: ScalarEnum::Date(s),
        }
    }
}

impl From<String> for Scalar {
    fn from(s: String) -> Self {
        Scalar {
            0: ScalarEnum::Str(s.into()),
        }
    }
}

impl<'a> From<&'a String> for Scalar {
    fn from(s: &String) -> Self {
        Scalar {
            0: ScalarEnum::Str(s.to_owned().into()),
        }
    }
}

impl From<&'static str> for Scalar {
    fn from(s: &'static str) -> Self {
        Scalar {
            0: ScalarEnum::Str(s.into()),
        }
    }
}

impl PartialEq<Scalar> for Scalar {
    fn eq(&self, other: &Self) -> bool {
        match (&self.0, &other.0) {
            (&ScalarEnum::Integer(x), &ScalarEnum::Integer(y)) => x == y,
            (&ScalarEnum::Integer(x), &ScalarEnum::Float(y)) => (f64::from(x)) == y,
            (&ScalarEnum::Float(x), &ScalarEnum::Integer(y)) => x == (f64::from(y)),
            (&ScalarEnum::Float(x), &ScalarEnum::Float(y)) => x == y,
            (&ScalarEnum::Bool(x), &ScalarEnum::Bool(y)) => x == y,
            (&ScalarEnum::Date(x), &ScalarEnum::Date(y)) => x == y,
            (&ScalarEnum::Str(ref x), &ScalarEnum::Str(ref y)) => x == y,
            // encode Ruby truthiness: all values except false and nil are true
            (_, &ScalarEnum::Bool(b)) | (&ScalarEnum::Bool(b), _) => b,
            _ => false,
        }
    }
}

impl Eq for Scalar {}

impl PartialOrd<Scalar> for Scalar {
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        match (&self.0, &other.0) {
            (&ScalarEnum::Integer(x), &ScalarEnum::Integer(y)) => x.partial_cmp(&y),
            (&ScalarEnum::Integer(x), &ScalarEnum::Float(y)) => (f64::from(x)).partial_cmp(&y),
            (&ScalarEnum::Float(x), &ScalarEnum::Integer(y)) => x.partial_cmp(&(f64::from(y))),
            (&ScalarEnum::Float(x), &ScalarEnum::Float(y)) => x.partial_cmp(&y),
            (&ScalarEnum::Bool(x), &ScalarEnum::Bool(y)) => x.partial_cmp(&y),
            (&ScalarEnum::Date(x), &ScalarEnum::Date(y)) => x.partial_cmp(&y),
            (&ScalarEnum::Str(ref x), &ScalarEnum::Str(ref y)) => x.partial_cmp(y),
            _ => None,
        }
    }
}

impl fmt::Display for Scalar {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let data = self.to_str();
        write!(f, "{}", data)
    }
}

const DATE_FORMAT: &str = "%Y-%m-%d %H:%M:%S %z";

mod friendly_date {
    use super::*;
    use serde::{self, Deserialize, Deserializer, Serializer};

    pub(crate) fn serialize<S>(date: &Date, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        let s = date.format(DATE_FORMAT).to_string();
        serializer.serialize_str(&s)
    }

    pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result<Date, D::Error>
    where
        D: Deserializer<'de>,
    {
        let s = String::deserialize(deserializer)?;
        Date::parse_from_str(&s, DATE_FORMAT).map_err(serde::de::Error::custom)
    }
}

fn parse_date(s: &str) -> Option<Date> {
    match s {
        "now" | "today" => {
            let now = chrono::offset::Utc::now();
            let now = now.naive_utc();
            let now = chrono::DateTime::from_utc(now, chrono::offset::FixedOffset::east(0));
            Some(now)
        }
        _ => {
            let formats = ["%d %B %Y %H:%M:%S %z", "%Y-%m-%d %H:%M:%S %z"];
            formats
                .iter()
                .filter_map(|f| Date::parse_from_str(s, f).ok())
                .next()
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    static TRUE: Scalar = Scalar(ScalarEnum::Bool(true));
    static FALSE: Scalar = Scalar(ScalarEnum::Bool(false));

    #[test]
    fn test_to_str_bool() {
        assert_eq!(TRUE.to_str(), "true");
    }

    #[test]
    fn test_to_str_integer() {
        let val: Scalar = 42i32.into();
        assert_eq!(val.to_str(), "42");
    }

    #[test]
    fn test_to_str_float() {
        let val: Scalar = 42f64.into();
        assert_eq!(val.to_str(), "42");

        let val: Scalar = 42.34.into();
        assert_eq!(val.to_str(), "42.34");
    }

    #[test]
    fn test_to_str_str() {
        let val: Scalar = "foobar".into();
        assert_eq!(val.to_str(), "foobar");
    }

    #[test]
    fn test_to_integer_bool() {
        assert_eq!(TRUE.to_integer(), None);
    }

    #[test]
    fn test_to_integer_integer() {
        let val: Scalar = 42i32.into();
        assert_eq!(val.to_integer(), Some(42i32));
    }

    #[test]
    fn test_to_integer_float() {
        let val: Scalar = 42f64.into();
        assert_eq!(val.to_integer(), None);

        let val: Scalar = 42.34.into();
        assert_eq!(val.to_integer(), None);
    }

    #[test]
    fn test_to_integer_str() {
        let val: Scalar = "foobar".into();
        assert_eq!(val.to_integer(), None);

        let val: Scalar = "42.34".into();
        assert_eq!(val.to_integer(), None);

        let val: Scalar = "42".into();
        assert_eq!(val.to_integer(), Some(42));
    }

    #[test]
    fn test_to_float_bool() {
        assert_eq!(TRUE.to_float(), None);
    }

    #[test]
    fn test_to_float_integer() {
        let val: Scalar = 42i32.into();
        assert_eq!(val.to_float(), Some(42f64));
    }

    #[test]
    fn test_to_float_float() {
        let val: Scalar = 42f64.into();
        assert_eq!(val.to_float(), Some(42f64));

        let val: Scalar = 42.34.into();
        assert_eq!(val.to_float(), Some(42.34));
    }

    #[test]
    fn test_to_float_str() {
        let val: Scalar = "foobar".into();
        assert_eq!(val.to_float(), None);

        let val: Scalar = "42.34".into();
        assert_eq!(val.to_float(), Some(42.34));

        let val: Scalar = "42".into();
        assert_eq!(val.to_float(), Some(42f64));
    }

    #[test]
    fn test_to_bool_bool() {
        assert_eq!(TRUE.to_bool(), Some(true));
    }

    #[test]
    fn test_to_bool_integer() {
        let val: Scalar = 42i32.into();
        assert_eq!(val.to_bool(), None);
    }

    #[test]
    fn test_to_bool_float() {
        let val: Scalar = 42f64.into();
        assert_eq!(val.to_bool(), None);

        let val: Scalar = 42.34.into();
        assert_eq!(val.to_bool(), None);
    }

    #[test]
    fn test_to_bool_str() {
        let val: Scalar = "foobar".into();
        assert_eq!(val.to_bool(), None);

        let val: Scalar = "true".into();
        assert_eq!(val.to_bool(), None);

        let val: Scalar = "false".into();
        assert_eq!(val.to_bool(), None);
    }

    #[test]
    fn integer_equality() {
        let val: Scalar = 42i32.into();
        let zero: Scalar = 0i32.into();
        assert_eq!(val, val);
        assert_eq!(zero, zero);
        assert!(val != zero);
        assert!(zero != val);
    }

    #[test]
    fn integers_have_ruby_truthiness() {
        let val: Scalar = 42i32.into();
        let zero: Scalar = 0i32.into();
        assert_eq!(TRUE, val);
        assert_eq!(val, TRUE);
        assert!(val.is_truthy());

        assert_eq!(TRUE, zero);
        assert_eq!(zero, TRUE);
        assert!(zero.is_truthy());
    }

    #[test]
    fn float_equality() {
        let val: Scalar = 42f64.into();
        let zero: Scalar = 0f64.into();
        assert_eq!(val, val);
        assert_eq!(zero, zero);
        assert!(val != zero);
        assert!(zero != val);
    }

    #[test]
    fn floats_have_ruby_truthiness() {
        let val: Scalar = 42f64.into();
        let zero: Scalar = 0f64.into();
        assert_eq!(TRUE, val);
        assert_eq!(val, TRUE);
        assert!(val.is_truthy());

        assert_eq!(TRUE, zero);
        assert_eq!(zero, TRUE);
        assert!(zero.is_truthy());
    }

    #[test]
    fn boolean_equality() {
        assert_eq!(TRUE, TRUE);
        assert_eq!(FALSE, FALSE);
        assert!(FALSE != TRUE);
        assert!(TRUE != FALSE);
    }

    #[test]
    fn booleans_have_ruby_truthiness() {
        assert!(TRUE.is_truthy());
        assert!(!FALSE.is_truthy());
    }

    #[test]
    fn string_equality() {
        let alpha: Scalar = "alpha".into();
        let beta: Scalar = "beta".into();
        let empty: Scalar = "".into();
        assert_eq!(alpha, alpha);
        assert_eq!(empty, empty);
        assert!(alpha != beta);
        assert!(beta != alpha);
    }

    #[test]
    fn strings_have_ruby_truthiness() {
        // all strings in ruby are true
        let alpha: Scalar = "alpha".into();
        let empty: Scalar = "".into();
        assert_eq!(TRUE, alpha);
        assert_eq!(alpha, TRUE);
        assert!(alpha.is_truthy());

        assert_eq!(TRUE, empty);
        assert_eq!(empty, TRUE);
        assert!(empty.is_truthy());
    }

    #[test]
    fn parse_date_empty_is_bad() {
        assert!(parse_date("").is_none());
    }

    #[test]
    fn parse_date_bad() {
        assert!(parse_date("aaaaa").is_none());
    }

    #[test]
    fn parse_date_now() {
        assert!(parse_date("now").is_some());
    }

    #[test]
    fn parse_date_today() {
        assert!(parse_date("today").is_some());
    }
}
<|endoftext|>


<commit_before>use std::rc::Rc;
use std::path::Path;
use std::cell::Cell;
use std::collections::BTreeMap;
use url::Url;
use regex::Regex;
use html5ever::tree_builder::TreeSink;
use html5ever::rcdom::Node;
use html5ever::rcdom::NodeData::{Element, Text};
use html5ever::rcdom::Handle;
use html5ever::rcdom::NodeData::{
    Document,
    Doctype,
    Comment,
    ProcessingInstruction
};
use html5ever:: rcdom::RcDom;
use html5ever::{QualName, LocalName};
use html5ever::tree_builder::{NodeOrText, ElementFlags};
use dom;

pub static PUNCTUATIONS: &'static str = r"([、。，．！？]|\.[^A-Za-z0-9]|,[^0-9]|!|\?)";
pub static POSITIVE: &'static str = "article|body|content|entry|hentry|main|page\
                                     |pagination|post|text|blog|story";
pub static NEGATIVE: &'static str = "combx|comment|com|contact|foot|footer|footnote\
                                     |masthead|media|meta|outbrain|promo|related\
                                     |scroll|shoutbox|sidebar|sponsor|shopping\
                                     |tags|tool|widget|form|textfield";
static BLOCK_CHILD_TAGS: [&'static str; 10] = [
    "a", "blockquote", "dl", "div", "img", "ol", "p", "pre", "table", "ul",
];

pub struct Candidate {
    pub node:  Rc<Node>,
    pub score: Cell<f32>,
}

pub fn fix_img_path(handle: Handle, url: &Url) -> bool {
    let src = dom::get_attr("src", handle.clone());
    if src.is_none() {
        return false
    }
    let s = src.unwrap();
    if !s.starts_with("//") && !s.starts_with("http://") && s.starts_with("https://") {
        match url.join(&s) {
            Ok(new_url) => dom::set_attr("src", new_url.as_str(), handle),
            Err(_)      => (),
        }
    }
    true
}

pub fn get_link_density(handle: Handle) -> f32 {
    let text_length = dom::text_len(handle.clone()) as f32;
    if text_length == 0.0 {
        return 0.0;
    }
    let mut link_length = 0.0;
    let mut links: Vec<Rc<Node>> = vec![];
    dom::find_node(handle.clone(), "a", &mut links);
    for link in links.iter() {
        link_length += dom::text_len(link.clone()) as f32;
    }
    link_length / text_length
}

pub fn is_candidate(handle: Handle) -> bool {
    let text_len = dom::text_len(handle.clone());
    if text_len < 20 {
        return false
    }
    let n: &str = &dom::get_tag_name(handle. clone()).unwrap_or("".to_string());
    match n {
        "p" => true,
        "div" | "article" | "center" | "section" => {
            if !dom::has_nodes(handle.clone(),
                               &BLOCK_CHILD_TAGS.iter().map(|t| *t).collect()) {
                true
            } else {
                dom::text_children_count(handle.clone()) > 5
            }
        },
        _ => false
    }
}

pub fn init_content_score(handle: Handle) -> f32 {
    let tag_name = dom::get_tag_name(handle.clone()).unwrap_or("".to_string());
    let score = match tag_name.as_ref() {
        "article"    => 10.0,
        "div"        => 5.0,
        "blockquote" => 3.0,
        "form"       => -3.0,
        "th"         => 5.0,
        _            => 0.0,
    };
    score + get_class_weight(handle.clone())
}

pub fn calc_content_score(handle: Handle) -> f32 {
    let mut score: f32 = 1.0;
    let mut text = String::new();
    dom::extract_text(handle.clone(), &mut text, true);
    let re = Regex::new(PUNCTUATIONS).unwrap();
    let mat = re.find_iter(&text);
    score += mat.count() as f32;
    score += f32::min(f32::floor(text.chars().count() as f32 / 100.0), 3.0);
    return score
}

pub fn get_class_weight(handle: Handle) -> f32 {
    let mut weight: f32 = 0.0;
    match handle.data {
        Element { name: _, ref attrs, .. } => {
            for prop in ["id", "class"].iter() {
                if let Some(class) = dom::attr(prop, &attrs.borrow()) {
                    if Regex::new(POSITIVE).unwrap().is_match(&class) {
                        weight += 25.0
                    };
                    if Regex::new(NEGATIVE).unwrap().is_match(&class) {
                        weight -= 25.0
                    }
                }
            }
        },
        _ => (),
    };
    weight
}

pub fn preprocess(mut dom: &mut RcDom,  handle: Handle) -> bool {
    match handle.clone().data {
        Element { ref name, .. } => {
            let tag_name = name.local.as_ref();
            match tag_name.to_lowercase().as_ref() {
                "script" | "link" | "style"  => {
                    return true
                },
                _     => (),
            }
        },
        _ => (),
    }
    let mut useless_nodes = vec![];
    let mut paragraph_nodes = vec![];
    let mut br_count = 0;
    for child in handle.children.borrow().iter() {
        if preprocess(&mut dom, child.clone()) {
            useless_nodes.push(child.clone());
        }
        let c = child.clone();
        match c.data {
            Element { ref name, .. } => {
                let tag_name = name.local.as_ref();
                if "br" == tag_name.to_lowercase() {
                    br_count += 1
                } else {
                    br_count = 0
                }
            },
            Text { ref contents } => {
                let s = contents.borrow();
                if br_count >= 2 && s.trim().len() > 0 {
                    paragraph_nodes.push(child.clone());
                    br_count = 0
                }
            },
            _ => ()
        }
    }
    for node in useless_nodes.iter() {
        dom.remove_from_parent(node);
    }
    for node in paragraph_nodes.iter() {
        let name = QualName::new(None, ns!(), LocalName::from("p"));
        let p = dom.create_element(name, vec![], ElementFlags::default());
        dom.append_before_sibling(node, NodeOrText::AppendNode(p.clone()));
        dom.remove_from_parent(node);
        match node.clone().data {
            Text { ref contents } => {
                let text = contents.clone().into_inner().clone();
                dom.append(&p, NodeOrText::AppendText(text))
            },
            _ => (),
        }
    }
    false
}

pub fn clean(mut dom: &mut RcDom, id: &Path, handle: Handle, url: &Url, candidates: &BTreeMap<String, Candidate>) -> bool {
    let mut useless = false;
    match handle.data {
        Document       => (),
        Doctype { .. } => (),
        Text { ref contents } => {
            let s = contents.borrow();
            if s.trim().len() == 0 {
                useless = true
            }
        },
        Comment { .. } => useless = true,
        Element { ref name, ref attrs, .. } => {
            let tag_name = name.local.as_ref();
            match tag_name.to_lowercase().as_ref() {
                "script" | "link" | "style" | "noscript" | "meta"
                    | "h1" | "object" | "header" | "footer" | "aside" => {
                    useless = true
                },
                "form" | "table" | "ul" | "div" => {
                    useless = is_useless(id, handle.clone(), candidates)
                },
                "img" => useless = fix_img_path(handle.clone(), url),
                _     => (),
            }
            dom::clean_attr("id"   , &mut *attrs.borrow_mut());
            dom::clean_attr("class", &mut *attrs.borrow_mut());
            dom::clean_attr("style", &mut *attrs.borrow_mut());
        },
        ProcessingInstruction { .. } => unreachable!()
    }
    let mut useless_nodes = vec![];
    for (i, child) in handle.children.borrow().iter().enumerate() {
        let pid = id.join(i.to_string());
        if clean(&mut dom, pid.as_path(), child.clone(), url, candidates) {
            useless_nodes.push(child.clone());
        }
    }
    for node in useless_nodes.iter() {
        dom.remove_from_parent(node);
    }
    if dom::is_empty(handle) {
        useless = true
    }
    useless
}

pub fn is_useless(id: &Path, handle: Handle, candidates: &BTreeMap<String, Candidate>) -> bool {
    let tag_name = &dom::get_tag_name(handle.clone()).unwrap_or("".to_string());
    let weight = get_class_weight(handle.clone());
    let score = id.to_str()
        .and_then(|id| candidates.get(id))
        .map(|c| c.score.get()).unwrap_or(0.0);
    if weight + score < 0.0 {
        return true
    }
    let text_nodes_len = dom::text_children_count(handle.clone());
    let mut p_nodes:     Vec<Rc<Node>> = vec![];
    let mut img_nodes:   Vec<Rc<Node>> = vec![];
    let mut li_nodes:    Vec<Rc<Node>> = vec![];
    let mut input_nodes: Vec<Rc<Node>> = vec![];
    let mut embed_nodes: Vec<Rc<Node>> = vec![];
    dom::find_node(handle.clone(), "p"     , &mut p_nodes);
    dom::find_node(handle.clone(), "img"   , &mut img_nodes);
    dom::find_node(handle.clone(), "li"    , &mut li_nodes);
    dom::find_node(handle.clone(), "input" , &mut input_nodes);
    dom::find_node(handle.clone(), "embed" , &mut embed_nodes);
    let p_count        = p_nodes.len();
    let img_count      = img_nodes.len();
    let li_count       = li_nodes.len() as i32 - 100;
    let input_count    = input_nodes.len();
    let embed_count    = embed_nodes.len();
    let link_density   = get_link_density(handle.clone());
    let content_length = dom::text_len(handle.clone());
    let para_count = text_nodes_len + p_count;

    if img_count > para_count + text_nodes_len {
        return true
    }
    if li_count > para_count as i32 && tag_name != "ul" && tag_name != "ol" {
        return true
    }
    if input_count as f32 > f32::floor(para_count as f32 / 3.0) {
        return true
    }
    if content_length < 25 && (img_count == 0 || img_count > 2) {
        return true
    }
    if weight < 25.0 && link_density > 0.2 {
        return true
    }
    if (embed_count == 1 && content_length < 35) || embed_count > 1 {
        return true
    }
    return false
}

pub fn is_link_list(handle: Handle) -> bool {
    let rate = evaluate_list(handle);
    rate > 7.5
}

fn evaluate_list(handle: Handle) -> f32 {
    let mut hit: i32 = 0;
    let mut len: i32 = 0;
    for child in handle.children.borrow().iter() {
        let c = child.clone();
        match c.data {
            Element { ref name, .. } => {
                let tag_name = name.local.as_ref();
                match tag_name.to_lowercase().as_ref() {
                    "li" | "dt" | "dd" => {
                        len += 1;
                        if dom::has_link(child.clone()) {
                            hit += 1
                        }
                    }
                    _ => (),
                }
            },
            _ => ()
        }
    }
    if len == 0 {
        0.0
    } else {
        9.0 * ((hit / len) as f32).powi(2) + 1.0
    }
}
<commit_msg>Improve preprocess: Remove unlikely nodes. And define regex with `lazy_static!`<commit_after>use std::rc::Rc;
use std::path::Path;
use std::cell::Cell;
use std::collections::BTreeMap;
use url::Url;
use regex::Regex;
use html5ever::tree_builder::TreeSink;
use html5ever::rcdom::Node;
use html5ever::rcdom::NodeData::{Element, Text};
use html5ever::rcdom::Handle;
use html5ever::rcdom::NodeData::{
    Document,
    Doctype,
    Comment,
    ProcessingInstruction
};
use html5ever:: rcdom::RcDom;
use html5ever::{QualName, LocalName};
use html5ever::tree_builder::{NodeOrText, ElementFlags};
use dom;

pub static PUNCTUATIONS_REGEX: &'static str = r"([、。，．！？]|\.[^A-Za-z0-9]|,[^0-9]|!|\?)";
pub static UNLIKELY_CANDIDATES: &'static str =
    "combx|comment|community|disqus|extra|foot|header|menu\
     |remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate\
     |pagination|pager|popup|tweet|twitter";
pub static LIKELY_CANDIDATES: &'static str = "and|article|body|column|main|shadow";
pub static POSITIVE_CANDIDATES: &'static str =
    "article|body|content|entry|hentry|main|page\
     |pagination|post|text|blog|story";
pub static NEGATIVE_CANDIDATES: &'static str =
    "combx|comment|com|contact|foot|footer|footnote\
     |masthead|media|meta|outbrain|promo|related\
     |scroll|shoutbox|sidebar|sponsor|shopping\
     |tags|tool|widget|form|textfield\
     |uiScale";// medium
static BLOCK_CHILD_TAGS: [&'static str; 10] = [
    "a", "blockquote", "dl", "div", "img", "ol", "p", "pre", "table", "ul",
];

lazy_static! {
    static ref PUNCTUATIONS: Regex = Regex::new(PUNCTUATIONS_REGEX).unwrap();
    static ref LIKELY:       Regex = Regex::new(LIKELY_CANDIDATES).unwrap();
    static ref UNLIKELY:     Regex = Regex::new(UNLIKELY_CANDIDATES).unwrap();
    static ref POSITIVE:     Regex = Regex::new(POSITIVE_CANDIDATES).unwrap();
    static ref NEGATIVE:     Regex = Regex::new(NEGATIVE_CANDIDATES).unwrap();
}

pub struct Candidate {
    pub node:  Rc<Node>,
    pub score: Cell<f32>,
}

pub fn fix_img_path(handle: Handle, url: &Url) -> bool {
    let src = dom::get_attr("src", handle.clone());
    if src.is_none() {
        return false
    }
    let s = src.unwrap();
    if !s.starts_with("//") && !s.starts_with("http://") && s.starts_with("https://") {
        match url.join(&s) {
            Ok(new_url) => dom::set_attr("src", new_url.as_str(), handle),
            Err(_)      => (),
        }
    }
    true
}

pub fn get_link_density(handle: Handle) -> f32 {
    let text_length = dom::text_len(handle.clone()) as f32;
    if text_length == 0.0 {
        return 0.0;
    }
    let mut link_length = 0.0;
    let mut links: Vec<Rc<Node>> = vec![];
    dom::find_node(handle.clone(), "a", &mut links);
    for link in links.iter() {
        link_length += dom::text_len(link.clone()) as f32;
    }
    link_length / text_length
}

pub fn is_candidate(handle: Handle) -> bool {
    let text_len = dom::text_len(handle.clone());
    if text_len < 20 {
        return false
    }
    let n: &str = &dom::get_tag_name(handle. clone()).unwrap_or("".to_string());
    match n {
        "p" => true,
        "div" | "article" | "center" | "section" => {
            if !dom::has_nodes(handle.clone(),
                               &BLOCK_CHILD_TAGS.iter().map(|t| *t).collect()) {
                true
            } else {
                dom::text_children_count(handle.clone()) > 5
            }
        },
        _ => false
    }
}

pub fn init_content_score(handle: Handle) -> f32 {
    let tag_name = dom::get_tag_name(handle.clone()).unwrap_or("".to_string());
    let score = match tag_name.as_ref() {
        "article"    => 10.0,
        "div"        => 5.0,
        "blockquote" => 3.0,
        "form"       => -3.0,
        "th"         => 5.0,
        _            => 0.0,
    };
    score + get_class_weight(handle.clone())
}

pub fn calc_content_score(handle: Handle) -> f32 {
    let mut score: f32 = 1.0;
    let mut text = String::new();
    dom::extract_text(handle.clone(), &mut text, true);
    let mat = PUNCTUATIONS.find_iter(&text);
    score += mat.count() as f32;
    score += f32::min(f32::floor(text.chars().count() as f32 / 100.0), 3.0);
    return score
}

pub fn get_class_weight(handle: Handle) -> f32 {
    let mut weight: f32 = 0.0;
    match handle.data {
        Element { name: _, ref attrs, .. } => {
            for name in ["id", "class"].iter() {
                if let Some(val) = dom::attr(name, &attrs.borrow()) {
                    if POSITIVE.is_match(&val) {
                        weight += 25.0
                    };
                    if NEGATIVE.is_match(&val) {
                        weight -= 25.0
                    }
                }
            }
        },
        _ => (),
    };
    weight
}

pub fn preprocess(mut dom: &mut RcDom,  handle: Handle) -> bool {
    match handle.clone().data {
        Element { ref name, ref attrs, .. } => {
            let tag_name = name.local.as_ref();
            match tag_name.to_lowercase().as_ref() {
                "script" | "link" | "style"  => {
                    return true
                },
                _     => (),
            }
            for name in ["id", "class"].iter() {
                if let Some(val) = dom::attr(name, &attrs.borrow()) {
                    if tag_name != "body" && UNLIKELY.is_match(&val) {
                        if !LIKELY.is_match(&val) {
                            return true
                        }
                    }
                }
            }
        },
        _ => (),
    }
    let mut useless_nodes = vec![];
    let mut paragraph_nodes = vec![];
    let mut br_count = 0;
    for child in handle.children.borrow().iter() {
        if preprocess(&mut dom, child.clone()) {
            useless_nodes.push(child.clone());
        }
        let c = child.clone();
        match c.data {
            Element { ref name, .. } => {
                let tag_name = name.local.as_ref();
                if "br" == tag_name.to_lowercase() {
                    br_count += 1
                } else {
                    br_count = 0
                }
            },
            Text { ref contents } => {
                let s = contents.borrow();
                if br_count >= 2 && s.trim().len() > 0 {
                    paragraph_nodes.push(child.clone());
                    br_count = 0
                }
            },
            _ => ()
        }
    }
    for node in useless_nodes.iter() {
        dom.remove_from_parent(node);
    }
    for node in paragraph_nodes.iter() {
        let name = QualName::new(None, ns!(), LocalName::from("p"));
        let p = dom.create_element(name, vec![], ElementFlags::default());
        dom.append_before_sibling(node, NodeOrText::AppendNode(p.clone()));
        dom.remove_from_parent(node);
        match node.clone().data {
            Text { ref contents } => {
                let text = contents.clone().into_inner().clone();
                dom.append(&p, NodeOrText::AppendText(text))
            },
            _ => (),
        }
    }
    false
}

pub fn clean(mut dom: &mut RcDom, id: &Path, handle: Handle, url: &Url, candidates: &BTreeMap<String, Candidate>) -> bool {
    let mut useless = false;
    match handle.data {
        Document       => (),
        Doctype { .. } => (),
        Text { ref contents } => {
            let s = contents.borrow();
            if s.trim().len() == 0 {
                useless = true
            }
        },
        Comment { .. } => useless = true,
        Element { ref name, ref attrs, .. } => {
            let tag_name = name.local.as_ref();
            match tag_name.to_lowercase().as_ref() {
                "script" | "link" | "style" | "noscript" | "meta"
                    | "h1" | "object" | "header" | "footer" | "aside" => {
                    useless = true
                },
                "form" | "table" | "ul" | "div" => {
                    useless = is_useless(id, handle.clone(), candidates)
                },
                "img" => useless = fix_img_path(handle.clone(), url),
                _     => (),
            }
            dom::clean_attr("id"   , &mut *attrs.borrow_mut());
            dom::clean_attr("class", &mut *attrs.borrow_mut());
            dom::clean_attr("style", &mut *attrs.borrow_mut());
        },
        ProcessingInstruction { .. } => unreachable!()
    }
    let mut useless_nodes = vec![];
    for (i, child) in handle.children.borrow().iter().enumerate() {
        let pid = id.join(i.to_string());
        if clean(&mut dom, pid.as_path(), child.clone(), url, candidates) {
            useless_nodes.push(child.clone());
        }
    }
    for node in useless_nodes.iter() {
        dom.remove_from_parent(node);
    }
    if dom::is_empty(handle) {
        useless = true
    }
    useless
}

pub fn is_useless(id: &Path, handle: Handle, candidates: &BTreeMap<String, Candidate>) -> bool {
    let tag_name = &dom::get_tag_name(handle.clone()).unwrap_or("".to_string());
    let weight = get_class_weight(handle.clone());
    let score = id.to_str()
        .and_then(|id| candidates.get(id))
        .map(|c| c.score.get()).unwrap_or(0.0);
    if weight + score < 0.0 {
        return true
    }
    let text_nodes_len = dom::text_children_count(handle.clone());
    let mut p_nodes:     Vec<Rc<Node>> = vec![];
    let mut img_nodes:   Vec<Rc<Node>> = vec![];
    let mut li_nodes:    Vec<Rc<Node>> = vec![];
    let mut input_nodes: Vec<Rc<Node>> = vec![];
    let mut embed_nodes: Vec<Rc<Node>> = vec![];
    dom::find_node(handle.clone(), "p"     , &mut p_nodes);
    dom::find_node(handle.clone(), "img"   , &mut img_nodes);
    dom::find_node(handle.clone(), "li"    , &mut li_nodes);
    dom::find_node(handle.clone(), "input" , &mut input_nodes);
    dom::find_node(handle.clone(), "embed" , &mut embed_nodes);
    let p_count        = p_nodes.len();
    let img_count      = img_nodes.len();
    let li_count       = li_nodes.len() as i32 - 100;
    let input_count    = input_nodes.len();
    let embed_count    = embed_nodes.len();
    let link_density   = get_link_density(handle.clone());
    let content_length = dom::text_len(handle.clone());
    let para_count = text_nodes_len + p_count;

    if img_count > para_count + text_nodes_len {
        return true
    }
    if li_count > para_count as i32 && tag_name != "ul" && tag_name != "ol" {
        return true
    }
    if input_count as f32 > f32::floor(para_count as f32 / 3.0) {
        return true
    }
    if content_length < 25 && (img_count == 0 || img_count > 2) {
        return true
    }
    if weight < 25.0 && link_density > 0.2 {
        return true
    }
    if (embed_count == 1 && content_length < 35) || embed_count > 1 {
        return true
    }
    return false
}

pub fn is_link_list(handle: Handle) -> bool {
    let rate = evaluate_list(handle);
    rate > 7.5
}

fn evaluate_list(handle: Handle) -> f32 {
    let mut hit: i32 = 0;
    let mut len: i32 = 0;
    for child in handle.children.borrow().iter() {
        let c = child.clone();
        match c.data {
            Element { ref name, .. } => {
                let tag_name = name.local.as_ref();
                match tag_name.to_lowercase().as_ref() {
                    "li" | "dt" | "dd" => {
                        len += 1;
                        if dom::has_link(child.clone()) {
                            hit += 1
                        }
                    }
                    _ => (),
                }
            },
            _ => ()
        }
    }
    if len == 0 {
        0.0
    } else {
        9.0 * ((hit / len) as f32).powi(2) + 1.0
    }
}
<|endoftext|>


<commit_before>use proc_macro2::{Span, TokenStream};
use syn::{Data, Field, Fields, Ident, Lit, Meta, NestedMeta, Variant};

use super::RustlerAttr;

///
/// A parsing context struct.
///
/// `Context` holds information usable for different codegen modules.
///
pub(crate) struct Context<'a> {
    pub attrs: Vec<RustlerAttr>,
    pub ident: &'a proc_macro2::Ident,
    pub ident_with_lifetime: proc_macro2::TokenStream,
    pub variants: Option<Vec<&'a Variant>>,
    pub struct_fields: Option<Vec<&'a Field>>,
    pub is_tuple_struct: bool,
}

impl<'a> Context<'a> {
    pub fn from_ast(ast: &'a syn::DeriveInput) -> Self {
        let mut attrs: Vec<_> = ast
            .attrs
            .iter()
            .map(Context::get_rustler_attrs)
            .flatten()
            .collect();

        //
        // Default: generate encoder and decoder
        //
        if !Context::encode_decode_attr_set(&attrs) {
            attrs.push(RustlerAttr::Encode);
            attrs.push(RustlerAttr::Decode);
        }

        let has_lifetime = match ast.generics.lifetimes().count() {
            0 => false,
            1 => true,
            _ => panic!("Struct can only have one lifetime argument"),
        };

        let ident = &ast.ident;
        let ident_with_lifetime = if has_lifetime {
            quote! { #ident <'a> }
        } else {
            quote! { #ident }
        };

        let variants = match ast.data {
            Data::Enum(ref data_enum) => Some(data_enum.variants.iter().collect()),
            _ => None,
        };

        let struct_fields = match ast.data {
            Data::Struct(ref data_struct) => Some(data_struct.fields.iter().collect()),
            _ => None,
        };

        let is_tuple_struct = match ast.data {
            Data::Struct(ref data_struct) => match data_struct.fields {
                Fields::Unnamed(_) => true,
                _ => false,
            },
            _ => false,
        };

        Self {
            attrs,
            ident,
            ident_with_lifetime,
            variants,
            struct_fields,
            is_tuple_struct,
        }
    }

    pub fn atoms_module_name(&self, span: Span) -> Ident {
        Ident::new(&format!("RUSTLER_ATOMS_{}", self.ident), span)
    }

    pub fn encode(&self) -> bool {
        self.attrs.iter().any(|attr| match attr {
            RustlerAttr::Encode => true,
            _ => false,
        })
    }

    pub fn decode(&self) -> bool {
        self.attrs.iter().any(|attr| match attr {
            RustlerAttr::Decode => true,
            _ => false,
        })
    }

    pub fn field_atoms(&self) -> Option<Vec<TokenStream>> {
        self.struct_fields.as_ref().map(|struct_fields| {
            struct_fields
                .iter()
                .map(|field| {
                    let atom_fun = Self::field_to_atom_fun(field);

                    let ident = field.ident.as_ref().unwrap();
                    let ident_str = ident.to_string();
                    let ident_str = Self::remove_raw(&ident_str);

                    quote! {
                        #atom_fun = #ident_str,
                    }
                })
                .collect()
        })
    }

    pub fn field_to_atom_fun(field: &Field) -> Ident {
        let ident = field.ident.as_ref().unwrap();
        let ident_str = ident.to_string();
        let ident_str = Self::remove_raw(&ident_str);

        Ident::new(&format!("atom_{}", ident_str), Span::call_site())
    }

    pub fn escape_ident_with_index(ident_str: &str, index: usize, infix: &str) -> Ident {
        Ident::new(
            &format!(
                "RUSTLER_{}_field_{}_{}",
                infix,
                index,
                Self::remove_raw(ident_str)
            ),
            Span::call_site(),
        )
    }

    pub fn escape_ident(ident_str: &str, infix: &str) -> Ident {
        Ident::new(
            &format!("RUSTLER_{}_field_{}", infix, Self::remove_raw(ident_str)),
            Span::call_site(),
        )
    }

    fn remove_raw(ident_str: &str) -> &str {
        ident_str
            .split("r#")
            .last()
            .expect("split has always at least one element")
    }

    fn encode_decode_attr_set(attrs: &[RustlerAttr]) -> bool {
        attrs.iter().any(|attr| match attr {
            RustlerAttr::Encode => true,
            RustlerAttr::Decode => true,
            _ => false,
        })
    }

    fn get_rustler_attrs(attr: &syn::Attribute) -> Vec<RustlerAttr> {
        attr.path
            .segments
            .iter()
            .filter_map(|segment| {
                let meta = attr.parse_meta().expect("can parse meta");
                match segment.ident.to_string().as_ref() {
                    "rustler" => Context::parse_rustler(&meta),
                    "tag" => Context::try_parse_tag(&meta),
                    "module" => Context::try_parse_module(&meta),
                    _ => None,
                }
            })
            .flatten()
            .collect()
    }

    fn parse_rustler(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::List(ref list) = meta {
            return Some(
                list.nested
                    .iter()
                    .map(Context::parse_nested_rustler)
                    .collect(),
            );
        }

        panic!("Expected encode and/or decode in rustler attribute");
    }

    fn parse_nested_rustler(nested: &NestedMeta) -> RustlerAttr {
        if let NestedMeta::Meta(ref meta) = nested {
            if let Meta::Path(ref path) = meta {
                match path.segments[0].ident.to_string().as_ref() {
                    "encode" => return RustlerAttr::Encode,
                    "decode" => return RustlerAttr::Decode,
                    other => panic!("Unexpected literal {}", other),
                }
            }
        }

        panic!("Expected encode and/or decode in rustler attribute");
    }

    fn try_parse_tag(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::NameValue(ref name_value) = meta {
            if let Lit::Str(ref tag) = name_value.lit {
                return Some(vec![RustlerAttr::Tag(tag.value())]);
            }
        }
        panic!("Cannot parse module")
    }

    fn try_parse_module(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::NameValue(name_value) = meta {
            if let Lit::Str(ref module) = name_value.lit {
                let ident = format!("Elixir.{}", module.value());
                return Some(vec![RustlerAttr::Module(ident)]);
            }
        }
        panic!("Cannot parse tag")
    }
}
<commit_msg>Allow clippy lint<commit_after>// TODO When we settle for a minimum version of Rust >= 1.42, remove this.
#![allow(clippy::match_like_matches_macro)]

use proc_macro2::{Span, TokenStream};
use syn::{Data, Field, Fields, Ident, Lit, Meta, NestedMeta, Variant};

use super::RustlerAttr;

///
/// A parsing context struct.
///
/// `Context` holds information usable for different codegen modules.
///
pub(crate) struct Context<'a> {
    pub attrs: Vec<RustlerAttr>,
    pub ident: &'a proc_macro2::Ident,
    pub ident_with_lifetime: proc_macro2::TokenStream,
    pub variants: Option<Vec<&'a Variant>>,
    pub struct_fields: Option<Vec<&'a Field>>,
    pub is_tuple_struct: bool,
}

impl<'a> Context<'a> {
    pub fn from_ast(ast: &'a syn::DeriveInput) -> Self {
        let mut attrs: Vec<_> = ast
            .attrs
            .iter()
            .map(Context::get_rustler_attrs)
            .flatten()
            .collect();

        //
        // Default: generate encoder and decoder
        //
        if !Context::encode_decode_attr_set(&attrs) {
            attrs.push(RustlerAttr::Encode);
            attrs.push(RustlerAttr::Decode);
        }

        let has_lifetime = match ast.generics.lifetimes().count() {
            0 => false,
            1 => true,
            _ => panic!("Struct can only have one lifetime argument"),
        };

        let ident = &ast.ident;
        let ident_with_lifetime = if has_lifetime {
            quote! { #ident <'a> }
        } else {
            quote! { #ident }
        };

        let variants = match ast.data {
            Data::Enum(ref data_enum) => Some(data_enum.variants.iter().collect()),
            _ => None,
        };

        let struct_fields = match ast.data {
            Data::Struct(ref data_struct) => Some(data_struct.fields.iter().collect()),
            _ => None,
        };

        let is_tuple_struct = match ast.data {
            Data::Struct(ref data_struct) => match data_struct.fields {
                Fields::Unnamed(_) => true,
                _ => false,
            },
            _ => false,
        };

        Self {
            attrs,
            ident,
            ident_with_lifetime,
            variants,
            struct_fields,
            is_tuple_struct,
        }
    }

    pub fn atoms_module_name(&self, span: Span) -> Ident {
        Ident::new(&format!("RUSTLER_ATOMS_{}", self.ident), span)
    }

    pub fn encode(&self) -> bool {
        self.attrs.iter().any(|attr| match attr {
            RustlerAttr::Encode => true,
            _ => false,
        })
    }

    pub fn decode(&self) -> bool {
        self.attrs.iter().any(|attr| match attr {
            RustlerAttr::Decode => true,
            _ => false,
        })
    }

    pub fn field_atoms(&self) -> Option<Vec<TokenStream>> {
        self.struct_fields.as_ref().map(|struct_fields| {
            struct_fields
                .iter()
                .map(|field| {
                    let atom_fun = Self::field_to_atom_fun(field);

                    let ident = field.ident.as_ref().unwrap();
                    let ident_str = ident.to_string();
                    let ident_str = Self::remove_raw(&ident_str);

                    quote! {
                        #atom_fun = #ident_str,
                    }
                })
                .collect()
        })
    }

    pub fn field_to_atom_fun(field: &Field) -> Ident {
        let ident = field.ident.as_ref().unwrap();
        let ident_str = ident.to_string();
        let ident_str = Self::remove_raw(&ident_str);

        Ident::new(&format!("atom_{}", ident_str), Span::call_site())
    }

    pub fn escape_ident_with_index(ident_str: &str, index: usize, infix: &str) -> Ident {
        Ident::new(
            &format!(
                "RUSTLER_{}_field_{}_{}",
                infix,
                index,
                Self::remove_raw(ident_str)
            ),
            Span::call_site(),
        )
    }

    pub fn escape_ident(ident_str: &str, infix: &str) -> Ident {
        Ident::new(
            &format!("RUSTLER_{}_field_{}", infix, Self::remove_raw(ident_str)),
            Span::call_site(),
        )
    }

    fn remove_raw(ident_str: &str) -> &str {
        ident_str
            .split("r#")
            .last()
            .expect("split has always at least one element")
    }

    fn encode_decode_attr_set(attrs: &[RustlerAttr]) -> bool {
        attrs.iter().any(|attr| match attr {
            RustlerAttr::Encode => true,
            RustlerAttr::Decode => true,
            _ => false,
        })
    }

    fn get_rustler_attrs(attr: &syn::Attribute) -> Vec<RustlerAttr> {
        attr.path
            .segments
            .iter()
            .filter_map(|segment| {
                let meta = attr.parse_meta().expect("can parse meta");
                match segment.ident.to_string().as_ref() {
                    "rustler" => Context::parse_rustler(&meta),
                    "tag" => Context::try_parse_tag(&meta),
                    "module" => Context::try_parse_module(&meta),
                    _ => None,
                }
            })
            .flatten()
            .collect()
    }

    fn parse_rustler(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::List(ref list) = meta {
            return Some(
                list.nested
                    .iter()
                    .map(Context::parse_nested_rustler)
                    .collect(),
            );
        }

        panic!("Expected encode and/or decode in rustler attribute");
    }

    fn parse_nested_rustler(nested: &NestedMeta) -> RustlerAttr {
        if let NestedMeta::Meta(ref meta) = nested {
            if let Meta::Path(ref path) = meta {
                match path.segments[0].ident.to_string().as_ref() {
                    "encode" => return RustlerAttr::Encode,
                    "decode" => return RustlerAttr::Decode,
                    other => panic!("Unexpected literal {}", other),
                }
            }
        }

        panic!("Expected encode and/or decode in rustler attribute");
    }

    fn try_parse_tag(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::NameValue(ref name_value) = meta {
            if let Lit::Str(ref tag) = name_value.lit {
                return Some(vec![RustlerAttr::Tag(tag.value())]);
            }
        }
        panic!("Cannot parse module")
    }

    fn try_parse_module(meta: &Meta) -> Option<Vec<RustlerAttr>> {
        if let Meta::NameValue(name_value) = meta {
            if let Lit::Str(ref module) = name_value.lit {
                let ident = format!("Elixir.{}", module.value());
                return Some(vec![RustlerAttr::Module(ident)]);
            }
        }
        panic!("Cannot parse tag")
    }
}
<|endoftext|>


<commit_before>//
// Copyright 2022 The Project Oak Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

use alloc::{vec, vec::Vec};
use anyhow::{anyhow, Context};
use log::{log, Level};

// TODO(#3297): Don't use null terminated and use `string_view` instead.
/// Prints `message` to the debug Restricted Kernel logs.
/// These logs are being sent outside of the Trusted Execution Environment.
#[no_mangle]
pub unsafe extern "C" fn oak_log_debug(message_ptr: *const core::ffi::c_char, message_len: usize) {
    let message_bytes =
        unsafe { alloc::slice::from_raw_parts(message_ptr as *const u8, message_len) };
    if let Ok(message_string) = core::str::from_utf8(message_bytes) {
        log!(Level::Debug, "tflite_micro: {}", message_string);
    } else {
        log!(
            Level::Debug,
            "tflite_micro: Couldn't parse a UTF-8 string: {:?}",
            message_bytes
        );
    }
}

// TODO(#3297): Decide the TensorFlow model format to be passed as `model_bytes`.
#[link(name = "tflite_micro", kind = "static")]
extern "C" {
    /// Initializes the TensorFlow model and returns a corresponding error code.
    ///
    /// Memory area that will be used by the TFLM memory manager is defined by `tensor_arena_bytes`:
    /// <https://github.com/tensorflow/tflite-micro/blob/main/tensorflow/lite/micro/docs/memory_management.md>
    ///
    /// This function returns a buffer size value in the `output_buffer_len_ptr`. This buffer should
    /// be allocated before calling `tflite_run` to store inference results.
    fn tflite_init(
        model_bytes_ptr: *const u8,
        model_bytes_len: usize,
        tensor_arena_bytes_ptr: *mut u8,
        tensor_arena_bytes_len: usize,
        output_buffer_len_ptr: *mut usize,
    ) -> i32;

    /// Performs inference on `input_bytes_ptr` and pass inference result to `output_bytes_ptr` and
    /// returns an error code.
    ///
    /// The value in the `output_bytes_len_ptr` cannot be bigger than the value in
    /// `output_buffer_len_ptr` from `tflite_init`.
    fn tflite_run(
        input_bytes_ptr: *const u8,
        input_bytes_len: usize,
        output_bytes_ptr: *mut u8,
        output_bytes_len_ptr: *mut usize,
    ) -> i32;
}

// TODO(#3297): Use 8GiB or 10GiB arena sizes.
const TENSOR_ARENA_SIZE: usize = 1024 * 1024 * 1024;

#[derive(Default)]
pub struct TfliteModel {
    tensor_arena: Vec<u8>,
    /// Defines a output buffer length that should be preallocated before calling `tflite_run`.
    /// This value is only initialized after `initialize` function because TFLite should return the
    /// `output_buffer_len`.
    /// If the value is `None` then `TfliteModel` hasn't been initialized yet.
    output_buffer_len: Option<usize>,
}

impl TfliteModel {
    pub fn new() -> Self {
        let tensor_arena = vec![0; TENSOR_ARENA_SIZE];
        Self {
            tensor_arena,
            output_buffer_len: None,
        }
    }

    pub fn initialize(&mut self, model_bytes: &[u8]) -> anyhow::Result<()> {
        log!(Level::Info, "Initializing the TFLite model");
        if self.output_buffer_len.is_some() {
            return Err(anyhow!("TFLite model has already been initialized"));
        }

        let model_bytes_len = model_bytes.len();
        let tensor_arena_len = self.tensor_arena.len();
        let mut output_buffer_len = 0usize;

        let result_code = unsafe {
            tflite_init(
                model_bytes.as_ptr(),
                model_bytes_len,
                self.tensor_arena.as_mut_ptr(),
                tensor_arena_len,
                &mut output_buffer_len,
            )
        };
        if result_code == 0 {
            self.output_buffer_len = Some(output_buffer_len);
            log!(
                Level::Info,
                "TFLite model has been successfully initialized"
            );
            Ok(())
        } else {
            Err(anyhow!(
                "couldn't initialize TFLite model, code: {}",
                result_code
            ))
        }
    }

    pub fn run(&mut self, input_bytes: &[u8]) -> anyhow::Result<Vec<u8>> {
        log!(Level::Info, "Running TFLite inference");
        let output_buffer_len = self
            .output_buffer_len
            .context("running inference on a non-initialized TensorFlow model")?;

        let input_bytes_len = input_bytes.len();
        let mut output_buffer = vec![0; output_buffer_len];
        let mut output_bytes_len = 0usize;

        let result_code = unsafe {
            tflite_run(
                input_bytes.as_ptr(),
                input_bytes_len,
                output_buffer.as_mut_ptr(),
                &mut output_bytes_len,
            )
        };
        if result_code == 0 {
            if output_bytes_len <= output_buffer_len {
                log!(Level::Info, "TFLite inference has run successfully");
                Ok(output_buffer[..output_bytes_len].to_vec())
            } else {
                // Panicking since if the output bytes length is bigger than the output buffer
                // length, then there is a potential for memory corruption and we can't rely on any
                // of the Rust memory safety assumptions.
                panic!("output bytes length is bigger than the output buffer length");
            }
        } else {
            Err(anyhow!(
                "couldn't run TFLite inference, code: {}",
                result_code
            ))
        }
    }
}
<commit_msg>Fix TFLM page fault (#3483)<commit_after>//
// Copyright 2022 The Project Oak Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

use alloc::{vec, vec::Vec};
use anyhow::{anyhow, Context};
use log::{log, Level};

// TODO(#3297): Don't use null terminated and use `string_view` instead.
/// Prints `message` to the debug Restricted Kernel logs.
/// These logs are being sent outside of the Trusted Execution Environment.
#[no_mangle]
pub unsafe extern "C" fn oak_log_debug(message_ptr: *const core::ffi::c_char, message_len: usize) {
    let message_bytes =
        unsafe { alloc::slice::from_raw_parts(message_ptr as *const u8, message_len) };
    if let Ok(message_string) = core::str::from_utf8(message_bytes) {
        log!(Level::Debug, "tflite_micro: {}", message_string);
    } else {
        log!(
            Level::Debug,
            "tflite_micro: Couldn't parse a UTF-8 string: {:?}",
            message_bytes
        );
    }
}

// TODO(#3297): Decide the TensorFlow model format to be passed as `model_bytes`.
#[link(name = "tflite_micro", kind = "static")]
extern "C" {
    /// Initializes the TensorFlow model and returns a corresponding error code.
    ///
    /// Memory area that will be used by the TFLM memory manager is defined by `tensor_arena_bytes`:
    /// <https://github.com/tensorflow/tflite-micro/blob/main/tensorflow/lite/micro/docs/memory_management.md>
    ///
    /// This function returns a buffer size value in the `output_buffer_len_ptr`. This buffer should
    /// be allocated before calling `tflite_run` to store inference results.
    fn tflite_init(
        model_bytes_ptr: *const u8,
        model_bytes_len: usize,
        tensor_arena_bytes_ptr: *mut u8,
        tensor_arena_bytes_len: usize,
        output_buffer_len_ptr: *mut usize,
    ) -> i32;

    /// Performs inference on `input_bytes_ptr` and pass inference result to `output_bytes_ptr` and
    /// returns an error code.
    ///
    /// The value in the `output_bytes_len_ptr` cannot be bigger than the value in
    /// `output_buffer_len_ptr` from `tflite_init`.
    fn tflite_run(
        input_bytes_ptr: *const u8,
        input_bytes_len: usize,
        output_bytes_ptr: *mut u8,
        output_bytes_len_ptr: *mut usize,
    ) -> i32;
}

// TODO(#3297): Use 8GiB or 10GiB arena sizes.
const TENSOR_ARENA_SIZE: usize = 1024 * 1024 * 1024;

#[derive(Default)]
pub struct TfliteModel {
    tensor_arena: Vec<u8>,
    /// TFLite model binary representation.
    /// Rust needs to keep ownership of the model, because the C++ code just uses it as a
    /// reference.
    model_bytes: Vec<u8>,
    /// Defines a output buffer length that should be preallocated before calling `tflite_run`.
    /// This value is only initialized after `initialize` function because TFLite should return the
    /// `output_buffer_len`.
    /// If the value is `None` then `TfliteModel` hasn't been initialized yet.
    output_buffer_len: Option<usize>,
}

impl TfliteModel {
    pub fn new() -> Self {
        Self {
            tensor_arena: vec![0; TENSOR_ARENA_SIZE],
            model_bytes: vec![],
            output_buffer_len: None,
        }
    }

    pub fn initialize(&mut self, model_bytes: &[u8]) -> anyhow::Result<()> {
        log!(Level::Info, "Initializing the TFLite model");
        if self.output_buffer_len.is_some() {
            return Err(anyhow!("TFLite model has already been initialized"));
        }

        // Save a copy of the TFLite model.
        self.model_bytes = model_bytes.to_vec();

        let mut output_buffer_len = 0usize;
        let result_code = unsafe {
            tflite_init(
                self.model_bytes.as_ptr(),
                self.model_bytes.len(),
                self.tensor_arena.as_mut_ptr(),
                self.tensor_arena.len(),
                &mut output_buffer_len,
            )
        };
        if result_code == 0 {
            self.output_buffer_len = Some(output_buffer_len);
            log!(
                Level::Info,
                "TFLite model has been successfully initialized"
            );
            Ok(())
        } else {
            Err(anyhow!(
                "couldn't initialize TFLite model, code: {}",
                result_code
            ))
        }
    }

    pub fn run(&mut self, input_bytes: &[u8]) -> anyhow::Result<Vec<u8>> {
        log!(Level::Info, "Running TFLite inference");
        let output_buffer_len = self
            .output_buffer_len
            .context("running inference on a non-initialized TensorFlow model")?;

        let mut output_buffer = vec![0; output_buffer_len];
        let mut output_bytes_len = 0usize;
        let result_code = unsafe {
            tflite_run(
                input_bytes.as_ptr(),
                input_bytes.len(),
                output_buffer.as_mut_ptr(),
                &mut output_bytes_len,
            )
        };
        if result_code == 0 {
            if output_bytes_len <= output_buffer_len {
                log!(Level::Info, "TFLite inference has run successfully");
                Ok(output_buffer[..output_bytes_len].to_vec())
            } else {
                // Panicking since if the output bytes length is bigger than the output buffer
                // length, then there is a potential for memory corruption and we can't rely on any
                // of the Rust memory safety assumptions.
                panic!("output bytes length is bigger than the output buffer length");
            }
        } else {
            Err(anyhow!(
                "couldn't run TFLite inference, code: {}",
                result_code
            ))
        }
    }
}
<|endoftext|>


<commit_before>#![feature(custom_test_frameworks)]
// Custom test runner, to avoid libtest being wrapped around compiletest which wraps libtest.
#![test_runner(test_runner)]

use std::path::PathBuf;
use std::env;

use compiletest_rs as compiletest;
use colored::*;

fn miri_path() -> PathBuf {
    if rustc_test_suite().is_some() {
        PathBuf::from(option_env!("MIRI_PATH").unwrap())
    } else {
        PathBuf::from(concat!("target/", env!("PROFILE"), "/miri"))
    }
}

fn rustc_test_suite() -> Option<PathBuf> {
    option_env!("RUSTC_TEST_SUITE").map(PathBuf::from)
}

fn rustc_lib_path() -> PathBuf {
    option_env!("RUSTC_LIB_PATH").unwrap().into()
}

fn run_tests(mode: &str, path: &str, target: &str, mut flags: Vec<String>) {
    let in_rustc_test_suite = rustc_test_suite().is_some();
    // Add some flags we always want.
    flags.push("--edition 2018".to_owned());
    if in_rustc_test_suite {
        // Less aggressive warnings to make the rustc toolstate management less painful.
        // (We often get warnings when e.g. a feature gets stabilized or some lint gets added/improved.)
        flags.push("-Astable-features".to_owned());
    } else {
        flags.push("-Dwarnings -Dunused".to_owned()); // overwrite the -Aunused in compiletest-rs
    }
    if let Ok(sysroot) = std::env::var("MIRI_SYSROOT") {
        flags.push(format!("--sysroot {}", sysroot));
    }

    // The rest of the configuration.
    let mut config = compiletest::Config::default().tempdir();
    config.mode = mode.parse().expect("Invalid mode");
    config.rustc_path = miri_path();
    if in_rustc_test_suite {
        config.run_lib_path = rustc_lib_path();
        config.compile_lib_path = rustc_lib_path();
    }
    config.filter = env::args().nth(1);
    config.host = get_host();
    config.src_base = PathBuf::from(path);
    config.target = target.to_owned();
    config.target_rustcflags = Some(flags.join(" "));
    compiletest::run_tests(&config);
}

fn compile_fail(path: &str, target: &str, opt: bool) {
    let opt_str = if opt { " with optimizations" } else { "" };
    eprintln!("{}", format!(
        "## Running compile-fail tests in {} against miri for target {}{}",
        path,
        target,
        opt_str
    ).green().bold());

    let mut flags = Vec::new();
    if opt {
        // Optimizing too aggressivley makes UB detection harder, but test at least
        // the default value.
        // FIXME: Opt level 3 ICEs during stack trace generation.
        flags.push("-Zmir-opt-level=1".to_owned());
    }

    run_tests("compile-fail", path, target, flags);
}

fn miri_pass(path: &str, target: &str, opt: bool) {
    let opt_str = if opt { " with optimizations" } else { "" };
    eprintln!("{}", format!(
        "## Running run-pass tests in {} against miri for target {}{}",
        path,
        target,
        opt_str
    ).green().bold());

    let mut flags = Vec::new();
    if opt {
        flags.push("-Zmir-opt-level=3".to_owned());
    }

    run_tests("ui", path, target, flags);
}

fn get_host() -> String {
    let rustc = rustc_test_suite().unwrap_or(PathBuf::from("rustc"));
    let rustc_version = std::process::Command::new(rustc)
        .arg("-vV")
        .output()
        .expect("rustc not found for -vV")
        .stdout;
    let rustc_version = std::str::from_utf8(&rustc_version).expect("rustc -vV is not utf8");
    let version_meta = rustc_version::version_meta_for(&rustc_version)
        .expect("failed to parse rustc version info");
    version_meta.host
}

fn get_target() -> String {
    std::env::var("MIRI_TEST_TARGET").unwrap_or_else(|_| get_host())
}

fn run_pass_miri(opt: bool) {
    miri_pass("tests/run-pass", &get_target(), opt);
}

fn compile_fail_miri(opt: bool) {
    compile_fail("tests/compile-fail", &get_target(), opt);
}

fn test_runner(_tests: &[&()]) {
    // Add a test env var to do environment communication tests
    std::env::set_var("MIRI_ENV_VAR_TEST", "0");

    run_pass_miri(false);
    // FIXME: hashmap ICEs with optimizations run_pass_miri(true);

    compile_fail_miri(false);
    compile_fail_miri(true);
}
<commit_msg>Auto merge of #943 - wesleywiser:turn_on_tests_again, r=RalfJung<commit_after>#![feature(custom_test_frameworks)]
// Custom test runner, to avoid libtest being wrapped around compiletest which wraps libtest.
#![test_runner(test_runner)]

use std::path::PathBuf;
use std::env;

use compiletest_rs as compiletest;
use colored::*;

fn miri_path() -> PathBuf {
    if rustc_test_suite().is_some() {
        PathBuf::from(option_env!("MIRI_PATH").unwrap())
    } else {
        PathBuf::from(concat!("target/", env!("PROFILE"), "/miri"))
    }
}

fn rustc_test_suite() -> Option<PathBuf> {
    option_env!("RUSTC_TEST_SUITE").map(PathBuf::from)
}

fn rustc_lib_path() -> PathBuf {
    option_env!("RUSTC_LIB_PATH").unwrap().into()
}

fn run_tests(mode: &str, path: &str, target: &str, mut flags: Vec<String>) {
    let in_rustc_test_suite = rustc_test_suite().is_some();
    // Add some flags we always want.
    flags.push("--edition 2018".to_owned());
    if in_rustc_test_suite {
        // Less aggressive warnings to make the rustc toolstate management less painful.
        // (We often get warnings when e.g. a feature gets stabilized or some lint gets added/improved.)
        flags.push("-Astable-features".to_owned());
    } else {
        flags.push("-Dwarnings -Dunused".to_owned()); // overwrite the -Aunused in compiletest-rs
    }
    if let Ok(sysroot) = std::env::var("MIRI_SYSROOT") {
        flags.push(format!("--sysroot {}", sysroot));
    }

    // The rest of the configuration.
    let mut config = compiletest::Config::default().tempdir();
    config.mode = mode.parse().expect("Invalid mode");
    config.rustc_path = miri_path();
    if in_rustc_test_suite {
        config.run_lib_path = rustc_lib_path();
        config.compile_lib_path = rustc_lib_path();
    }
    config.filter = env::args().nth(1);
    config.host = get_host();
    config.src_base = PathBuf::from(path);
    config.target = target.to_owned();
    config.target_rustcflags = Some(flags.join(" "));
    compiletest::run_tests(&config);
}

fn compile_fail(path: &str, target: &str, opt: bool) {
    let opt_str = if opt { " with optimizations" } else { "" };
    eprintln!("{}", format!(
        "## Running compile-fail tests in {} against miri for target {}{}",
        path,
        target,
        opt_str
    ).green().bold());

    let mut flags = Vec::new();
    if opt {
        // Optimizing too aggressivley makes UB detection harder, but test at least
        // the default value.
        // FIXME: Opt level 3 ICEs during stack trace generation.
        flags.push("-Zmir-opt-level=1".to_owned());
    }

    run_tests("compile-fail", path, target, flags);
}

fn miri_pass(path: &str, target: &str, opt: bool) {
    let opt_str = if opt { " with optimizations" } else { "" };
    eprintln!("{}", format!(
        "## Running run-pass tests in {} against miri for target {}{}",
        path,
        target,
        opt_str
    ).green().bold());

    let mut flags = Vec::new();
    if opt {
        flags.push("-Zmir-opt-level=3".to_owned());
    }

    run_tests("ui", path, target, flags);
}

fn get_host() -> String {
    let rustc = rustc_test_suite().unwrap_or(PathBuf::from("rustc"));
    let rustc_version = std::process::Command::new(rustc)
        .arg("-vV")
        .output()
        .expect("rustc not found for -vV")
        .stdout;
    let rustc_version = std::str::from_utf8(&rustc_version).expect("rustc -vV is not utf8");
    let version_meta = rustc_version::version_meta_for(&rustc_version)
        .expect("failed to parse rustc version info");
    version_meta.host
}

fn get_target() -> String {
    std::env::var("MIRI_TEST_TARGET").unwrap_or_else(|_| get_host())
}

fn run_pass_miri(opt: bool) {
    miri_pass("tests/run-pass", &get_target(), opt);
}

fn compile_fail_miri(opt: bool) {
    compile_fail("tests/compile-fail", &get_target(), opt);
}

fn test_runner(_tests: &[&()]) {
    // Add a test env var to do environment communication tests
    std::env::set_var("MIRI_ENV_VAR_TEST", "0");

    run_pass_miri(false);
    run_pass_miri(true);

    compile_fail_miri(false);
    compile_fail_miri(true);
}
<|endoftext|>


<commit_before>#![feature(slice_concat_ext)]

extern crate compiletest_rs as compiletest;

use std::slice::SliceConcatExt;
use std::path::{PathBuf, Path};
use std::io::Write;

macro_rules! eprintln {
    ($($arg:tt)*) => {
        let stderr = std::io::stderr();
        writeln!(stderr.lock(), $($arg)*).unwrap();
    }
}

fn miri_path() -> String {
    format!("target/{}/miri", env!("PROFILE"))
}

fn compile_fail(sysroot: &Path, path: &str, target: &str, host: &str, fullmir: bool) {
    eprintln!("## Running compile-fail tests in {} against miri for target {}", path, target);
    let mut config = compiletest::default_config();
    config.mode = "compile-fail".parse().expect("Invalid mode");
    config.rustc_path = miri_path().into();
    if fullmir {
        if host != target {
            // skip fullmir on nonhost
            return;
        }
        let sysroot = Path::new(&std::env::var("HOME").unwrap()).join(".xargo").join("HOST");
        config.target_rustcflags = Some(format!("--sysroot {}", sysroot.to_str().unwrap()));
        config.src_base = PathBuf::from(path.to_string());
    } else {
        config.target_rustcflags = Some(format!("--sysroot {}", sysroot.to_str().unwrap()));
        config.src_base = PathBuf::from(path.to_string());
    }
    config.target = target.to_owned();
    compiletest::run_tests(&config);
}

fn run_pass(path: &str) {
    eprintln!("## Running run-pass tests in {} against rustc", path);
    let mut config = compiletest::default_config();
    config.mode = "run-pass".parse().expect("Invalid mode");
    config.src_base = PathBuf::from(path);
    config.target_rustcflags = Some("-Dwarnings".to_string());
    config.host_rustcflags = Some("-Dwarnings".to_string());
    compiletest::run_tests(&config);
}

fn miri_pass(path: &str, target: &str, host: &str, fullmir: bool, opt: bool) {
    let opt_str = if opt {
        " with optimizations"
    } else {
        ""
    };
    eprintln!("## Running run-pass tests in {} against miri for target {}{}", path, target, opt_str);
    let mut config = compiletest::default_config();
    config.mode = "mir-opt".parse().expect("Invalid mode");
    config.src_base = PathBuf::from(path);
    config.target = target.to_owned();
    config.host = host.to_owned();
    config.rustc_path = miri_path().into();
    let mut flags = Vec::new();
    if fullmir {
        if host != target {
            // skip fullmir on nonhost
            return;
        }
        let sysroot = Path::new(&std::env::var("HOME").unwrap()).join(".xargo").join("HOST");
        flags.push(format!("--sysroot {}", sysroot.to_str().unwrap()));
    }
    if opt {
        flags.push("-Zmir-opt-level=3".to_owned());
    } else {
        flags.push("-Zmir-opt-level=0".to_owned());
    }
    config.target_rustcflags = Some(flags.join(" "));
    // don't actually execute the final binary, it might be for other targets and we only care
    // about running miri, not the binary.
    config.runtool = Some("echo \"\" || ".to_owned());
    if target == host {
        std::env::set_var("MIRI_HOST_TARGET", "yes");
    }
    compiletest::run_tests(&config);
    std::env::set_var("MIRI_HOST_TARGET", "");
}

fn is_target_dir<P: Into<PathBuf>>(path: P) -> bool {
    let mut path = path.into();
    path.push("lib");
    path.metadata().map(|m| m.is_dir()).unwrap_or(false)
}

fn for_all_targets<F: FnMut(String)>(sysroot: &Path, mut f: F) {
    let target_dir = sysroot.join("lib").join("rustlib");
    for entry in std::fs::read_dir(target_dir).expect("invalid sysroot") {
        let entry = entry.unwrap();
        if !is_target_dir(entry.path()) { continue; }
        let target = entry.file_name().into_string().unwrap();
        f(target);
    }
}

fn get_sysroot() -> PathBuf {
    let sysroot = std::env::var("MIRI_SYSROOT").unwrap_or_else(|_| {
        let sysroot = std::process::Command::new("rustc")
            .arg("--print")
            .arg("sysroot")
            .output()
            .expect("rustc not found")
            .stdout;
        String::from_utf8(sysroot).expect("sysroot is not utf8")
    });
    PathBuf::from(sysroot.trim())
}

fn get_host() -> String {
    let host = std::process::Command::new("rustc")
        .arg("-vV")
        .output()
        .expect("rustc not found for -vV")
        .stdout;
    let host = std::str::from_utf8(&host).expect("sysroot is not utf8");
    let host = host.split("\nhost: ").nth(1).expect("no host: part in rustc -vV");
    let host = host.split('\n').next().expect("no \n after host");
    String::from(host)
}

#[test]
fn run_pass_miri() {
    let sysroot = get_sysroot();
    let host = get_host();

    for &opt in [false, true].iter() {
        for_all_targets(&sysroot, |target| {
            miri_pass("tests/run-pass", &target, &host, false, opt);
        });
        miri_pass("tests/run-pass-fullmir", &host, &host, true, opt);
    }
}

#[test]
fn run_pass_rustc() {
    run_pass("tests/run-pass");
    run_pass("tests/run-pass-fullmir");
}

#[test]
fn compile_fail_miri() {
    let sysroot = get_sysroot();
    let host = get_host();

    for_all_targets(&sysroot, |target| {
        compile_fail(&sysroot, "tests/compile-fail", &target, &host, false);
    });
    compile_fail(&sysroot, "tests/compile-fail-fullmir", &host, &host, true);
}
<commit_msg>no need to do run-time formatting<commit_after>#![feature(slice_concat_ext, const_fn)]

extern crate compiletest_rs as compiletest;

use std::slice::SliceConcatExt;
use std::path::{PathBuf, Path};
use std::io::Write;

macro_rules! eprintln {
    ($($arg:tt)*) => {
        let stderr = std::io::stderr();
        writeln!(stderr.lock(), $($arg)*).unwrap();
    }
}

const fn miri_path() -> &'static str {
    concat!("target/", env!("PROFILE"), "/miri")
}

fn compile_fail(sysroot: &Path, path: &str, target: &str, host: &str, fullmir: bool) {
    eprintln!("## Running compile-fail tests in {} against miri for target {}", path, target);
    let mut config = compiletest::default_config();
    config.mode = "compile-fail".parse().expect("Invalid mode");
    config.rustc_path = miri_path().into();
    if fullmir {
        if host != target {
            // skip fullmir on nonhost
            return;
        }
        let sysroot = Path::new(&std::env::var("HOME").unwrap()).join(".xargo").join("HOST");
        config.target_rustcflags = Some(format!("--sysroot {}", sysroot.to_str().unwrap()));
        config.src_base = PathBuf::from(path.to_string());
    } else {
        config.target_rustcflags = Some(format!("--sysroot {}", sysroot.to_str().unwrap()));
        config.src_base = PathBuf::from(path.to_string());
    }
    config.target = target.to_owned();
    compiletest::run_tests(&config);
}

fn run_pass(path: &str) {
    eprintln!("## Running run-pass tests in {} against rustc", path);
    let mut config = compiletest::default_config();
    config.mode = "run-pass".parse().expect("Invalid mode");
    config.src_base = PathBuf::from(path);
    config.target_rustcflags = Some("-Dwarnings".to_string());
    config.host_rustcflags = Some("-Dwarnings".to_string());
    compiletest::run_tests(&config);
}

fn miri_pass(path: &str, target: &str, host: &str, fullmir: bool, opt: bool) {
    let opt_str = if opt {
        " with optimizations"
    } else {
        ""
    };
    eprintln!("## Running run-pass tests in {} against miri for target {}{}", path, target, opt_str);
    let mut config = compiletest::default_config();
    config.mode = "mir-opt".parse().expect("Invalid mode");
    config.src_base = PathBuf::from(path);
    config.target = target.to_owned();
    config.host = host.to_owned();
    config.rustc_path = miri_path().into();
    let mut flags = Vec::new();
    if fullmir {
        if host != target {
            // skip fullmir on nonhost
            return;
        }
        let sysroot = Path::new(&std::env::var("HOME").unwrap()).join(".xargo").join("HOST");
        flags.push(format!("--sysroot {}", sysroot.to_str().unwrap()));
    }
    if opt {
        flags.push("-Zmir-opt-level=3".to_owned());
    } else {
        flags.push("-Zmir-opt-level=0".to_owned());
    }
    config.target_rustcflags = Some(flags.join(" "));
    // don't actually execute the final binary, it might be for other targets and we only care
    // about running miri, not the binary.
    config.runtool = Some("echo \"\" || ".to_owned());
    if target == host {
        std::env::set_var("MIRI_HOST_TARGET", "yes");
    }
    compiletest::run_tests(&config);
    std::env::set_var("MIRI_HOST_TARGET", "");
}

fn is_target_dir<P: Into<PathBuf>>(path: P) -> bool {
    let mut path = path.into();
    path.push("lib");
    path.metadata().map(|m| m.is_dir()).unwrap_or(false)
}

fn for_all_targets<F: FnMut(String)>(sysroot: &Path, mut f: F) {
    let target_dir = sysroot.join("lib").join("rustlib");
    for entry in std::fs::read_dir(target_dir).expect("invalid sysroot") {
        let entry = entry.unwrap();
        if !is_target_dir(entry.path()) { continue; }
        let target = entry.file_name().into_string().unwrap();
        f(target);
    }
}

fn get_sysroot() -> PathBuf {
    let sysroot = std::env::var("MIRI_SYSROOT").unwrap_or_else(|_| {
        let sysroot = std::process::Command::new("rustc")
            .arg("--print")
            .arg("sysroot")
            .output()
            .expect("rustc not found")
            .stdout;
        String::from_utf8(sysroot).expect("sysroot is not utf8")
    });
    PathBuf::from(sysroot.trim())
}

fn get_host() -> String {
    let host = std::process::Command::new("rustc")
        .arg("-vV")
        .output()
        .expect("rustc not found for -vV")
        .stdout;
    let host = std::str::from_utf8(&host).expect("sysroot is not utf8");
    let host = host.split("\nhost: ").nth(1).expect("no host: part in rustc -vV");
    let host = host.split('\n').next().expect("no \n after host");
    String::from(host)
}

#[test]
fn run_pass_miri() {
    let sysroot = get_sysroot();
    let host = get_host();

    for &opt in [false, true].iter() {
        for_all_targets(&sysroot, |target| {
            miri_pass("tests/run-pass", &target, &host, false, opt);
        });
        miri_pass("tests/run-pass-fullmir", &host, &host, true, opt);
    }
}

#[test]
fn run_pass_rustc() {
    run_pass("tests/run-pass");
    run_pass("tests/run-pass-fullmir");
}

#[test]
fn compile_fail_miri() {
    let sysroot = get_sysroot();
    let host = get_host();

    for_all_targets(&sysroot, |target| {
        compile_fail(&sysroot, "tests/compile-fail", &target, &host, false);
    });
    compile_fail(&sysroot, "tests/compile-fail-fullmir", &host, &host, true);
}
<|endoftext|>