<commit_before>// Format validated types representing lexical tokens defined in
// RFC 5322 (as well as some referred from RFC 5234)
// in order to support SMTP (RFC 5321)

// Macro for defining sequences of characters within a character class
macro_rules! def_cclass {
    ( $typ:ident, $test:ident) => {
        #[derive(Debug, Clone, PartialEq)]
        pub struct $typ(pub Vec<u8>);
        impl Parsable for $typ {
            fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
                let mut pos: usize = 0;
                let mut output: Vec<u8> = Vec::new();
                while pos < input.len() && $test(input[pos]) {
                    output.push(input[pos]);
                    pos += 1;
                }
                if output.len() > 0 {
                    Ok( ($typ(output), &input[pos..]) )
                }
                else {
                    if pos >= input.len() { Err( ParseError::Eof ) }
                    else { Err( ParseError::NotFound ) }
                }
            }
        }
        impl Streamable for $typ {
            fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
                Ok(try!(w.write(&self.0[..])))
            }
        }
    };
}

// Macro for assigning the returned remaining input of a parse function to an existing
// variable
macro_rules! parse {
    ($pth:ident, $rem:ident) => {
        {
            $pth::parse($rem).map(|(value, r)| { $rem = r; value })
        }
    };
}

macro_rules! req {
    ($rem:ident, $bytes:expr, $input:ident) => {
        let len: usize = $bytes.len();
        if $rem.len() < len {
            return Err(ParseError::Eof);
        }
        if &$rem[0..len] != $bytes {
            return Err(ParseError::Expected($bytes.to_vec()));
        }
        $rem = &$rem[len..];
    };
}

pub mod error;
pub use self::error::ParseError;
pub mod types;
pub mod headers;

use std::io::Write;
use std::io::Error as IoError;
use self::headers::{Return, Received};
use self::headers::{ResentDate, ResentFrom, ResentSender, ResentTo, ResentCc, ResentBcc,
                    ResentMessageId};
use self::headers::{OrigDate, From, Sender, ReplyTo, To, Cc, Bcc, MessageId, InReplyTo,
                    References, Subject, Comments, Keywords, OptionalField};

pub trait Parsable: Sized {
    /// Parse the object off of the beginning of the `input`.  If found, returns Some object,
    /// and a slice containing the remainer of the input.
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError>;
}

pub trait Streamable {
    /// Serializes and sends the content out to `w`, returning the number of bytes written.
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError>;
}

// 3.6.7
// trace           =   [return]
//                     1*received
#[derive(Debug, Clone, PartialEq)]
pub struct Trace {
    return_path: Option<Return>,
    received: Vec<Received>
}
impl Parsable for Trace {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        let maybe_return = parse!(Return, rem).ok();
        let mut received: Vec<Received> = Vec::new();
        while let Ok(r) = parse!(Received, rem) {
            received.push(r);
        }
        if received.len() < 1 { return Err(ParseError::NotFound); }
        Ok((Trace {
            return_path: maybe_return,
            received: received,
        }, rem))
    }
}
impl Streamable for Trace {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        let mut count: usize = 0;
        if let Some(ref rp) = self.return_path {
            count += try!(rp.stream(w));
        }
        for r in &self.received {
            count += try!(r.stream(w));
        }
        Ok(count)
    }
}

#[derive(Debug, Clone, PartialEq)]
pub enum ResentField {
    Date(ResentDate),
    From(ResentFrom),
    Sender(ResentSender),
    To(ResentTo),
    Cc(ResentCc),
    Bcc(ResentBcc),
    MessageId(ResentMessageId),
}
impl Parsable for ResentField {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        if let Ok(x) = parse!(ResentDate, rem) {
            return Ok((ResentField::Date(x), rem));
        }
        if let Ok(x) = parse!(ResentFrom, rem) {
            return Ok((ResentField::From(x), rem));
        }
        if let Ok(x) = parse!(ResentSender, rem) {
            return Ok((ResentField::Sender(x), rem));
        }
        if let Ok(x) = parse!(ResentTo, rem) {
            return Ok((ResentField::To(x), rem));
        }
        if let Ok(x) = parse!(ResentCc, rem) {
            return Ok((ResentField::Cc(x), rem));
        }
        if let Ok(x) = parse!(ResentBcc, rem) {
            return Ok((ResentField::Bcc(x), rem));
        }
        if let Ok(x) = parse!(ResentMessageId, rem) {
            return Ok((ResentField::MessageId(x), rem));
        }
        Err(ParseError::NotFound)
    }
}
impl Streamable for ResentField {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        match *self {
            ResentField::Date(ref x) => x.stream(w),
            ResentField::From(ref x) => x.stream(w),
            ResentField::Sender(ref x) => x.stream(w),
            ResentField::To(ref x) => x.stream(w),
            ResentField::Cc(ref x) => x.stream(w),
            ResentField::Bcc(ref x) => x.stream(w),
            ResentField::MessageId(ref x) => x.stream(w),
        }
    }
}

// 3.6
// a sub part of the Fields definition
#[derive(Debug, Clone, PartialEq)]
pub enum Field {
    OrigDate(OrigDate),
    From(From),
    Sender(Sender),
    ReplyTo(ReplyTo),
    To(To),
    Cc(Cc),
    Bcc(Bcc),
    MessageId(MessageId),
    InReplyTo(InReplyTo),
    References(References),
    Subject(Subject),
    Comments(Comments),
    Keywords(Keywords),
    OptionalField(OptionalField),
}
impl Parsable for Field {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        if let Ok(x) = parse!(OrigDate, rem) {
            return Ok((Field::OrigDate(x), rem));
        }
        if let Ok(x) = parse!(From, rem) {
            return Ok((Field::From(x), rem));
        }
        if let Ok(x) = parse!(Sender, rem) {
            return Ok((Field::Sender(x), rem));
        }
        if let Ok(x) = parse!(ReplyTo, rem) {
            return Ok((Field::ReplyTo(x), rem));
        }
        if let Ok(x) = parse!(To, rem) {
            return Ok((Field::To(x), rem));
        }
        if let Ok(x) = parse!(Cc, rem) {
            return Ok((Field::Cc(x), rem));
        }
        if let Ok(x) = parse!(Bcc, rem) {
            return Ok((Field::Bcc(x), rem));
        }
        if let Ok(x) = parse!(MessageId, rem) {
            return Ok((Field::MessageId(x), rem));
        }
        if let Ok(x) = parse!(InReplyTo, rem) {
            return Ok((Field::InReplyTo(x), rem));
        }
        if let Ok(x) = parse!(References, rem) {
            return Ok((Field::References(x), rem));
        }
        if let Ok(x) = parse!(Subject, rem) {
            return Ok((Field::Subject(x), rem));
        }
        if let Ok(x) = parse!(Comments, rem) {
            return Ok((Field::Comments(x), rem));
        }
        if let Ok(x) = parse!(Keywords, rem) {
            return Ok((Field::Keywords(x), rem));
        }
        if let Ok(x) = parse!(OptionalField, rem) {
            return Ok((Field::OptionalField(x), rem));
        }
        Err(ParseError::NotFound)
    }
}
impl Streamable for Field {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        match *self {
            Field::OrigDate(ref x) => x.stream(w),
            Field::From(ref x) => x.stream(w),
            Field::Sender(ref x) => x.stream(w),
            Field::ReplyTo(ref x) => x.stream(w),
            Field::To(ref x) => x.stream(w),
            Field::Cc(ref x) => x.stream(w),
            Field::Bcc(ref x) => x.stream(w),
            Field::MessageId(ref x) => x.stream(w),
            Field::InReplyTo(ref x) => x.stream(w),
            Field::References(ref x) => x.stream(w),
            Field::Subject(ref x) => x.stream(w),
            Field::Comments(ref x) => x.stream(w),
            Field::Keywords(ref x) => x.stream(w),
            Field::OptionalField(ref x) => x.stream(w),
        }
    }
}

// 3.6
// a sub part of the Fields definition
#[derive(Debug, Clone, PartialEq)]
pub struct ResentTraceBlock {
    pub trace: Trace,
    pub resent_fields: Vec<ResentField>,
}
impl Parsable for ResentTraceBlock {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        if let Ok(t) = parse!(Trace, rem) {
            let mut fields: Vec<ResentField> = Vec::new();
            while let Ok(f) = parse!(ResentField, rem) {
                fields.push(f);
            }
            if fields.len() == 0 {
                Err(ParseError::ExpectedType("Resent Field"))
            } else {
                Ok((ResentTraceBlock {
                    trace: t,
                    resent_fields: fields
                }, rem))
            }
        } else {
            Err(ParseError::NotFound)
        }
    }
}
impl Streamable for ResentTraceBlock {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        let mut count: usize = 0;
        count += try!(self.trace.stream(w));
        for field in &self.resent_fields {
            count += try!(field.stream(w));
        }
        Ok(count)
    }
}

// 3.6
// a sub part of the Fields definition
#[derive(Debug, Clone, PartialEq)]
pub struct OptTraceBlock {
    pub trace: Trace,
    pub opt_fields: Vec<OptionalField>,
}
impl Parsable for OptTraceBlock {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        if let Ok(t) = parse!(Trace, rem) {
            let mut fields: Vec<OptionalField> = Vec::new();
            while let Ok(f) = parse!(OptionalField, rem) {
                fields.push(f);
            }
            if fields.len() == 0 {
                Err(ParseError::ExpectedType("Optional Field"))
            } else {
                Ok((OptTraceBlock {
                    trace: t,
                    opt_fields: fields
                }, rem))
            }
        } else {
            Err(ParseError::NotFound)
        }
    }
}
impl Streamable for OptTraceBlock {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        let mut count: usize = 0;
        count += try!(self.trace.stream(w));
        for field in &self.opt_fields {
            count += try!(field.stream(w));
        }
        Ok(count)
    }
}
<commit_msg>rfc5322: TraceBlock<commit_after>// Format validated types representing lexical tokens defined in
// RFC 5322 (as well as some referred from RFC 5234)
// in order to support SMTP (RFC 5321)

// Macro for defining sequences of characters within a character class
macro_rules! def_cclass {
    ( $typ:ident, $test:ident) => {
        #[derive(Debug, Clone, PartialEq)]
        pub struct $typ(pub Vec<u8>);
        impl Parsable for $typ {
            fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
                let mut pos: usize = 0;
                let mut output: Vec<u8> = Vec::new();
                while pos < input.len() && $test(input[pos]) {
                    output.push(input[pos]);
                    pos += 1;
                }
                if output.len() > 0 {
                    Ok( ($typ(output), &input[pos..]) )
                }
                else {
                    if pos >= input.len() { Err( ParseError::Eof ) }
                    else { Err( ParseError::NotFound ) }
                }
            }
        }
        impl Streamable for $typ {
            fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
                Ok(try!(w.write(&self.0[..])))
            }
        }
    };
}

// Macro for assigning the returned remaining input of a parse function to an existing
// variable
macro_rules! parse {
    ($pth:ident, $rem:ident) => {
        {
            $pth::parse($rem).map(|(value, r)| { $rem = r; value })
        }
    };
}

macro_rules! req {
    ($rem:ident, $bytes:expr, $input:ident) => {
        let len: usize = $bytes.len();
        if $rem.len() < len {
            return Err(ParseError::Eof);
        }
        if &$rem[0..len] != $bytes {
            return Err(ParseError::Expected($bytes.to_vec()));
        }
        $rem = &$rem[len..];
    };
}

pub mod error;
pub use self::error::ParseError;
pub mod types;
pub mod headers;

use std::io::Write;
use std::io::Error as IoError;
use self::headers::{Return, Received};
use self::headers::{ResentDate, ResentFrom, ResentSender, ResentTo, ResentCc, ResentBcc,
                    ResentMessageId};
use self::headers::{OrigDate, From, Sender, ReplyTo, To, Cc, Bcc, MessageId, InReplyTo,
                    References, Subject, Comments, Keywords, OptionalField};

pub trait Parsable: Sized {
    /// Parse the object off of the beginning of the `input`.  If found, returns Some object,
    /// and a slice containing the remainer of the input.
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError>;
}

pub trait Streamable {
    /// Serializes and sends the content out to `w`, returning the number of bytes written.
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError>;
}

// 3.6.7
// trace           =   [return]
//                     1*received
#[derive(Debug, Clone, PartialEq)]
pub struct Trace {
    return_path: Option<Return>,
    received: Vec<Received>
}
impl Parsable for Trace {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        let maybe_return = parse!(Return, rem).ok();
        let mut received: Vec<Received> = Vec::new();
        while let Ok(r) = parse!(Received, rem) {
            received.push(r);
        }
        if received.len() < 1 { return Err(ParseError::NotFound); }
        Ok((Trace {
            return_path: maybe_return,
            received: received,
        }, rem))
    }
}
impl Streamable for Trace {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        let mut count: usize = 0;
        if let Some(ref rp) = self.return_path {
            count += try!(rp.stream(w));
        }
        for r in &self.received {
            count += try!(r.stream(w));
        }
        Ok(count)
    }
}

#[derive(Debug, Clone, PartialEq)]
pub enum ResentField {
    Date(ResentDate),
    From(ResentFrom),
    Sender(ResentSender),
    To(ResentTo),
    Cc(ResentCc),
    Bcc(ResentBcc),
    MessageId(ResentMessageId),
}
impl Parsable for ResentField {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        if let Ok(x) = parse!(ResentDate, rem) {
            return Ok((ResentField::Date(x), rem));
        }
        if let Ok(x) = parse!(ResentFrom, rem) {
            return Ok((ResentField::From(x), rem));
        }
        if let Ok(x) = parse!(ResentSender, rem) {
            return Ok((ResentField::Sender(x), rem));
        }
        if let Ok(x) = parse!(ResentTo, rem) {
            return Ok((ResentField::To(x), rem));
        }
        if let Ok(x) = parse!(ResentCc, rem) {
            return Ok((ResentField::Cc(x), rem));
        }
        if let Ok(x) = parse!(ResentBcc, rem) {
            return Ok((ResentField::Bcc(x), rem));
        }
        if let Ok(x) = parse!(ResentMessageId, rem) {
            return Ok((ResentField::MessageId(x), rem));
        }
        Err(ParseError::NotFound)
    }
}
impl Streamable for ResentField {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        match *self {
            ResentField::Date(ref x) => x.stream(w),
            ResentField::From(ref x) => x.stream(w),
            ResentField::Sender(ref x) => x.stream(w),
            ResentField::To(ref x) => x.stream(w),
            ResentField::Cc(ref x) => x.stream(w),
            ResentField::Bcc(ref x) => x.stream(w),
            ResentField::MessageId(ref x) => x.stream(w),
        }
    }
}

// 3.6
// a sub part of the Fields definition
#[derive(Debug, Clone, PartialEq)]
pub enum Field {
    OrigDate(OrigDate),
    From(From),
    Sender(Sender),
    ReplyTo(ReplyTo),
    To(To),
    Cc(Cc),
    Bcc(Bcc),
    MessageId(MessageId),
    InReplyTo(InReplyTo),
    References(References),
    Subject(Subject),
    Comments(Comments),
    Keywords(Keywords),
    OptionalField(OptionalField),
}
impl Parsable for Field {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        if let Ok(x) = parse!(OrigDate, rem) {
            return Ok((Field::OrigDate(x), rem));
        }
        if let Ok(x) = parse!(From, rem) {
            return Ok((Field::From(x), rem));
        }
        if let Ok(x) = parse!(Sender, rem) {
            return Ok((Field::Sender(x), rem));
        }
        if let Ok(x) = parse!(ReplyTo, rem) {
            return Ok((Field::ReplyTo(x), rem));
        }
        if let Ok(x) = parse!(To, rem) {
            return Ok((Field::To(x), rem));
        }
        if let Ok(x) = parse!(Cc, rem) {
            return Ok((Field::Cc(x), rem));
        }
        if let Ok(x) = parse!(Bcc, rem) {
            return Ok((Field::Bcc(x), rem));
        }
        if let Ok(x) = parse!(MessageId, rem) {
            return Ok((Field::MessageId(x), rem));
        }
        if let Ok(x) = parse!(InReplyTo, rem) {
            return Ok((Field::InReplyTo(x), rem));
        }
        if let Ok(x) = parse!(References, rem) {
            return Ok((Field::References(x), rem));
        }
        if let Ok(x) = parse!(Subject, rem) {
            return Ok((Field::Subject(x), rem));
        }
        if let Ok(x) = parse!(Comments, rem) {
            return Ok((Field::Comments(x), rem));
        }
        if let Ok(x) = parse!(Keywords, rem) {
            return Ok((Field::Keywords(x), rem));
        }
        if let Ok(x) = parse!(OptionalField, rem) {
            return Ok((Field::OptionalField(x), rem));
        }
        Err(ParseError::NotFound)
    }
}
impl Streamable for Field {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        match *self {
            Field::OrigDate(ref x) => x.stream(w),
            Field::From(ref x) => x.stream(w),
            Field::Sender(ref x) => x.stream(w),
            Field::ReplyTo(ref x) => x.stream(w),
            Field::To(ref x) => x.stream(w),
            Field::Cc(ref x) => x.stream(w),
            Field::Bcc(ref x) => x.stream(w),
            Field::MessageId(ref x) => x.stream(w),
            Field::InReplyTo(ref x) => x.stream(w),
            Field::References(ref x) => x.stream(w),
            Field::Subject(ref x) => x.stream(w),
            Field::Comments(ref x) => x.stream(w),
            Field::Keywords(ref x) => x.stream(w),
            Field::OptionalField(ref x) => x.stream(w),
        }
    }
}

// 3.6
// a sub part of the Fields definition
#[derive(Debug, Clone, PartialEq)]
pub struct ResentTraceBlock {
    pub trace: Trace,
    pub resent_fields: Vec<ResentField>,
}
impl Parsable for ResentTraceBlock {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        if let Ok(t) = parse!(Trace, rem) {
            let mut fields: Vec<ResentField> = Vec::new();
            while let Ok(f) = parse!(ResentField, rem) {
                fields.push(f);
            }
            if fields.len() == 0 {
                Err(ParseError::ExpectedType("Resent Field"))
            } else {
                Ok((ResentTraceBlock {
                    trace: t,
                    resent_fields: fields
                }, rem))
            }
        } else {
            Err(ParseError::NotFound)
        }
    }
}
impl Streamable for ResentTraceBlock {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        let mut count: usize = 0;
        count += try!(self.trace.stream(w));
        for field in &self.resent_fields {
            count += try!(field.stream(w));
        }
        Ok(count)
    }
}

// 3.6
// a sub part of the Fields definition
#[derive(Debug, Clone, PartialEq)]
pub struct OptTraceBlock {
    pub trace: Trace,
    pub opt_fields: Vec<OptionalField>,
}
impl Parsable for OptTraceBlock {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        if let Ok(t) = parse!(Trace, rem) {
            let mut fields: Vec<OptionalField> = Vec::new();
            while let Ok(f) = parse!(OptionalField, rem) {
                fields.push(f);
            }
            if fields.len() == 0 {
                Err(ParseError::ExpectedType("Optional Field"))
            } else {
                Ok((OptTraceBlock {
                    trace: t,
                    opt_fields: fields
                }, rem))
            }
        } else {
            Err(ParseError::NotFound)
        }
    }
}
impl Streamable for OptTraceBlock {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        let mut count: usize = 0;
        count += try!(self.trace.stream(w));
        for field in &self.opt_fields {
            count += try!(field.stream(w));
        }
        Ok(count)
    }
}

// 3.6
// a sub part of the Fields definition
#[derive(Debug, Clone, PartialEq)]
pub enum TraceBlock {
    Resent(ResentTraceBlock),
    Opt(OptTraceBlock),
}
impl Parsable for TraceBlock {
    fn parse(input: &[u8]) -> Result<(Self, &[u8]), ParseError> {
        let mut rem = input;
        if let Ok(block) = parse!(ResentTraceBlock, rem) {
            Ok((TraceBlock::Resent(block), rem))
        }
        else if let Ok(block) = parse!(OptTraceBlock, rem) {
            Ok((TraceBlock::Opt(block), rem))
        }
        else {
            Err(ParseError::NotFound)
        }
    }
}
impl Streamable for TraceBlock {
    fn stream<W: Write>(&self, w: &mut W) -> Result<usize, IoError> {
        match *self {
            TraceBlock::Resent(ref block) => block.stream(w),
            TraceBlock::Opt(ref block) => block.stream(w),
        }
    }
}
<|endoftext|>


<commit_before>extern crate redis;

use std::env;
use std::fmt;
use std::error::Error;
use std::default::Default;
use std::time::{SystemTime, UNIX_EPOCH};

use Value;
use rand::{Rng, thread_rng};
use serde::{Serialize, Serializer};
use serde::ser::SerializeStruct;
use serde_json;
use r2d2_redis::RedisConnectionManager;
use r2d2::{Config, Pool, PooledConnection, GetTimeout, InitializationError};

const REDIS_URL_DEFAULT: &'static str = "redis://127.0.0.1/";
pub type RedisPooledConnection = PooledConnection<RedisConnectionManager>;
pub type RedisPool = Pool<RedisConnectionManager>;

#[derive(Debug)]
pub struct ClientError {
    kind: ErrorKind,
}

#[derive(Debug)]
enum ErrorKind {
    Redis(redis::RedisError),
    PoolTimeout(GetTimeout),
    PoolInit(InitializationError),
}

pub fn create_redis_pool() -> Result<RedisPool, ClientError> {
    let config = Config::builder().build();
    let redis_url = &env::var("REDIS_URL").unwrap_or(REDIS_URL_DEFAULT.to_owned());
    let url = redis::parse_redis_url(redis_url).unwrap();
    let manager = RedisConnectionManager::new(url).unwrap();
    Pool::new(config, manager).map_err(|err| ClientError { kind: ErrorKind::PoolInit(err) })
}

pub struct Job {
    pub class: String,
    pub args: Vec<Value>,
    pub retry: i64,
    pub queue: String,
    pub jid: String,
    pub created_at: u64,
    pub enqueued_at: u64,
}

impl fmt::Display for ClientError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self.kind {
            ErrorKind::Redis(ref err) => err.fmt(f),
            ErrorKind::PoolTimeout(ref err) => err.fmt(f),
            ErrorKind::PoolInit(ref err) => err.fmt(f),
        }
    }
}

impl Error for ClientError {
    fn description(&self) -> &str {
        match self.kind {
            ErrorKind::Redis(ref err) => err.description(),
            ErrorKind::PoolTimeout(ref err) => err.description(),
            ErrorKind::PoolInit(ref err) => err.description(),
        }
    }

    fn cause(&self) -> Option<&Error> {
        match self.kind {
            ErrorKind::Redis(ref err) => Some(err),
            ErrorKind::PoolTimeout(ref err) => Some(err),
            ErrorKind::PoolInit(ref err) => Some(err),
        }
    }
}

impl From<redis::RedisError> for ClientError {
    fn from(error: redis::RedisError) -> ClientError {
        ClientError { kind: ErrorKind::Redis(error) }
    }
}

impl From<GetTimeout> for ClientError {
    fn from(error: GetTimeout) -> ClientError {
        ClientError { kind: ErrorKind::PoolTimeout(error) }
    }
}

impl From<InitializationError> for ClientError {
    fn from(error: InitializationError) -> ClientError {
        ClientError { kind: ErrorKind::PoolInit(error) }
    }
}

impl Default for JobOpts {
    fn default() -> JobOpts {
        let now = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap()
            .as_secs() as u64;
        let jid = thread_rng().gen_ascii_chars().take(24).collect::<String>();
        JobOpts {
            retry: 25,
            queue: "default".to_string(),
            jid: jid,
            created_at: now,
            enqueued_at: now,
        }
    }
}

pub struct JobOpts {
    pub retry: i64,
    pub queue: String,
    pub jid: String,
    pub created_at: u64,
    pub enqueued_at: u64,
}

/// # Examples
///
/// ```
/// use std::default::Default;
/// use sidekiq::Value;
/// use sidekiq::{Job, JobOpts};
///
/// // Create a job
/// let class = "MyClass".to_string();
/// let job_opts = JobOpts {
///     queue: "test".to_string(),
///     ..Default::default()
/// };
/// let job = Job::new(class, vec![sidekiq::Value::Null], job_opts);
/// ```
impl Job {
    pub fn new(class: String, args: Vec<Value>, opts: JobOpts) -> Job {
        Job {
            class: class,
            args: args,
            retry: opts.retry,
            queue: opts.queue,
            jid: opts.jid,
            created_at: opts.created_at,
            enqueued_at: opts.enqueued_at,
        }
    }
}

impl Serialize for Job {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: Serializer
    {
        let mut s = try!(serializer.serialize_struct("Job", 7));
        try!(s.serialize_field("class", &self.class));
        try!(s.serialize_field("args", &self.args));
        try!(s.serialize_field("retry", &self.retry));
        try!(s.serialize_field("queue", &self.queue));
        try!(s.serialize_field("jid", &self.jid));
        try!(s.serialize_field("created_at", &self.created_at));
        try!(s.serialize_field("enqueued_at", &self.enqueued_at));
        s.end()
    }
}

pub struct ClientOpts {
    pub namespace: Option<String>,
}

impl Default for ClientOpts {
    fn default() -> ClientOpts {
        ClientOpts { namespace: None }
    }
}

pub struct Client {
    pub redis_pool: RedisPool,
    pub namespace: Option<String>,
}

/// # Examples
///
/// ```
///
/// use sidekiq::{Job, Value};
/// use sidekiq::{Client, ClientOpts, create_redis_pool};
///
/// let ns = "test";
/// let client_opts = ClientOpts {
///     namespace: Some(ns.to_string()),
///     ..Default::default()
/// };
/// let pool = create_redis_pool().unwrap();
/// let client = Client::new(pool, client_opts);
/// let class = "MyClass".to_string();
/// let job = Job::new(class, vec![sidekiq::Value::Null], Default::default());
/// match client.push(job) {
///     Ok(_) => {},
///     Err(err) => {
///         println!("Sidekiq push failed: {}", err);
///     },
/// }
/// ```
impl Client {
    pub fn new(redis_pool: RedisPool, opts: ClientOpts) -> Client {
        Client {
            redis_pool: redis_pool,
            namespace: opts.namespace,
        }
    }

    fn connect(&self) -> Result<RedisPooledConnection, ClientError> {
        match self.redis_pool.get() {
            Ok(conn) => Ok(conn),
            Err(err) => Err(ClientError { kind: ErrorKind::PoolTimeout(err) }),
        }
    }

    pub fn push(&self, job: Job) -> Result<(), ClientError> {
        self.raw_push(vec![job])
    }

    pub fn push_bulk(&self, jobs: Vec<Job>) -> Result<(), ClientError> {
        self.raw_push(jobs)
    }

    fn raw_push(&self, payloads: Vec<Job>) -> Result<(), ClientError> {
        let ref p = payloads[0];
        let to_push =
            payloads.iter().map(|entry| serde_json::to_string(&entry).unwrap()).collect::<Vec<_>>();
        match self.connect() {
            Ok(conn) => {
                redis::pipe()
                    .atomic()
                    .cmd("SADD")
                    .arg("queues")
                    .arg(p.queue.to_string())
                    .ignore()
                    .cmd("LPUSH")
                    .arg(self.queue_name(&p.queue))
                    .arg(to_push)
                    .query(&*conn)
                    .map_err(|err| ClientError { kind: ErrorKind::Redis(err) })
            }
            Err(err) => Err(err),
        }
    }

    fn queue_name(&self, queue: &str) -> String {
        if let Some(ref ns) = self.namespace {
            format!("{}:queue:{}", ns, queue)
        } else {
            format!("queue:{}", queue)
        }
    }
}
<commit_msg>Fix use of unwrap_or followed by a function call<commit_after>extern crate redis;

use std::env;
use std::fmt;
use std::error::Error;
use std::default::Default;
use std::time::{SystemTime, UNIX_EPOCH};

use Value;
use rand::{Rng, thread_rng};
use serde::{Serialize, Serializer};
use serde::ser::SerializeStruct;
use serde_json;
use r2d2_redis::RedisConnectionManager;
use r2d2::{Config, Pool, PooledConnection, GetTimeout, InitializationError};

const REDIS_URL_DEFAULT: &'static str = "redis://127.0.0.1/";
pub type RedisPooledConnection = PooledConnection<RedisConnectionManager>;
pub type RedisPool = Pool<RedisConnectionManager>;

#[derive(Debug)]
pub struct ClientError {
    kind: ErrorKind,
}

#[derive(Debug)]
enum ErrorKind {
    Redis(redis::RedisError),
    PoolTimeout(GetTimeout),
    PoolInit(InitializationError),
}

pub fn create_redis_pool() -> Result<RedisPool, ClientError> {
    let config = Config::builder().build();
    let redis_url = &env::var("REDIS_URL").unwrap_or_else(|_| REDIS_URL_DEFAULT.to_owned());
    let url = redis::parse_redis_url(redis_url).unwrap();
    let manager = RedisConnectionManager::new(url).unwrap();
    Pool::new(config, manager).map_err(|err| ClientError { kind: ErrorKind::PoolInit(err) })
}

pub struct Job {
    pub class: String,
    pub args: Vec<Value>,
    pub retry: i64,
    pub queue: String,
    pub jid: String,
    pub created_at: u64,
    pub enqueued_at: u64,
}

impl fmt::Display for ClientError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self.kind {
            ErrorKind::Redis(ref err) => err.fmt(f),
            ErrorKind::PoolTimeout(ref err) => err.fmt(f),
            ErrorKind::PoolInit(ref err) => err.fmt(f),
        }
    }
}

impl Error for ClientError {
    fn description(&self) -> &str {
        match self.kind {
            ErrorKind::Redis(ref err) => err.description(),
            ErrorKind::PoolTimeout(ref err) => err.description(),
            ErrorKind::PoolInit(ref err) => err.description(),
        }
    }

    fn cause(&self) -> Option<&Error> {
        match self.kind {
            ErrorKind::Redis(ref err) => Some(err),
            ErrorKind::PoolTimeout(ref err) => Some(err),
            ErrorKind::PoolInit(ref err) => Some(err),
        }
    }
}

impl From<redis::RedisError> for ClientError {
    fn from(error: redis::RedisError) -> ClientError {
        ClientError { kind: ErrorKind::Redis(error) }
    }
}

impl From<GetTimeout> for ClientError {
    fn from(error: GetTimeout) -> ClientError {
        ClientError { kind: ErrorKind::PoolTimeout(error) }
    }
}

impl From<InitializationError> for ClientError {
    fn from(error: InitializationError) -> ClientError {
        ClientError { kind: ErrorKind::PoolInit(error) }
    }
}

impl Default for JobOpts {
    fn default() -> JobOpts {
        let now = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap()
            .as_secs() as u64;
        let jid = thread_rng().gen_ascii_chars().take(24).collect::<String>();
        JobOpts {
            retry: 25,
            queue: "default".to_string(),
            jid: jid,
            created_at: now,
            enqueued_at: now,
        }
    }
}

pub struct JobOpts {
    pub retry: i64,
    pub queue: String,
    pub jid: String,
    pub created_at: u64,
    pub enqueued_at: u64,
}

/// # Examples
///
/// ```
/// use std::default::Default;
/// use sidekiq::Value;
/// use sidekiq::{Job, JobOpts};
///
/// // Create a job
/// let class = "MyClass".to_string();
/// let job_opts = JobOpts {
///     queue: "test".to_string(),
///     ..Default::default()
/// };
/// let job = Job::new(class, vec![sidekiq::Value::Null], job_opts);
/// ```
impl Job {
    pub fn new(class: String, args: Vec<Value>, opts: JobOpts) -> Job {
        Job {
            class: class,
            args: args,
            retry: opts.retry,
            queue: opts.queue,
            jid: opts.jid,
            created_at: opts.created_at,
            enqueued_at: opts.enqueued_at,
        }
    }
}

impl Serialize for Job {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: Serializer
    {
        let mut s = try!(serializer.serialize_struct("Job", 7));
        try!(s.serialize_field("class", &self.class));
        try!(s.serialize_field("args", &self.args));
        try!(s.serialize_field("retry", &self.retry));
        try!(s.serialize_field("queue", &self.queue));
        try!(s.serialize_field("jid", &self.jid));
        try!(s.serialize_field("created_at", &self.created_at));
        try!(s.serialize_field("enqueued_at", &self.enqueued_at));
        s.end()
    }
}

pub struct ClientOpts {
    pub namespace: Option<String>,
}

impl Default for ClientOpts {
    fn default() -> ClientOpts {
        ClientOpts { namespace: None }
    }
}

pub struct Client {
    pub redis_pool: RedisPool,
    pub namespace: Option<String>,
}

/// # Examples
///
/// ```
///
/// use sidekiq::{Job, Value};
/// use sidekiq::{Client, ClientOpts, create_redis_pool};
///
/// let ns = "test";
/// let client_opts = ClientOpts {
///     namespace: Some(ns.to_string()),
///     ..Default::default()
/// };
/// let pool = create_redis_pool().unwrap();
/// let client = Client::new(pool, client_opts);
/// let class = "MyClass".to_string();
/// let job = Job::new(class, vec![sidekiq::Value::Null], Default::default());
/// match client.push(job) {
///     Ok(_) => {},
///     Err(err) => {
///         println!("Sidekiq push failed: {}", err);
///     },
/// }
/// ```
impl Client {
    pub fn new(redis_pool: RedisPool, opts: ClientOpts) -> Client {
        Client {
            redis_pool: redis_pool,
            namespace: opts.namespace,
        }
    }

    fn connect(&self) -> Result<RedisPooledConnection, ClientError> {
        match self.redis_pool.get() {
            Ok(conn) => Ok(conn),
            Err(err) => Err(ClientError { kind: ErrorKind::PoolTimeout(err) }),
        }
    }

    pub fn push(&self, job: Job) -> Result<(), ClientError> {
        self.raw_push(vec![job])
    }

    pub fn push_bulk(&self, jobs: Vec<Job>) -> Result<(), ClientError> {
        self.raw_push(jobs)
    }

    fn raw_push(&self, payloads: Vec<Job>) -> Result<(), ClientError> {
        let ref p = payloads[0];
        let to_push =
            payloads.iter().map(|entry| serde_json::to_string(&entry).unwrap()).collect::<Vec<_>>();
        match self.connect() {
            Ok(conn) => {
                redis::pipe()
                    .atomic()
                    .cmd("SADD")
                    .arg("queues")
                    .arg(p.queue.to_string())
                    .ignore()
                    .cmd("LPUSH")
                    .arg(self.queue_name(&p.queue))
                    .arg(to_push)
                    .query(&*conn)
                    .map_err(|err| ClientError { kind: ErrorKind::Redis(err) })
            }
            Err(err) => Err(err),
        }
    }

    fn queue_name(&self, queue: &str) -> String {
        if let Some(ref ns) = self.namespace {
            format!("{}:queue:{}", ns, queue)
        } else {
            format!("queue:{}", queue)
        }
    }
}
<|endoftext|>


<commit_before>use htmlescape::encode_minimal;
use schema::FieldValue;
use schema::Value;
use std::collections::BTreeMap;
use tokenizer::BoxedTokenizer;
use tokenizer::{Token, TokenStream, Tokenizer};
use Document;
use Index;
use Term;

#[derive(Debug)]
pub struct HighlightSection {
    start: usize,
    stop: usize,
}

impl HighlightSection {
    fn new(start: usize, stop: usize) -> HighlightSection {
        HighlightSection { start, stop }
    }
}

#[derive(Debug)]
pub struct FragmentCandidate {
    score: f32,
    start_offset: usize,
    stop_offset: usize,
    num_chars: usize,
    highlighted: Vec<HighlightSection>,
}

impl FragmentCandidate {
    /// Create a basic `FragmentCandidate`
    ///
    /// `score`, `num_chars` are set to 0
    /// and `highlighted` is set to empty vec
    /// stop_offset is set to start_offset, which is taken as a param.
    fn new(start_offset: usize) -> FragmentCandidate {
        FragmentCandidate {
            score: 0.0,
            start_offset: start_offset,
            stop_offset: start_offset,
            num_chars: 0,
            highlighted: vec![],
        }
    }

    /// Updates `score` and `highlighted` fields of the objects.
    ///
    /// taking the token and terms, the token is added to the fragment.
    /// if the token is one of the terms, the score
    /// and highlighted fields are updated in the fragment.
    fn try_add_token(&mut self, token: &Token, terms: &BTreeMap<String, f32>) {
        self.stop_offset = token.offset_to;

        if let Some(score) = terms.get(&token.text.to_lowercase()) {
            self.score += score;
            self.highlighted.push(HighlightSection {
                start: token.offset_from,
                stop: token.offset_to,
            });
        }
    }
}

#[derive(Debug)]
pub struct Snippet {
    fragments: String,
    highlighted: Vec<HighlightSection>,
}

const HIGHLIGHTEN_PREFIX: &str = "";
const HIGHLIGHTEN_POSTFIX: &str = "";

impl Snippet {
    /// Returns a hignlightned html from the `Snippet`.
    pub fn to_html(&self) -> String {
        let mut html = String::new();
        let mut start_from: usize = 0;

        for item in self.highlighted.iter() {
            html.push_str(&encode_minimal(&self.fragments[start_from..item.start]));
            html.push_str(HIGHLIGHTEN_PREFIX);
            html.push_str(&encode_minimal(&self.fragments[item.start..item.stop]));
            html.push_str(HIGHLIGHTEN_POSTFIX);
            start_from = item.stop;
        }
        html.push_str(&encode_minimal(
            &self.fragments[start_from..self.fragments.len()],
        ));
        html
    }
}

/// Returns a non-empty list of "good" fragments.
///
/// If no target term is within the text, then the function
/// should return an empty Vec.
///
/// If a target term is within the text, then the returned
/// list is required to be non-empty.
///
/// The returned list is non-empty and contain less
/// than 12 possibly overlapping fragments.
///
/// All fragments should contain at least one target term
/// and have at most `max_num_chars` characters (not bytes).
///
/// It is ok to emit non-overlapping fragments, for instance,
/// one short and one long containing the same keyword, in order
/// to leave optimization opportunity to the fragment selector
/// upstream.
///
/// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
/// has to be a valid string.
fn search_fragments<'a>(
    tokenizer: Box<BoxedTokenizer>,
    text: &'a str,
    terms: BTreeMap<String, f32>,
    max_num_chars: usize,
) -> Vec<FragmentCandidate> {
    let mut token_stream = tokenizer.token_stream(text);
    let mut fragment = FragmentCandidate::new(0);
    let mut fragments: Vec<FragmentCandidate> = vec![];

    while let Some(next) = token_stream.next() {
        if (next.offset_to - fragment.start_offset) > max_num_chars {
            if fragment.score > 0.0 {
                fragments.push(fragment)
            };
            fragment = FragmentCandidate::new(next.offset_from);
        }
        fragment.try_add_token(next, &terms);
    }
    if fragment.score > 0.0 {
        fragments.push(fragment)
    }

    fragments
}

/// Returns a Snippet
///
/// Takes a vector of `FragmentCandidate`s and the text.
/// Figures out the best fragment from it and creates a snippet.
fn select_best_fragment_combination<'a>(
    fragments: Vec<FragmentCandidate>,
    text: &'a str,
) -> Snippet {
    if let Some(init) = fragments.iter().nth(0) {
        let fragment =
            fragments.iter().skip(1).fold(
                init,
                |acc, item| {
                    if item.score > acc.score {
                        item
                    } else {
                        acc
                    }
                },
            );
        let fragment_text = &text[fragment.start_offset..fragment.stop_offset];
        let highlighted = fragment
            .highlighted
            .iter()
            .map(|item| HighlightSection {
                start: item.start - fragment.start_offset,
                stop: item.stop - fragment.start_offset,
            }).collect();
        Snippet {
            fragments: fragment_text.to_owned(),
            highlighted: highlighted,
        }
    } else {
        // when there no fragments to chose from,
        // for now create a empty snippet
        Snippet {
            fragments: String::new(),
            highlighted: vec![],
        }
    }
}

pub fn generate_snippet<'a>(
    doc: &'a [FieldValue],
    index: &Index,
    terms: Vec<Term>,
    max_num_chars: usize,
) -> Snippet {
    unimplemented!();
}

#[cfg(test)]
mod tests {
    use super::{search_fragments, select_best_fragment_combination};
    use std::collections::BTreeMap;
    use std::iter::Iterator;
    use tokenizer::{box_tokenizer, SimpleTokenizer};

    const TOKENIZER: SimpleTokenizer = SimpleTokenizer;

    #[test]
    fn test_snippet() {
        let boxed_tokenizer = box_tokenizer(TOKENIZER);

        let text = "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide better memory safety while still maintaining performance.

Rust is free and open-source software, released under an MIT License, or Apache License 2.0. Its designers have refined the language through the experiences of writing the Servo web browser layout engine[14] and the Rust compiler. A large proportion of current commits to the project are from community members.[15]

Rust won first place for \"most loved programming language\" in the Stack Overflow Developer Survey in 2016, 2017, and 2018.
";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("rust"), 1.0);
        terms.insert(String::from("language"), 0.9);

        let fragments = search_fragments(boxed_tokenizer, &text, terms, 100);
        assert_eq!(fragments.len(), 7);
        {
            let first = fragments.iter().nth(0).unwrap();
            assert_eq!(first.score, 1.9);
            assert_eq!(first.stop_offset, 89);
        }
        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe".to_owned());
        assert_eq!(snippet.to_html(), "Rust is a systems programming language sponsored by Mozilla which describes it as a "safe".to_owned())
    }

    #[test]
    fn test_snippet_in_second_fragment() {
        let boxed_tokenizer = box_tokenizer(TOKENIZER);

        let text = "a b c d e f g";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("c"), 1.0);

        let fragments = search_fragments(boxed_tokenizer, &text, terms, 3);

        assert_eq!(fragments.len(), 1);
        {
            let first = fragments.iter().nth(0).unwrap();
            assert_eq!(first.score, 1.0);
            assert_eq!(first.start_offset, 4);
            assert_eq!(first.stop_offset, 7);
        }

        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "c d");
        assert_eq!(snippet.to_html(), "c d");
    }

    #[test]
    fn test_snippet_with_term_at_the_end_of_fragment() {
        let boxed_tokenizer = box_tokenizer(TOKENIZER);

        let text = "a b c d e f f g";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("f"), 1.0);

        let fragments = search_fragments(boxed_tokenizer, &text, terms, 3);

        assert_eq!(fragments.len(), 2);
        {
            let first = fragments.iter().nth(0).unwrap();
            assert_eq!(first.score, 1.0);
            assert_eq!(first.stop_offset, 11);
            assert_eq!(first.start_offset, 8);
        }

        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "e f");
        assert_eq!(snippet.to_html(), "e f");
    }

    #[test]
    fn test_snippet_with_term_not_in_text() {
        let boxed_tokenizer = box_tokenizer(TOKENIZER);

        let text = "a b c d";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("z"), 1.0);

        let fragments = search_fragments(boxed_tokenizer, &text, terms, 3);

        assert_eq!(fragments.len(), 0);

        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "");
        assert_eq!(snippet.to_html(), "");
    }

    fn test_snippet_with_no_terms() {
        let boxed_tokenizer = box_tokenizer(TOKENIZER);

        let text = "a b c d";

        let mut terms = BTreeMap::new();
        let fragments = search_fragments(boxed_tokenizer, &text, terms, 3);
        assert_eq!(fragments.len(), 0);

        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "");
        assert_eq!(snippet.to_html(), "");
    }
}
<commit_msg>Use HighlightSection::new rather than just directly creating the object<commit_after>use htmlescape::encode_minimal;
use schema::FieldValue;
use schema::Value;
use std::collections::BTreeMap;
use tokenizer::BoxedTokenizer;
use tokenizer::{Token, TokenStream, Tokenizer};
use Document;
use Index;
use Term;

#[derive(Debug)]
pub struct HighlightSection {
    start: usize,
    stop: usize,
}

impl HighlightSection {
    fn new(start: usize, stop: usize) -> HighlightSection {
        HighlightSection { start, stop }
    }
}

#[derive(Debug)]
pub struct FragmentCandidate {
    score: f32,
    start_offset: usize,
    stop_offset: usize,
    num_chars: usize,
    highlighted: Vec<HighlightSection>,
}

impl FragmentCandidate {
    /// Create a basic `FragmentCandidate`
    ///
    /// `score`, `num_chars` are set to 0
    /// and `highlighted` is set to empty vec
    /// stop_offset is set to start_offset, which is taken as a param.
    fn new(start_offset: usize) -> FragmentCandidate {
        FragmentCandidate {
            score: 0.0,
            start_offset: start_offset,
            stop_offset: start_offset,
            num_chars: 0,
            highlighted: vec![],
        }
    }

    /// Updates `score` and `highlighted` fields of the objects.
    ///
    /// taking the token and terms, the token is added to the fragment.
    /// if the token is one of the terms, the score
    /// and highlighted fields are updated in the fragment.
    fn try_add_token(&mut self, token: &Token, terms: &BTreeMap<String, f32>) {
        self.stop_offset = token.offset_to;

        if let Some(score) = terms.get(&token.text.to_lowercase()) {
            self.score += score;
            self.highlighted
                .push(HighlightSection::new(token.offset_from, token.offset_to));
        }
    }
}

#[derive(Debug)]
pub struct Snippet {
    fragments: String,
    highlighted: Vec<HighlightSection>,
}

const HIGHLIGHTEN_PREFIX: &str = "";
const HIGHLIGHTEN_POSTFIX: &str = "";

impl Snippet {
    /// Returns a hignlightned html from the `Snippet`.
    pub fn to_html(&self) -> String {
        let mut html = String::new();
        let mut start_from: usize = 0;

        for item in self.highlighted.iter() {
            html.push_str(&encode_minimal(&self.fragments[start_from..item.start]));
            html.push_str(HIGHLIGHTEN_PREFIX);
            html.push_str(&encode_minimal(&self.fragments[item.start..item.stop]));
            html.push_str(HIGHLIGHTEN_POSTFIX);
            start_from = item.stop;
        }
        html.push_str(&encode_minimal(
            &self.fragments[start_from..self.fragments.len()],
        ));
        html
    }
}

/// Returns a non-empty list of "good" fragments.
///
/// If no target term is within the text, then the function
/// should return an empty Vec.
///
/// If a target term is within the text, then the returned
/// list is required to be non-empty.
///
/// The returned list is non-empty and contain less
/// than 12 possibly overlapping fragments.
///
/// All fragments should contain at least one target term
/// and have at most `max_num_chars` characters (not bytes).
///
/// It is ok to emit non-overlapping fragments, for instance,
/// one short and one long containing the same keyword, in order
/// to leave optimization opportunity to the fragment selector
/// upstream.
///
/// Fragments must be valid in the sense that `&text[fragment.start..fragment.stop]`\
/// has to be a valid string.
fn search_fragments<'a>(
    tokenizer: Box<BoxedTokenizer>,
    text: &'a str,
    terms: BTreeMap<String, f32>,
    max_num_chars: usize,
) -> Vec<FragmentCandidate> {
    let mut token_stream = tokenizer.token_stream(text);
    let mut fragment = FragmentCandidate::new(0);
    let mut fragments: Vec<FragmentCandidate> = vec![];

    while let Some(next) = token_stream.next() {
        if (next.offset_to - fragment.start_offset) > max_num_chars {
            if fragment.score > 0.0 {
                fragments.push(fragment)
            };
            fragment = FragmentCandidate::new(next.offset_from);
        }
        fragment.try_add_token(next, &terms);
    }
    if fragment.score > 0.0 {
        fragments.push(fragment)
    }

    fragments
}

/// Returns a Snippet
///
/// Takes a vector of `FragmentCandidate`s and the text.
/// Figures out the best fragment from it and creates a snippet.
fn select_best_fragment_combination<'a>(
    fragments: Vec<FragmentCandidate>,
    text: &'a str,
) -> Snippet {
    if let Some(init) = fragments.iter().nth(0) {
        let fragment =
            fragments.iter().skip(1).fold(
                init,
                |acc, item| {
                    if item.score > acc.score {
                        item
                    } else {
                        acc
                    }
                },
            );
        let fragment_text = &text[fragment.start_offset..fragment.stop_offset];
        let highlighted = fragment
            .highlighted
            .iter()
            .map(|item| {
                HighlightSection::new(
                    item.start - fragment.start_offset,
                    item.stop - fragment.start_offset,
                )
            }).collect();
        Snippet {
            fragments: fragment_text.to_owned(),
            highlighted: highlighted,
        }
    } else {
        // when there no fragments to chose from,
        // for now create a empty snippet
        Snippet {
            fragments: String::new(),
            highlighted: vec![],
        }
    }
}

pub fn generate_snippet<'a>(
    doc: &'a [FieldValue],
    index: &Index,
    terms: Vec<Term>,
    max_num_chars: usize,
) -> Snippet {
    unimplemented!();
}

#[cfg(test)]
mod tests {
    use super::{search_fragments, select_best_fragment_combination};
    use std::collections::BTreeMap;
    use std::iter::Iterator;
    use tokenizer::{box_tokenizer, SimpleTokenizer};

    const TOKENIZER: SimpleTokenizer = SimpleTokenizer;

    #[test]
    fn test_snippet() {
        let boxed_tokenizer = box_tokenizer(TOKENIZER);

        let text = "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its designers intend it to provide better memory safety while still maintaining performance.

Rust is free and open-source software, released under an MIT License, or Apache License 2.0. Its designers have refined the language through the experiences of writing the Servo web browser layout engine[14] and the Rust compiler. A large proportion of current commits to the project are from community members.[15]

Rust won first place for \"most loved programming language\" in the Stack Overflow Developer Survey in 2016, 2017, and 2018.
";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("rust"), 1.0);
        terms.insert(String::from("language"), 0.9);

        let fragments = search_fragments(boxed_tokenizer, &text, terms, 100);
        assert_eq!(fragments.len(), 7);
        {
            let first = fragments.iter().nth(0).unwrap();
            assert_eq!(first.score, 1.9);
            assert_eq!(first.stop_offset, 89);
        }
        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe".to_owned());
        assert_eq!(snippet.to_html(), "Rust is a systems programming language sponsored by Mozilla which describes it as a "safe".to_owned())
    }

    #[test]
    fn test_snippet_in_second_fragment() {
        let boxed_tokenizer = box_tokenizer(TOKENIZER);

        let text = "a b c d e f g";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("c"), 1.0);

        let fragments = search_fragments(boxed_tokenizer, &text, terms, 3);

        assert_eq!(fragments.len(), 1);
        {
            let first = fragments.iter().nth(0).unwrap();
            assert_eq!(first.score, 1.0);
            assert_eq!(first.start_offset, 4);
            assert_eq!(first.stop_offset, 7);
        }

        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "c d");
        assert_eq!(snippet.to_html(), "c d");
    }

    #[test]
    fn test_snippet_with_term_at_the_end_of_fragment() {
        let boxed_tokenizer = box_tokenizer(TOKENIZER);

        let text = "a b c d e f f g";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("f"), 1.0);

        let fragments = search_fragments(boxed_tokenizer, &text, terms, 3);

        assert_eq!(fragments.len(), 2);
        {
            let first = fragments.iter().nth(0).unwrap();
            assert_eq!(first.score, 1.0);
            assert_eq!(first.stop_offset, 11);
            assert_eq!(first.start_offset, 8);
        }

        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "e f");
        assert_eq!(snippet.to_html(), "e f");
    }

    #[test]
    fn test_snippet_with_term_not_in_text() {
        let boxed_tokenizer = box_tokenizer(TOKENIZER);

        let text = "a b c d";

        let mut terms = BTreeMap::new();
        terms.insert(String::from("z"), 1.0);

        let fragments = search_fragments(boxed_tokenizer, &text, terms, 3);

        assert_eq!(fragments.len(), 0);

        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "");
        assert_eq!(snippet.to_html(), "");
    }

    fn test_snippet_with_no_terms() {
        let boxed_tokenizer = box_tokenizer(TOKENIZER);

        let text = "a b c d";

        let mut terms = BTreeMap::new();
        let fragments = search_fragments(boxed_tokenizer, &text, terms, 3);
        assert_eq!(fragments.len(), 0);

        let snippet = select_best_fragment_combination(fragments, &text);
        assert_eq!(snippet.fragments, "");
        assert_eq!(snippet.to_html(), "");
    }
}
<|endoftext|>


<commit_before>//! Data consumers
//!
//! The goal of data producers is to parse data depending on the previous result.
//! It can be used to selectively seek in a file.
//!
//! ## Example
//!
//! This consumer will take 4 samples from the input, print them, then stop
//!
//! ```rust
//!  use nom::{MemProducer,Consumer,ConsumerState};
//!  use std::str;
//!
//!  struct TestPrintConsumer {
//!    counter: usize
//!  }
//!
//!  impl TestPrintConsumer {
//!    fn new() -> TestPrintConsumer {
//!      TestPrintConsumer { counter: 0 }
//!    }
//!  }
//!
//!  // Return ConsumerState::Await if it needs more data, or ConsumerDone when it ends
//!  impl Consumer for TestPrintConsumer {
//!    fn consume(&mut self, input: &[u8]) -> ConsumerState {
//!      println!("{} -> {}", self.counter, str::from_utf8(input).unwrap());
//!      self.counter = self.counter + 1;
//!      if self.counter <=4 {
//!        ConsumerState::Await
//!      } else {
//!        ConsumerState::ConsumerDone
//!      }
//!    }
//!
//!    fn end(&mut self) {
//!      println!("finished");
//!    }
//!  }
//!
//!  // It can consume data directly from a producer
//!  let mut p = MemProducer::new("abcdefghijklmnopqrstuvwx".as_bytes(), 4);
//!  let mut c = TestPrintConsumer::new();
//!  c.run(&mut p);
//! ```

use self::ConsumerState::*;
use producer::Producer;
use producer::ProducerState::*;
use internal::Err;

/// Holds the current state of the consumer
///
/// * Await if more data is needed
///
/// * ConsumerDone if the consumer does not need anymore data to be parsed
///
/// * ConsumerError when something went wrong
#[derive(Debug,PartialEq,Eq,Copy)]
pub enum ConsumerState {
  Await,
  Incomplete,
  ConsumerDone,
  ConsumerError(Err)
}

/// Implement the consume method, taking a byte array as input and returning a consumer state
///
/// The run function takes care of continuing or not
pub trait Consumer {
  fn consume(&mut self, input: &[u8]) -> ConsumerState;
  fn end(&mut self);

  fn run(&mut self, producer: &mut Producer) {
    let mut acc: Vec<u8> = Vec::new();
    //let mut v2: Vec<u8>  = Vec::new();
    loop {
      let state   = producer.produce();
      let mut eof = false;
      match state {
        Data(v) => {
          println!("got data");
          acc.push_all(v);
        },
        Eof([])  => {
          println!("eof empty");
          eof = true;
          break;
        }
        Eof(v) => {
          println!("eof with {} bytes", v.len());
          eof = true;
          acc.push_all(v);
        }
        _ => {break;}
      }
      //v2.push_all(acc.as_slice());
      //match consumer.consume(v2.as_slice()) {
      match self.consume(&acc[]) {
        ConsumerError(e) => {
          println!("consumer error, stopping: {}", e);
        },
        ConsumerDone => {
          println!("data, done");
          acc.clear();
          self.end();
          //acc.push_all(i);
          break;
        },
        Await => {
          println!("await");
          acc.clear();
          //acc.push_all(i);
        },
        Incomplete => {
          println!("incomplete");
        }
      }
    if eof { self.end(); }
    }
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use super::ConsumerState::*;
  use producer::MemProducer;
  use std::str;

  struct TestPrintConsumer {
    counter: usize,
    ended:   bool
  }

  impl TestPrintConsumer {
    fn new() -> TestPrintConsumer {
      TestPrintConsumer { counter: 0, ended: false }
    }
  }

  impl Consumer for TestPrintConsumer {
    fn consume(&mut self, input: &[u8]) -> ConsumerState {
      println!("{} -> {}", self.counter, str::from_utf8(input).unwrap());
      self.counter = self.counter + 1;
      if self.counter <= 4 {
        Await
      } else {
        ConsumerDone
      }
    }

    fn end(&mut self) {
      assert_eq!(self.counter, 5);
      self.ended = true;
    }
  }

  #[test]
  fn pull() {
    let mut p = MemProducer::new("abcdefghijklmnopqrstuvwx".as_bytes(), 4);
    let mut c = TestPrintConsumer::new();
    c.run(&mut p);

    assert!(c.ended);
  }
}
<commit_msg>Await returns the number of bytes remaining<commit_after>//! Data consumers
//!
//! The goal of data producers is to parse data depending on the previous result.
//! It can be used to selectively seek in a file.
//!
//! ## Example
//!
//! This consumer will take 4 samples from the input, print them, then stop
//!
//! ```rust
//!  use nom::{MemProducer,Consumer,ConsumerState};
//!  use std::str;
//!
//!  struct TestPrintConsumer {
//!    counter: usize
//!  }
//!
//!  impl TestPrintConsumer {
//!    fn new() -> TestPrintConsumer {
//!      TestPrintConsumer { counter: 0 }
//!    }
//!  }
//!
//!  // Return ConsumerState::Await if it needs more data, or ConsumerDone when it ends
//!  impl Consumer for TestPrintConsumer {
//!    fn consume(&mut self, input: &[u8]) -> ConsumerState {
//!      println!("{} -> {}", self.counter, str::from_utf8(input).unwrap());
//!      self.counter = self.counter + 1;
//!      if self.counter <=4 {
//!        ConsumerState::Await
//!      } else {
//!        ConsumerState::ConsumerDone
//!      }
//!    }
//!
//!    fn end(&mut self) {
//!      println!("finished");
//!    }
//!  }
//!
//!  // It can consume data directly from a producer
//!  let mut p = MemProducer::new("abcdefghijklmnopqrstuvwx".as_bytes(), 4);
//!  let mut c = TestPrintConsumer::new();
//!  c.run(&mut p);
//! ```

use self::ConsumerState::*;
use producer::Producer;
use producer::ProducerState::*;
use internal::Err;

/// Holds the current state of the consumer
///
/// * Await if more data is needed
///
/// * ConsumerDone if the consumer does not need anymore data to be parsed
///
/// * ConsumerError when something went wrong
#[derive(Debug,PartialEq,Eq,Copy)]
pub enum ConsumerState {
  Await(usize),
  Incomplete,
  ConsumerDone,
  ConsumerError(Err)
}

/// Implement the consume method, taking a byte array as input and returning a consumer state
///
/// The run function takes care of continuing or not
pub trait Consumer {
  fn consume(&mut self, input: &[u8]) -> ConsumerState;
  fn end(&mut self);

  fn run(&mut self, producer: &mut Producer) {
    let mut acc: Vec<u8> = Vec::new();
    //let mut v2: Vec<u8>  = Vec::new();
    loop {
      let state   = producer.produce();
      let mut eof = false;
      let mut end = false;
      match state {
        Data(v) => {
          println!("got data");
          acc.push_all(v);
        },
        Eof([])  => {
          println!("eof empty");
          eof = true;
          self.end();
          return
        }
        Eof(v) => {
          println!("eof with {} bytes", v.len());
          eof = true;
          acc.push_all(v);
        }
        _ => {break;}
      }
      //v2.push_all(acc.as_slice());
      //match consumer.consume(v2.as_slice()) {
      match self.consume(&acc[]) {
        ConsumerError(e) => {
          println!("consumer error, stopping: {}", e);
        },
        ConsumerDone => {
          println!("data, done");
          acc.clear();
          end = true;
          //acc.push_all(i);
          break;
        },
        Await(i) => {
          println!("await: remains {} bytes", i);
          let mut tmp = Vec::new();
          tmp.push_all(&acc[(acc.len()-i)..acc.len()]);
          acc.clear();
          acc = tmp;
          println!("acc size: {}", acc.len());
        },
        Incomplete => {
          println!("incomplete");
        }
      }
    if eof { self.end(); }
    if end { self.end(); }
    }
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use super::ConsumerState::*;
  use producer::MemProducer;
  use std::str;

  struct TestPrintConsumer {
    counter: usize,
    ended:   bool
  }

  impl TestPrintConsumer {
    fn new() -> TestPrintConsumer {
      TestPrintConsumer { counter: 0, ended: false }
    }
  }

  impl Consumer for TestPrintConsumer {
    fn consume(&mut self, input: &[u8]) -> ConsumerState {
      println!("{} -> {}", self.counter, str::from_utf8(input).unwrap());
      self.counter = self.counter + 1;
      if self.counter <= 4 {
        Await(0)
      } else {
        ConsumerDone
      }
    }

    fn end(&mut self) {
      assert_eq!(self.counter, 5);
      self.ended = true;
    }
  }

  #[test]
  fn pull() {
    let mut p = MemProducer::new("abcdefghijklmnopqrstuvwx".as_bytes(), 4);
    let mut c = TestPrintConsumer::new();
    c.run(&mut p);

    assert!(c.ended);
  }
}
<|endoftext|>


<commit_before>use std::{collections::BTreeMap, convert::TryFrom};

use ruma::{
    events::{
        from_raw_json_value,
        pdu::{Pdu, PduStub},
        room::member::{MemberEventContent, MembershipState},
        EventDeHelper, EventType,
    },
    identifiers::{EventId, RoomId, ServerName, UserId},
};
use serde::{de, Serialize};
use serde_json::value::RawValue as RawJsonValue;
use std::time::SystemTime;

pub struct Requester<'a> {
    pub prev_event_ids: Vec<EventId>,
    pub room_id: &'a RoomId,
    pub content: &'a serde_json::Value,
    pub state_key: Option<String>,
    pub sender: &'a UserId,
}

#[derive(Clone, Debug, Serialize)]
#[serde(untagged)]
pub enum StateEvent {
    Full(Pdu),
    Sync(PduStub),
}

impl StateEvent {
    pub fn to_requester(&self) -> Requester<'_> {
        Requester {
            prev_event_ids: self.prev_event_ids(),
            room_id: self.room_id().unwrap(),
            content: self.content(),
            state_key: self.state_key(),
            sender: self.sender(),
        }
    }

    pub fn is_power_event(&self) -> bool {
        match self {
            Self::Full(any_event) => match any_event {
                Pdu::RoomV1Pdu(event) => match event.kind {
                    EventType::RoomPowerLevels
                    | EventType::RoomJoinRules
                    | EventType::RoomCreate => event.state_key == Some("".into()),
                    EventType::RoomMember => {
                        if let Ok(content) =
                            // TODO fix clone
                            serde_json::from_value::<MemberEventContent>(event.content.clone())
                        {
                            if [MembershipState::Leave, MembershipState::Ban]
                                .contains(&content.membership)
                            {
                                return event.sender.as_str()
                                    // TODO is None here a failure
                                    != event.state_key.as_deref().unwrap_or("NOT A STATE KEY");
                            }
                        }

                        false
                    }
                    _ => false,
                },
                Pdu::RoomV3Pdu(event) => event.state_key == Some("".into()),
            },
            Self::Sync(any_event) => match any_event {
                PduStub::RoomV1PduStub(event) => match event.kind {
                    EventType::RoomPowerLevels
                    | EventType::RoomJoinRules
                    | EventType::RoomCreate => event.state_key == Some("".into()),
                    EventType::RoomMember => {
                        if let Ok(content) =
                            serde_json::from_value::<MemberEventContent>(event.content.clone())
                        {
                            if [MembershipState::Leave, MembershipState::Ban]
                                .contains(&content.membership)
                            {
                                return event.sender.as_str()
                                    // TODO does None here mean the same as state_key = ""
                                    != event.state_key.as_deref().unwrap_or("");
                            }
                        }

                        false
                    }
                    _ => false,
                },
                PduStub::RoomV3PduStub(event) => event.state_key == Some("".into()),
            },
        }
    }
    pub fn deserialize_content<C: serde::de::DeserializeOwned>(
        &self,
    ) -> Result<C, serde_json::Error> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => serde_json::from_value(ev.content.clone()),
                Pdu::RoomV3Pdu(ev) => serde_json::from_value(ev.content.clone()),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => serde_json::from_value(ev.content.clone()),
                PduStub::RoomV3PduStub(ev) => serde_json::from_value(ev.content.clone()),
            },
        }
    }
    pub fn origin_server_ts(&self) -> &SystemTime {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => &ev.origin_server_ts,
                Pdu::RoomV3Pdu(ev) => &ev.origin_server_ts,
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => &ev.origin_server_ts,
                PduStub::RoomV3PduStub(ev) => &ev.origin_server_ts,
            },
        }
    }
    pub fn event_id(&self) -> EventId {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.event_id.clone(),
                Pdu::RoomV3Pdu(_) => EventId::try_from(&*format!(
                    "${}",
                    ruma::signatures::reference_hash(
                        &serde_json::to_value(&ev).expect("event is valid, we just created it")
                    )
                    .expect("ruma can calculate reference hashes")
                ))
                .expect("ruma's reference hashes are valid event ids"),
            },
            Self::Sync(ev) => EventId::try_from(&*format!(
                "${}",
                ruma::signatures::reference_hash(
                    &serde_json::to_value(&ev).expect("event is valid, we just created it")
                )
                .expect("ruma can calculate reference hashes")
            ))
            .expect("ruma's reference hashes are valid event ids"),
        }
    }

    pub fn sender(&self) -> &UserId {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => &ev.sender,
                Pdu::RoomV3Pdu(ev) => &ev.sender,
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => &ev.sender,
                PduStub::RoomV3PduStub(ev) => &ev.sender,
            },
        }
    }

    pub fn redacts(&self) -> Option<&EventId> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.redacts.as_ref(),
                Pdu::RoomV3Pdu(ev) => ev.redacts.as_ref(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => ev.redacts.as_ref(),
                PduStub::RoomV3PduStub(ev) => ev.redacts.as_ref(),
            },
        }
    }

    pub fn room_id(&self) -> Option<&RoomId> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => Some(&ev.room_id),
                Pdu::RoomV3Pdu(ev) => Some(&ev.room_id),
            },
            Self::Sync(_) => None,
        }
    }
    pub fn kind(&self) -> EventType {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.kind.clone(),
                Pdu::RoomV3Pdu(ev) => ev.kind.clone(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => ev.kind.clone(),
                PduStub::RoomV3PduStub(ev) => ev.kind.clone(),
            },
        }
    }
    pub fn state_key(&self) -> Option<String> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.state_key.clone(),
                Pdu::RoomV3Pdu(ev) => ev.state_key.clone(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => ev.state_key.clone(),
                PduStub::RoomV3PduStub(ev) => ev.state_key.clone(),
            },
        }
    }

    pub fn prev_event_ids(&self) -> Vec<EventId> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.prev_events.iter().map(|(id, _)| id).cloned().collect(),
                Pdu::RoomV3Pdu(ev) => ev.prev_events.clone(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => {
                    ev.prev_events.iter().map(|(id, _)| id).cloned().collect()
                }
                PduStub::RoomV3PduStub(ev) => ev.prev_events.to_vec(),
            },
        }
    }

    pub fn auth_events(&self) -> Vec<EventId> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.auth_events.iter().map(|(id, _)| id).cloned().collect(),
                Pdu::RoomV3Pdu(ev) => ev.auth_events.to_vec(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => {
                    ev.auth_events.iter().map(|(id, _)| id).cloned().collect()
                }
                PduStub::RoomV3PduStub(ev) => ev.auth_events.to_vec(),
            },
        }
    }

    pub fn content(&self) -> &serde_json::Value {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => &ev.content,
                Pdu::RoomV3Pdu(ev) => &ev.content,
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => &ev.content,
                PduStub::RoomV3PduStub(ev) => &ev.content,
            },
        }
    }

    pub fn unsigned(&self) -> &BTreeMap<String, serde_json::Value> {
        // CONFIRM: The only way this would fail is if we got bad json, it should fail in ruma
        // before it fails here.
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => &ev.unsigned,
                Pdu::RoomV3Pdu(ev) => &ev.unsigned,
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => &ev.unsigned,
                PduStub::RoomV3PduStub(ev) => &ev.unsigned,
            },
        }
    }

    pub fn signatures(&self) -> BTreeMap<Box<ServerName>, BTreeMap<String, String>> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(_) => maplit::btreemap! {},
                Pdu::RoomV3Pdu(ev) => ev.signatures.clone(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => ev.signatures.clone(),
                PduStub::RoomV3PduStub(ev) => ev.signatures.clone(),
            },
        }
    }

    pub fn is_type_and_key(&self, ev_type: EventType, state_key: &str) -> bool {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => {
                    ev.kind == ev_type && ev.state_key.as_deref() == Some(state_key)
                }
                Pdu::RoomV3Pdu(ev) => {
                    ev.kind == ev_type && ev.state_key.as_deref() == Some(state_key)
                }
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => {
                    ev.kind == ev_type && ev.state_key.as_deref() == Some(state_key)
                }
                PduStub::RoomV3PduStub(ev) => {
                    ev.kind == ev_type && ev.state_key.as_deref() == Some(state_key)
                }
            },
        }
    }
}

impl<'de> de::Deserialize<'de> for StateEvent {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: de::Deserializer<'de>,
    {
        let json = Box::<RawJsonValue>::deserialize(deserializer)?;
        let EventDeHelper {
            room_id, unsigned, ..
        } = from_raw_json_value(&json)?;

        // Determine whether the event is a full or sync
        // based on the fields present.
        if room_id.is_some() {
            Ok(match unsigned {
                Some(unsigned) if unsigned.redacted_because.is_some() => {
                    panic!("TODO deal with redacted events")
                }
                _ => StateEvent::Full(Pdu::RoomV1Pdu(from_raw_json_value(&json)?)),
            })
        } else {
            Ok(match unsigned {
                Some(unsigned) if unsigned.redacted_because.is_some() => {
                    panic!("TODO deal with redacted events")
                }
                _ => StateEvent::Sync(from_raw_json_value(&json)?),
            })
        }
    }
}
<commit_msg>Update signatures since Cargo.lock seemed to be using cached ruma<commit_after>use std::{collections::BTreeMap, convert::TryFrom};

use ruma::{
    events::{
        from_raw_json_value,
        pdu::{Pdu, PduStub},
        room::member::{MemberEventContent, MembershipState},
        EventDeHelper, EventType,
    },
    identifiers::{EventId, RoomId, ServerName, UserId},
};
use serde::{de, Serialize};
use serde_json::value::RawValue as RawJsonValue;
use std::time::SystemTime;

pub struct Requester<'a> {
    pub prev_event_ids: Vec<EventId>,
    pub room_id: &'a RoomId,
    pub content: &'a serde_json::Value,
    pub state_key: Option<String>,
    pub sender: &'a UserId,
}

#[derive(Clone, Debug, Serialize)]
#[serde(untagged)]
pub enum StateEvent {
    Full(Pdu),
    Sync(PduStub),
}

impl StateEvent {
    pub fn to_requester(&self) -> Requester<'_> {
        Requester {
            prev_event_ids: self.prev_event_ids(),
            room_id: self.room_id().unwrap(),
            content: self.content(),
            state_key: self.state_key(),
            sender: self.sender(),
        }
    }

    pub fn is_power_event(&self) -> bool {
        match self {
            Self::Full(any_event) => match any_event {
                Pdu::RoomV1Pdu(event) => match event.kind {
                    EventType::RoomPowerLevels
                    | EventType::RoomJoinRules
                    | EventType::RoomCreate => event.state_key == Some("".into()),
                    EventType::RoomMember => {
                        if let Ok(content) =
                            // TODO fix clone
                            serde_json::from_value::<MemberEventContent>(event.content.clone())
                        {
                            if [MembershipState::Leave, MembershipState::Ban]
                                .contains(&content.membership)
                            {
                                return event.sender.as_str()
                                    // TODO is None here a failure
                                    != event.state_key.as_deref().unwrap_or("NOT A STATE KEY");
                            }
                        }

                        false
                    }
                    _ => false,
                },
                Pdu::RoomV3Pdu(event) => event.state_key == Some("".into()),
            },
            Self::Sync(any_event) => match any_event {
                PduStub::RoomV1PduStub(event) => match event.kind {
                    EventType::RoomPowerLevels
                    | EventType::RoomJoinRules
                    | EventType::RoomCreate => event.state_key == Some("".into()),
                    EventType::RoomMember => {
                        if let Ok(content) =
                            serde_json::from_value::<MemberEventContent>(event.content.clone())
                        {
                            if [MembershipState::Leave, MembershipState::Ban]
                                .contains(&content.membership)
                            {
                                return event.sender.as_str()
                                    // TODO does None here mean the same as state_key = ""
                                    != event.state_key.as_deref().unwrap_or("");
                            }
                        }

                        false
                    }
                    _ => false,
                },
                PduStub::RoomV3PduStub(event) => event.state_key == Some("".into()),
            },
        }
    }
    pub fn deserialize_content<C: serde::de::DeserializeOwned>(
        &self,
    ) -> Result<C, serde_json::Error> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => serde_json::from_value(ev.content.clone()),
                Pdu::RoomV3Pdu(ev) => serde_json::from_value(ev.content.clone()),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => serde_json::from_value(ev.content.clone()),
                PduStub::RoomV3PduStub(ev) => serde_json::from_value(ev.content.clone()),
            },
        }
    }
    pub fn origin_server_ts(&self) -> &SystemTime {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => &ev.origin_server_ts,
                Pdu::RoomV3Pdu(ev) => &ev.origin_server_ts,
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => &ev.origin_server_ts,
                PduStub::RoomV3PduStub(ev) => &ev.origin_server_ts,
            },
        }
    }
    pub fn event_id(&self) -> EventId {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.event_id.clone(),
                Pdu::RoomV3Pdu(_) => EventId::try_from(&*format!(
                    "${}",
                    ruma::signatures::reference_hash(
                        &serde_json::to_value(&ev).expect("event is valid, we just created it")
                    )
                    .expect("ruma can calculate reference hashes")
                ))
                .expect("ruma's reference hashes are valid event ids"),
            },
            Self::Sync(ev) => EventId::try_from(&*format!(
                "${}",
                ruma::signatures::reference_hash(
                    &serde_json::to_value(&ev).expect("event is valid, we just created it")
                )
                .expect("ruma can calculate reference hashes")
            ))
            .expect("ruma's reference hashes are valid event ids"),
        }
    }

    pub fn sender(&self) -> &UserId {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => &ev.sender,
                Pdu::RoomV3Pdu(ev) => &ev.sender,
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => &ev.sender,
                PduStub::RoomV3PduStub(ev) => &ev.sender,
            },
        }
    }

    pub fn redacts(&self) -> Option<&EventId> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.redacts.as_ref(),
                Pdu::RoomV3Pdu(ev) => ev.redacts.as_ref(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => ev.redacts.as_ref(),
                PduStub::RoomV3PduStub(ev) => ev.redacts.as_ref(),
            },
        }
    }

    pub fn room_id(&self) -> Option<&RoomId> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => Some(&ev.room_id),
                Pdu::RoomV3Pdu(ev) => Some(&ev.room_id),
            },
            Self::Sync(_) => None,
        }
    }
    pub fn kind(&self) -> EventType {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.kind.clone(),
                Pdu::RoomV3Pdu(ev) => ev.kind.clone(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => ev.kind.clone(),
                PduStub::RoomV3PduStub(ev) => ev.kind.clone(),
            },
        }
    }
    pub fn state_key(&self) -> Option<String> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.state_key.clone(),
                Pdu::RoomV3Pdu(ev) => ev.state_key.clone(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => ev.state_key.clone(),
                PduStub::RoomV3PduStub(ev) => ev.state_key.clone(),
            },
        }
    }

    pub fn prev_event_ids(&self) -> Vec<EventId> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.prev_events.iter().map(|(id, _)| id).cloned().collect(),
                Pdu::RoomV3Pdu(ev) => ev.prev_events.clone(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => {
                    ev.prev_events.iter().map(|(id, _)| id).cloned().collect()
                }
                PduStub::RoomV3PduStub(ev) => ev.prev_events.to_vec(),
            },
        }
    }

    pub fn auth_events(&self) -> Vec<EventId> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => ev.auth_events.iter().map(|(id, _)| id).cloned().collect(),
                Pdu::RoomV3Pdu(ev) => ev.auth_events.to_vec(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => {
                    ev.auth_events.iter().map(|(id, _)| id).cloned().collect()
                }
                PduStub::RoomV3PduStub(ev) => ev.auth_events.to_vec(),
            },
        }
    }

    pub fn content(&self) -> &serde_json::Value {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => &ev.content,
                Pdu::RoomV3Pdu(ev) => &ev.content,
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => &ev.content,
                PduStub::RoomV3PduStub(ev) => &ev.content,
            },
        }
    }

    pub fn unsigned(&self) -> &BTreeMap<String, serde_json::Value> {
        // CONFIRM: The only way this would fail is if we got bad json, it should fail in ruma
        // before it fails here.
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => &ev.unsigned,
                Pdu::RoomV3Pdu(ev) => &ev.unsigned,
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => &ev.unsigned,
                PduStub::RoomV3PduStub(ev) => &ev.unsigned,
            },
        }
    }

    pub fn signatures(&self) -> BTreeMap<Box<ServerName>, BTreeMap<ruma::ServerKeyId, String>> {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(_) => maplit::btreemap! {},
                Pdu::RoomV3Pdu(ev) => ev.signatures.clone(),
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => ev.signatures.clone(),
                PduStub::RoomV3PduStub(ev) => ev.signatures.clone(),
            },
        }
    }

    pub fn is_type_and_key(&self, ev_type: EventType, state_key: &str) -> bool {
        match self {
            Self::Full(ev) => match ev {
                Pdu::RoomV1Pdu(ev) => {
                    ev.kind == ev_type && ev.state_key.as_deref() == Some(state_key)
                }
                Pdu::RoomV3Pdu(ev) => {
                    ev.kind == ev_type && ev.state_key.as_deref() == Some(state_key)
                }
            },
            Self::Sync(ev) => match ev {
                PduStub::RoomV1PduStub(ev) => {
                    ev.kind == ev_type && ev.state_key.as_deref() == Some(state_key)
                }
                PduStub::RoomV3PduStub(ev) => {
                    ev.kind == ev_type && ev.state_key.as_deref() == Some(state_key)
                }
            },
        }
    }
}

impl<'de> de::Deserialize<'de> for StateEvent {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: de::Deserializer<'de>,
    {
        let json = Box::<RawJsonValue>::deserialize(deserializer)?;
        let EventDeHelper {
            room_id, unsigned, ..
        } = from_raw_json_value(&json)?;

        // Determine whether the event is a full or sync
        // based on the fields present.
        if room_id.is_some() {
            Ok(match unsigned {
                Some(unsigned) if unsigned.redacted_because.is_some() => {
                    panic!("TODO deal with redacted events")
                }
                _ => StateEvent::Full(Pdu::RoomV1Pdu(from_raw_json_value(&json)?)),
            })
        } else {
            Ok(match unsigned {
                Some(unsigned) if unsigned.redacted_because.is_some() => {
                    panic!("TODO deal with redacted events")
                }
                _ => StateEvent::Sync(from_raw_json_value(&json)?),
            })
        }
    }
}
<|endoftext|>


<commit_before>use common::err::{Error, TResult, Void, void_ok};
use rows::RowBlock;
use rows::vector::Vector1;
use common::types::{DataType, TypeKind, HasTypeKind};
use common::schema::{Column, Schema};
use expr::{Datum, Expr};

pub trait Eval {
  fn bind(&mut self, schema: &Schema) -> Void;

  fn datatype(&self) -> &DataType;

  // fn eval(&self, RowBlock) -> TResult<&Vector1>;
  fn is_const(&self) -> bool;
}

// Unary Expressions
struct Not {child: Box<Eval>}
struct IsNull {child: Box<Eval>}

// Binary Comparison Expressions
struct And {lhs: Box<Eval>, rhs: Box<Eval>}
struct Or {lhs: Box<Eval>, rhs: Box<Eval>}
struct Equal {lhs: Box<Eval>, rhs: Box<Eval>}
struct NotEqual {lhs: Box<Eval>, rhs: Box<Eval>}
struct LessThan {lhs: Box<Eval>, rhs: Box<Eval>}
struct LessThanOrEqual {lhs: Box<Eval>, rhs: Box<Eval>}
struct GreaterThan {lhs: Box<Eval>, rhs: Box<Eval>}
struct GreaterThanOrEqual {lhs: Box<Eval>, rhs: Box<Eval>}

// Binary Arithmetic Evalessions
struct Plus {lhs: Box<Eval>, rhs: Box<Eval>}
struct Minus {lhs: Box<Eval>, rhs: Box<Eval>}
struct Multiply {lhs: Box<Eval>, rhs: Box<Eval>}
struct Divide {lhs: Box<Eval>, rhs: Box<Eval>}
struct Modular {lhs: Box<Eval>, rhs: Box<Eval>}

// String operators or pattern matching predicates
struct Like {pattern: String, child: Box<Eval>}
struct SimilarTo {pattern: String, child: Box<Eval>}
struct RegexMatch {pattern: String, child: Box<Eval>}
struct Concatenate {lhs: Box<Eval>, rhs: Box<Eval>}

struct Between {lhs: Box<Eval>, mid: Box<Eval>, rhs: Box<Eval>}

struct Field {column: Column, field_id: Option<usize>}
struct Const {datum: Datum, res_type: DataType}

impl Eval for Plus {
  fn bind(&mut self, schema: &Schema) -> Void {
    //self.field_id = schema.get_

    void_ok()
  }

  fn datatype(&self) -> &DataType {
    self.lhs.datatype()
  }

  fn is_const(&self) -> bool { false }
}

impl Field {
  fn new(column: Column) -> Field {
    Field {
      column: column,
      field_id: None
    }
  }
}

impl Eval for Field {
  fn bind(&mut self, schema: &Schema) -> Void {
    //self.field_id = schema.get_

    void_ok()
  }

  fn datatype(&self) -> &DataType {
    &self.column.data_type
  }

  fn is_const(&self) -> bool { false }

  // fn eval(&self, RowBlock) -> TResult<&Vector1>;
}

impl Const {
  fn new(datum: &Datum) -> Const {
    Const {
        datum: datum.clone(),
        res_type: DataType::new(datum.type_kind())
    }
  }
}

impl Eval for Const {

  fn bind(&mut self, schema: &Schema) -> Void {
    self.res_type = DataType::new(self.datum.type_kind());
    void_ok()
  }

  fn datatype(&self) -> &DataType {
    &self.res_type
  }

  fn is_const(&self) -> bool { true }
}

// pub fn walk_expr<'a>(expr: &'a Expr) {
//   match *expr {
//     Expr::Plus (ref lhs, ref rhs) => {
//       walk_expr(&**lhs);
//       walk_expr(&**rhs);
//     },
//     _ => {}
//   };
// }

pub fn compile<'a>(expr: &'a Expr) -> TResult<Box<Eval>> {
  match *expr {

    // Expr::Between {lhs, mid, rhs} => {
    //   Ok(Box::new(
    //     Between {
    //       lhs: try!(compile(lhs)),
    //       mid: try!(compile(mid)),
    //       rhs: try!(compile(rhs))
    //     })
    //   )
    // },

    // Expr::Plus (ref lhs, ref rhs) => {
    //   Ok(Box::new(
    //     Plus {
    //       lhs: try!(compile(&**lhs)),
    //       rhs: try!(compile(&**rhs)),
    //     }
    //   ))
    // },

    // Expr::Field {column} => Ok(Box::new(Field::new(column))),

    // Expr::Const {datum} => Ok(Box::new(Const::new(&datum))),

    _ => Err(Error::InvalidExpression)
  }
}

// pub trait ExprVisitor<'v> {
//   type Value;

//   fn visit(&mut self, )
// }

<commit_msg>Define common/map/filter eval traits.<commit_after>//!
//! Expression Evaluator Trait and Implementation
//!

use common::err::{Error, TResult, Void, void_ok};
use rows::RowBlock;
use rows::vector::Vector1;
use common::types::{DataType, TypeKind, HasTypeKind};
use common::schema::{Column, Schema};
use expr::{Datum, Expr};

/// Common Trait of All Expression Evaluators
pub trait Eval {
  fn bind(&mut self, schema: &Schema) -> Void;

  fn ty(&self) -> &DataType;

  fn is_const(&self) -> bool;
}

/// Map Expression Evaluator Trait
pub trait MapEval {
  fn eval(&self, RowBlock) -> TResult<&Vector1>;
}

/// Filter Expression Evaluator Trait
pub trait FilterEval {
  fn filter(&self, RowBlock) -> Void;
}

// Unary Expressions
pub struct Not {child: Box<Eval>}
pub struct IsNull {child: Box<Eval>}

// Binary Comparison Expressions
pub struct And {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct Or {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct Equal {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct NotEqual {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct LessThan {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct LessThanOrEqual {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct GreaterThan {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct GreaterThanOrEqual {lhs: Box<Eval>, rhs: Box<Eval>}

// Binary Arithmetic Evalessions
pub struct Plus {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct Minus {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct Multiply {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct Divide {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct Modular {lhs: Box<Eval>, rhs: Box<Eval>}

// String operators or pattern matching predicates
pub struct Concatenate {lhs: Box<Eval>, rhs: Box<Eval>}
pub struct Like {pattern: String, child: Box<Eval>}
pub struct SimilarTo {pattern: String, child: Box<Eval>}
pub struct RegexMatch {pattern: String, child: Box<Eval>}

pub struct Between {pred: Box<Eval>, begin: Box<Eval>, end: Box<Eval>}
pub struct In {pred: Box<Eval>, row: Box<Row>}

pub struct Row {values: Vec<Box<Eval>>}
pub struct Field {column: Column, field_id: Option<usize>}
pub struct Const {datum: Datum, res_type: DataType}

impl Eval for Plus {
  fn bind(&mut self, schema: &Schema) -> Void {
    //self.field_id = schema.get_

    void_ok()
  }

  fn ty(&self) -> &DataType {
    self.lhs.ty()
  }

  fn is_const(&self) -> bool { false }
}

impl Field {
  fn new(column: Column) -> Field {
    Field {
      column: column,
      field_id: None
    }
  }
}

impl Eval for Field {
  fn bind(&mut self, schema: &Schema) -> Void {
    //self.field_id = schema.get_

    void_ok()
  }

  fn ty(&self) -> &DataType {
    &self.column.data_type
  }

  fn is_const(&self) -> bool { false }

  // fn eval(&self, RowBlock) -> TResult<&Vector1>;
}

impl Const {
  fn new(datum: &Datum) -> Const {
    Const {
        datum: datum.clone(),
        res_type: DataType::new(datum.type_kind())
    }
  }
}

impl Eval for Const {

  fn bind(&mut self, schema: &Schema) -> Void {
    self.res_type = DataType::new(self.datum.type_kind());
    void_ok()
  }

  fn ty(&self) -> &DataType {
    &self.res_type
  }

  fn is_const(&self) -> bool { true }
}

// pub fn walk_expr<'a>(expr: &'a Expr) {
//   match *expr {
//     Expr::Plus (ref lhs, ref rhs) => {
//       walk_expr(&**lhs);
//       walk_expr(&**rhs);
//     },
//     _ => {}
//   };
// }

pub fn compile<'a>(expr: &'a Expr) -> TResult<Box<Eval>> {
  match *expr {

    // Expr::Between {lhs, mid, rhs} => {
    //   Ok(Box::new(
    //     Between {
    //       lhs: try!(compile(lhs)),
    //       mid: try!(compile(mid)),
    //       rhs: try!(compile(rhs))
    //     })
    //   )
    // },

    // Expr::Plus (ref lhs, ref rhs) => {
    //   Ok(Box::new(
    //     Plus {
    //       lhs: try!(compile(&**lhs)),
    //       rhs: try!(compile(&**rhs)),
    //     }
    //   ))
    // },

    // Expr::Field {column} => Ok(Box::new(Field::new(column))),

    // Expr::Const {datum} => Ok(Box::new(Const::new(&datum))),

    _ => Err(Error::InvalidExpression)
  }
}

pub fn map_plus_vec_vec<RES, L, R>(lhs: Vec<L>, rhs: Vec<R>) {
}<|endoftext|>