<commit_before>use nom::{self, IResult};
use path;
use std::str;

#[derive(Debug)]
pub enum Expr<'a> {
    NumLit(&'a str),
    StrLit(&'a str),
    Var(&'a str),
    Attr(Box<Expr<'a>>, &'a str),
    Filter(&'a str, Vec<Expr<'a>>),
    BinOp(&'a str, Box<Expr<'a>>, Box<Expr<'a>>),
    Group(Box<Expr<'a>>),
    Call(Box<Expr<'a>>, &'a str, Vec<Expr<'a>>),
}

#[derive(Debug)]
pub enum Target<'a> {
    Name(&'a str),
}

#[derive(Clone, Copy, Debug)]
pub struct WS(pub bool, pub bool);

#[derive(Debug)]
pub enum Node<'a> {
    Lit(&'a str, &'a str, &'a str),
    Comment(),
    Expr(WS, Expr<'a>),
    Cond(Vec<(WS, Option<Expr<'a>>, Vec<Node<'a>>)>, WS),
    Loop(WS, Target<'a>, Expr<'a>, Vec<Node<'a>>, WS),
    Extends(Expr<'a>),
    BlockDef(WS, &'a str, Vec<Node<'a>>, WS),
    Block(WS, &'a str, WS),
    Include(WS, String),
}

pub type Cond<'a> = (WS, Option<Expr<'a>>, Vec<Node<'a>>);

fn split_ws_parts(s: &[u8]) -> Node {
    if s.is_empty() {
        let rs = str::from_utf8(s).unwrap();
        return Node::Lit(rs, rs, rs);
    }
    let is_ws = |c: &u8| {
        *c != b' ' && *c != b'\t' && *c != b'\r' && *c != b'\n'
    };
    let start = s.iter().position(&is_ws);
    let res = if start.is_none() {
            (s, &s[0..0], &s[0..0])
        } else {
            let start = start.unwrap();
            let end = s.iter().rposition(&is_ws);
            if end.is_none() {
                (&s[..start], &s[start..], &s[0..0])
            } else {
                let end = end.unwrap();
                (&s[..start], &s[start..end + 1], &s[end + 1..])
            }
        };
    Node::Lit(str::from_utf8(res.0).unwrap(),
              str::from_utf8(res.1).unwrap(),
              str::from_utf8(res.2).unwrap())
}

enum ContentState {
    Any,
    Brace(usize),
    End(usize),
}

fn take_content(i: &[u8]) -> IResult<&[u8], Node> {
    use parser::ContentState::*;
    let mut state = Any;
    for (idx, c) in i.iter().enumerate() {
        state = match (state, *c) {
            (Any, b'{') => Brace(idx),
            (Any, _) => Any,
            (Brace(start), b'{') |
            (Brace(start), b'%') |
            (Brace(start), b'#') => End(start),
            (Brace(_), _) => Any,
            (End(_), _) => panic!("cannot happen"),
        };
        if let End(_) = state {
            break;
        }
    }
    match state {
        Any |
        Brace(_) => IResult::Done(&i[..0], split_ws_parts(i)),
        End(0) => IResult::Error(nom::ErrorKind::Custom(0)),
        End(start) => IResult::Done(&i[start..], split_ws_parts(&i[..start])),
    }
}

fn identifier(input: &[u8]) -> IResult<&[u8], &str> {
    if !nom::is_alphabetic(input[0]) && input[0] != b'_' {
        return IResult::Error(nom::ErrorKind::Custom(0));
    }
    for (i, ch) in input.iter().enumerate() {
        if i == 0 || nom::is_alphanumeric(*ch) || *ch == b'_' {
            continue;
        }
        return IResult::Done(&input[i..],
                             str::from_utf8(&input[..i]).unwrap());
    }
    IResult::Done(&input[1..], str::from_utf8(&input[..1]).unwrap())
}

named!(expr_num_lit<Expr>, map!(nom::digit,
    |s| Expr::NumLit(str::from_utf8(s).unwrap())
));

named!(expr_str_lit<Expr>, map!(
    delimited!(char!('"'), is_not!("\""), char!('"')),
    |s| Expr::StrLit(str::from_utf8(s).unwrap())
));

named!(expr_var<Expr>, map!(identifier,
    |s| Expr::Var(s))
);

named!(target_single<Target>, map!(identifier,
    |s| Target::Name(s)
));

named!(arguments<Option<Vec<Expr>>>, opt!(
    do_parse!(
        tag_s!("(") >>
        arg0: ws!(opt!(expr_any)) >>
        args: many0!(do_parse!(
            tag_s!(",") >>
            argn: ws!(expr_any) >>
            (argn)
        )) >>
        tag_s!(")") >>
        ({
            let mut res = Vec::new();
            if arg0.is_some() {
                res.push(arg0.unwrap());
            }
            res.extend(args);
            res
        })
    )
));

named!(expr_group<Expr>, map!(
    delimited!(char!('('), expr_any, char!(')')),
    |s| Expr::Group(Box::new(s))
));

named!(expr_single<Expr>, alt!(
    expr_num_lit |
    expr_str_lit |
    expr_var |
    expr_group
));

named!(attr<(&str, Option<Vec<Expr>>)>, do_parse!(
    tag_s!(".") >>
    attr: identifier >>
    args: arguments >>
    (attr, args)
));

named!(expr_attr<Expr>, do_parse!(
    obj: expr_single >>
    attrs: many0!(attr) >>
    ({
        let mut res = obj;
        for (aname, args) in attrs {
            res = if args.is_some() {
                Expr::Call(Box::new(res), aname, args.unwrap())
            } else {
                Expr::Attr(Box::new(res), aname)
            };
        }
        res
    })
));

named!(filter<(&str, Option<Vec<Expr>>)>, do_parse!(
    tag_s!("|") >>
    fname: identifier >>
    args: arguments >>
    (fname, args)
));

named!(expr_filtered<Expr>, do_parse!(
    obj: expr_attr >>
    filters: many0!(filter) >>
    ({
       let mut res = obj;
       for (fname, args) in filters {
           res = Expr::Filter(fname, {
               let mut args = match args {
                   Some(inner) => inner,
                   None => Vec::new(),
               };
               args.insert(0, res);
               args
           });
       }
       res
    })
));

macro_rules! expr_prec_layer {
    ( $name:ident, $inner:ident, $( $op:expr ),* ) => {
        named!($name<Expr>, alt!(
            do_parse!(
                left: $inner >>
                op: ws!(alt!($( tag_s!($op) )|*)) >>
                right: $inner >>
                (Expr::BinOp(str::from_utf8(op).unwrap(),
                             Box::new(left), Box::new(right)))
            ) | $inner
        ));
    }
}

expr_prec_layer!(expr_muldivmod, expr_filtered, "*", "/", "%");
expr_prec_layer!(expr_addsub, expr_muldivmod, "+", "-");
expr_prec_layer!(expr_shifts, expr_addsub, ">>", "<<");
expr_prec_layer!(expr_band, expr_shifts, "&");
expr_prec_layer!(expr_bxor, expr_band, "^");
expr_prec_layer!(expr_bor, expr_bxor, "|");
expr_prec_layer!(expr_compare, expr_bor,
    "==", "!=", ">=", ">", "<=", "<"
);
expr_prec_layer!(expr_and, expr_compare, "&&");
expr_prec_layer!(expr_any, expr_and, "||");

named!(expr_node<Node>, do_parse!(
    tag_s!("{{") >>
    pws: opt!(tag_s!("-")) >>
    expr: ws!(expr_any) >>
    nws: opt!(tag_s!("-")) >>
    tag_s!("}}") >>
    (Node::Expr(WS(pws.is_some(), nws.is_some()), expr))
));

named!(cond_if<Expr>, do_parse!(
    ws!(tag_s!("if")) >>
    cond: ws!(expr_any) >>
    (cond)
));

named!(cond_block<Cond>, do_parse!(
    tag_s!("{%") >>
    pws: opt!(tag_s!("-")) >>
    ws!(tag_s!("else")) >>
    cond: opt!(cond_if) >>
    nws: opt!(tag_s!("-")) >>
    tag_s!("%}") >>
    block: parse_template >>
    (WS(pws.is_some(), nws.is_some()), cond, block)
));

named!(block_if<Node>, do_parse!(
    pws1: opt!(tag_s!("-")) >>
    cond: ws!(cond_if) >>
    nws1: opt!(tag_s!("-")) >>
    tag_s!("%}") >>
    block: parse_template >>
    elifs: many0!(cond_block) >>
    tag_s!("{%") >>
    pws2: opt!(tag_s!("-")) >>
    ws!(tag_s!("endif")) >>
    nws2: opt!(tag_s!("-")) >>
    ({
       let mut res = Vec::new();
       res.push((WS(pws1.is_some(), nws1.is_some()), Some(cond), block));
       res.extend(elifs);
       Node::Cond(res, WS(pws2.is_some(), nws2.is_some()))
    })
));

named!(block_for<Node>, do_parse!(
    pws1: opt!(tag_s!("-")) >>
    ws!(tag_s!("for")) >>
    var: ws!(target_single) >>
    ws!(tag_s!("in")) >>
    iter: ws!(expr_any) >>
    nws1: opt!(tag_s!("-")) >>
    tag_s!("%}") >>
    block: parse_template >>
    tag_s!("{%") >>
    pws2: opt!(tag_s!("-")) >>
    ws!(tag_s!("endfor")) >>
    nws2: opt!(tag_s!("-")) >>
    (Node::Loop(WS(pws1.is_some(), nws1.is_some()),
                var, iter, block,
                WS(pws2.is_some(), pws2.is_some())))
));

named!(block_extends<Node>, do_parse!(
    ws!(tag_s!("extends")) >>
    name: ws!(expr_str_lit) >>
    (Node::Extends(name))
));

named!(block_block<Node>, do_parse!(
    pws1: opt!(tag_s!("-")) >>
    ws!(tag_s!("block")) >>
    name: ws!(identifier) >>
    nws1: opt!(tag_s!("-")) >>
    tag_s!("%}") >>
    contents: parse_template >>
    tag_s!("{%") >>
    pws2: opt!(tag_s!("-")) >>
    ws!(tag_s!("endblock")) >>
    nws2: opt!(tag_s!("-")) >>
    (Node::BlockDef(WS(pws1.is_some(), nws1.is_some()),
                    name, contents,
                    WS(pws2.is_some(), pws2.is_some())))
));

named!(block_include<Node>, do_parse!(
    pws: opt!(tag_s!("-")) >>
    ws!(tag_s!("include")) >>
    name: ws!(expr_str_lit) >>
    nws: opt!(tag_s!("-")) >>
    ({
        let mut src = match name {
            Expr::StrLit(s) => path::get_template_source(s),
            _ => panic!("include path must be a string literal"),
        };
        if src.ends_with('\n') {
            let _ = src.pop();
        }
        Node::Include(WS(pws.is_some(), nws.is_some()), src)
    })
));

named!(block_node<Node>, do_parse!(
    tag_s!("{%") >>
    contents: alt!(
        block_if |
        block_for |
        block_extends |
        block_include |
        block_block
    ) >>
    tag_s!("%}") >>
    (contents)
));

named!(block_comment<Node>, do_parse!(
    tag_s!("{#") >>
    take_until_s!("#}") >>
    tag_s!("#}") >>
    (Node::Comment())
));

named!(parse_template<Vec<Node<'a>>>, many0!(alt!(
    take_content |
    block_comment |
    expr_node |
    block_node
)));

pub fn parse(src: &str) -> Vec<Node> {
    match parse_template(src.as_bytes()) {
        IResult::Done(left, res) => {
            if left.len() > 0 {
                let s = str::from_utf8(left).unwrap();
                panic!("unable to parse template:\n\n{:?}", s);
            } else {
                res
            }
        },
        IResult::Error(err) => panic!("problems parsing template source: {}", err),
        IResult::Incomplete(_) => panic!("parsing incomplete"),
    }
}

#[cfg(test)]
mod tests {
    fn check_ws_split(s: &str, res: &(&str, &str, &str)) {
        let node = super::split_ws_parts(s.as_bytes());
        match node {
            super::Node::Lit(lws, s, rws) => {
                assert_eq!(lws, res.0);
                assert_eq!(s, res.1);
                assert_eq!(rws, res.2);
            },
            _ => { panic!("fail"); },
        }
    }
    #[test]
    fn test_ws_splitter() {
        check_ws_split("", &("", "", ""));
        check_ws_split("a", &("", "a", ""));
        check_ws_split("\ta", &("\t", "a", ""));
        check_ws_split("b\n", &("", "b", "\n"));
        check_ws_split(" \t\r\n", &(" \t\r\n", "", ""));
    }
}
<commit_msg>Add test for panic on invalid blocks (see #27)<commit_after>use nom::{self, IResult};
use path;
use std::str;

#[derive(Debug)]
pub enum Expr<'a> {
    NumLit(&'a str),
    StrLit(&'a str),
    Var(&'a str),
    Attr(Box<Expr<'a>>, &'a str),
    Filter(&'a str, Vec<Expr<'a>>),
    BinOp(&'a str, Box<Expr<'a>>, Box<Expr<'a>>),
    Group(Box<Expr<'a>>),
    Call(Box<Expr<'a>>, &'a str, Vec<Expr<'a>>),
}

#[derive(Debug)]
pub enum Target<'a> {
    Name(&'a str),
}

#[derive(Clone, Copy, Debug)]
pub struct WS(pub bool, pub bool);

#[derive(Debug)]
pub enum Node<'a> {
    Lit(&'a str, &'a str, &'a str),
    Comment(),
    Expr(WS, Expr<'a>),
    Cond(Vec<(WS, Option<Expr<'a>>, Vec<Node<'a>>)>, WS),
    Loop(WS, Target<'a>, Expr<'a>, Vec<Node<'a>>, WS),
    Extends(Expr<'a>),
    BlockDef(WS, &'a str, Vec<Node<'a>>, WS),
    Block(WS, &'a str, WS),
    Include(WS, String),
}

pub type Cond<'a> = (WS, Option<Expr<'a>>, Vec<Node<'a>>);

fn split_ws_parts(s: &[u8]) -> Node {
    if s.is_empty() {
        let rs = str::from_utf8(s).unwrap();
        return Node::Lit(rs, rs, rs);
    }
    let is_ws = |c: &u8| {
        *c != b' ' && *c != b'\t' && *c != b'\r' && *c != b'\n'
    };
    let start = s.iter().position(&is_ws);
    let res = if start.is_none() {
            (s, &s[0..0], &s[0..0])
        } else {
            let start = start.unwrap();
            let end = s.iter().rposition(&is_ws);
            if end.is_none() {
                (&s[..start], &s[start..], &s[0..0])
            } else {
                let end = end.unwrap();
                (&s[..start], &s[start..end + 1], &s[end + 1..])
            }
        };
    Node::Lit(str::from_utf8(res.0).unwrap(),
              str::from_utf8(res.1).unwrap(),
              str::from_utf8(res.2).unwrap())
}

enum ContentState {
    Any,
    Brace(usize),
    End(usize),
}

fn take_content(i: &[u8]) -> IResult<&[u8], Node> {
    use parser::ContentState::*;
    let mut state = Any;
    for (idx, c) in i.iter().enumerate() {
        state = match (state, *c) {
            (Any, b'{') => Brace(idx),
            (Any, _) => Any,
            (Brace(start), b'{') |
            (Brace(start), b'%') |
            (Brace(start), b'#') => End(start),
            (Brace(_), _) => Any,
            (End(_), _) => panic!("cannot happen"),
        };
        if let End(_) = state {
            break;
        }
    }
    match state {
        Any |
        Brace(_) => IResult::Done(&i[..0], split_ws_parts(i)),
        End(0) => IResult::Error(nom::ErrorKind::Custom(0)),
        End(start) => IResult::Done(&i[start..], split_ws_parts(&i[..start])),
    }
}

fn identifier(input: &[u8]) -> IResult<&[u8], &str> {
    if !nom::is_alphabetic(input[0]) && input[0] != b'_' {
        return IResult::Error(nom::ErrorKind::Custom(0));
    }
    for (i, ch) in input.iter().enumerate() {
        if i == 0 || nom::is_alphanumeric(*ch) || *ch == b'_' {
            continue;
        }
        return IResult::Done(&input[i..],
                             str::from_utf8(&input[..i]).unwrap());
    }
    IResult::Done(&input[1..], str::from_utf8(&input[..1]).unwrap())
}

named!(expr_num_lit<Expr>, map!(nom::digit,
    |s| Expr::NumLit(str::from_utf8(s).unwrap())
));

named!(expr_str_lit<Expr>, map!(
    delimited!(char!('"'), is_not!("\""), char!('"')),
    |s| Expr::StrLit(str::from_utf8(s).unwrap())
));

named!(expr_var<Expr>, map!(identifier,
    |s| Expr::Var(s))
);

named!(target_single<Target>, map!(identifier,
    |s| Target::Name(s)
));

named!(arguments<Option<Vec<Expr>>>, opt!(
    do_parse!(
        tag_s!("(") >>
        arg0: ws!(opt!(expr_any)) >>
        args: many0!(do_parse!(
            tag_s!(",") >>
            argn: ws!(expr_any) >>
            (argn)
        )) >>
        tag_s!(")") >>
        ({
            let mut res = Vec::new();
            if arg0.is_some() {
                res.push(arg0.unwrap());
            }
            res.extend(args);
            res
        })
    )
));

named!(expr_group<Expr>, map!(
    delimited!(char!('('), expr_any, char!(')')),
    |s| Expr::Group(Box::new(s))
));

named!(expr_single<Expr>, alt!(
    expr_num_lit |
    expr_str_lit |
    expr_var |
    expr_group
));

named!(attr<(&str, Option<Vec<Expr>>)>, do_parse!(
    tag_s!(".") >>
    attr: identifier >>
    args: arguments >>
    (attr, args)
));

named!(expr_attr<Expr>, do_parse!(
    obj: expr_single >>
    attrs: many0!(attr) >>
    ({
        let mut res = obj;
        for (aname, args) in attrs {
            res = if args.is_some() {
                Expr::Call(Box::new(res), aname, args.unwrap())
            } else {
                Expr::Attr(Box::new(res), aname)
            };
        }
        res
    })
));

named!(filter<(&str, Option<Vec<Expr>>)>, do_parse!(
    tag_s!("|") >>
    fname: identifier >>
    args: arguments >>
    (fname, args)
));

named!(expr_filtered<Expr>, do_parse!(
    obj: expr_attr >>
    filters: many0!(filter) >>
    ({
       let mut res = obj;
       for (fname, args) in filters {
           res = Expr::Filter(fname, {
               let mut args = match args {
                   Some(inner) => inner,
                   None => Vec::new(),
               };
               args.insert(0, res);
               args
           });
       }
       res
    })
));

macro_rules! expr_prec_layer {
    ( $name:ident, $inner:ident, $( $op:expr ),* ) => {
        named!($name<Expr>, alt!(
            do_parse!(
                left: $inner >>
                op: ws!(alt!($( tag_s!($op) )|*)) >>
                right: $inner >>
                (Expr::BinOp(str::from_utf8(op).unwrap(),
                             Box::new(left), Box::new(right)))
            ) | $inner
        ));
    }
}

expr_prec_layer!(expr_muldivmod, expr_filtered, "*", "/", "%");
expr_prec_layer!(expr_addsub, expr_muldivmod, "+", "-");
expr_prec_layer!(expr_shifts, expr_addsub, ">>", "<<");
expr_prec_layer!(expr_band, expr_shifts, "&");
expr_prec_layer!(expr_bxor, expr_band, "^");
expr_prec_layer!(expr_bor, expr_bxor, "|");
expr_prec_layer!(expr_compare, expr_bor,
    "==", "!=", ">=", ">", "<=", "<"
);
expr_prec_layer!(expr_and, expr_compare, "&&");
expr_prec_layer!(expr_any, expr_and, "||");

named!(expr_node<Node>, do_parse!(
    tag_s!("{{") >>
    pws: opt!(tag_s!("-")) >>
    expr: ws!(expr_any) >>
    nws: opt!(tag_s!("-")) >>
    tag_s!("}}") >>
    (Node::Expr(WS(pws.is_some(), nws.is_some()), expr))
));

named!(cond_if<Expr>, do_parse!(
    ws!(tag_s!("if")) >>
    cond: ws!(expr_any) >>
    (cond)
));

named!(cond_block<Cond>, do_parse!(
    tag_s!("{%") >>
    pws: opt!(tag_s!("-")) >>
    ws!(tag_s!("else")) >>
    cond: opt!(cond_if) >>
    nws: opt!(tag_s!("-")) >>
    tag_s!("%}") >>
    block: parse_template >>
    (WS(pws.is_some(), nws.is_some()), cond, block)
));

named!(block_if<Node>, do_parse!(
    pws1: opt!(tag_s!("-")) >>
    cond: ws!(cond_if) >>
    nws1: opt!(tag_s!("-")) >>
    tag_s!("%}") >>
    block: parse_template >>
    elifs: many0!(cond_block) >>
    tag_s!("{%") >>
    pws2: opt!(tag_s!("-")) >>
    ws!(tag_s!("endif")) >>
    nws2: opt!(tag_s!("-")) >>
    ({
       let mut res = Vec::new();
       res.push((WS(pws1.is_some(), nws1.is_some()), Some(cond), block));
       res.extend(elifs);
       Node::Cond(res, WS(pws2.is_some(), nws2.is_some()))
    })
));

named!(block_for<Node>, do_parse!(
    pws1: opt!(tag_s!("-")) >>
    ws!(tag_s!("for")) >>
    var: ws!(target_single) >>
    ws!(tag_s!("in")) >>
    iter: ws!(expr_any) >>
    nws1: opt!(tag_s!("-")) >>
    tag_s!("%}") >>
    block: parse_template >>
    tag_s!("{%") >>
    pws2: opt!(tag_s!("-")) >>
    ws!(tag_s!("endfor")) >>
    nws2: opt!(tag_s!("-")) >>
    (Node::Loop(WS(pws1.is_some(), nws1.is_some()),
                var, iter, block,
                WS(pws2.is_some(), pws2.is_some())))
));

named!(block_extends<Node>, do_parse!(
    ws!(tag_s!("extends")) >>
    name: ws!(expr_str_lit) >>
    (Node::Extends(name))
));

named!(block_block<Node>, do_parse!(
    pws1: opt!(tag_s!("-")) >>
    ws!(tag_s!("block")) >>
    name: ws!(identifier) >>
    nws1: opt!(tag_s!("-")) >>
    tag_s!("%}") >>
    contents: parse_template >>
    tag_s!("{%") >>
    pws2: opt!(tag_s!("-")) >>
    ws!(tag_s!("endblock")) >>
    nws2: opt!(tag_s!("-")) >>
    (Node::BlockDef(WS(pws1.is_some(), nws1.is_some()),
                    name, contents,
                    WS(pws2.is_some(), pws2.is_some())))
));

named!(block_include<Node>, do_parse!(
    pws: opt!(tag_s!("-")) >>
    ws!(tag_s!("include")) >>
    name: ws!(expr_str_lit) >>
    nws: opt!(tag_s!("-")) >>
    ({
        let mut src = match name {
            Expr::StrLit(s) => path::get_template_source(s),
            _ => panic!("include path must be a string literal"),
        };
        if src.ends_with('\n') {
            let _ = src.pop();
        }
        Node::Include(WS(pws.is_some(), nws.is_some()), src)
    })
));

named!(block_node<Node>, do_parse!(
    tag_s!("{%") >>
    contents: alt!(
        block_if |
        block_for |
        block_extends |
        block_include |
        block_block
    ) >>
    tag_s!("%}") >>
    (contents)
));

named!(block_comment<Node>, do_parse!(
    tag_s!("{#") >>
    take_until_s!("#}") >>
    tag_s!("#}") >>
    (Node::Comment())
));

named!(parse_template<Vec<Node<'a>>>, many0!(alt!(
    take_content |
    block_comment |
    expr_node |
    block_node
)));

pub fn parse(src: &str) -> Vec<Node> {
    match parse_template(src.as_bytes()) {
        IResult::Done(left, res) => {
            if left.len() > 0 {
                let s = str::from_utf8(left).unwrap();
                panic!("unable to parse template:\n\n{:?}", s);
            } else {
                res
            }
        },
        IResult::Error(err) => panic!("problems parsing template source: {}", err),
        IResult::Incomplete(_) => panic!("parsing incomplete"),
    }
}

#[cfg(test)]
mod tests {
    fn check_ws_split(s: &str, res: &(&str, &str, &str)) {
        let node = super::split_ws_parts(s.as_bytes());
        match node {
            super::Node::Lit(lws, s, rws) => {
                assert_eq!(lws, res.0);
                assert_eq!(s, res.1);
                assert_eq!(rws, res.2);
            },
            _ => { panic!("fail"); },
        }
    }
    #[test]
    fn test_ws_splitter() {
        check_ws_split("", &("", "", ""));
        check_ws_split("a", &("", "a", ""));
        check_ws_split("\ta", &("\t", "a", ""));
        check_ws_split("b\n", &("", "b", "\n"));
        check_ws_split(" \t\r\n", &(" \t\r\n", "", ""));
    }
    #[test]
    #[should_panic]
    fn test_invalid_block() {
        super::parse("{% extend \"blah\" %}");
    }
}
<|endoftext|>


<commit_before>//! General purpose state transition operator.
use std::hash::Hash;
use std::collections::HashMap;

use ::Data;
use dataflow::{Stream, Scope};
use dataflow::operators::unary::Unary;
use dataflow::channels::pact::Exchange;

/// Generic state-transition machinery: each key has a state, and receives a sequence of events.
/// Events are applied in time-order, but no other promises are made. Each state transition can
/// produce output, which is sent.
///
/// state_machine will buffer inputs if earlier inputs may still arrive. it will directly apply
/// updates for the current time reflected in the notificator, though. In the case of partially
/// ordered times, the only guarantee is that updates are not applied out of order, not that there
/// is some total order on times respecting the total order (updates may be interleaved).

/// Provides the `state_machine` method.
pub trait StateMachine<S: Scope, K: Data+Hash+Eq, V: Data> {
	/// Tracks a state for each presented key, using user-supplied state transition logic.
	///
	/// The transition logic `fold` may mutate the state, and produce both output records and
	/// a `bool` indicating that it is appropriate to deregister the state, cleaning up once
	/// the state is no longer helpful.
	fn state_machine<
		R: Data, 									// output type
		D: Default+'static, 						// per-key state (data)
		I: IntoIterator<Item=R>,					// type of output iterator
		F: Fn(&K, V, &mut D)->(bool, I)+'static, 	// state update logic
		H: Fn(&K)->u64+'static,						// "hash" function for keys
	>(&self, fold: F, hash: H) -> Stream<S, R> where S::Timestamp : Hash+Eq ;
}

impl<S: Scope, K: Data+Hash+Eq, V: Data> StateMachine<S, K, V> for Stream<S, (K, V)> {
	fn state_machine<
			R: Data, 									// output type
			D: Default+'static, 						// per-key state (data)
			I: IntoIterator<Item=R>,					// type of output iterator
			F: Fn(&K, V, &mut D)->(bool, I)+'static, 	// state update logic
			H: Fn(&K)->u64+'static,						// "hash" function for keys
		>(&self, fold: F, hash: H) -> Stream<S, R> where S::Timestamp : Hash+Eq {

		let mut pending = HashMap::new(); 	// times -> (keys -> state)
		let mut states = HashMap::new();	// keys -> state

		self.unary_notify(Exchange::new(move |&(ref k, _)| hash(k)), "StateMachine", vec![], move |input, output, notificator| {

			// stash each input and request a notification when ready
			while let Some((time, data)) = input.next() {
				// stash if not time yet
				if notificator.frontier(0).iter().any(|x| x.lt(&time.time())) {
					pending.entry(time.time()).or_insert(Vec::new()).extend(data.drain(..));
					notificator.notify_at(time);
				}
				else {
					// else we can process immediately
					let mut session = output.session(&time);
					for (key, val) in data.drain(..) {
						let (remove, output) = {
							let state = states.entry(key.clone()).or_insert(Default::default());
							fold(&key, val, state)
						};
						if remove { states.remove(&key); }
						session.give_iterator(output.into_iter());
					}
				}
			}

			// go through each time with data, process each (key, val) pair.
			while let Some((time, _)) = notificator.next() {
				if let Some(pend) = pending.remove(&time.time()) {
					let mut session = output.session(&time);
					for (key, val) in pend {
						let (remove, output) = {
							let state = states.entry(key.clone()).or_insert(Default::default());
							fold(&key, val, state)
						};
						if remove { states.remove(&key); }
						session.give_iterator(output.into_iter());
					}
				}
			}

		})
	}
}

#[cfg(test)]
mod test {

	use dataflow::operators::{ToStream, Inspect, Map};
	use super::StateMachine;

    #[test]
    fn it_works() {

    	::example(|scope| {

    		let result = vec![(0,0), (0,2), (0,6), (0,12), (0,20), (1,1), (1,4), (1,9), (1,16), (1,25)];

    		(0..10).to_stream(scope)
    			   .map(|x| (x % 2, x))
    			   .state_machine(|_key, val, agg| { *agg += val; (false, Some((*_key, *agg))) }, |key| *key as u64)
    			   .inspect(move |x| assert!(result.contains(x)));

    	})

    }
}

<commit_msg>updated docs with examples<commit_after>//! General purpose state transition operator.
use std::hash::Hash;
use std::collections::HashMap;

use ::Data;
use dataflow::{Stream, Scope};
use dataflow::operators::unary::Unary;
use dataflow::channels::pact::Exchange;

/// Generic state-transition machinery: each key has a state, and receives a sequence of events.
/// Events are applied in time-order, but no other promises are made. Each state transition can
/// produce output, which is sent.
///
/// state_machine will buffer inputs if earlier inputs may still arrive. it will directly apply
/// updates for the current time reflected in the notificator, though. In the case of partially
/// ordered times, the only guarantee is that updates are not applied out of order, not that there
/// is some total order on times respecting the total order (updates may be interleaved).

/// Provides the `state_machine` method.
pub trait StateMachine<S: Scope, K: Data+Hash+Eq, V: Data> {
	/// Tracks a state for each presented key, using user-supplied state transition logic.
	///
	/// The transition logic `fold` may mutate the state, and produce both output records and
	/// a `bool` indicating that it is appropriate to deregister the state, cleaning up once
	/// the state is no longer helpful.
	///
    /// #Examples
    /// ```
    /// use timely::dataflow::operators::{ToStream, Map, Inspect};
    /// use timely::dataflow::operators::aggregation::StateMachine;
    ///
    ///	timely::example(|scope| {
	///
	///     // these results happen to be right, but aren't guaranteed.
	///     // the system is at liberty to re-order within a timestamp.
    ///     let result = vec![(0,0), (0,2), (0,6), (0,12), (0,20),
    ///                       (1,1), (1,4), (1,9), (1,16), (1,25)];
    ///
    ///		(0..10).to_stream(scope)
    ///			   .map(|x| (x % 2, x))
    ///			   .state_machine(
    ///                |_key, val, agg| { *agg += val; (false, Some((*_key, *agg))) },
    ///                |key| *key as u64
    ///            )
    ///            .inspect(move |x| assert!(result.contains(x)));
    /// });
    /// ```
	fn state_machine<
		R: Data, 									// output type
		D: Default+'static, 						// per-key state (data)
		I: IntoIterator<Item=R>,					// type of output iterator
		F: Fn(&K, V, &mut D)->(bool, I)+'static, 	// state update logic
		H: Fn(&K)->u64+'static,						// "hash" function for keys
	>(&self, fold: F, hash: H) -> Stream<S, R> where S::Timestamp : Hash+Eq ;
}

impl<S: Scope, K: Data+Hash+Eq, V: Data> StateMachine<S, K, V> for Stream<S, (K, V)> {
	fn state_machine<
			R: Data, 									// output type
			D: Default+'static, 						// per-key state (data)
			I: IntoIterator<Item=R>,					// type of output iterator
			F: Fn(&K, V, &mut D)->(bool, I)+'static, 	// state update logic
			H: Fn(&K)->u64+'static,						// "hash" function for keys
		>(&self, fold: F, hash: H) -> Stream<S, R> where S::Timestamp : Hash+Eq {

		let mut pending = HashMap::new(); 	// times -> (keys -> state)
		let mut states = HashMap::new();	// keys -> state

		self.unary_notify(Exchange::new(move |&(ref k, _)| hash(k)), "StateMachine", vec![], move |input, output, notificator| {

			// stash each input and request a notification when ready
			while let Some((time, data)) = input.next() {
				// stash if not time yet
				if notificator.frontier(0).iter().any(|x| x.lt(&time.time())) {
					pending.entry(time.time()).or_insert(Vec::new()).extend(data.drain(..));
					notificator.notify_at(time);
				}
				else {
					// else we can process immediately
					let mut session = output.session(&time);
					for (key, val) in data.drain(..) {
						let (remove, output) = {
							let state = states.entry(key.clone()).or_insert(Default::default());
							fold(&key, val, state)
						};
						if remove { states.remove(&key); }
						session.give_iterator(output.into_iter());
					}
				}
			}

			// go through each time with data, process each (key, val) pair.
			while let Some((time, _)) = notificator.next() {
				if let Some(pend) = pending.remove(&time.time()) {
					let mut session = output.session(&time);
					for (key, val) in pend {
						let (remove, output) = {
							let state = states.entry(key.clone()).or_insert(Default::default());
							fold(&key, val, state)
						};
						if remove { states.remove(&key); }
						session.give_iterator(output.into_iter());
					}
				}
			}

		})
	}
}<|endoftext|>


<commit_before>use crate::compiler::Code;
use crate::driver::cmd::AsmSyntax;
use crate::ty::SourceTypeArray;
use crate::vm::{Fct, VM};

pub fn supported() -> bool {
    false
}

pub fn disassemble(
    _vm: &VM,
    _fct: &Fct,
    _type_params: &SourceTypeArray,
    _code: &Code,
    _asm_syntax: AsmSyntax,
) {
    unreachable!();
}
<commit_msg>fix build without disassembler<commit_after>use crate::compiler::Code;
use crate::driver::cmd::AsmSyntax;
use crate::ty::SourceTypeArray;
use crate::vm::{FctDefinition, VM};

pub fn supported() -> bool {
    false
}

pub fn disassemble(
    _vm: &VM,
    _fct: &FctDefinition,
    _type_params: &SourceTypeArray,
    _code: &Code,
    _asm_syntax: AsmSyntax,
) {
    unreachable!();
}
<|endoftext|>


<commit_before>//! Parse a format description into a standardized representation.

use crate::format_description::{
    error::InvalidFormatDescription, helper, modifier, Component, Description,
};
use alloc::vec::Vec;

/// The item parsed and remaining chunk of the format description after one
/// iteration.
#[derive(Debug)]
struct ParsedItem<'a> {
    /// The item that was parsed.
    item: Description<'a>,
    /// What is left of the input string after the item was parsed.
    remaining: &'a str,
}

/// Parse a component from the format description. Neither the leading nor
/// trailing bracket should be present in the parameter.
fn parse_component<'a>(
    mut s: &'a str,
    index: &mut usize,
) -> Result<Component, InvalidFormatDescription<'a>> {
    // Trim any whitespace between the opening bracket and the component name.
    s = helper::consume_whitespace(s, index);

    // Everything before the first whitespace is the component name.
    let component_name;
    let component_index = *index;
    if let Some(whitespace_loc) = s.find(char::is_whitespace) {
        *index += whitespace_loc;
        component_name = &s[..whitespace_loc];
        s = &s[whitespace_loc..];
        // Trim any whitespace between the component name and the first
        // modifier.
        s = helper::consume_whitespace(s, index);
    } else {
        component_name = s;
        // There is no whitespace remaining, so the full input is the component
        // name.
        s = "";
    }

    match component_name {
        "day" | "hour" | "minute" | "month" | "offset_hour" | "offset_minute" | "offset_second"
        | "ordinal" | "period" | "second" | "subsecond" | "weekday" | "week_number" | "year" => {}
        name => {
            return Err(InvalidFormatDescription::InvalidComponentName {
                name,
                index: component_index,
            })
        }
    }

    let modifiers = modifier::ParsedModifiers::parse(component_name, s, index)?;
    Ok(match component_name {
        "day" => Component::Day {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "hour" => Component::Hour {
            padding: modifiers.padding.unwrap_or_default(),
            is_12_hour_clock: modifiers.hour_is_12_hour_clock.unwrap_or_default(),
        },
        "minute" => Component::Minute {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "month" => Component::Month {
            padding: modifiers.padding.unwrap_or_default(),
            repr: modifiers.month_repr.unwrap_or_default(),
        },
        "offset_hour" => Component::OffsetHour {
            sign_is_mandatory: modifiers.sign_is_mandatory.unwrap_or_default(),
            padding: modifiers.padding.unwrap_or_default(),
        },
        "offset_minute" => Component::OffsetMinute {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "offset_second" => Component::OffsetSecond {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "ordinal" => Component::Ordinal {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "period" => Component::Period {
            is_uppercase: modifiers.period_is_uppercase.unwrap_or(true),
        },
        "second" => Component::Second {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "subsecond" => Component::Subsecond {
            digits: modifiers.subsecond_digits.unwrap_or_default(),
        },
        "weekday" => Component::Weekday {
            repr: modifiers.weekday_repr.unwrap_or_default(),
            one_indexed: modifiers.weekday_is_one_indexed.unwrap_or(true),
        },
        "week_number" => Component::WeekNumber {
            padding: modifiers.padding.unwrap_or_default(),
            repr: modifiers.week_number_repr.unwrap_or_default(),
        },
        "year" => Component::Year {
            padding: modifiers.padding.unwrap_or_default(),
            repr: modifiers.year_repr.unwrap_or_default(),
            iso_week_based: modifiers.year_is_iso_week_based.unwrap_or_default(),
            sign_is_mandatory: modifiers.sign_is_mandatory.unwrap_or_default(),
        },
        _ => unreachable!(
            "All valid component names should be caught in the above `matches!` clause."
        ),
    })
}

/// Parse a literal string from the format description.
#[allow(clippy::option_if_let_else)] // I think this style is better here.
fn parse_literal<'a>(s: &'a str, index: &mut usize) -> ParsedItem<'a> {
    if let Some(loc) = s.find('[') {
        *index += loc;
        ParsedItem {
            item: Description::Literal(&s[..loc]),
            remaining: &s[loc..],
        }
    } else {
        *index += s.len();
        ParsedItem {
            item: Description::Literal(s),
            remaining: "",
        }
    }
}

/// Parse either a literal or a component from the format description.
#[allow(clippy::manual_strip)] // lint was not designed for this case
fn parse_item<'a>(
    s: &'a str,
    index: &mut usize,
) -> Result<ParsedItem<'a>, InvalidFormatDescription<'a>> {
    if s.starts_with("[[") {
        *index += 2;
        return Ok(ParsedItem {
            item: Description::Literal(&s[..1]),
            remaining: &s[2..],
        });
    }

    if s.starts_with('[') {
        if let Some(bracket_index) = s.find(']') {
            *index += 1;
            Ok(ParsedItem {
                item: Description::Component(parse_component(&s[1..bracket_index], index)?),
                remaining: &s[bracket_index + 1..],
            })
        } else {
            Err(InvalidFormatDescription::UnclosedOpeningBracket { index: *index })
        }
    } else {
        Ok(parse_literal(s, index))
    }
}

/// Parse a sequence of items from the format description.
#[cfg_attr(__time_formatting_01_docs, doc(cfg(feature = "alloc")))]
pub fn parse_format_description(
    mut s: &str,
) -> Result<Vec<Description<'_>>, InvalidFormatDescription<'_>> {
    let mut compound = Vec::new();
    let mut loc = 0;

    while !s.is_empty() {
        let ParsedItem { item, remaining } = parse_item(s, &mut loc)?;
        s = remaining;
        compound.push(item);
    }

    Ok(compound)
}
<commit_msg>Simplify parsing of literals<commit_after>//! Parse a format description into a standardized representation.

use crate::format_description::{
    error::InvalidFormatDescription, helper, modifier, Component, Description,
};
use alloc::vec::Vec;

/// The item parsed and remaining chunk of the format description after one
/// iteration.
#[derive(Debug)]
struct ParsedItem<'a> {
    /// The item that was parsed.
    item: Description<'a>,
    /// What is left of the input string after the item was parsed.
    remaining: &'a str,
}

/// Parse a component from the format description. Neither the leading nor
/// trailing bracket should be present in the parameter.
fn parse_component<'a>(
    mut s: &'a str,
    index: &mut usize,
) -> Result<Component, InvalidFormatDescription<'a>> {
    // Trim any whitespace between the opening bracket and the component name.
    s = helper::consume_whitespace(s, index);

    // Everything before the first whitespace is the component name.
    let component_name;
    let component_index = *index;
    if let Some(whitespace_loc) = s.find(char::is_whitespace) {
        *index += whitespace_loc;
        component_name = &s[..whitespace_loc];
        s = &s[whitespace_loc..];
        // Trim any whitespace between the component name and the first
        // modifier.
        s = helper::consume_whitespace(s, index);
    } else {
        component_name = s;
        // There is no whitespace remaining, so the full input is the component
        // name.
        s = "";
    }

    match component_name {
        "day" | "hour" | "minute" | "month" | "offset_hour" | "offset_minute" | "offset_second"
        | "ordinal" | "period" | "second" | "subsecond" | "weekday" | "week_number" | "year" => {}
        name => {
            return Err(InvalidFormatDescription::InvalidComponentName {
                name,
                index: component_index,
            })
        }
    }

    let modifiers = modifier::ParsedModifiers::parse(component_name, s, index)?;
    Ok(match component_name {
        "day" => Component::Day {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "hour" => Component::Hour {
            padding: modifiers.padding.unwrap_or_default(),
            is_12_hour_clock: modifiers.hour_is_12_hour_clock.unwrap_or_default(),
        },
        "minute" => Component::Minute {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "month" => Component::Month {
            padding: modifiers.padding.unwrap_or_default(),
            repr: modifiers.month_repr.unwrap_or_default(),
        },
        "offset_hour" => Component::OffsetHour {
            sign_is_mandatory: modifiers.sign_is_mandatory.unwrap_or_default(),
            padding: modifiers.padding.unwrap_or_default(),
        },
        "offset_minute" => Component::OffsetMinute {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "offset_second" => Component::OffsetSecond {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "ordinal" => Component::Ordinal {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "period" => Component::Period {
            is_uppercase: modifiers.period_is_uppercase.unwrap_or(true),
        },
        "second" => Component::Second {
            padding: modifiers.padding.unwrap_or_default(),
        },
        "subsecond" => Component::Subsecond {
            digits: modifiers.subsecond_digits.unwrap_or_default(),
        },
        "weekday" => Component::Weekday {
            repr: modifiers.weekday_repr.unwrap_or_default(),
            one_indexed: modifiers.weekday_is_one_indexed.unwrap_or(true),
        },
        "week_number" => Component::WeekNumber {
            padding: modifiers.padding.unwrap_or_default(),
            repr: modifiers.week_number_repr.unwrap_or_default(),
        },
        "year" => Component::Year {
            padding: modifiers.padding.unwrap_or_default(),
            repr: modifiers.year_repr.unwrap_or_default(),
            iso_week_based: modifiers.year_is_iso_week_based.unwrap_or_default(),
            sign_is_mandatory: modifiers.sign_is_mandatory.unwrap_or_default(),
        },
        _ => unreachable!(
            "All valid component names should be caught in the above `matches!` clause."
        ),
    })
}

/// Parse a literal string from the format description.
#[allow(clippy::option_if_let_else)] // I think this style is better here.
fn parse_literal<'a>(s: &'a str, index: &mut usize) -> ParsedItem<'a> {
    let loc = s.find('[').unwrap_or_else(|| s.len());
    *index += loc;
    ParsedItem {
        item: Description::Literal(&s[..loc]),
        remaining: &s[loc..],
    }
}

/// Parse either a literal or a component from the format description.
#[allow(clippy::manual_strip)] // lint was not designed for this case
fn parse_item<'a>(
    s: &'a str,
    index: &mut usize,
) -> Result<ParsedItem<'a>, InvalidFormatDescription<'a>> {
    if s.starts_with("[[") {
        *index += 2;
        return Ok(ParsedItem {
            item: Description::Literal(&s[..1]),
            remaining: &s[2..],
        });
    }

    if s.starts_with('[') {
        if let Some(bracket_index) = s.find(']') {
            *index += 1;
            Ok(ParsedItem {
                item: Description::Component(parse_component(&s[1..bracket_index], index)?),
                remaining: &s[bracket_index + 1..],
            })
        } else {
            Err(InvalidFormatDescription::UnclosedOpeningBracket { index: *index })
        }
    } else {
        Ok(parse_literal(s, index))
    }
}

/// Parse a sequence of items from the format description.
#[cfg_attr(__time_formatting_01_docs, doc(cfg(feature = "alloc")))]
pub fn parse_format_description(
    mut s: &str,
) -> Result<Vec<Description<'_>>, InvalidFormatDescription<'_>> {
    let mut compound = Vec::new();
    let mut loc = 0;

    while !s.is_empty() {
        let ParsedItem { item, remaining } = parse_item(s, &mut loc)?;
        s = remaining;
        compound.push(item);
    }

    Ok(compound)
}
<|endoftext|>


<commit_before>// Copyright (C) 2018 Sebastian Dröge <sebastian@centricular.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.

extern crate glib;
use glib::prelude::*;

extern crate gstreamer as gst;
extern crate gstreamer_check as gst_check;

extern crate gstthreadshare;

use gstthreadshare::{RTPJitterBuffer, RTPJitterBufferItem};

fn init() {
    use std::sync::Once;
    static INIT: Once = Once::new();

    INIT.call_once(|| {
        gst::init().unwrap();
        gstthreadshare::plugin_register_static().expect("gstthreadshare appsrc test");
    });
}

#[test]
fn test_push() {
    init();

    let jb = RTPJitterBuffer::new();
    let item = RTPJitterBufferItem::new(
        gst::Buffer::new(),
        gst::ClockTime(Some(57)),
        gst::ClockTime(Some(72)),
        135,
        142,
    );
    jb.insert(item);
    let (item, percent) = jb.pop();

    let buffer = item.get_buffer();
    eprintln!("Buffer: {:?}", buffer);

    eprintln!("Percent, pts: {} , {}", percent, item.get_pts());

    let mut h = gst_check::Harness::new("ts-appsrc");

    let caps = gst::Caps::new_simple("foo/bar", &[]);
    {
        let appsrc = h.get_element().unwrap();
        appsrc.set_property("caps", &caps).unwrap();
        appsrc.set_property("do-timestamp", &true).unwrap();
        appsrc.set_property("context", &"test-push").unwrap();
    }

    h.play();

    {
        let appsrc = h.get_element().unwrap();
        for _ in 0..3 {
            assert!(appsrc
                .emit("push-buffer", &[&gst::Buffer::new()])
                .unwrap()
                .unwrap()
                .get_some::<bool>()
                .unwrap());
        }

        assert!(appsrc
            .emit("end-of-stream", &[])
            .unwrap()
            .unwrap()
            .get_some::<bool>()
            .unwrap());
    }

    for _ in 0..3 {
        let _buffer = h.pull().unwrap();
    }

    let mut n_events = 0;
    loop {
        use gst::EventView;

        let event = h.pull_event().unwrap();
        match event.view() {
            EventView::StreamStart(..) => {
                assert_eq!(n_events, 0);
            }
            EventView::Caps(ev) => {
                assert_eq!(n_events, 1);
                let event_caps = ev.get_caps();
                assert_eq!(caps.as_ref(), event_caps);
            }
            EventView::Segment(..) => {
                assert_eq!(n_events, 2);
            }
            EventView::Eos(..) => {
                break;
            }
            _ => (),
        }
        n_events += 1;
    }
    assert!(n_events >= 3);
}
<commit_msg>tests/appsrc: remove unrelated jitterbuffer "tests"<commit_after>// Copyright (C) 2018 Sebastian Dröge <sebastian@centricular.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 51 Franklin Street, Suite 500,
// Boston, MA 02110-1335, USA.

extern crate glib;
use glib::prelude::*;

extern crate gstreamer as gst;
extern crate gstreamer_check as gst_check;

extern crate gstthreadshare;

fn init() {
    use std::sync::Once;
    static INIT: Once = Once::new();

    INIT.call_once(|| {
        gst::init().unwrap();
        gstthreadshare::plugin_register_static().expect("gstthreadshare appsrc test");
    });
}

#[test]
fn test_push() {
    init();

    let mut h = gst_check::Harness::new("ts-appsrc");

    let caps = gst::Caps::new_simple("foo/bar", &[]);
    {
        let appsrc = h.get_element().unwrap();
        appsrc.set_property("caps", &caps).unwrap();
        appsrc.set_property("do-timestamp", &true).unwrap();
        appsrc.set_property("context", &"test-push").unwrap();
    }

    h.play();

    {
        let appsrc = h.get_element().unwrap();
        for _ in 0..3 {
            assert!(appsrc
                .emit("push-buffer", &[&gst::Buffer::new()])
                .unwrap()
                .unwrap()
                .get_some::<bool>()
                .unwrap());
        }

        assert!(appsrc
            .emit("end-of-stream", &[])
            .unwrap()
            .unwrap()
            .get_some::<bool>()
            .unwrap());
    }

    for _ in 0..3 {
        let _buffer = h.pull().unwrap();
    }

    let mut n_events = 0;
    loop {
        use gst::EventView;

        let event = h.pull_event().unwrap();
        match event.view() {
            EventView::StreamStart(..) => {
                assert_eq!(n_events, 0);
            }
            EventView::Caps(ev) => {
                assert_eq!(n_events, 1);
                let event_caps = ev.get_caps();
                assert_eq!(caps.as_ref(), event_caps);
            }
            EventView::Segment(..) => {
                assert_eq!(n_events, 2);
            }
            EventView::Eos(..) => {
                break;
            }
            _ => (),
        }
        n_events += 1;
    }
    assert!(n_events >= 3);
}
<|endoftext|>


<commit_before>use actix_http::HttpService;
use actix_server::Server;
use actix_service::map_config;
use actix_web::{dev::AppConfig, get, App};

#[get("/")]
async fn index() -> &'static str {
    "Hello, world. From Actix Web!"
}

#[tokio::main(flavor = "current_thread")]
async fn main() -> std::io::Result<()> {
    Server::build()
        .bind("hello-world", "127.0.0.1:8080", || {
            // construct actix-web app
            let app = App::new().service(index);

            HttpService::build()
                // pass the app to service builder
                // map_config is used to map App's configuration to ServiceBuilder
                .finish(map_config(app, |_| AppConfig::default()))
                .tcp()
        })?
        .run()
        .await
}
<commit_msg>update actix-web-in-http example<commit_after>use actix_http::HttpService;
use actix_server::Server;
use actix_service::map_config;
use actix_web::{dev::AppConfig, get, App};

#[get("/")]
async fn index() -> &'static str {
    "Hello, world. From Actix Web!"
}

#[tokio::main(flavor = "current_thread")]
async fn main() -> std::io::Result<()> {
    Server::build()
        .bind("hello-world", "127.0.0.1:8080", || {
            // construct actix-web app
            let app = App::new().service(index);

            HttpService::build()
                // pass the app to service builder
                // map_config is used to map App's configuration to ServiceBuilder
                // h1 will configure server to only use HTTP/1.1
                .h1(map_config(app, |_| AppConfig::default()))
                .tcp()
        })?
        .run()
        .await
}
<|endoftext|>


<commit_before>// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
// Modified from https://github.com/rust-lang/cargo/blob/426fae51f39ebf6c545a2c12f78bc09fbfdb7aa9/src/cargo/util/cpu.rs
// TODO: Maybe use https://github.com/heim-rs/heim is better after https://github.com/heim-rs/heim/issues/233 is fixed.

use derive_more::{Add, Sub};
use std::io;

#[derive(Debug, Clone, Copy, Add, Sub)]
pub struct LiunxStyleCpuTime {
    pub user: u64,
    pub nice: u64,
    pub system: u64,
    pub idle: u64,
    pub iowait: u64,
    pub irq: u64,
    pub softirq: u64,
    pub steal: u64,
    pub guest: u64,
    pub guest_nice: u64,
}

impl LiunxStyleCpuTime {
    pub fn total(&self) -> u64 {
        // Note: guest(_nice) is not counted, since it is already in user.
        // See https://unix.stackexchange.com/questions/178045/proc-stat-is-guest-counted-into-user-time
        self.user
            + self.system
            + self.idle
            + self.nice
            + self.iowait
            + self.irq
            + self.softirq
            + self.steal
    }

    pub fn current() -> io::Result<LiunxStyleCpuTime> {
        imp::current()
    }
}

#[cfg(target_os = "linux")]
mod imp {
    use std::fs::File;
    use std::io::{self, Read};

    pub fn current() -> io::Result<super::LiunxStyleCpuTime> {
        let mut state = String::new();
        File::open("/proc/stat")?.read_to_string(&mut state)?;

        (|| {
            let mut parts = state.lines().next()?.split_whitespace();
            if parts.next()? != "cpu" {
                return None;
            }
            Some(super::LiunxStyleCpuTime {
                user: parts.next()?.parse::<u64>().ok()?,
                nice: parts.next()?.parse::<u64>().ok()?,
                system: parts.next()?.parse::<u64>().ok()?,
                idle: parts.next()?.parse::<u64>().ok()?,
                iowait: parts.next()?.parse::<u64>().ok()?,
                irq: parts.next()?.parse::<u64>().ok()?,
                softirq: parts.next()?.parse::<u64>().ok()?,
                steal: parts.next()?.parse::<u64>().ok()?,
                guest: parts.next()?.parse::<u64>().ok()?,
                guest_nice: parts.next()?.parse::<u64>().ok()?,
            })
        })()
        .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "first line of /proc/stat malformed"))
    }
}

#[cfg(target_os = "macos")]
#[allow(bad_style)]
mod imp {
    use std::io;
    use std::ptr;

    type host_t = u32;
    type mach_port_t = u32;
    type vm_map_t = mach_port_t;
    type vm_offset_t = usize;
    type vm_size_t = usize;
    type vm_address_t = vm_offset_t;
    type processor_flavor_t = i32;
    type natural_t = u32;
    type processor_info_array_t = *mut i32;
    type mach_msg_type_number_t = i32;
    type kern_return_t = i32;

    const PROESSOR_CPU_LOAD_INFO: processor_flavor_t = 2;
    const CPU_STATE_USER: usize = 0;
    const CPU_STATE_SYSTEM: usize = 1;
    const CPU_STATE_IDLE: usize = 2;
    const CPU_STATE_NICE: usize = 3;
    const CPU_STATE_MAX: usize = 4;

    extern "C" {
        static mut mach_task_self_: mach_port_t;

        fn mach_host_self() -> mach_port_t;
        fn host_processor_info(
            host: host_t,
            flavor: processor_flavor_t,
            out_processor_count: *mut natural_t,
            out_processor_info: *mut processor_info_array_t,
            out_processor_infoCnt: *mut mach_msg_type_number_t,
        ) -> kern_return_t;
        fn vm_deallocate(
            target_task: vm_map_t,
            address: vm_address_t,
            size: vm_size_t,
        ) -> kern_return_t;
    }

    #[repr(C)]
    struct processor_cpu_load_info_data_t {
        cpu_ticks: [u32; CPU_STATE_MAX],
    }

    pub fn current() -> io::Result<super::LiunxStyleCpuTime> {
        // There's scant little documentation on `host_processor_info`
        // throughout the internet, so this is just modeled after what everyone
        // else is doing. For now this is modeled largely after libuv.

        unsafe {
            let mut num_cpus_u = 0;
            let mut cpu_info = ptr::null_mut();
            let mut msg_type = 0;
            let err = host_processor_info(
                mach_host_self(),
                PROESSOR_CPU_LOAD_INFO,
                &mut num_cpus_u,
                &mut cpu_info,
                &mut msg_type,
            );
            if err != 0 {
                return Err(io::Error::last_os_error());
            }
            let mut ret = super::LiunxStyleCpuTime {
                user: 0,
                system: 0,
                idle: 0,
                iowait: 0,
                irq: 0,
                softirq: 0,
                steal: 0,
                guest: 0,
                nice: 0,
                guest_nice: 0,
            };
            let mut current = cpu_info as *const processor_cpu_load_info_data_t;
            for _ in 0..num_cpus_u {
                ret.user += current.cpu_ticks[CPU_STATE_USER] as u64;
                ret.system += current.cpu_ticks[CPU_STATE_SYSTEM] as u64;
                ret.idle += current.cpu_ticks[CPU_STATE_IDLE] as u64;
                ret.nice += current.cpu_ticks[CPU_STATE_NICE] as u64;
                current = current.offset(1);
            }
            vm_deallocate(mach_task_self_, cpu_info as vm_address_t, msg_type as usize);
            Ok(ret)
        }
    }
}

#[cfg(not(any(target_os = "linux", target_os = "macos")))]
mod imp {
    use std::io;

    pub fn current() -> io::Result<super::LiunxStyleCpuTime> {
        Err(io::Error::new(
            io::ErrorKind::Other,
            "unsupported platform to learn CPU state",
        ))
    }
}
<commit_msg>tikv_util: fix compile errors on macOS (#10039)<commit_after>// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
// Modified from https://github.com/rust-lang/cargo/blob/426fae51f39ebf6c545a2c12f78bc09fbfdb7aa9/src/cargo/util/cpu.rs
// TODO: Maybe use https://github.com/heim-rs/heim is better after https://github.com/heim-rs/heim/issues/233 is fixed.

use derive_more::{Add, Sub};
use std::io;

#[derive(Debug, Clone, Copy, Add, Sub)]
pub struct LiunxStyleCpuTime {
    pub user: u64,
    pub nice: u64,
    pub system: u64,
    pub idle: u64,
    pub iowait: u64,
    pub irq: u64,
    pub softirq: u64,
    pub steal: u64,
    pub guest: u64,
    pub guest_nice: u64,
}

impl LiunxStyleCpuTime {
    pub fn total(&self) -> u64 {
        // Note: guest(_nice) is not counted, since it is already in user.
        // See https://unix.stackexchange.com/questions/178045/proc-stat-is-guest-counted-into-user-time
        self.user
            + self.system
            + self.idle
            + self.nice
            + self.iowait
            + self.irq
            + self.softirq
            + self.steal
    }

    pub fn current() -> io::Result<LiunxStyleCpuTime> {
        imp::current()
    }
}

#[cfg(target_os = "linux")]
mod imp {
    use std::fs::File;
    use std::io::{self, Read};

    pub fn current() -> io::Result<super::LiunxStyleCpuTime> {
        let mut state = String::new();
        File::open("/proc/stat")?.read_to_string(&mut state)?;

        (|| {
            let mut parts = state.lines().next()?.split_whitespace();
            if parts.next()? != "cpu" {
                return None;
            }
            Some(super::LiunxStyleCpuTime {
                user: parts.next()?.parse::<u64>().ok()?,
                nice: parts.next()?.parse::<u64>().ok()?,
                system: parts.next()?.parse::<u64>().ok()?,
                idle: parts.next()?.parse::<u64>().ok()?,
                iowait: parts.next()?.parse::<u64>().ok()?,
                irq: parts.next()?.parse::<u64>().ok()?,
                softirq: parts.next()?.parse::<u64>().ok()?,
                steal: parts.next()?.parse::<u64>().ok()?,
                guest: parts.next()?.parse::<u64>().ok()?,
                guest_nice: parts.next()?.parse::<u64>().ok()?,
            })
        })()
        .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "first line of /proc/stat malformed"))
    }
}

#[cfg(target_os = "macos")]
#[allow(bad_style)]
mod imp {
    use std::io;
    use std::ptr;

    type host_t = u32;
    type mach_port_t = u32;
    type vm_map_t = mach_port_t;
    type vm_offset_t = usize;
    type vm_size_t = usize;
    type vm_address_t = vm_offset_t;
    type processor_flavor_t = i32;
    type natural_t = u32;
    type processor_info_array_t = *mut i32;
    type mach_msg_type_number_t = i32;
    type kern_return_t = i32;

    const PROESSOR_CPU_LOAD_INFO: processor_flavor_t = 2;
    const CPU_STATE_USER: usize = 0;
    const CPU_STATE_SYSTEM: usize = 1;
    const CPU_STATE_IDLE: usize = 2;
    const CPU_STATE_NICE: usize = 3;
    const CPU_STATE_MAX: usize = 4;

    extern "C" {
        static mut mach_task_self_: mach_port_t;

        fn mach_host_self() -> mach_port_t;
        fn host_processor_info(
            host: host_t,
            flavor: processor_flavor_t,
            out_processor_count: *mut natural_t,
            out_processor_info: *mut processor_info_array_t,
            out_processor_infoCnt: *mut mach_msg_type_number_t,
        ) -> kern_return_t;
        fn vm_deallocate(
            target_task: vm_map_t,
            address: vm_address_t,
            size: vm_size_t,
        ) -> kern_return_t;
    }

    #[repr(C)]
    struct processor_cpu_load_info_data_t {
        cpu_ticks: [u32; CPU_STATE_MAX],
    }

    pub fn current() -> io::Result<super::LiunxStyleCpuTime> {
        // There's scant little documentation on `host_processor_info`
        // throughout the internet, so this is just modeled after what everyone
        // else is doing. For now this is modeled largely after libuv.

        unsafe {
            let mut num_cpus_u = 0;
            let mut cpu_info = ptr::null_mut();
            let mut msg_type = 0;
            let err = host_processor_info(
                mach_host_self(),
                PROESSOR_CPU_LOAD_INFO,
                &mut num_cpus_u,
                &mut cpu_info,
                &mut msg_type,
            );
            if err != 0 {
                return Err(io::Error::last_os_error());
            }
            let mut ret = super::LiunxStyleCpuTime {
                user: 0,
                system: 0,
                idle: 0,
                iowait: 0,
                irq: 0,
                softirq: 0,
                steal: 0,
                guest: 0,
                nice: 0,
                guest_nice: 0,
            };
            let mut current = cpu_info as *const processor_cpu_load_info_data_t;
            for _ in 0..num_cpus_u {
                ret.user += (*current).cpu_ticks[CPU_STATE_USER] as u64;
                ret.system += (*current).cpu_ticks[CPU_STATE_SYSTEM] as u64;
                ret.idle += (*current).cpu_ticks[CPU_STATE_IDLE] as u64;
                ret.nice += (*current).cpu_ticks[CPU_STATE_NICE] as u64;
                current = current.offset(1);
            }
            vm_deallocate(mach_task_self_, cpu_info as vm_address_t, msg_type as usize);
            Ok(ret)
        }
    }
}

#[cfg(not(any(target_os = "linux", target_os = "macos")))]
mod imp {
    use std::io;

    pub fn current() -> io::Result<super::LiunxStyleCpuTime> {
        Err(io::Error::new(
            io::ErrorKind::Other,
            "unsupported platform to learn CPU state",
        ))
    }
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::FuzzTargetImpl;
use libra_proptest_helpers::ValueGenerator;
use libra_secure_json_rpc::fuzzing::{
    fuzz_account_state_response, fuzz_submit_transaction_response, generate_account_state_corpus,
    generate_submit_transaction_corpus,
};

#[derive(Clone, Debug, Default)]
pub struct SecureJsonRpcSubmitTransaction;

/// This implementation will fuzz the submit_transaction() JSON RPC response returned to the
/// secure client.
impl FuzzTargetImpl for SecureJsonRpcSubmitTransaction {
    fn description(&self) -> &'static str {
        "Secure JSON RPC submit_transaction() response"
    }

    fn generate(&self, _idx: usize, _gen: &mut ValueGenerator) -> Option<Vec<u8>> {
        Some(generate_submit_transaction_corpus())
    }

    fn fuzz(&self, data: &[u8]) {
        fuzz_submit_transaction_response(data);
    }
}

#[derive(Clone, Debug, Default)]
pub struct SecureJsonRpcGetAccountState;

/// This implementation will fuzz the get_account_state_with_proof() JSON RPC response returned to
/// the secure client.
impl FuzzTargetImpl for SecureJsonRpcGetAccountState {
    fn description(&self) -> &'static str {
        "Secure JSON RPC get_account_state_with_proof() response"
    }

    fn generate(&self, _idx: usize, _gen: &mut ValueGenerator) -> Option<Vec<u8>> {
        Some(generate_account_state_corpus())
    }

    fn fuzz(&self, data: &[u8]) {
        fuzz_account_state_response(data);
    }
}

#[derive(Clone, Debug, Default)]
pub struct SecureJsonRpcGetAccountTransaction;

/// This implementation will fuzz the get_account_transaction() JSON RPC response returned to the
/// secure client.
impl FuzzTargetImpl for SecureJsonRpcGetAccountTransaction {
    fn description(&self) -> &'static str {
        "Secure JSON RPC get_account_transaction() response"
    }

    fn generate(&self, _idx: usize, _gen: &mut ValueGenerator) -> Option<Vec<u8>> {
        Some(generate_submit_transaction_corpus())
    }

    fn fuzz(&self, data: &[u8]) {
        fuzz_submit_transaction_response(data);
    }
}
<commit_msg>[Fuzzing] Fix a bug in the secure JSON RPC client fuzzer: SecureJsonRpcGetAccountTransaction target.<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::FuzzTargetImpl;
use libra_proptest_helpers::ValueGenerator;
use libra_secure_json_rpc::fuzzing::{
    fuzz_account_state_response, fuzz_submit_transaction_response,
    fuzz_transaction_status_response, generate_account_state_corpus,
    generate_submit_transaction_corpus, generate_transaction_status_corpus,
};

#[derive(Clone, Debug, Default)]
pub struct SecureJsonRpcSubmitTransaction;

/// This implementation will fuzz the submit_transaction() JSON RPC response returned to the
/// secure client.
impl FuzzTargetImpl for SecureJsonRpcSubmitTransaction {
    fn description(&self) -> &'static str {
        "Secure JSON RPC submit_transaction() response"
    }

    fn generate(&self, _idx: usize, _gen: &mut ValueGenerator) -> Option<Vec<u8>> {
        Some(generate_submit_transaction_corpus())
    }

    fn fuzz(&self, data: &[u8]) {
        fuzz_submit_transaction_response(data);
    }
}

#[derive(Clone, Debug, Default)]
pub struct SecureJsonRpcGetAccountState;

/// This implementation will fuzz the get_account_state_with_proof() JSON RPC response returned to
/// the secure client.
impl FuzzTargetImpl for SecureJsonRpcGetAccountState {
    fn description(&self) -> &'static str {
        "Secure JSON RPC get_account_state_with_proof() response"
    }

    fn generate(&self, _idx: usize, _gen: &mut ValueGenerator) -> Option<Vec<u8>> {
        Some(generate_account_state_corpus())
    }

    fn fuzz(&self, data: &[u8]) {
        fuzz_account_state_response(data);
    }
}

#[derive(Clone, Debug, Default)]
pub struct SecureJsonRpcGetAccountTransaction;

/// This implementation will fuzz the get_account_transaction() JSON RPC response returned to the
/// secure client.
impl FuzzTargetImpl for SecureJsonRpcGetAccountTransaction {
    fn description(&self) -> &'static str {
        "Secure JSON RPC get_account_transaction() response"
    }

    fn generate(&self, _idx: usize, _gen: &mut ValueGenerator) -> Option<Vec<u8>> {
        Some(generate_transaction_status_corpus())
    }

    fn fuzz(&self, data: &[u8]) {
        fuzz_transaction_status_response(data);
    }
}
<|endoftext|>


<commit_before>use crate::registry::{self, alt_api_path};
use crate::{find_json_mismatch, lines_match};
use flate2::read::GzDecoder;
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::{self, prelude::*, SeekFrom};
use std::path::{Path, PathBuf};
use tar::Archive;

fn read_le_u32<R>(mut reader: R) -> io::Result<u32>
where
    R: Read,
{
    let mut buf = [0; 4];
    reader.read_exact(&mut buf)?;
    Ok(u32::from_le_bytes(buf))
}

/// Checks the result of a crate publish.
pub fn validate_upload(expected_json: &str, expected_crate_name: &str, expected_files: &[&str]) {
    let new_path = registry::api_path().join("api/v1/crates/new");
    _validate_upload(
        &new_path,
        expected_json,
        expected_crate_name,
        expected_files,
        &[],
    );
}

/// Checks the result of a crate publish, along with the contents of the files.
pub fn validate_upload_with_contents(
    expected_json: &str,
    expected_crate_name: &str,
    expected_files: &[&str],
    expected_contents: &[(&str, &str)],
) {
    let new_path = registry::api_path().join("api/v1/crates/new");
    _validate_upload(
        &new_path,
        expected_json,
        expected_crate_name,
        expected_files,
        expected_contents,
    );
}

/// Checks the result of a crate publish to an alternative registry.
pub fn validate_alt_upload(
    expected_json: &str,
    expected_crate_name: &str,
    expected_files: &[&str],
) {
    let new_path = alt_api_path().join("api/v1/crates/new");
    _validate_upload(
        &new_path,
        expected_json,
        expected_crate_name,
        expected_files,
        &[],
    );
}

fn _validate_upload(
    new_path: &Path,
    expected_json: &str,
    expected_crate_name: &str,
    expected_files: &[&str],
    expected_contents: &[(&str, &str)],
) {
    let mut f = File::open(new_path).unwrap();
    // 32-bit little-endian integer of length of JSON data.
    let json_sz = read_le_u32(&mut f).expect("read json length");
    let mut json_bytes = vec![0; json_sz as usize];
    f.read_exact(&mut json_bytes).expect("read JSON data");
    let actual_json = serde_json::from_slice(&json_bytes).expect("uploaded JSON should be valid");
    let expected_json = serde_json::from_str(expected_json).expect("expected JSON does not parse");
    find_json_mismatch(&expected_json, &actual_json)
        .expect("uploaded JSON did not match expected JSON");

    // 32-bit little-endian integer of length of crate file.
    let crate_sz = read_le_u32(&mut f).expect("read crate length");
    let mut krate_bytes = vec![0; crate_sz as usize];
    f.read_exact(&mut krate_bytes).expect("read crate data");
    // Check at end.
    let current = f.seek(SeekFrom::Current(0)).unwrap();
    assert_eq!(f.seek(SeekFrom::End(0)).unwrap(), current);

    // Verify the tarball.
    validate_crate_contents(
        &krate_bytes[..],
        expected_crate_name,
        expected_files,
        expected_contents,
    );
}

/// Checks the contents of a `.crate` file.
///
/// - `expected_crate_name` should be something like `foo-0.0.1.crate`.
/// - `expected_files` should be a complete list of files in the crate
///   (relative to expected_crate_name).
/// - `expected_contents` should be a list of `(file_name, contents)` tuples
///   to validate the contents of the given file. Only the listed files will
///   be checked (others will be ignored).
pub fn validate_crate_contents(
    reader: impl Read,
    expected_crate_name: &str,
    expected_files: &[&str],
    expected_contents: &[(&str, &str)],
) {
    let mut rdr = GzDecoder::new(reader);
    assert_eq!(
        rdr.header().unwrap().filename().unwrap(),
        expected_crate_name.as_bytes()
    );
    let mut contents = Vec::new();
    rdr.read_to_end(&mut contents).unwrap();
    let mut ar = Archive::new(&contents[..]);
    let files: HashMap<PathBuf, String> = ar
        .entries()
        .unwrap()
        .map(|entry| {
            let mut entry = entry.unwrap();
            let name = entry.path().unwrap().into_owned();
            let mut contents = String::new();
            entry.read_to_string(&mut contents).unwrap();
            (name, contents)
        })
        .collect();
    assert!(expected_crate_name.ends_with(".crate"));
    let base_crate_name = Path::new(&expected_crate_name[..expected_crate_name.len() - 6]);
    let actual_files: HashSet<PathBuf> = files.keys().cloned().collect();
    let expected_files: HashSet<PathBuf> = expected_files
        .iter()
        .map(|name| base_crate_name.join(name))
        .collect();
    let missing: Vec<&PathBuf> = expected_files.difference(&actual_files).collect();
    let extra: Vec<&PathBuf> = actual_files.difference(&expected_files).collect();
    if !missing.is_empty() || !extra.is_empty() {
        panic!(
            "uploaded archive does not match.\nMissing: {:?}\nExtra: {:?}\n",
            missing, extra
        );
    }
    if !expected_contents.is_empty() {
        for (e_file_name, e_file_contents) in expected_contents {
            let full_e_name = base_crate_name.join(e_file_name);
            let actual_contents = files
                .get(&full_e_name)
                .unwrap_or_else(|| panic!("file `{}` missing in archive", e_file_name));
            if !lines_match(e_file_contents, actual_contents) {
                panic!(
                    "Crate contents mismatch for {:?}:\n\
                     --- expected\n\
                     {}\n\
                     --- actual \n\
                     {}\n",
                    e_file_name, e_file_contents, actual_contents
                );
            }
        }
    }
}
<commit_msg>Display formatted output for JSON diffing in tests.<commit_after>use crate::registry::{self, alt_api_path};
use crate::{find_json_mismatch, lines_match};
use flate2::read::GzDecoder;
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::{self, prelude::*, SeekFrom};
use std::path::{Path, PathBuf};
use tar::Archive;

fn read_le_u32<R>(mut reader: R) -> io::Result<u32>
where
    R: Read,
{
    let mut buf = [0; 4];
    reader.read_exact(&mut buf)?;
    Ok(u32::from_le_bytes(buf))
}

/// Checks the result of a crate publish.
pub fn validate_upload(expected_json: &str, expected_crate_name: &str, expected_files: &[&str]) {
    let new_path = registry::api_path().join("api/v1/crates/new");
    _validate_upload(
        &new_path,
        expected_json,
        expected_crate_name,
        expected_files,
        &[],
    );
}

/// Checks the result of a crate publish, along with the contents of the files.
pub fn validate_upload_with_contents(
    expected_json: &str,
    expected_crate_name: &str,
    expected_files: &[&str],
    expected_contents: &[(&str, &str)],
) {
    let new_path = registry::api_path().join("api/v1/crates/new");
    _validate_upload(
        &new_path,
        expected_json,
        expected_crate_name,
        expected_files,
        expected_contents,
    );
}

/// Checks the result of a crate publish to an alternative registry.
pub fn validate_alt_upload(
    expected_json: &str,
    expected_crate_name: &str,
    expected_files: &[&str],
) {
    let new_path = alt_api_path().join("api/v1/crates/new");
    _validate_upload(
        &new_path,
        expected_json,
        expected_crate_name,
        expected_files,
        &[],
    );
}

fn _validate_upload(
    new_path: &Path,
    expected_json: &str,
    expected_crate_name: &str,
    expected_files: &[&str],
    expected_contents: &[(&str, &str)],
) {
    let mut f = File::open(new_path).unwrap();
    // 32-bit little-endian integer of length of JSON data.
    let json_sz = read_le_u32(&mut f).expect("read json length");
    let mut json_bytes = vec![0; json_sz as usize];
    f.read_exact(&mut json_bytes).expect("read JSON data");
    let actual_json = serde_json::from_slice(&json_bytes).expect("uploaded JSON should be valid");
    let expected_json = serde_json::from_str(expected_json).expect("expected JSON does not parse");

    if let Err(e) = find_json_mismatch(&expected_json, &actual_json) {
        panic!("{}", e);
    }

    // 32-bit little-endian integer of length of crate file.
    let crate_sz = read_le_u32(&mut f).expect("read crate length");
    let mut krate_bytes = vec![0; crate_sz as usize];
    f.read_exact(&mut krate_bytes).expect("read crate data");
    // Check at end.
    let current = f.seek(SeekFrom::Current(0)).unwrap();
    assert_eq!(f.seek(SeekFrom::End(0)).unwrap(), current);

    // Verify the tarball.
    validate_crate_contents(
        &krate_bytes[..],
        expected_crate_name,
        expected_files,
        expected_contents,
    );
}

/// Checks the contents of a `.crate` file.
///
/// - `expected_crate_name` should be something like `foo-0.0.1.crate`.
/// - `expected_files` should be a complete list of files in the crate
///   (relative to expected_crate_name).
/// - `expected_contents` should be a list of `(file_name, contents)` tuples
///   to validate the contents of the given file. Only the listed files will
///   be checked (others will be ignored).
pub fn validate_crate_contents(
    reader: impl Read,
    expected_crate_name: &str,
    expected_files: &[&str],
    expected_contents: &[(&str, &str)],
) {
    let mut rdr = GzDecoder::new(reader);
    assert_eq!(
        rdr.header().unwrap().filename().unwrap(),
        expected_crate_name.as_bytes()
    );
    let mut contents = Vec::new();
    rdr.read_to_end(&mut contents).unwrap();
    let mut ar = Archive::new(&contents[..]);
    let files: HashMap<PathBuf, String> = ar
        .entries()
        .unwrap()
        .map(|entry| {
            let mut entry = entry.unwrap();
            let name = entry.path().unwrap().into_owned();
            let mut contents = String::new();
            entry.read_to_string(&mut contents).unwrap();
            (name, contents)
        })
        .collect();
    assert!(expected_crate_name.ends_with(".crate"));
    let base_crate_name = Path::new(&expected_crate_name[..expected_crate_name.len() - 6]);
    let actual_files: HashSet<PathBuf> = files.keys().cloned().collect();
    let expected_files: HashSet<PathBuf> = expected_files
        .iter()
        .map(|name| base_crate_name.join(name))
        .collect();
    let missing: Vec<&PathBuf> = expected_files.difference(&actual_files).collect();
    let extra: Vec<&PathBuf> = actual_files.difference(&expected_files).collect();
    if !missing.is_empty() || !extra.is_empty() {
        panic!(
            "uploaded archive does not match.\nMissing: {:?}\nExtra: {:?}\n",
            missing, extra
        );
    }
    if !expected_contents.is_empty() {
        for (e_file_name, e_file_contents) in expected_contents {
            let full_e_name = base_crate_name.join(e_file_name);
            let actual_contents = files
                .get(&full_e_name)
                .unwrap_or_else(|| panic!("file `{}` missing in archive", e_file_name));
            if !lines_match(e_file_contents, actual_contents) {
                panic!(
                    "Crate contents mismatch for {:?}:\n\
                     --- expected\n\
                     {}\n\
                     --- actual \n\
                     {}\n",
                    e_file_name, e_file_contents, actual_contents
                );
            }
        }
    }
}
<|endoftext|>


<commit_before>use super::*;

pub(super) const PATTERN_FIRST: TokenSet = expressions::LITERAL_FIRST
    .union(paths::PATH_FIRST)
    .union(token_set![BOX_KW, REF_KW, MUT_KW, L_PAREN, L_BRACK, AMP, UNDERSCORE, MINUS]);

pub(super) fn pattern(p: &mut Parser) {
    pattern_r(p, PAT_RECOVERY_SET);
}

/// Parses a pattern list separated by pipes `|`
pub(super) fn pattern_list(p: &mut Parser) {
    pattern_list_r(p, PAT_RECOVERY_SET)
}

/// Parses a pattern list separated by pipes `|`
/// using the given `recovery_set`
pub(super) fn pattern_list_r(p: &mut Parser, recovery_set: TokenSet) {
    p.eat(T![|]);
    pattern_r(p, recovery_set);

    while p.eat(T![|]) {
        pattern_r(p, recovery_set);
    }
}

pub(super) fn pattern_r(p: &mut Parser, recovery_set: TokenSet) {
    if let Some(lhs) = atom_pat(p, recovery_set) {
        // test range_pat
        // fn main() {
        //     match 92 {
        //         0 ... 100 => (),
        //         101 ..= 200 => (),
        //         200 .. 301=> (),
        //     }
        // }
        if p.at(T![...]) || p.at(T![..=]) || p.at(T![..]) {
            let m = lhs.precede(p);
            p.bump();
            atom_pat(p, recovery_set);
            m.complete(p, RANGE_PAT);
        }
        // test marco_pat
        // fn main() {
        //     let m!(x) = 0;
        // }
        else if lhs.kind() == PATH_PAT && p.at(T![!]) {
            let m = lhs.precede(p);
            items::macro_call_after_excl(p);
            m.complete(p, MACRO_CALL);
        }
    }
}

const PAT_RECOVERY_SET: TokenSet =
    token_set![LET_KW, IF_KW, WHILE_KW, LOOP_KW, MATCH_KW, R_PAREN, COMMA];

fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> {
    let la0 = p.nth(0);
    let la1 = p.nth(1);
    if la0 == T![ref]
        || la0 == T![mut]
        || la0 == T![box]
        || (la0 == IDENT && !(la1 == T![::] || la1 == T!['('] || la1 == T!['{'] || la1 == T![!]))
    {
        return Some(bind_pat(p, true));
    }
    if paths::is_use_path_start(p) {
        return Some(path_pat(p));
    }

    if is_literal_pat_start(p) {
        return Some(literal_pat(p));
    }

    let m = match la0 {
        T![_] => placeholder_pat(p),
        T![&] => ref_pat(p),
        T!['('] => tuple_pat(p),
        T!['['] => slice_pat(p),
        _ => {
            p.err_recover("expected pattern", recovery_set);
            return None;
        }
    };
    Some(m)
}

fn is_literal_pat_start(p: &mut Parser) -> bool {
    p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER)
        || p.at_ts(expressions::LITERAL_FIRST)
}

// test literal_pattern
// fn main() {
//     match () {
//         -1 => (),
//         92 => (),
//         'c' => (),
//         "hello" => (),
//     }
// }
fn literal_pat(p: &mut Parser) -> CompletedMarker {
    assert!(is_literal_pat_start(p));
    let m = p.start();
    if p.at(T![-]) {
        p.bump();
    }
    expressions::literal(p);
    m.complete(p, LITERAL_PAT)
}

// test path_part
// fn foo() {
//     let foo::Bar = ();
//     let ::Bar = ();
//     let Bar { .. } = ();
//     let Bar(..) = ();
// }
fn path_pat(p: &mut Parser) -> CompletedMarker {
    assert!(paths::is_use_path_start(p));
    let m = p.start();
    paths::expr_path(p);
    let kind = match p.current() {
        T!['('] => {
            tuple_pat_fields(p);
            TUPLE_STRUCT_PAT
        }
        T!['{'] => {
            record_field_pat_list(p);
            RECORD_PAT
        }
        _ => PATH_PAT,
    };
    m.complete(p, kind)
}

// test tuple_pat_fields
// fn foo() {
//     let S() = ();
//     let S(_) = ();
//     let S(_,) = ();
//     let S(_, .. , x) = ();
// }
fn tuple_pat_fields(p: &mut Parser) {
    assert!(p.at(T!['(']));
    p.bump();
    pat_list(p, T![')']);
    p.expect(T![')']);
}

// test record_field_pat_list
// fn foo() {
//     let S {} = ();
//     let S { f, ref mut g } = ();
//     let S { h: _, ..} = ();
//     let S { h: _, } = ();
// }
fn record_field_pat_list(p: &mut Parser) {
    assert!(p.at(T!['{']));
    let m = p.start();
    p.bump();
    while !p.at(EOF) && !p.at(T!['}']) {
        match p.current() {
            T![..] => p.bump(),
            IDENT if p.nth(1) == T![:] => record_field_pat(p),
            T!['{'] => error_block(p, "expected ident"),
            _ => {
                bind_pat(p, false);
            }
        }
        if !p.at(T!['}']) {
            p.expect(T![,]);
        }
    }
    p.expect(T!['}']);
    m.complete(p, RECORD_FIELD_PAT_LIST);
}

fn record_field_pat(p: &mut Parser) {
    assert!(p.at(IDENT));
    assert!(p.nth(1) == T![:]);

    let m = p.start();
    name(p);
    p.bump();
    pattern(p);
    m.complete(p, RECORD_FIELD_PAT);
}

// test placeholder_pat
// fn main() { let _ = (); }
fn placeholder_pat(p: &mut Parser) -> CompletedMarker {
    assert!(p.at(T![_]));
    let m = p.start();
    p.bump();
    m.complete(p, PLACEHOLDER_PAT)
}

// test ref_pat
// fn main() {
//     let &a = ();
//     let &mut b = ();
// }
fn ref_pat(p: &mut Parser) -> CompletedMarker {
    assert!(p.at(T![&]));
    let m = p.start();
    p.bump();
    p.eat(T![mut]);
    pattern(p);
    m.complete(p, REF_PAT)
}

// test tuple_pat
// fn main() {
//     let (a, b, ..) = ();
// }
fn tuple_pat(p: &mut Parser) -> CompletedMarker {
    assert!(p.at(T!['(']));
    let m = p.start();
    tuple_pat_fields(p);
    m.complete(p, TUPLE_PAT)
}

// test slice_pat
// fn main() {
//     let [a, b, ..] = [];
// }
fn slice_pat(p: &mut Parser) -> CompletedMarker {
    assert!(p.at(T!['[']));
    let m = p.start();
    p.bump();
    pat_list(p, T![']']);
    p.expect(T![']']);
    m.complete(p, SLICE_PAT)
}

fn pat_list(p: &mut Parser, ket: SyntaxKind) {
    while !p.at(EOF) && !p.at(ket) {
        match p.current() {
            T![..] => p.bump(),
            _ => {
                if !p.at_ts(PATTERN_FIRST) {
                    p.error("expected a pattern");
                    break;
                }
                pattern(p)
            }
        }
        if !p.at(ket) {
            p.expect(T![,]);
        }
    }
}

// test bind_pat
// fn main() {
//     let a = ();
//     let mut b = ();
//     let ref c = ();
//     let ref mut d = ();
//     let e @ _ = ();
//     let ref mut f @ g @ _ = ();
//     let box i = Box::new(1i32);
// }
fn bind_pat(p: &mut Parser, with_at: bool) -> CompletedMarker {
    let m = p.start();
    p.eat(T![box]);
    p.eat(T![ref]);
    p.eat(T![mut]);
    name(p);
    if with_at && p.eat(T![@]) {
        pattern(p);
    }
    m.complete(p, BIND_PAT)
}
<commit_msg>Parse `BoxPat`<commit_after>use super::*;

pub(super) const PATTERN_FIRST: TokenSet = expressions::LITERAL_FIRST
    .union(paths::PATH_FIRST)
    .union(token_set![BOX_KW, REF_KW, MUT_KW, L_PAREN, L_BRACK, AMP, UNDERSCORE, MINUS]);

pub(super) fn pattern(p: &mut Parser) {
    pattern_r(p, PAT_RECOVERY_SET);
}

/// Parses a pattern list separated by pipes `|`
pub(super) fn pattern_list(p: &mut Parser) {
    pattern_list_r(p, PAT_RECOVERY_SET)
}

/// Parses a pattern list separated by pipes `|`
/// using the given `recovery_set`
pub(super) fn pattern_list_r(p: &mut Parser, recovery_set: TokenSet) {
    p.eat(T![|]);
    pattern_r(p, recovery_set);

    while p.eat(T![|]) {
        pattern_r(p, recovery_set);
    }
}

pub(super) fn pattern_r(p: &mut Parser, recovery_set: TokenSet) {
    if let Some(lhs) = atom_pat(p, recovery_set) {
        // test range_pat
        // fn main() {
        //     match 92 {
        //         0 ... 100 => (),
        //         101 ..= 200 => (),
        //         200 .. 301=> (),
        //     }
        // }
        if p.at(T![...]) || p.at(T![..=]) || p.at(T![..]) {
            let m = lhs.precede(p);
            p.bump();
            atom_pat(p, recovery_set);
            m.complete(p, RANGE_PAT);
        }
        // test marco_pat
        // fn main() {
        //     let m!(x) = 0;
        // }
        else if lhs.kind() == PATH_PAT && p.at(T![!]) {
            let m = lhs.precede(p);
            items::macro_call_after_excl(p);
            m.complete(p, MACRO_CALL);
        }
    }
}

const PAT_RECOVERY_SET: TokenSet =
    token_set![LET_KW, IF_KW, WHILE_KW, LOOP_KW, MATCH_KW, R_PAREN, COMMA];

fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> {
    // Checks the token after an IDENT to see if a pattern is a path (Struct { .. }) or macro
    // (T![x]).
    let is_path_or_macro_pat =
        |la1| la1 == T![::] || la1 == T!['('] || la1 == T!['{'] || la1 == T![!];

    let m = match p.nth(0) {
        T![box] => box_pat(p),
        T![ref] | T![mut] | IDENT if !is_path_or_macro_pat(p.nth(1)) => bind_pat(p, true),

        _ if paths::is_use_path_start(p) => path_pat(p),
        _ if is_literal_pat_start(p) => literal_pat(p),

        T![_] => placeholder_pat(p),
        T![&] => ref_pat(p),
        T!['('] => tuple_pat(p),
        T!['['] => slice_pat(p),

        _ => {
            p.err_recover("expected pattern", recovery_set);
            return None;
        }
    };

    Some(m)
}

fn is_literal_pat_start(p: &Parser) -> bool {
    p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER)
        || p.at_ts(expressions::LITERAL_FIRST)
}

// test literal_pattern
// fn main() {
//     match () {
//         -1 => (),
//         92 => (),
//         'c' => (),
//         "hello" => (),
//     }
// }
fn literal_pat(p: &mut Parser) -> CompletedMarker {
    assert!(is_literal_pat_start(p));
    let m = p.start();
    if p.at(T![-]) {
        p.bump();
    }
    expressions::literal(p);
    m.complete(p, LITERAL_PAT)
}

// test path_part
// fn foo() {
//     let foo::Bar = ();
//     let ::Bar = ();
//     let Bar { .. } = ();
//     let Bar(..) = ();
// }
fn path_pat(p: &mut Parser) -> CompletedMarker {
    assert!(paths::is_use_path_start(p));
    let m = p.start();
    paths::expr_path(p);
    let kind = match p.current() {
        T!['('] => {
            tuple_pat_fields(p);
            TUPLE_STRUCT_PAT
        }
        T!['{'] => {
            record_field_pat_list(p);
            RECORD_PAT
        }
        _ => PATH_PAT,
    };
    m.complete(p, kind)
}

// test tuple_pat_fields
// fn foo() {
//     let S() = ();
//     let S(_) = ();
//     let S(_,) = ();
//     let S(_, .. , x) = ();
// }
fn tuple_pat_fields(p: &mut Parser) {
    assert!(p.at(T!['(']));
    p.bump();
    pat_list(p, T![')']);
    p.expect(T![')']);
}

// test record_field_pat_list
// fn foo() {
//     let S {} = ();
//     let S { f, ref mut g } = ();
//     let S { h: _, ..} = ();
//     let S { h: _, } = ();
// }
fn record_field_pat_list(p: &mut Parser) {
    assert!(p.at(T!['{']));
    let m = p.start();
    p.bump();
    while !p.at(EOF) && !p.at(T!['}']) {
        match p.current() {
            T![..] => p.bump(),
            IDENT if p.nth(1) == T![:] => record_field_pat(p),
            T!['{'] => error_block(p, "expected ident"),
            _ => {
                bind_pat(p, false);
            }
        }
        if !p.at(T!['}']) {
            p.expect(T![,]);
        }
    }
    p.expect(T!['}']);
    m.complete(p, RECORD_FIELD_PAT_LIST);
}

fn record_field_pat(p: &mut Parser) {
    assert!(p.at(IDENT));
    assert!(p.nth(1) == T![:]);

    let m = p.start();
    name(p);
    p.bump();
    pattern(p);
    m.complete(p, RECORD_FIELD_PAT);
}

// test placeholder_pat
// fn main() { let _ = (); }
fn placeholder_pat(p: &mut Parser) -> CompletedMarker {
    assert!(p.at(T![_]));
    let m = p.start();
    p.bump();
    m.complete(p, PLACEHOLDER_PAT)
}

// test ref_pat
// fn main() {
//     let &a = ();
//     let &mut b = ();
// }
fn ref_pat(p: &mut Parser) -> CompletedMarker {
    assert!(p.at(T![&]));
    let m = p.start();
    p.bump();
    p.eat(T![mut]);
    pattern(p);
    m.complete(p, REF_PAT)
}

// test tuple_pat
// fn main() {
//     let (a, b, ..) = ();
// }
fn tuple_pat(p: &mut Parser) -> CompletedMarker {
    assert!(p.at(T!['(']));
    let m = p.start();
    tuple_pat_fields(p);
    m.complete(p, TUPLE_PAT)
}

// test slice_pat
// fn main() {
//     let [a, b, ..] = [];
// }
fn slice_pat(p: &mut Parser) -> CompletedMarker {
    assert!(p.at(T!['[']));
    let m = p.start();
    p.bump();
    pat_list(p, T![']']);
    p.expect(T![']']);
    m.complete(p, SLICE_PAT)
}

fn pat_list(p: &mut Parser, ket: SyntaxKind) {
    while !p.at(EOF) && !p.at(ket) {
        match p.current() {
            T![..] => p.bump(),
            _ => {
                if !p.at_ts(PATTERN_FIRST) {
                    p.error("expected a pattern");
                    break;
                }
                pattern(p)
            }
        }
        if !p.at(ket) {
            p.expect(T![,]);
        }
    }
}

// test bind_pat
// fn main() {
//     let a = ();
//     let mut b = ();
//     let ref c = ();
//     let ref mut d = ();
//     let e @ _ = ();
//     let ref mut f @ g @ _ = ();
// }
fn bind_pat(p: &mut Parser, with_at: bool) -> CompletedMarker {
    let m = p.start();
    p.eat(T![ref]);
    p.eat(T![mut]);
    name(p);
    if with_at && p.eat(T![@]) {
        pattern(p);
    }
    m.complete(p, BIND_PAT)
}

// test_err ref_box_pat
// fn main() {
//     let ref box i = ();
// }

// test box_pat
// fn main() {
//     let box i = ();
//     let box Outer { box i, j: box Inner(box &x) } = ();
//     let box ref mut i = ();
// }
fn box_pat(p: &mut Parser) -> CompletedMarker {
    assert!(p.at(T![box]));
    let m = p.start();
    p.bump();
    pattern(p);
    m.complete(p, BOX_PAT)
}
<|endoftext|>