<commit_before>#![macro_use]

// For testing purposes, we want to generate valid `Ast`s,
//  but `ast!` is clunky and makes it *super* easy to leave off `Quote*` and `ExtendEnv`
//  and actually running the parser is a big dependency (and requires huge string literals).
//  so we introduce a `u!` macro that looks up forms but infers the internal structure.

// It's weird to be relying on the grammar while ignoring parts of it, hence "flimsy",
//  but errors are much more likely to be compile-time than inscrutable test problems.

// It's not unsafe to use `u!` for runtime operations, but there's a runtime cost, so don't do it.

use crate::{
    ast::Ast::{self, *},
    grammar::FormPat,
    name::*,
    util::mbe::EnvMBE,
};
use std::iter::{Iterator, Peekable};

// First, transforms from `[a b c; d e f; g h i]` to `[g h i] {[a b c] [d e f]}`
//   to get around a Rust macro parsing restriction,
//  then makes a REP flimsy shape for it.
macro_rules! u_rep {
    ([; $( $ts:tt )*]   $acc_cur:tt { $( $acc_rest:tt )* }) => {
        u_rep!( [ $( $ts )* ] [] {  $( $acc_rest )*  $acc_cur })
    };
    ([$t:tt $( $ts:tt )*]  [ $( $acc_cur:tt )* ] { $( $acc_rest:tt )* }) => {
        u_rep!( [ $( $ts )* ] [$($acc_cur)* $t] { $( $acc_rest )* })
    };
    ([] [] {}) => {
        // Empty repeat
        crate::ast::Shape(vec![crate::ast::Atom(n("REP"))])
    };
    ([]  [ $( $acc_cur:tt )* ] { $( [ $( $acc_rest:tt )* ] )* }) => {
        crate::ast::Shape(vec![
            crate::ast::Atom(n("REP")),
            $( u_shape_if_many!(  $($acc_rest)* ), )*
            u_shape_if_many!(  $($acc_cur)* )
        ])
    };
}

macro_rules! u_shape_if_many {
    ($t:tt) => {
        u!($t)
    };
    () => {
        compile_error!("empty u!")
    };
    ( $($ts:tt)* ) => {
        u!((~ $($ts)* ))
    };
}

thread_local! {
    pub static default_nt: std::cell::RefCell<String> = std::cell::RefCell::new("Expr".to_owned());
}

macro_rules! u {
    ($atom:ident) => {
        // Default to this, because `Call` will use whatever it's given, without a grammar:
        crate::ast::VariableReference(n(stringify!($atom)))
    };
    ( [ , $seq:expr ] ) => {
        {
            let mut contents: Vec<Ast> = $seq;
            contents.insert(0, crate::ast::Atom(n("REP")));
            crate::ast::Shape(contents)
        }
    };
    ( [ $( $ts:tt )*  ] ) => {
        u_rep!( [$($ts)*] [] {} )
    };
    ( { $form:ident : $( $ts:tt )*} ) => {
        {
            let f = crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                crate::core_forms::find_core_form(&def_nt.borrow(), stringify!($form))
            });
            crate::ast::Node(f.clone(),
                crate::macros::flimsy_syntax::parse_flimsy_mbe(&u!( (~ $($ts)* ) ), &f.grammar)
                    .unwrap_or_else(crate::util::mbe::EnvMBE::new),
                crate::beta::ExportBeta::Nothing)
        }
    };
    ( { $nt:ident $form:ident : $( $ts:tt )*} ) => {
        {
            let mut old_default_nt = "".to_owned();
            let f = crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                old_default_nt = def_nt.borrow().clone();
                let nt = stringify!($nt);
                *def_nt.borrow_mut() = nt.to_string();

                crate::core_forms::find_core_form(&nt, stringify!($form))
            });
            let res = crate::ast::Node(f.clone(),
                crate::macros::flimsy_syntax::parse_flimsy_mbe(&u!( (~ $($ts)* ) ), &f.grammar)
                    .unwrap_or_else(crate::util::mbe::EnvMBE::new),
                crate::beta::ExportBeta::Nothing);
            crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                *def_nt.borrow_mut() = old_default_nt;
            });
            res
        }
    };
    // The need for explicit exports is unfortunate;
    //  that information is part of `Scope`, not `Form` (maybe we should change that?)
    ( { $form:ident => $ebeta:tt : $( $ts:tt )*} ) => {
        {
            let f = crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                crate::core_forms::find_core_form(&def_nt.borrow(), stringify!($form))
            });
            crate::ast::Node(f.clone(),
                crate::macros::flimsy_syntax::parse_flimsy_mbe(&u!( (~ $($ts)* ) ), &f.grammar)
                    .unwrap_or_else(crate::util::mbe::EnvMBE::new),
                ebeta!($ebeta))
        }
    };
    ( { $nt:ident $form:ident => $ebeta:tt : $( $ts:tt )*} ) => {
        {
            // code duplication from above ) :
            let mut old_default_nt = "".to_owned();
            let f = crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                old_default_nt = def_nt.borrow().clone();
                let nt = stringify!($nt);
                *def_nt.borrow_mut() = nt.to_string();

                crate::core_forms::find_core_form(&nt, stringify!($form))
            });

            let res =crate::ast::Node(f.clone(),
                    crate::macros::flimsy_syntax::parse_flimsy_mbe(&u!( (~ $($ts)* ) ), &f.grammar)
                        .unwrap_or_else(crate::util::mbe::EnvMBE::new),
                    ebeta!($ebeta));
            crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                *def_nt.borrow_mut() = old_default_nt;
            });
            res
        }
    };
    ( { $form:expr ; $( $ts:tt )*} ) => {
        {
            let f = $form;
            crate::ast::Node(f.clone(),
                crate::macros::flimsy_syntax::parse_flimsy_mbe(&u!( (~ $($ts)* ) ), &f.grammar)
                    .unwrap_or_else(crate::util::mbe::EnvMBE::new),
                crate::beta::ExportBeta::Nothing)
        }
    };
    ({ $( $anything:tt )* }) => {
        compile_error!("Needed a : or ; in u!");
    };
    // Currently, nested `Seq`s need to correspond to nested `SEQ`s, so this creates one explicitly:
    ((~ $($ts:tt)*)) => {
        crate::ast::Shape(vec![
            crate::ast::Atom(n("SEQ")),
            $( u!( $ts ) ),*
        ])
    };
    ((at $t:tt)) => {
        crate::ast::Atom(n(stringify!($t)))
    };
    ((prim $t:tt)) => {
        crate::core_type_forms::get__primitive_type(n(stringify!($t)))
    };
    ((, $interpolate:expr)) => {
        $interpolate
    };
    // Two or more token trees (avoid infinite regress by not handling the one-element case)
    ( $t_first:tt $t_second:tt $( $t:tt )* ) => {
        ::ast::Shape(vec![
            ::ast::Atom(n("SEQ")),
            u!( $t_first ), u!( $t_second ), $( u!( $t ) ),*
        ])
    };
}

macro_rules! uty {
    ($( $ts:tt )*) => {
        {
            let mut old_default_nt = "".to_owned();
            crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                old_default_nt = def_nt.borrow().clone();
                *def_nt.borrow_mut() = "Type".to_owned();
            });
            let res = u!( $($ts)* );

            crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                *def_nt.borrow_mut() = old_default_nt;
            });
            res
        }
    };
}

fn parse_flimsy_seq<'a, I>(flimsy_seq: &mut Peekable<I>, grammar: &FormPat) -> EnvMBE<Ast>
where I: Iterator<Item = &'a Ast> {
    use crate::grammar::FormPat::*;

    match grammar {
        Seq(ref grammar_parts) => {
            let mut result = EnvMBE::new();

            for grammar_part in grammar_parts {
                result = result.combine_overriding(&parse_flimsy_seq(flimsy_seq, grammar_part));
            }
            result
        }
        _ => {
            let flimsy = *match flimsy_seq.peek() {
                None => return EnvMBE::new(), // Or is this an error?
                Some(f) => f,
            };
            match parse_flimsy_mbe(flimsy, grammar) {
                None => EnvMBE::new(),
                Some(res) => {
                    // `Anyways`es shouldn't consume anything (and they'll always be `Named`):
                    let consuming = match grammar {
                        Named(_, ref body) => match **body {
                            Anyways(_) => false,
                            _ => true,
                        },
                        _ => true,
                    };
                    if consuming {
                        let _ = flimsy_seq.next();
                    }
                    res
                }
            }
        }
    }
}

pub fn parse_flimsy_mbe(flimsy: &Ast, grammar: &FormPat) -> Option<EnvMBE<Ast>> {
    use crate::grammar::FormPat::*;

    match grammar {
        Literal(_, _) => None,
        Call(_) => None,
        Scan(_) => None,
        Seq(_) => match flimsy {
            Shape(flimsy_parts) => {
                if flimsy_parts[0] != Atom(n("SEQ")) {
                    panic!("Needed a SEQ, got {}", flimsy)
                }
                let mut fpi = flimsy_parts[1..].iter().peekable();

                Some(parse_flimsy_seq(&mut fpi, grammar))
            }
            _ => panic!("Needed a SEQ shape, got {}", flimsy),
        },
        Star(ref body) | Plus(ref body) => match flimsy {
            Shape(flimsy_parts) => {
                if flimsy_parts[0] != Atom(n("REP")) {
                    panic!("Need a REP, got {}", flimsy_parts[0])
                }

                let mut reps = vec![];
                for flimsy_part in flimsy_parts[1..].iter() {
                    reps.push(parse_flimsy_mbe(flimsy_part, &*body).unwrap());
                }
                Some(EnvMBE::new_from_anon_repeat(reps))
            }
            _ => panic!("Needed a REP shape, got {}", flimsy),
        },
        Alt(ref subs) => {
            // HACK: always pick the first branch of the `Alt`
            // (mainly affects unquotation, where it skips the type annotation)
            parse_flimsy_mbe(flimsy, &*subs[0])
        }
        Named(name, ref body) => Some(EnvMBE::new_from_leaves(
            crate::util::assoc::Assoc::new().set(*name, parse_flimsy_ast(flimsy, &*body)),
        )),
        SynImport(_, _, _) => panic!("`SynImport` can't work without a real parser"),
        NameImport(_, _) => panic!("`NameImport` should live underneath `Named`: {:?}", grammar),
        _ => unimplemented!("Can't handle {:?}", grammar),
    }
}

fn parse_flimsy_ast(flimsy: &Ast, grammar: &FormPat) -> Ast {
    use crate::grammar::FormPat::*;

    match grammar {
        Anyways(ref a) => a.clone(),
        Impossible => unimplemented!(),
        Scan(_) => flimsy.clone(),
        Literal(_, _) => Trivial,
        VarRef(_) => match flimsy {
            VariableReference(a) => VariableReference(*a),
            non_atom => panic!("Needed an atom, got {}", non_atom),
        },
        NameImport(body, beta) => {
            ExtendEnv(Box::new(parse_flimsy_ast(flimsy, &*body)), beta.clone())
        }
        NameImportPhaseless(body, beta) => {
            ExtendEnvPhaseless(Box::new(parse_flimsy_ast(flimsy, &*body)), beta.clone())
        }
        QuoteDeepen(body, pos) => QuoteMore(Box::new(parse_flimsy_ast(flimsy, &*body)), *pos),
        QuoteEscape(body, depth) => QuoteLess(Box::new(parse_flimsy_ast(flimsy, &*body)), *depth),

        Call(name) => {
            // HACK: don't descend into `Call(n("DefaultAtom"))
            if *name == n("DefaultAtom") || *name == n("AtomNotInPat") {
                match flimsy {
                    VariableReference(a) => Atom(*a),
                    non_atom => panic!("Needed an atom, got {}", non_atom),
                }
            } else {
                flimsy.clone()
            }
        }
        _ => unimplemented!("Can't handle {:?}", grammar),
    }
}
<commit_msg>Improve `flimsy_syntax`'s handling of `Anyways`. It's still hacky, though.<commit_after>#![macro_use]

// For testing purposes, we want to generate valid `Ast`s,
//  but `ast!` is clunky and makes it *super* easy to leave off `Quote*` and `ExtendEnv`
//  and actually running the parser is a big dependency (and requires huge string literals).
//  so we introduce a `u!` macro that looks up forms but infers the internal structure.

// It's weird to be relying on the grammar while ignoring parts of it, hence "flimsy",
//  but errors are much more likely to be compile-time than inscrutable test problems.

// It's not unsafe to use `u!` for runtime operations, but there's a runtime cost, so don't do it.

use crate::{
    ast::Ast::{self, *},
    grammar::FormPat,
    name::*,
    util::mbe::EnvMBE,
};
use std::iter::{Iterator, Peekable};

// First, transforms from `[a b c; d e f; g h i]` to `[g h i] {[a b c] [d e f]}`
//   to get around a Rust macro parsing restriction,
//  then makes a REP flimsy shape for it.
macro_rules! u_rep {
    ([; $( $ts:tt )*]   $acc_cur:tt { $( $acc_rest:tt )* }) => {
        u_rep!( [ $( $ts )* ] [] {  $( $acc_rest )*  $acc_cur })
    };
    ([$t:tt $( $ts:tt )*]  [ $( $acc_cur:tt )* ] { $( $acc_rest:tt )* }) => {
        u_rep!( [ $( $ts )* ] [$($acc_cur)* $t] { $( $acc_rest )* })
    };
    ([] [] {}) => {
        // Empty repeat
        crate::ast::Shape(vec![crate::ast::Atom(n("REP"))])
    };
    ([]  [ $( $acc_cur:tt )* ] { $( [ $( $acc_rest:tt )* ] )* }) => {
        crate::ast::Shape(vec![
            crate::ast::Atom(n("REP")),
            $( u_shape_if_many!(  $($acc_rest)* ), )*
            u_shape_if_many!(  $($acc_cur)* )
        ])
    };
}

macro_rules! u_shape_if_many {
    ($t:tt) => {
        u!($t)
    };
    () => {
        compile_error!("empty u!")
    };
    ( $($ts:tt)* ) => {
        u!((~ $($ts)* ))
    };
}

thread_local! {
    pub static default_nt: std::cell::RefCell<String> = std::cell::RefCell::new("Expr".to_owned());
}

macro_rules! u {
    ($atom:ident) => {
        // Default to this, because `Call` will use whatever it's given, without a grammar:
        crate::ast::VariableReference(n(stringify!($atom)))
    };
    ( [ , $seq:expr ] ) => {
        {
            let mut contents: Vec<Ast> = $seq;
            contents.insert(0, crate::ast::Atom(n("REP")));
            crate::ast::Shape(contents)
        }
    };
    ( [ $( $ts:tt )*  ] ) => {
        u_rep!( [$($ts)*] [] {} )
    };
    ( { $form:ident : $( $ts:tt )*} ) => {
        {
            let f = crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                crate::core_forms::find_core_form(&def_nt.borrow(), stringify!($form))
            });
            crate::ast::Node(f.clone(),
                crate::macros::flimsy_syntax::parse_flimsy_mbe(&u!( (~ $($ts)* ) ), &f.grammar)
                    .unwrap_or_else(crate::util::mbe::EnvMBE::new),
                crate::beta::ExportBeta::Nothing)
        }
    };
    ( { $nt:ident $form:ident : $( $ts:tt )*} ) => {
        {
            let mut old_default_nt = "".to_owned();
            let f = crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                old_default_nt = def_nt.borrow().clone();
                let nt = stringify!($nt);
                *def_nt.borrow_mut() = nt.to_string();

                crate::core_forms::find_core_form(&nt, stringify!($form))
            });
            let res = crate::ast::Node(f.clone(),
                crate::macros::flimsy_syntax::parse_flimsy_mbe(&u!( (~ $($ts)* ) ), &f.grammar)
                    .unwrap_or_else(crate::util::mbe::EnvMBE::new),
                crate::beta::ExportBeta::Nothing);
            crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                *def_nt.borrow_mut() = old_default_nt;
            });
            res
        }
    };
    // The need for explicit exports is unfortunate;
    //  that information is part of `Scope`, not `Form` (maybe we should change that?)
    ( { $form:ident => $ebeta:tt : $( $ts:tt )*} ) => {
        {
            let f = crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                crate::core_forms::find_core_form(&def_nt.borrow(), stringify!($form))
            });
            crate::ast::Node(f.clone(),
                crate::macros::flimsy_syntax::parse_flimsy_mbe(&u!( (~ $($ts)* ) ), &f.grammar)
                    .unwrap_or_else(crate::util::mbe::EnvMBE::new),
                ebeta!($ebeta))
        }
    };
    ( { $nt:ident $form:ident => $ebeta:tt : $( $ts:tt )*} ) => {
        {
            // code duplication from above ) :
            let mut old_default_nt = "".to_owned();
            let f = crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                old_default_nt = def_nt.borrow().clone();
                let nt = stringify!($nt);
                *def_nt.borrow_mut() = nt.to_string();

                crate::core_forms::find_core_form(&nt, stringify!($form))
            });

            let res =crate::ast::Node(f.clone(),
                    crate::macros::flimsy_syntax::parse_flimsy_mbe(&u!( (~ $($ts)* ) ), &f.grammar)
                        .unwrap_or_else(crate::util::mbe::EnvMBE::new),
                    ebeta!($ebeta));
            crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                *def_nt.borrow_mut() = old_default_nt;
            });
            res
        }
    };
    ( { $form:expr ; $( $ts:tt )*} ) => {
        {
            let f = $form;
            crate::ast::Node(f.clone(),
                crate::macros::flimsy_syntax::parse_flimsy_mbe(&u!( (~ $($ts)* ) ), &f.grammar)
                    .unwrap_or_else(crate::util::mbe::EnvMBE::new),
                crate::beta::ExportBeta::Nothing)
        }
    };
    ({ $( $anything:tt )* }) => {
        compile_error!("Needed a : or ; in u!");
    };
    // Currently, nested `Seq`s need to correspond to nested `SEQ`s, so this creates one explicitly:
    ((~ $($ts:tt)*)) => {
        crate::ast::Shape(vec![
            crate::ast::Atom(n("SEQ")),
            $( u!( $ts ) ),*
        ])
    };
    ((at $t:tt)) => {
        crate::ast::Atom(n(stringify!($t)))
    };
    ((prim $t:tt)) => {
        crate::core_type_forms::get__primitive_type(n(stringify!($t)))
    };
    ((, $interpolate:expr)) => {
        $interpolate
    };
    // Two or more token trees (avoid infinite regress by not handling the one-element case)
    ( $t_first:tt $t_second:tt $( $t:tt )* ) => {
        ::ast::Shape(vec![
            ::ast::Atom(n("SEQ")),
            u!( $t_first ), u!( $t_second ), $( u!( $t ) ),*
        ])
    };
}

macro_rules! uty {
    ($( $ts:tt )*) => {
        {
            let mut old_default_nt = "".to_owned();
            crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                old_default_nt = def_nt.borrow().clone();
                *def_nt.borrow_mut() = "Type".to_owned();
            });
            let res = u!( $($ts)* );

            crate::macros::flimsy_syntax::default_nt.with(|def_nt| {
                *def_nt.borrow_mut() = old_default_nt;
            });
            res
        }
    };
}

fn parse_flimsy_seq<'a, I>(flimsy_seq: &mut Peekable<I>, grammar: &FormPat) -> EnvMBE<Ast>
where I: Iterator<Item = &'a Ast> {
    use crate::grammar::FormPat::*;

    match grammar {
        Seq(ref grammar_parts) => {
            let mut result = EnvMBE::new();

            for grammar_part in grammar_parts {
                result = result.combine_overriding(&parse_flimsy_seq(flimsy_seq, grammar_part));
            }
            result
        }
        _ => {
            // `Anyways`es shouldn't consume anything (and they'll always be `Named`):
            let consuming = match grammar {
                Named(_, ref body) => match **body {
                    Anyways(_) => false,
                    _ => true,
                },
                _ => true,
            };
            let flimsy = *match flimsy_seq.peek() {
                None if consuming => return EnvMBE::new(), // Or is this an error?
                None => &&Ast::Trivial,
                Some(f) => f,
            };
            match parse_flimsy_mbe(flimsy, grammar) {
                None => EnvMBE::new(),
                Some(res) => {
                    if consuming {
                        let _ = flimsy_seq.next();
                    }
                    res
                }
            }
        }
    }
}

pub fn parse_flimsy_mbe(flimsy: &Ast, grammar: &FormPat) -> Option<EnvMBE<Ast>> {
    use crate::grammar::FormPat::*;

    match grammar {
        Literal(_, _) => None,
        Call(_) => None,
        Scan(_) => None,
        Seq(_) => match flimsy {
            Shape(flimsy_parts) => {
                if flimsy_parts[0] != Atom(n("SEQ")) {
                    panic!("Needed a SEQ, got {}", flimsy)
                }
                let mut fpi = flimsy_parts[1..].iter().peekable();

                Some(parse_flimsy_seq(&mut fpi, grammar))
            }
            _ => panic!("Needed a SEQ shape, got {}", flimsy),
        },
        Star(ref body) | Plus(ref body) => match flimsy {
            Shape(flimsy_parts) => {
                if flimsy_parts[0] != Atom(n("REP")) {
                    panic!("Need a REP, got {}", flimsy_parts[0])
                }

                let mut reps = vec![];
                for flimsy_part in flimsy_parts[1..].iter() {
                    reps.push(parse_flimsy_mbe(flimsy_part, &*body).unwrap());
                }
                Some(EnvMBE::new_from_anon_repeat(reps))
            }
            _ => panic!("Needed a REP shape, got {}", flimsy),
        },
        Alt(ref subs) => {
            // HACK: always pick the first branch of the `Alt`
            // (mainly affects unquotation, where it skips the type annotation)
            parse_flimsy_mbe(flimsy, &*subs[0])
        }
        Named(name, ref body) => Some(EnvMBE::new_from_leaves(
            crate::util::assoc::Assoc::new().set(*name, parse_flimsy_ast(flimsy, &*body)),
        )),
        SynImport(_, _, _) => panic!("`SynImport` can't work without a real parser"),
        NameImport(_, _) => panic!("`NameImport` should live underneath `Named`: {:?}", grammar),
        _ => unimplemented!("Can't handle {:?}", grammar),
    }
}

fn parse_flimsy_ast(flimsy: &Ast, grammar: &FormPat) -> Ast {
    use crate::grammar::FormPat::*;

    match grammar {
        Anyways(ref a) => a.clone(),
        Impossible => unimplemented!(),
        Scan(_) => flimsy.clone(),
        Literal(_, _) => Trivial,
        VarRef(_) => match flimsy {
            VariableReference(a) => VariableReference(*a),
            non_atom => panic!("Needed an atom, got {}", non_atom),
        },
        NameImport(body, beta) => {
            ExtendEnv(Box::new(parse_flimsy_ast(flimsy, &*body)), beta.clone())
        }
        NameImportPhaseless(body, beta) => {
            ExtendEnvPhaseless(Box::new(parse_flimsy_ast(flimsy, &*body)), beta.clone())
        }
        QuoteDeepen(body, pos) => QuoteMore(Box::new(parse_flimsy_ast(flimsy, &*body)), *pos),
        QuoteEscape(body, depth) => QuoteLess(Box::new(parse_flimsy_ast(flimsy, &*body)), *depth),

        Call(name) => {
            // HACK: don't descend into `Call(n("DefaultAtom"))
            if *name == n("DefaultAtom") || *name == n("AtomNotInPat") {
                match flimsy {
                    VariableReference(a) => Atom(*a),
                    non_atom => panic!("Needed an atom, got {}", non_atom),
                }
            } else {
                flimsy.clone()
            }
        }
        _ => unimplemented!("Can't handle {:?}", grammar),
    }
}
<|endoftext|>


<commit_before>use network::buffer::Buffer;

#[derive(Debug,PartialEq)]
pub enum Element {
  /// start, end in the stream
  Slice(usize, usize),
  Insert(Vec<u8>),
  // Splice(usize) // should copy x bytes in kernel
}

impl Element {
  pub fn consume(&self, size: usize) -> (usize,Option<Element>) {
    println!("consuming {} bytes of {:?}", size, self);
    match *self {
      Element::Slice(begin, end) => {
        if size >= end - begin {
          (size - (end - begin), None)
        } else {
          (0, Some(Element::Slice(begin + size, end)))
        }
      },
      Element::Insert(ref v)     => {
        if size >= v.len() {
          (size - v.len(), None)
        } else {
          (0, Some(Element::Insert(Vec::from(&v[size..]))))
        }
      }
    }
  }

  pub fn available_data(&self) -> usize {
    match self {
      &Element::Slice(begin, end) => end - begin,
      &Element::Insert(ref data)  => data.len(),
    }
  }
}

/// The BufferQueue has two roles: holding incoming data, and indicating
/// which data will go out. When new data arrives, it is added at the
/// end of the internal buffer. This new data is then eventually parsed or
/// handled in some way by external code. The external code then adds
/// element to the queue, indicating what to do with the data:
///   - copy a subset of the input data (and advance if needed)
///   - insert external data, like a HTTP header
///   - splice out of the kernel some data that was spliced in
///
/// position is the index in the stream of data already handled.
/// it corresponds to the beginning of available data in the Buffer
/// a Slice(begin, end) would point to buffer.data()[begin-position..end-position]
/// (in the easiest case)
///
/// The buffer's available data may be smaller than `end - begin`.
/// It can happen if the parser indicated we need to copy more data than is available,
/// like with a content length
///
/// should the buffer queue indicate how much data it needs?
#[derive(Debug,PartialEq)]
pub struct BufferQueue {
  pub position:  usize,
  pub buffer:    Buffer,
  pub queue:     Vec<Element>,
}

impl BufferQueue {
  pub fn available_data(&self) -> usize {
    self.queue.iter().fold(0, |acc, ref el| acc + el.available_data())
  }

  //FIXME: implement this
  pub fn unparsed_data(&self) {}
  //FIXME: implement this
  pub fn consumed_unparsed_data(&mut self) {}
  //FIXME: implement this
  pub fn needed_data(&self)  {}

  pub fn next_buffer(&self) -> Option<&[u8]> {
    self.queue.get(0).map(|el| {
      match el {
        &Element::Slice(begin, end) => self.calculate_slice(begin, end),
        &Element::Insert(ref v)     => &v[..]
      }
    })
  }

  /// begin should not be smaller than position
  /// begin can be larger than position+buffer_available_size
  /// end must be larger than begin
  /// bufferqueue.position => position in the stream (may be larger than the underlying buffer)
  /// buffer.position      => position in the buffer (no larger than buffer.capacity)
  /// buffer.end           => end of available data in the buffer (no larger than buffer.capacity)
  /// slice.begin          => position of slice beginning in the stream (larger or equal to bufferqueue.position,
  ///                           can be larger than buffer position, end or capacity)
  /// slice.end            => position of slice end in the stream (larger or equal to slice.begin,
  ///                           can be larger than buffer position, end or capacity)
  ///
  ///  position in input stream or output stream (where do we count the insert?)
  pub fn calculate_slice(&self, begin:usize, end: usize) -> &[u8] {
    &self.buffer.data()[(begin - self.position)..(end - self.position)]
  }

  pub fn consume(&mut self, size: usize) {
    let mut to_consume = size;
    let mut consumed   = 0;
    while to_consume > 0 {
      let new_first_element = match self.queue.first() {
        None => {
          assert!(to_consume == 0, "no more element in queue, we should not ask to consume {} more bytes", to_consume);
          break;
        },
        Some(el) => match el {
          &Element::Slice(begin, end) => {
            if to_consume >= end - begin {
              consumed  += end - begin;
              to_consume = to_consume - (end - begin);
              None
            } else {
              consumed += to_consume;
              let new_element = Element::Slice(begin + to_consume, end);
              to_consume = 0;
              Some(new_element)
            }
          },
          &Element::Insert(ref v)     => {
            if to_consume >= v.len() {
              to_consume = to_consume - v.len();
              None
            } else {
              let new_element = Element::Insert(Vec::from(&v[to_consume..]));
              to_consume = 0;
              Some(new_element)
            }
          }
        }
      };

      match new_first_element {
        None     => { self.queue.remove(0); },
        // FIXME: do we test for the presence of the first element here?
        Some(el) => { self.queue[0] = el; },
      };
    };

    println!("consumed: {}", consumed);
    self.buffer.consume(consumed);
    self.position += consumed;
  }

  pub fn next(&self) -> Option<&Element> {
    self.queue.get(0)
  }

  // FIXME: implement Write to get incoming data
  pub fn write(&mut self, input: &[u8]) {}
}

#[cfg(test)]
mod tests {
  use super::*;
  use network::buffer::Buffer;
  use nom::HexDisplay;

  #[test]
  fn consume() {
    let mut b = BufferQueue {
      position: 0,
      buffer:   Buffer::from_slice(b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"),
      queue:    vec!(
        Element::Slice(0, 3),
        Element::Insert(Vec::from(&b"test"[..])),
        Element::Slice(5, 10)
      )
    };

    assert_eq!(b.next_buffer(), Some(&b"ABC"[..]));
    b.consume(3);
    assert_eq!(b.position, 3);
    assert_eq!(b.queue, vec!(
        Element::Insert(Vec::from(&b"test"[..])),
        Element::Slice(5, 10))
    );

    assert_eq!(b.next_buffer(), Some(&b"test"[..]));
    b.consume(2);
    assert_eq!(b.position, 3);
    assert_eq!(b.queue, vec!(
        Element::Insert(Vec::from(&b"st"[..])),
        Element::Slice(5, 10))
    );

    assert_eq!(b.next_buffer(), Some(&b"st"[..]));
    b.consume(2);
    assert_eq!(b.position, 3);
    assert_eq!(b.queue, vec!(
        Element::Slice(5, 10))
    );

    assert_eq!(b.next_buffer(), Some(&b"FGHIJ"[..]));
    b.consume(2);
    assert_eq!(b.position, 5);
    assert_eq!(b.queue, vec!(Element::Slice(7, 10)));

    assert_eq!(b.next_buffer(), Some(&b"HIJ"[..]));
    b.consume(3);
    assert_eq!(b.position, 8);
    assert_eq!(b.queue, vec!());

    assert_eq!(b.next_buffer(), None);
  }
}
<commit_msg>better BufferQueue implementation<commit_after>use network::buffer::Buffer;
use pool::Reset;
use std::io::{self,Write};
use nom::HexDisplay;

#[derive(Debug,PartialEq,Clone)]
pub enum InputElement {
  /// length in the stream
  Slice(usize),
  Splice(usize), // x bytes copied in kernel
}

#[derive(Debug,PartialEq,Clone)]
pub enum OutputElement {
  /// length in the stream
  Slice(usize),
  Delete(usize),
  Insert(Vec<u8>),
  Splice(usize), // should copy x bytes from kernel to socket
}

/// The BufferQueue has two roles: holding incoming data, and indicating
/// which data will go out. When new data arrives, it is added at the
/// end of the internal buffer. This new data is then eventually parsed or
/// handled in some way by external code. The external code then adds
/// element to the queue, indicating what to do with the data:
///   - copy a subset of the input data (and advance if needed)
///   - insert external data, like a HTTP header
///   - splice out of the kernel some data that was spliced in
///
/// position is the index in the stream of data already handled.
/// it corresponds to the beginning of available data in the Buffer
/// a Slice(begin, end) would point to buffer.data()[begin-position..end-position]
/// (in the easiest case)
///
/// unparsed_position is the index in the stream of data that was
/// not parsed yet
///
/// The buffer's available data may be smaller than `end - begin`.
/// It can happen if the parser indicated we need to copy more data than is available,
/// like with a content length
///
/// should the buffer queue indicate how much data it needs?
//#[derive(Debug,PartialEq,Clone)]
pub struct BufferQueue {
  /// position of buffer start in stream
  pub buffer_position:        usize,
  pub parsed_position:        usize,
  pub output_position:        usize,
  pub start_parsing_position: usize,
  pub buffer:                 Buffer,
  /// Vec<(start, length)>
  pub input_queue:            Vec<InputElement>,
  pub output_queue:           Vec<OutputElement>,
}

impl BufferQueue {
  pub fn with_capacity(capacity: usize) -> BufferQueue {
    BufferQueue {
      buffer_position:        0,
      parsed_position:        0,
      output_position:        0,
      start_parsing_position: 0,
      buffer:                 Buffer::with_capacity(capacity),
      input_queue:            Vec::with_capacity(8),
      output_queue:           Vec::with_capacity(8),
    }
  }

  pub fn available_input_data(&self) -> usize {
    self.input_queue.iter().fold(0, |acc, el| {
      acc + match el {
        &InputElement::Slice(sz) | &InputElement::Splice(sz) => sz
      }
    })
  }

  pub fn spliced_input(&mut self, count: usize) {
    self.input_queue.push(InputElement::Splice(count));
  }

  pub fn merge_input_slices(&self) -> usize {
    let mut acc = 0usize;
    for el in self.input_queue.iter() {
      match el {
        &InputElement::Splice(_) => break,
        &InputElement::Slice(sz) => acc += sz,
      }
    }

    assert!(acc <= self.buffer.available_data(), "the merged input slices can't be larger than current data in buffer");
    acc
  }

  pub fn input_data_size(&self) -> usize {
    let mut acc = 0usize;
    for el in self.input_queue.iter() {
      match el {
        &InputElement::Splice(sz) => acc += sz,
        &InputElement::Slice(sz)  => acc += sz,
      }
    }
    acc
  }

  pub fn unparsed_data(&self) -> &[u8] {
    let largest_size = self.merge_input_slices();
    &self.buffer.data()[..largest_size]
  }

  /// should only be called with a count inferior to self.input_data_size()
  pub fn consume_parsed_data(&mut self, size: usize) {
    //FIXME: to_consume must contain unparsed_position - parsed_position ?
    let mut to_consume = size;
    while to_consume > 0 {
      let new_first_element = match self.input_queue.first() {
        None => {
          //assert!(to_consume == 0, "no more element in queue, we should not ask to consume {} more bytes", to_consume);
          break;
        },
        Some(&InputElement::Slice(sz)) => {
          if to_consume >= sz {
            to_consume -= sz;
            None
          } else {
            let new_element = InputElement::Slice(sz - to_consume);
            to_consume = 0;
            Some(new_element)
          }
        },
        Some(&InputElement::Splice(sz)) => {
          if to_consume >= sz {
            to_consume -= sz;
            None
          } else {
            panic!("we should not start parsing from inside a splicing buffer. But what if consume_parsed_data was called during a parsing loop? Should only call consume_parsed_data after the parsing loop finished");
          }
        },
      };

      match new_first_element {
        None     => { self.input_queue.remove(0); },
        Some(el) => { self.input_queue[0] = el; },
      };
    }

    self.parsed_position        += size - to_consume;
    self.start_parsing_position += size;
  }

  pub fn slice_output(&mut self, count: usize) {
    self.output_queue.push(OutputElement::Slice(count));
  }

  pub fn delete_output(&mut self, count: usize) {
    self.output_queue.push(OutputElement::Delete(count));
  }

  pub fn splice_output(&mut self, count: usize) {
    self.output_queue.push(OutputElement::Splice(count));
  }

  pub fn insert_output(&mut self, v: Vec<u8>) {
    self.output_queue.push(OutputElement::Insert(v));
  }

  pub fn output_data_size(&self) -> usize {
    let mut acc = 0usize;
    for el in self.output_queue.iter() {
      match el {
        &OutputElement::Splice(sz)    => acc += sz,
        &OutputElement::Slice(sz)     => acc += sz,
        &OutputElement::Insert(ref v) => acc += v.len(),
        &OutputElement::Delete(_)     => {},
      }
    }
    acc
  }

  pub fn merge_output_slices(&self) -> usize {
    let mut acc = 0usize;
    for el in self.output_queue.iter() {
      match el {
        &OutputElement::Slice(sz) => acc += sz,
        _ => break,
      }
    }

    assert!(acc <= self.buffer.available_data(), "the merged output slices can't be larger than current data in buffer");
    acc
  }

  pub fn merge_output_deletes(&self) -> usize {
    let mut acc = 0usize;
    for el in self.output_queue.iter() {
      match el {
        &OutputElement::Delete(sz) => acc += sz,
        _ => break,
      }
    }

    assert!(acc <= self.buffer.available_data(), "the merged output deletes can't be larger than current data in buffer");
    acc
  }

  pub fn next_output_data(&self) -> &[u8] {
    let mut it = self.output_queue.iter();
    //first, calculate how many bytes we need to jump
    let mut start         = 0usize;
    let mut largest_size  = 0usize;
    let mut delete_ended  = false;
    for el in it {
      println!("start={}, length={}, el = {:?}", start, largest_size, el);
      if !delete_ended {
        match el {
          &OutputElement::Delete(sz) => start += sz,
          _ => {
            delete_ended = true;
            match el {
              &OutputElement::Slice(sz)     => largest_size += sz,
              &OutputElement::Insert(ref v) => return &v[..],
              _ => break,
            }
          },
        }
      } else {
        match el {
          &OutputElement::Slice(sz) => largest_size += sz,
          _ => break,
        }
      }
    }

    println!("buffer data: {:?}", self.buffer.data());
    println!("calculated start={}, length={}", start, largest_size);
    &self.buffer.data()[start..start+largest_size]
  }

  /// should only be called with a count inferior to self.input_data_size()
  pub fn consume_output_data(&mut self, size: usize) {
    let mut to_consume = size;
    while to_consume > 0 {
      let new_first_element = match self.output_queue.first() {
        None => {
          assert!(to_consume == 0, "no more element in queue, we should not ask to consume {} more bytes", to_consume);
          break;
        },
        Some(&OutputElement::Slice(sz)) => {
          self.buffer_position += to_consume;
          self.buffer.consume(to_consume);

          if to_consume >= sz {
            to_consume -= sz;
            None
          } else {
            let new_element = OutputElement::Slice(sz - to_consume);
            to_consume = 0;
            Some(new_element)
          }
        },
        Some(&OutputElement::Delete(sz)) => {
          self.buffer_position += sz;
          //FIXME: what if we can't delete that much data?
          self.buffer.consume(sz);
          None
        },
        Some(&OutputElement::Splice(sz)) => {
          if to_consume >= sz {
            to_consume -= sz;
            None
          } else {
            let new_element = OutputElement::Splice(sz - to_consume);
            to_consume = 0;
            Some(new_element)
          }
        },
        Some(&OutputElement::Insert(ref v)) => {
          if to_consume >= v.len() {
            to_consume = to_consume - v.len();
            None
          } else {
            let new_element = OutputElement::Insert(Vec::from(&v[to_consume..]));
            to_consume = 0;
            Some(new_element)
          }
        },
      };

      match new_first_element {
        None     => { self.output_queue.remove(0); },
        Some(el) => { self.output_queue[0] = el; },
      };
    }
  }
}

impl Write for BufferQueue {
  fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
    match self.buffer.write(buf) {
      Err(e) => Err(e),
      Ok(sz) => {
        self.input_queue.push(InputElement::Slice(sz));
        Ok(sz)
      }
    }
  }

  fn flush(&mut self) -> io::Result<()> {
    Ok(())
  }
}

impl Reset for BufferQueue {
  fn reset(&mut self) {
    self.parsed_position = 0;
    self.output_position = 0;
    self.buffer_position = 0;
    self.start_parsing_position = 0;
    self.buffer.reset();
    self.input_queue.clear();
    self.output_queue.clear();
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use network::buffer::Buffer;
  use nom::HexDisplay;
  use std::io::Write;

  #[test]
  fn consume() {
    let mut b = BufferQueue {
      parsed_position:        0,
      output_position:        0,
      buffer_position:        0,
      start_parsing_position: 0,
      buffer:                 Buffer::from_slice(b"ABCDEFGHIJ"),
      input_queue:            vec!(InputElement::Slice(10)),
      output_queue:           vec!()
    };

    assert_eq!(b.unparsed_data(), &b"ABCDEFGHIJ"[..]);
    b.consume_parsed_data(4);
    assert_eq!(b.parsed_position, 4);
    assert_eq!(b.start_parsing_position, 4);
    assert_eq!(b.input_queue, vec!(InputElement::Slice(6)));

    b.slice_output(4);
    assert_eq!(b.output_queue, vec!(OutputElement::Slice(4)));

    b.insert_output(Vec::from(&b"test"[..]));
    assert_eq!(b.output_queue, vec!(
        OutputElement::Slice(4),
        OutputElement::Insert(Vec::from(&b"test"[..]))
      )
    );
    assert_eq!(b.next_output_data(), &b"ABCD"[..]);

    b.consume_output_data(2);
    assert_eq!(b.next_output_data(), &b"CD"[..]);

    b.consume_parsed_data(8);
    assert_eq!(b.parsed_position, 10);
    assert_eq!(b.start_parsing_position, 12);
    assert_eq!(b.input_queue, vec!());

    println!("**test**");
    b.consume_output_data(2);
    assert_eq!(b.next_output_data(), &b"test"[..]);
    b.consume_output_data(2);
    assert_eq!(b.next_output_data(), &b"st"[..]);

    b.delete_output(2);
    b.slice_output(4);
    assert_eq!(
      b.output_queue,
      vec!(
        OutputElement::Insert(Vec::from(&b"st"[..])),
        OutputElement::Delete(2),
        OutputElement::Slice(4)
      )
    );

    b.consume_output_data(2);
    assert_eq!(
      b.output_queue,
      vec!(
        OutputElement::Delete(2),
        OutputElement::Slice(4)
      )
    );
    assert_eq!(b.next_output_data(), &b"GHIJ"[..]);

    b.consume_output_data(1);
    assert_eq!(
      b.output_queue,
      vec!(
        OutputElement::Slice(3)
      )
    );
    assert_eq!(b.next_output_data(), &b"HIJ"[..]);

    b.write(&b"KLMNOP"[..]);
  }
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io
#![allow(
    clippy::unused_unit,
    clippy::needless_return,
    clippy::never_loop,
    clippy::option_map_unit_fn
)]

use crate::Error;
use osmpbfreader::{OsmId, OsmObj, StoreObjs};
use slog_scope::info;

#[cfg(feature = "db-storage")]
use crate::settings::osm2mimir::Database;

#[cfg(feature = "db-storage")]
use slog_scope::error;

#[cfg(feature = "db-storage")]
use std::fs;

#[cfg(feature = "db-storage")]
use std::path::PathBuf;

#[cfg(feature = "db-storage")]
use std::collections::HashMap;

use std::borrow::Cow;
use std::collections::BTreeMap;

use super::street::Kind;

#[cfg(feature = "db-storage")]
use rusqlite::{Connection, DropBehavior, ToSql, NO_PARAMS};

#[cfg(feature = "db-storage")]
macro_rules! err_logger {
    ($obj:expr, $err_msg:expr) => {
        match $obj {
            Ok(x) => Some(x),
            Err(e) => {
                error!("{}: {}", $err_msg, e);
                None
            }
        }?
    };
    ($obj:expr, $err_msg:expr, $ret:expr) => {
        match $obj {
            Ok(x) => x,
            Err(e) => {
                error!("{}: {}", $err_msg, e);
                return $ret;
            }
        }
    };
}

macro_rules! get_kind {
    ($obj:expr) => {
        if $obj.is_node() {
            &0
        } else if $obj.is_way() {
            &1
        } else if $obj.is_relation() {
            &2
        } else {
            panic!("Unknown OSM object kind!")
        }
    };
}

pub trait Getter {
    fn get(&self, key: &OsmId) -> Option<Cow<OsmObj>>;
}

impl Getter for BTreeMap<OsmId, OsmObj> {
    fn get(&self, key: &OsmId) -> Option<Cow<OsmObj>> {
        self.get(key).map(|x| Cow::Borrowed(x))
    }
}

#[cfg(feature = "db-storage")]
pub struct Db<'a> {
    conn: Connection,
    db_file: &'a PathBuf,
    buffer: HashMap<OsmId, OsmObj>,
    db_buffer_size: usize,
}

#[cfg(feature = "db-storage")]
impl<'a> Db<'a> {
    fn new(db_file: &'a PathBuf, db_buffer_size: usize) -> Result<Db<'a>, String> {
        let _ = fs::remove_file(db_file); // we ignore any potential error
        let conn = Connection::open(&db_file)
            .map_err(|e| format!("failed to open SQLITE connection: {}", e))?;

        conn.execute(
            "CREATE TABLE ids (
                id   INTEGER NOT NULL,
                obj  BLOB NOT NULL,
                kind INTEGER NOT NULL,
                UNIQUE(id, kind)
             )",
            NO_PARAMS,
        )
        .map_err(|e| format!("failed to create table: {}", e))?;
        Ok(Db {
            conn,
            db_file,
            buffer: HashMap::with_capacity(db_buffer_size),
            db_buffer_size,
        })
    }

    fn get_from_id(&self, id: &OsmId) -> Option<Cow<OsmObj>> {
        if let Some(obj) = self.buffer.get(id) {
            return Some(Cow::Borrowed(obj));
        }
        let mut stmt = err_logger!(
            self.conn
                .prepare("SELECT obj FROM ids WHERE id=?1 AND kind=?2"),
            "Db::get_from_id: prepare failed"
        );
        let mut iter = err_logger!(
            stmt.query(&[&id.inner_id() as &dyn ToSql, get_kind!(id)]),
            "Db::get_from_id: query_map failed"
        );
        while let Some(row) = err_logger!(iter.next(), "Db::get_from_id: next failed") {
            let obj: Vec<u8> = err_logger!(row.get(0), "Db::get_from_id: failed to get obj field");
            let obj: OsmObj = err_logger!(
                bincode::deserialize(&obj),
                "Db::for_each: serde conversion failed",
                None
            );
            return Some(Cow::Owned(obj));
        }
        None
    }

    #[allow(dead_code)]
    fn for_each<F: FnMut(Cow<OsmObj>)>(&self, mut f: F) {
        for value in self.buffer.values() {
            f(Cow::Borrowed(value));
        }
        let mut stmt = err_logger!(
            self.conn.prepare("SELECT obj FROM ids"),
            "Db::for_each: prepare failed",
            ()
        );
        let mut rows = err_logger!(stmt.query(NO_PARAMS), "Db::for_each: query_map failed", ());
        while let Some(row) = err_logger!(rows.next(), "Db::for_each: next failed", ()) {
            let obj: Vec<u8> = err_logger!(row.get(0), "Db::for_each: failed to get obj field", ());

            let obj: OsmObj = err_logger!(
                bincode::deserialize(&obj),
                "Db::for_each: serde conversion failed",
                ()
            );
            f(Cow::Owned(obj));
        }
    }

    fn for_each_filter<F: FnMut(Cow<OsmObj>)>(&self, filter: Kind, mut f: F) {
        self.buffer
            .values()
            .filter(|e| *get_kind!(e) == filter as i32)
            .for_each(|value| f(Cow::Borrowed(value)));
        let mut stmt = err_logger!(
            self.conn.prepare("SELECT obj FROM ids WHERE kind=?1"),
            "Db::for_each: prepare failed",
            ()
        );
        let mut rows = err_logger!(
            stmt.query(&[&(filter as i32) as &dyn ToSql]),
            "Db::for_each: query_map failed",
            ()
        );
        while let Some(row) = err_logger!(rows.next(), "Db::for_each: next failed", ()) {
            let obj: Vec<u8> = err_logger!(row.get(0), "Db::for_each: failed to get obj field", ());

            let obj: OsmObj = err_logger!(
                bincode::deserialize(&obj),
                "Db::for_each: serde conversion failed",
                ()
            );
            f(Cow::Owned(obj));
        }
    }

    fn flush_buffer(&mut self) {
        if self.buffer.is_empty() {
            return;
        }
        let mut tx = err_logger!(
            self.conn.transaction(),
            "Db::flush_buffer: transaction creation failed",
            ()
        );
        tx.set_drop_behavior(DropBehavior::Ignore);

        {
            let mut stmt = err_logger!(
                tx.prepare("INSERT OR IGNORE INTO ids(id, obj, kind) VALUES (?1, ?2, ?3)"),
                "Db::flush_buffer: prepare failed",
                ()
            );
            for (id, obj) in self.buffer.drain() {
                let ser_obj = match bincode::serialize(&obj) {
                    Ok(s) => s,
                    Err(e) => {
                        error!("Db::flush_buffer: failed to convert to json: {}", e);
                        continue;
                    }
                };
                let kind = get_kind!(obj);
                if let Err(e) = stmt.execute(&[&id.inner_id() as &dyn ToSql, &ser_obj, kind]) {
                    error!("Db::flush_buffer: insert failed: {}", e);
                }
            }
        }
        err_logger!(
            tx.commit(),
            "Db::flush_buffer: transaction commit failed",
            ()
        );
    }
}

#[cfg(feature = "db-storage")]
impl<'a> StoreObjs for Db<'a> {
    fn insert(&mut self, id: OsmId, obj: OsmObj) {
        if self.buffer.len() >= self.db_buffer_size {
            self.flush_buffer();
        }
        self.buffer.insert(id, obj);
    }

    fn contains_key(&self, id: &OsmId) -> bool {
        if self.buffer.contains_key(id) {
            return true;
        }
        let mut stmt = err_logger!(
            self.conn
                .prepare("SELECT id FROM ids WHERE id=?1 AND kind=?2"),
            "Db::contains_key: prepare failed",
            false
        );
        let mut iter = err_logger!(
            stmt.query(&[&id.inner_id() as &dyn ToSql, get_kind!(id)]),
            "Db::contains_key: query_map failed",
            false
        );
        err_logger!(iter.next(), "Db::contains_key: no row", false).is_some()
    }
}

#[cfg(feature = "db-storage")]
impl<'a> Getter for Db<'a> {
    fn get(&self, key: &OsmId) -> Option<Cow<OsmObj>> {
        self.get_from_id(key)
    }
}

#[cfg(feature = "db-storage")]
impl<'a> Drop for Db<'a> {
    fn drop(&mut self) {
        let _ = fs::remove_file(self.db_file); // we ignore any potential error
    }
}

#[cfg(feature = "db-storage")]
pub enum ObjWrapper<'a> {
    Map(BTreeMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>),
    Db(Db<'a>),
}

#[cfg(not(feature = "db-storage"))]
pub enum ObjWrapper {
    Map(BTreeMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>),
}

#[cfg(feature = "db-storage")]
impl<'a> ObjWrapper<'a> {
    pub fn new(db: &'a Option<Database>) -> Result<ObjWrapper<'a>, Error> {
        Ok(if let Some(ref db) = db {
            info!("Running with Db storage");
            ObjWrapper::Db(Db::new(&db.file, db.buffer_size).map_err(failure::err_msg)?)
        } else {
            info!("Running with BTreeMap (RAM) storage");
            ObjWrapper::Map(BTreeMap::new())
        })
    }

    #[allow(dead_code)]
    pub fn for_each<F: FnMut(Cow<OsmObj>)>(&self, mut f: F) {
        match *self {
            ObjWrapper::Map(ref m) => {
                for value in m.values() {
                    f(Cow::Borrowed(value));
                }
            }
            ObjWrapper::Db(ref db) => db.for_each(f),
        }
    }

    pub fn for_each_filter<F: FnMut(Cow<OsmObj>)>(&self, filter: Kind, mut f: F) {
        match *self {
            ObjWrapper::Map(ref m) => {
                m.values()
                    .filter(|e| *get_kind!(e) == filter as i32)
                    .for_each(|value| f(Cow::Borrowed(value)));
            }
            ObjWrapper::Db(ref db) => db.for_each_filter(filter, f),
        }
    }
}

#[cfg(feature = "db-storage")]
impl<'a> Getter for ObjWrapper<'a> {
    fn get(&self, key: &OsmId) -> Option<Cow<OsmObj>> {
        match *self {
            ObjWrapper::Map(ref m) => m.get(key).map(|x| Cow::Borrowed(x)),
            ObjWrapper::Db(ref db) => db.get(key),
        }
    }
}

#[cfg(feature = "db-storage")]
impl<'a> StoreObjs for ObjWrapper<'a> {
    fn insert(&mut self, id: OsmId, obj: OsmObj) {
        match *self {
            ObjWrapper::Map(ref mut m) => {
                m.insert(id, obj);
            }
            ObjWrapper::Db(ref mut db) => db.insert(id, obj),
        }
    }

    fn contains_key(&self, id: &OsmId) -> bool {
        match *self {
            ObjWrapper::Map(ref m) => m.contains_key(id),
            ObjWrapper::Db(ref db) => db.contains_key(id),
        }
    }
}

#[cfg(not(feature = "db-storage"))]
impl ObjWrapper {
    pub fn new() -> Result<ObjWrapper, Error> {
        info!("Running with BTreeMap (RAM) storage");
        Ok(ObjWrapper::Map(BTreeMap::new()))
    }

    #[allow(dead_code)]
    pub fn for_each<F: FnMut(Cow<OsmObj>)>(&self, mut f: F) {
        match *self {
            ObjWrapper::Map(ref m) => {
                for value in m.values() {
                    f(Cow::Borrowed(value));
                }
            }
        }
    }

    pub fn for_each_filter<F: FnMut(Cow<OsmObj>)>(&self, filter: Kind, mut f: F) {
        match *self {
            ObjWrapper::Map(ref m) => {
                m.values()
                    .filter(|e| *get_kind!(e) == filter as i32)
                    .for_each(|value| f(Cow::Borrowed(value)));
            }
        }
    }
}

#[cfg(not(feature = "db-storage"))]
impl Getter for ObjWrapper {
    fn get(&self, key: &OsmId) -> Option<Cow<OsmObj>> {
        match *self {
            ObjWrapper::Map(ref m) => m.get(key).map(|x| Cow::Borrowed(x)),
        }
    }
}

#[cfg(not(feature = "db-storage"))]
impl StoreObjs for ObjWrapper {
    fn insert(&mut self, id: OsmId, obj: OsmObj) {
        match *self {
            ObjWrapper::Map(ref mut m) => {
                m.insert(id, obj);
            }
        }
    }

    fn contains_key(&self, id: &OsmId) -> bool {
        match *self {
            ObjWrapper::Map(ref m) => m.contains_key(id),
        }
    }
}
<commit_msg>Fixed linting<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io
#![allow(
    clippy::unused_unit,
    clippy::needless_return,
    clippy::never_loop,
    clippy::option_map_unit_fn
)]

use crate::Error;
use osmpbfreader::{OsmId, OsmObj, StoreObjs};
use slog_scope::info;

#[cfg(feature = "db-storage")]
use crate::settings::osm2mimir::Database;

#[cfg(feature = "db-storage")]
use slog_scope::error;

#[cfg(feature = "db-storage")]
use std::fs;

#[cfg(feature = "db-storage")]
use std::path::Path;

#[cfg(feature = "db-storage")]
use std::collections::HashMap;

use std::borrow::Cow;
use std::collections::BTreeMap;

use super::street::Kind;

#[cfg(feature = "db-storage")]
use rusqlite::{Connection, DropBehavior, ToSql, NO_PARAMS};

#[cfg(feature = "db-storage")]
macro_rules! err_logger {
    ($obj:expr, $err_msg:expr) => {
        match $obj {
            Ok(x) => Some(x),
            Err(e) => {
                error!("{}: {}", $err_msg, e);
                None
            }
        }?
    };
    ($obj:expr, $err_msg:expr, $ret:expr) => {
        match $obj {
            Ok(x) => x,
            Err(e) => {
                error!("{}: {}", $err_msg, e);
                return $ret;
            }
        }
    };
}

macro_rules! get_kind {
    ($obj:expr) => {
        if $obj.is_node() {
            &0
        } else if $obj.is_way() {
            &1
        } else if $obj.is_relation() {
            &2
        } else {
            panic!("Unknown OSM object kind!")
        }
    };
}

pub trait Getter {
    fn get(&self, key: &OsmId) -> Option<Cow<OsmObj>>;
}

impl Getter for BTreeMap<OsmId, OsmObj> {
    fn get(&self, key: &OsmId) -> Option<Cow<OsmObj>> {
        self.get(key).map(|x| Cow::Borrowed(x))
    }
}

#[cfg(feature = "db-storage")]
pub struct Db<'a> {
    conn: Connection,
    db_file: &'a Path,
    buffer: HashMap<OsmId, OsmObj>,
    db_buffer_size: usize,
}

#[cfg(feature = "db-storage")]
impl<'a> Db<'a> {
    fn new(db_file: &'a Path, db_buffer_size: usize) -> Result<Db<'a>, String> {
        let _ = fs::remove_file(db_file); // we ignore any potential error
        let conn = Connection::open(&db_file)
            .map_err(|e| format!("failed to open SQLITE connection: {}", e))?;

        conn.execute(
            "CREATE TABLE ids (
                id   INTEGER NOT NULL,
                obj  BLOB NOT NULL,
                kind INTEGER NOT NULL,
                UNIQUE(id, kind)
             )",
            NO_PARAMS,
        )
        .map_err(|e| format!("failed to create table: {}", e))?;
        Ok(Db {
            conn,
            db_file,
            buffer: HashMap::with_capacity(db_buffer_size),
            db_buffer_size,
        })
    }

    fn get_from_id(&self, id: &OsmId) -> Option<Cow<OsmObj>> {
        if let Some(obj) = self.buffer.get(id) {
            return Some(Cow::Borrowed(obj));
        }
        let mut stmt = err_logger!(
            self.conn
                .prepare("SELECT obj FROM ids WHERE id=?1 AND kind=?2"),
            "Db::get_from_id: prepare failed"
        );
        let mut iter = err_logger!(
            stmt.query(&[&id.inner_id() as &dyn ToSql, get_kind!(id)]),
            "Db::get_from_id: query_map failed"
        );
        while let Some(row) = err_logger!(iter.next(), "Db::get_from_id: next failed") {
            let obj: Vec<u8> = err_logger!(row.get(0), "Db::get_from_id: failed to get obj field");
            let obj: OsmObj = err_logger!(
                bincode::deserialize(&obj),
                "Db::for_each: serde conversion failed",
                None
            );
            return Some(Cow::Owned(obj));
        }
        None
    }

    #[allow(dead_code)]
    fn for_each<F: FnMut(Cow<OsmObj>)>(&self, mut f: F) {
        for value in self.buffer.values() {
            f(Cow::Borrowed(value));
        }
        let mut stmt = err_logger!(
            self.conn.prepare("SELECT obj FROM ids"),
            "Db::for_each: prepare failed",
            ()
        );
        let mut rows = err_logger!(stmt.query(NO_PARAMS), "Db::for_each: query_map failed", ());
        while let Some(row) = err_logger!(rows.next(), "Db::for_each: next failed", ()) {
            let obj: Vec<u8> = err_logger!(row.get(0), "Db::for_each: failed to get obj field", ());

            let obj: OsmObj = err_logger!(
                bincode::deserialize(&obj),
                "Db::for_each: serde conversion failed",
                ()
            );
            f(Cow::Owned(obj));
        }
    }

    fn for_each_filter<F: FnMut(Cow<OsmObj>)>(&self, filter: Kind, mut f: F) {
        self.buffer
            .values()
            .filter(|e| *get_kind!(e) == filter as i32)
            .for_each(|value| f(Cow::Borrowed(value)));
        let mut stmt = err_logger!(
            self.conn.prepare("SELECT obj FROM ids WHERE kind=?1"),
            "Db::for_each: prepare failed",
            ()
        );
        let mut rows = err_logger!(
            stmt.query(&[&(filter as i32) as &dyn ToSql]),
            "Db::for_each: query_map failed",
            ()
        );
        while let Some(row) = err_logger!(rows.next(), "Db::for_each: next failed", ()) {
            let obj: Vec<u8> = err_logger!(row.get(0), "Db::for_each: failed to get obj field", ());

            let obj: OsmObj = err_logger!(
                bincode::deserialize(&obj),
                "Db::for_each: serde conversion failed",
                ()
            );
            f(Cow::Owned(obj));
        }
    }

    fn flush_buffer(&mut self) {
        if self.buffer.is_empty() {
            return;
        }
        let mut tx = err_logger!(
            self.conn.transaction(),
            "Db::flush_buffer: transaction creation failed",
            ()
        );
        tx.set_drop_behavior(DropBehavior::Ignore);

        {
            let mut stmt = err_logger!(
                tx.prepare("INSERT OR IGNORE INTO ids(id, obj, kind) VALUES (?1, ?2, ?3)"),
                "Db::flush_buffer: prepare failed",
                ()
            );
            for (id, obj) in self.buffer.drain() {
                let ser_obj = match bincode::serialize(&obj) {
                    Ok(s) => s,
                    Err(e) => {
                        error!("Db::flush_buffer: failed to convert to json: {}", e);
                        continue;
                    }
                };
                let kind = get_kind!(obj);
                if let Err(e) = stmt.execute(&[&id.inner_id() as &dyn ToSql, &ser_obj, kind]) {
                    error!("Db::flush_buffer: insert failed: {}", e);
                }
            }
        }
        err_logger!(
            tx.commit(),
            "Db::flush_buffer: transaction commit failed",
            ()
        );
    }
}

#[cfg(feature = "db-storage")]
impl<'a> StoreObjs for Db<'a> {
    fn insert(&mut self, id: OsmId, obj: OsmObj) {
        if self.buffer.len() >= self.db_buffer_size {
            self.flush_buffer();
        }
        self.buffer.insert(id, obj);
    }

    fn contains_key(&self, id: &OsmId) -> bool {
        if self.buffer.contains_key(id) {
            return true;
        }
        let mut stmt = err_logger!(
            self.conn
                .prepare("SELECT id FROM ids WHERE id=?1 AND kind=?2"),
            "Db::contains_key: prepare failed",
            false
        );
        let mut iter = err_logger!(
            stmt.query(&[&id.inner_id() as &dyn ToSql, get_kind!(id)]),
            "Db::contains_key: query_map failed",
            false
        );
        err_logger!(iter.next(), "Db::contains_key: no row", false).is_some()
    }
}

#[cfg(feature = "db-storage")]
impl<'a> Getter for Db<'a> {
    fn get(&self, key: &OsmId) -> Option<Cow<OsmObj>> {
        self.get_from_id(key)
    }
}

#[cfg(feature = "db-storage")]
impl<'a> Drop for Db<'a> {
    fn drop(&mut self) {
        let _ = fs::remove_file(self.db_file); // we ignore any potential error
    }
}

#[cfg(feature = "db-storage")]
pub enum ObjWrapper<'a> {
    Map(BTreeMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>),
    Db(Db<'a>),
}

#[cfg(not(feature = "db-storage"))]
pub enum ObjWrapper {
    Map(BTreeMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>),
}

#[cfg(feature = "db-storage")]
impl<'a> ObjWrapper<'a> {
    pub fn new(db: &'a Option<Database>) -> Result<ObjWrapper<'a>, Error> {
        Ok(if let Some(ref db) = db {
            info!("Running with Db storage");
            ObjWrapper::Db(Db::new(&db.file, db.buffer_size).map_err(failure::err_msg)?)
        } else {
            info!("Running with BTreeMap (RAM) storage");
            ObjWrapper::Map(BTreeMap::new())
        })
    }

    #[allow(dead_code)]
    pub fn for_each<F: FnMut(Cow<OsmObj>)>(&self, mut f: F) {
        match *self {
            ObjWrapper::Map(ref m) => {
                for value in m.values() {
                    f(Cow::Borrowed(value));
                }
            }
            ObjWrapper::Db(ref db) => db.for_each(f),
        }
    }

    pub fn for_each_filter<F: FnMut(Cow<OsmObj>)>(&self, filter: Kind, mut f: F) {
        match *self {
            ObjWrapper::Map(ref m) => {
                m.values()
                    .filter(|e| *get_kind!(e) == filter as i32)
                    .for_each(|value| f(Cow::Borrowed(value)));
            }
            ObjWrapper::Db(ref db) => db.for_each_filter(filter, f),
        }
    }
}

#[cfg(feature = "db-storage")]
impl<'a> Getter for ObjWrapper<'a> {
    fn get(&self, key: &OsmId) -> Option<Cow<OsmObj>> {
        match *self {
            ObjWrapper::Map(ref m) => m.get(key).map(|x| Cow::Borrowed(x)),
            ObjWrapper::Db(ref db) => db.get(key),
        }
    }
}

#[cfg(feature = "db-storage")]
impl<'a> StoreObjs for ObjWrapper<'a> {
    fn insert(&mut self, id: OsmId, obj: OsmObj) {
        match *self {
            ObjWrapper::Map(ref mut m) => {
                m.insert(id, obj);
            }
            ObjWrapper::Db(ref mut db) => db.insert(id, obj),
        }
    }

    fn contains_key(&self, id: &OsmId) -> bool {
        match *self {
            ObjWrapper::Map(ref m) => m.contains_key(id),
            ObjWrapper::Db(ref db) => db.contains_key(id),
        }
    }
}

#[cfg(not(feature = "db-storage"))]
impl ObjWrapper {
    pub fn new() -> Result<ObjWrapper, Error> {
        info!("Running with BTreeMap (RAM) storage");
        Ok(ObjWrapper::Map(BTreeMap::new()))
    }

    #[allow(dead_code)]
    pub fn for_each<F: FnMut(Cow<OsmObj>)>(&self, mut f: F) {
        match *self {
            ObjWrapper::Map(ref m) => {
                for value in m.values() {
                    f(Cow::Borrowed(value));
                }
            }
        }
    }

    pub fn for_each_filter<F: FnMut(Cow<OsmObj>)>(&self, filter: Kind, mut f: F) {
        match *self {
            ObjWrapper::Map(ref m) => {
                m.values()
                    .filter(|e| *get_kind!(e) == filter as i32)
                    .for_each(|value| f(Cow::Borrowed(value)));
            }
        }
    }
}

#[cfg(not(feature = "db-storage"))]
impl Getter for ObjWrapper {
    fn get(&self, key: &OsmId) -> Option<Cow<OsmObj>> {
        match *self {
            ObjWrapper::Map(ref m) => m.get(key).map(|x| Cow::Borrowed(x)),
        }
    }
}

#[cfg(not(feature = "db-storage"))]
impl StoreObjs for ObjWrapper {
    fn insert(&mut self, id: OsmId, obj: OsmObj) {
        match *self {
            ObjWrapper::Map(ref mut m) => {
                m.insert(id, obj);
            }
        }
    }

    fn contains_key(&self, id: &OsmId) -> bool {
        match *self {
            ObjWrapper::Map(ref m) => m.contains_key(id),
        }
    }
}
<|endoftext|>


<commit_before>extern crate char_iter;
#[macro_use]
extern crate cripes;

mod test_util;
mod test_pattern;
<commit_msg>test/mod.rs: add missing `inclusive_range_syntax` feature decl.<commit_after>#![feature(inclusive_range_syntax)]

extern crate char_iter;
#[macro_use]
extern crate cripes;

mod test_util;
mod test_pattern;
<|endoftext|>


<commit_before>//! Shader module.
//!
//! Shader functions and declarations can be grouped in so-called *modules*. Modules structure is
//! inherently tied to the filesystem’s tree.
//!
//! You’re not supposed to use modules at the Rust level, even though you can. You’re supposed to
//! actually write modules that will be used by shader programs.

use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};

use render::shader::lang::parser;
use render::shader::lang::syntax::{Module as SyntaxModule, ModulePath};
use sys::resource::{CacheKey, Load, LoadError, LoadResult, Store, StoreKey};

/// Shader module.
///
/// A shader module is a piece of GLSL code with optional import lists (dependencies).
///
/// You’re not supposed to directly manipulate any object of this type. You just write modules on
/// disk and let everyting happen automatically for you.
#[derive(Clone, Debug, PartialEq)]
pub struct Module(SyntaxModule);

impl Module {
  /// Retrieve all the modules this module depends on, without duplicates.
  pub fn deps(&self, store: &mut Store, key: &ModuleKey) -> Result<Vec<ModuleKey>, DepsError> {
    let mut deps = Vec::new();
    self.deps_no_cycle(store, &key, &mut Vec::new(), &mut deps).map(|_| deps)
  }

  fn deps_no_cycle(&self, store: &mut Store, key: &ModuleKey, parents: &mut Vec<ModuleKey>, deps: &mut Vec<ModuleKey>) -> Result<(), DepsError> {
    let imports = self.0.imports.iter().map(|il| &il.module);

    parents.push(key.clone());

    for module_path in imports {
      let module_key = ModuleKey(module_path.path.join("."));

      // check whether it’s already in the deps
      if deps.contains(&module_key) {
        continue;
      }

      // check whether the module was already visited
      if parents.contains(&module_key) {
        return Err(DepsError::Cycle(module_key.clone(), module_key.clone()));
      }

      // get the dependency module
      let module = store.get(&module_key).ok_or_else(|| DepsError::LoadError(module_key.clone()))?;
      let r = module.borrow().deps_no_cycle(store, &module_key, parents, deps)?;

      deps.push(module_key.clone());
      parents.pop();
    }

    Ok(())
  }
}

/// Class of errors that can happen in dependencies.
#[derive(Clone, Debug, PartialEq)]
pub enum DepsError {
  /// If a module’s dependencies has any cycle, the dependencies are unusable and the cycle is
  /// returned.
  Cycle(ModuleKey, ModuleKey),
  /// There was a loading error of a module.
  LoadError(ModuleKey)
}

#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct ModuleKey(String);

impl ModuleKey {
  pub fn new(key: &str) -> Self {
    ModuleKey(key.to_owned())
  }
}

impl CacheKey for ModuleKey {
  type Target = Module;
}

impl StoreKey for ModuleKey {
  fn key_to_path(&self) -> PathBuf {
    PathBuf::from(self.0.replace(".", "/") + ".spsl")
  }
}

impl Load for Module {
  fn load<P>(path: P, _: &mut Store) -> Result<LoadResult<Self>, LoadError> where P: AsRef<Path> {
    let path = path.as_ref();

    let mut fh = File::open(path).map_err(|_| LoadError::FileNotFound(path.to_owned()))?;
    let mut src = String::new();
    let _ = fh.read_to_string(&mut src);

    match parser::parse_str(&src[..], parser::module) {
      parser::ParseResult::Ok(module) => {
        Ok(Module(module).into())
      }
      parser::ParseResult::Err(e) => Err(LoadError::ConversionFailed(format!("{:?}", e))),
      _ => Err(LoadError::ConversionFailed("incomplete input".to_owned()))
    }
  }
}

<commit_msg>Module::gather implemented.<commit_after>//! Shader module.
//!
//! Shader functions and declarations can be grouped in so-called *modules*. Modules structure is
//! inherently tied to the filesystem’s tree.
//!
//! You’re not supposed to use modules at the Rust level, even though you can. You’re supposed to
//! actually write modules that will be used by shader programs.

use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};

use render::shader::lang::parser;
use render::shader::lang::syntax::Module as SyntaxModule;
use sys::resource::{CacheKey, Load, LoadError, LoadResult, Store, StoreKey};

/// Shader module.
///
/// A shader module is a piece of GLSL code with optional import lists (dependencies).
///
/// You’re not supposed to directly manipulate any object of this type. You just write modules on
/// disk and let everyting happen automatically for you.
#[derive(Clone, Debug, PartialEq)]
pub struct Module(pub SyntaxModule); // FIXME: remove the pub

impl Module {
  /// Retrieve all the modules this module depends on, without duplicates.
  pub fn deps(&self, store: &mut Store, key: &ModuleKey) -> Result<Vec<ModuleKey>, DepsError> {
    let mut deps = Vec::new();
    self.deps_no_cycle(store, &key, &mut Vec::new(), &mut deps).map(|_| deps)
  }

  fn deps_no_cycle(&self, store: &mut Store, key: &ModuleKey, parents: &mut Vec<ModuleKey>, deps: &mut Vec<ModuleKey>) -> Result<(), DepsError> {
    let imports = self.0.imports.iter().map(|il| &il.module);

    parents.push(key.clone());

    for module_path in imports {
      let module_key = ModuleKey(module_path.path.join("."));

      // check whether it’s already in the deps
      if deps.contains(&module_key) {
        continue;
      }

      // check whether the module was already visited
      if parents.contains(&module_key) {
        return Err(DepsError::Cycle(module_key.clone(), module_key.clone()));
      }

      // get the dependency module
      let module = store.get(&module_key).ok_or_else(|| DepsError::LoadError(module_key.clone()))?;
      module.borrow().deps_no_cycle(store, &module_key, parents, deps)?;

      deps.push(module_key.clone());
      parents.pop();
    }

    Ok(())
  }

  /// Fold a module and its dependencies into a single module. The list of dependencies is also
  /// returned.
  pub fn gather(&self, store: &mut Store, k: &ModuleKey) -> Result<(Self, Vec<ModuleKey>), DepsError> {
    let deps = self.deps(store, k)?;
    let glsl =
      deps.iter()
          .flat_map(|kd| {
              let m = store.get(kd).unwrap();
              let g = m.borrow().0.glsl.clone();
              g
            })
          .chain(self.0.glsl.clone())
          .collect();

    let module = Module(SyntaxModule {
      imports: Vec::new(),
      glsl
    });

    Ok((module, deps))
  }
}

/// Class of errors that can happen in dependencies.
#[derive(Clone, Debug, PartialEq)]
pub enum DepsError {
  /// If a module’s dependencies has any cycle, the dependencies are unusable and the cycle is
  /// returned.
  Cycle(ModuleKey, ModuleKey),
  /// There was a loading error of a module.
  LoadError(ModuleKey)
}

#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct ModuleKey(String);

impl ModuleKey {
  pub fn new(key: &str) -> Self {
    ModuleKey(key.to_owned())
  }
}

impl CacheKey for ModuleKey {
  type Target = Module;
}

impl StoreKey for ModuleKey {
  fn key_to_path(&self) -> PathBuf {
    PathBuf::from(self.0.replace(".", "/") + ".spsl")
  }
}

impl Load for Module {
  fn load<P>(path: P, _: &mut Store) -> Result<LoadResult<Self>, LoadError> where P: AsRef<Path> {
    let path = path.as_ref();

    let mut fh = File::open(path).map_err(|_| LoadError::FileNotFound(path.to_owned()))?;
    let mut src = String::new();
    let _ = fh.read_to_string(&mut src);

    match parser::parse_str(&src[..], parser::module) {
      parser::ParseResult::Ok(module) => {
        Ok(Module(module).into())
      }
      parser::ParseResult::Err(e) => Err(LoadError::ConversionFailed(format!("{:?}", e))),
      _ => Err(LoadError::ConversionFailed("incomplete input".to_owned()))
    }
  }
}

<|endoftext|>


<commit_before>//! Database used for testing `hir`.

use std::{
    fmt, panic,
    sync::{Arc, Mutex},
};

use base_db::{
    salsa, AnchoredPath, CrateId, FileId, FileLoader, FileLoaderDelegate, SourceDatabase, Upcast,
};
use hir_def::{db::DefDatabase, ModuleId};
use hir_expand::db::AstDatabase;
use rustc_hash::{FxHashMap, FxHashSet};
use syntax::TextRange;
use test_utils::extract_annotations;

#[salsa::database(
    base_db::SourceDatabaseExtStorage,
    base_db::SourceDatabaseStorage,
    hir_expand::db::AstDatabaseStorage,
    hir_def::db::InternDatabaseStorage,
    hir_def::db::DefDatabaseStorage,
    crate::db::HirDatabaseStorage
)]
#[derive(Default)]
pub(crate) struct TestDB {
    storage: salsa::Storage<TestDB>,
    events: Mutex<Option<Vec<salsa::Event>>>,
}
impl fmt::Debug for TestDB {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("TestDB").finish()
    }
}

impl Upcast<dyn AstDatabase> for TestDB {
    fn upcast(&self) -> &(dyn AstDatabase + 'static) {
        &*self
    }
}

impl Upcast<dyn DefDatabase> for TestDB {
    fn upcast(&self) -> &(dyn DefDatabase + 'static) {
        &*self
    }
}

impl salsa::Database for TestDB {
    fn salsa_event(&self, event: salsa::Event) {
        let mut events = self.events.lock().unwrap();
        if let Some(events) = &mut *events {
            events.push(event);
        }
    }
}

impl salsa::ParallelDatabase for TestDB {
    fn snapshot(&self) -> salsa::Snapshot<TestDB> {
        salsa::Snapshot::new(TestDB {
            storage: self.storage.snapshot(),
            events: Default::default(),
        })
    }
}

impl panic::RefUnwindSafe for TestDB {}

impl FileLoader for TestDB {
    fn file_text(&self, file_id: FileId) -> Arc<String> {
        FileLoaderDelegate(self).file_text(file_id)
    }
    fn resolve_path(&self, path: AnchoredPath) -> Option<FileId> {
        FileLoaderDelegate(self).resolve_path(path)
    }
    fn relevant_crates(&self, file_id: FileId) -> Arc<FxHashSet<CrateId>> {
        FileLoaderDelegate(self).relevant_crates(file_id)
    }
}

impl TestDB {
    pub(crate) fn module_for_file(&self, file_id: FileId) -> ModuleId {
        for &krate in self.relevant_crates(file_id).iter() {
            let crate_def_map = self.crate_def_map(krate);
            for (local_id, data) in crate_def_map.modules() {
                if data.origin.file_id() == Some(file_id) {
                    return crate_def_map.module_id(local_id);
                }
            }
        }
        panic!("Can't find module for file")
    }

    pub(crate) fn extract_annotations(&self) -> FxHashMap<FileId, Vec<(TextRange, String)>> {
        let mut files = Vec::new();
        let crate_graph = self.crate_graph();
        for krate in crate_graph.iter() {
            let crate_def_map = self.crate_def_map(krate);
            for (module_id, _) in crate_def_map.modules() {
                let file_id = crate_def_map[module_id].origin.file_id();
                files.extend(file_id)
            }
        }
        files
            .into_iter()
            .filter_map(|file_id| {
                let text = self.file_text(file_id);
                let annotations = extract_annotations(&text);
                if annotations.is_empty() {
                    return None;
                }
                Some((file_id, annotations))
            })
            .collect()
    }
}

impl TestDB {
    pub(crate) fn log(&self, f: impl FnOnce()) -> Vec<salsa::Event> {
        *self.events.lock().unwrap() = Some(Vec::new());
        f();
        self.events.lock().unwrap().take().unwrap()
    }

    pub(crate) fn log_executed(&self, f: impl FnOnce()) -> Vec<String> {
        let events = self.log(f);
        events
            .into_iter()
            .filter_map(|e| match e.kind {
                // This pretty horrible, but `Debug` is the only way to inspect
                // QueryDescriptor at the moment.
                salsa::EventKind::WillExecute { database_key } => {
                    Some(format!("{:?}", database_key.debug(self)))
                }
                _ => None,
            })
            .collect()
    }
}
<commit_msg>Set enable_proc_attr_macros in hir_ty TestDB<commit_after>//! Database used for testing `hir`.

use std::{
    fmt, panic,
    sync::{Arc, Mutex},
};

use base_db::{
    salsa, AnchoredPath, CrateId, FileId, FileLoader, FileLoaderDelegate, SourceDatabase, Upcast,
};
use hir_def::{db::DefDatabase, ModuleId};
use hir_expand::db::AstDatabase;
use rustc_hash::{FxHashMap, FxHashSet};
use syntax::TextRange;
use test_utils::extract_annotations;

#[salsa::database(
    base_db::SourceDatabaseExtStorage,
    base_db::SourceDatabaseStorage,
    hir_expand::db::AstDatabaseStorage,
    hir_def::db::InternDatabaseStorage,
    hir_def::db::DefDatabaseStorage,
    crate::db::HirDatabaseStorage
)]
pub(crate) struct TestDB {
    storage: salsa::Storage<TestDB>,
    events: Mutex<Option<Vec<salsa::Event>>>,
}

impl Default for TestDB {
    fn default() -> Self {
        let mut this = Self { storage: Default::default(), events: Default::default() };
        this.set_enable_proc_attr_macros(true);
        this
    }
}

impl fmt::Debug for TestDB {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("TestDB").finish()
    }
}

impl Upcast<dyn AstDatabase> for TestDB {
    fn upcast(&self) -> &(dyn AstDatabase + 'static) {
        &*self
    }
}

impl Upcast<dyn DefDatabase> for TestDB {
    fn upcast(&self) -> &(dyn DefDatabase + 'static) {
        &*self
    }
}

impl salsa::Database for TestDB {
    fn salsa_event(&self, event: salsa::Event) {
        let mut events = self.events.lock().unwrap();
        if let Some(events) = &mut *events {
            events.push(event);
        }
    }
}

impl salsa::ParallelDatabase for TestDB {
    fn snapshot(&self) -> salsa::Snapshot<TestDB> {
        salsa::Snapshot::new(TestDB {
            storage: self.storage.snapshot(),
            events: Default::default(),
        })
    }
}

impl panic::RefUnwindSafe for TestDB {}

impl FileLoader for TestDB {
    fn file_text(&self, file_id: FileId) -> Arc<String> {
        FileLoaderDelegate(self).file_text(file_id)
    }
    fn resolve_path(&self, path: AnchoredPath) -> Option<FileId> {
        FileLoaderDelegate(self).resolve_path(path)
    }
    fn relevant_crates(&self, file_id: FileId) -> Arc<FxHashSet<CrateId>> {
        FileLoaderDelegate(self).relevant_crates(file_id)
    }
}

impl TestDB {
    pub(crate) fn module_for_file(&self, file_id: FileId) -> ModuleId {
        for &krate in self.relevant_crates(file_id).iter() {
            let crate_def_map = self.crate_def_map(krate);
            for (local_id, data) in crate_def_map.modules() {
                if data.origin.file_id() == Some(file_id) {
                    return crate_def_map.module_id(local_id);
                }
            }
        }
        panic!("Can't find module for file")
    }

    pub(crate) fn extract_annotations(&self) -> FxHashMap<FileId, Vec<(TextRange, String)>> {
        let mut files = Vec::new();
        let crate_graph = self.crate_graph();
        for krate in crate_graph.iter() {
            let crate_def_map = self.crate_def_map(krate);
            for (module_id, _) in crate_def_map.modules() {
                let file_id = crate_def_map[module_id].origin.file_id();
                files.extend(file_id)
            }
        }
        files
            .into_iter()
            .filter_map(|file_id| {
                let text = self.file_text(file_id);
                let annotations = extract_annotations(&text);
                if annotations.is_empty() {
                    return None;
                }
                Some((file_id, annotations))
            })
            .collect()
    }
}

impl TestDB {
    pub(crate) fn log(&self, f: impl FnOnce()) -> Vec<salsa::Event> {
        *self.events.lock().unwrap() = Some(Vec::new());
        f();
        self.events.lock().unwrap().take().unwrap()
    }

    pub(crate) fn log_executed(&self, f: impl FnOnce()) -> Vec<String> {
        let events = self.log(f);
        events
            .into_iter()
            .filter_map(|e| match e.kind {
                // This pretty horrible, but `Debug` is the only way to inspect
                // QueryDescriptor at the moment.
                salsa::EventKind::WillExecute { database_key } => {
                    Some(format!("{:?}", database_key.debug(self)))
                }
                _ => None,
            })
            .collect()
    }
}
<|endoftext|>


<commit_before>use crate::ty::BuiltinType;

pub static PARAM_OFFSET: i32 = 16;

pub fn next_param_offset(param_offset: i32, _: BuiltinType) -> i32 {
    param_offset + 8
}
<commit_msg>fix build on arm64<commit_after>use crate::ty::SourceType;

pub static PARAM_OFFSET: i32 = 16;

pub fn next_param_offset(param_offset: i32, _: SourceType) -> i32 {
    param_offset + 8
}
<|endoftext|>