<commit_before>#![feature(alloc)]
#![feature(const_fn)]
#![feature(heap_api)]
#![feature(libc)]
#![feature(raw)]

///
/// # An overview of Execution Operators
///
/// ## Our goals
///  * Maximize CPU efficiency
///    * less branch misprediction
///    * less function call overhead
///    * more CPU pipelining
///  * Maximize memory bandwidth (less memory copy)
///
/// # Execution Model
///
/// ## Columnar and Vectorized Processing
///
/// ## Input and output characteristics of operators
///  * Input source - no input / output page required / reuse read buffer itself if possible.
///  * Filter       - Just bypass the page and only sets the selected rows IDs to the page.
///  * Others       - Both input and output pages are required.
///

extern crate alloc;
extern crate itertools;
extern crate libc;

extern crate common;
extern crate plan;
extern crate storage;

pub mod driver;
pub mod split;
pub mod task;
pub mod planner;
pub mod processor;

pub mod filter;
pub mod hash_join;
pub mod scan;

use std::rc::Rc;

use common::err::{Void, Result, Error};
use common::input::InputSource;
use common::rows::{Page, PageBuilder};
use common::types::Ty;

use driver::DriverContext;

pub trait Executor
{
  fn init      (&mut self) -> Void;
  fn need_input(&self) -> bool;
  fn add_input(&mut self, &Page) -> Void;
  fn next(&mut self) -> Result<&Page>;
  fn close     (&mut self) -> Void;
}

pub trait ExecutorFactory
{
  fn create(&self, ctx: &DriverContext) -> Option<Box<Executor>>;

  fn types(&self) -> &Vec<Ty>;
}

pub trait Processor
{
  fn process(
    &self,
    input: &Page,
    start_pos: usize,
    end_pos: usize,
    builder: &mut PageBuilder) -> Void;
}

/*
#[test]
pub fn test_pipeline() {
  let page_builder: Box<PageBuilder> = Box::new(DefaultPageBuilder);

  let input_page = page_builder.create(&vec![
    Box::new(Int4Type),
    Box::new(Float4Type),
  ]);

  let mut output_page = page_builder.create(&vec![
    Box::new(Int4Type),
    Box::new(Float4Type),
  ]);
}
*/<commit_msg>Fixed compilation warnings of exec module.<commit_after>#![feature(alloc)]
#![feature(const_fn)]
#![feature(heap_api)]
#![feature(libc)]
#![feature(raw)]

///
/// # An overview of Execution Operators
///
/// ## Our goals
///  * Maximize CPU efficiency
///    * less branch misprediction
///    * less function call overhead
///    * more CPU pipelining
///  * Maximize memory bandwidth (less memory copy)
///
/// # Execution Model
///
/// ## Columnar and Vectorized Processing
///
/// ## Input and output characteristics of operators
///  * Input source - no input / output page required / reuse read buffer itself if possible.
///  * Filter       - Just bypass the page and only sets the selected rows IDs to the page.
///  * Others       - Both input and output pages are required.
///

extern crate alloc;
extern crate itertools;
extern crate libc;

extern crate common;
extern crate plan;
extern crate storage;

pub mod driver;
pub mod split;
pub mod task;
pub mod planner;
pub mod processor;

pub mod filter;
pub mod hash_join;
pub mod scan;

use common::err::{Void, Result};
use common::rows::{Page, PageBuilder};
use common::types::Ty;

use driver::DriverContext;

pub trait Executor
{
  fn init      (&mut self) -> Void;
  fn need_input(&self) -> bool;
  fn add_input(&mut self, &Page) -> Void;
  fn next(&mut self) -> Result<&Page>;
  fn close     (&mut self) -> Void;
}

pub trait ExecutorFactory
{
  fn create(&self, ctx: &DriverContext) -> Option<Box<Executor>>;

  fn types(&self) -> &Vec<Ty>;
}

pub trait Processor
{
  fn process(
    &self,
    input: &Page,
    start_pos: usize,
    end_pos: usize,
    builder: &mut PageBuilder) -> Void;
}

/*
#[test]
pub fn test_pipeline() {
  let page_builder: Box<PageBuilder> = Box::new(DefaultPageBuilder);

  let input_page = page_builder.create(&vec![
    Box::new(Int4Type),
    Box::new(Float4Type),
  ]);

  let mut output_page = page_builder.create(&vec![
    Box::new(Int4Type),
    Box::new(Float4Type),
  ]);
}
*/<|endoftext|>


<commit_before>//! Compile-time generated maps and sets.
//!
//! The `phf::Map` and `phf::Set` types have roughly comparable performance to
//! a standard hash table, but can be generated as compile-time static values.
//!
//! # Usage
//!
//! If the `macros` Cargo feature is enabled, the `phf_map`, `phf_set`,
//! `phf_ordered_map`, and `phf_ordered_set` macros can be used to construct
//! the PHF type. This method can be used with a stable compiler
//! (`rustc` version 1.30+)
//!
//! ```toml
//! [dependencies]
//! phf = { version = "0.7.24", features = ["macros"] }
//! ```
//!
//! ```
//! use phf::{phf_map, phf_set};
//!
//! static MY_MAP: phf::Map<&'static str, u32> = phf_map! {
//!     "hello" => 1,
//!     "world" => 2,
//! };
//!
//! static MY_SET: phf::Set<&'static str> = phf_set! {
//!     "hello world",
//!     "hola mundo",
//! };
//!
//! fn main() {
//!     assert_eq!(MY_MAP["hello"], 1);
//!     assert!(MY_SET.contains("hello world"));
//! }
//! ```
//!
//! (Alternatively, you can use the phf_codegen crate to generate PHF datatypes
//! in a build script)
#![doc(html_root_url="https://docs.rs/phf/0.7")]
#![warn(missing_docs)]
#![cfg_attr(not(feature = "std"), no_std)]

#[cfg(feature = "std")]
extern crate std as core;

#[cfg(feature = "macros")]
/// Macro to create a `static` (compile-time) [`Map`].
///
/// Requires the `"macros"` feature.
///
/// # Example
///
/// ```rust,edition2018
/// use ::phf::{phf_map, Map};
///
/// static MY_MAP: Map<&'static str, u32> = phf_map! {
///     "hello" => 1,
///     "world" => 2,
/// };
///
/// fn main ()
/// {
///     assert_eq!(MY_MAP["hello"], 1);
/// }
/// ```
#[::proc_macro_hack::proc_macro_hack]
pub use phf_macros:: phf_map;

#[cfg(feature = "macros")]
/// Macro to create a `static` (compile-time) [`Set`].
///
/// Requires the `"macros"` feature.
///
/// # Example
///
/// ```rust,edition2018
/// use ::phf::{phf_set, Set};
///
/// static MY_SET: Set<&'static str> = phf_set! {
///     "hello world",
///     "hola mundo",
/// };
///
/// fn main ()
/// {
///     assert!(MY_SET.contains("hello world"));
/// }
/// ```
#[::proc_macro_hack::proc_macro_hack]
pub use phf_macros::phf_set;

use core::ops::Deref;

pub use phf_shared::PhfHash;
#[doc(inline)]
pub use self::map::Map;
#[doc(inline)]
pub use self::set::Set;

pub mod map;
pub mod set;

// WARNING: this is not considered part of phf's public API and is subject to
// change at any time.
//
// Basically Cow, but with the Owned version conditionally compiled
#[doc(hidden)]
pub enum Slice<T: 'static> {
    Static(&'static [T]),
    #[cfg(feature = "std")]
    Dynamic(Vec<T>),
}

impl<T> Deref for Slice<T> {
    type Target = [T];

    fn deref(&self) -> &[T] {
        match *self {
            Slice::Static(t) => t,
            #[cfg(feature = "std")]
            Slice::Dynamic(ref t) => t,
        }
    }
}
<commit_msg>Fix style in doc<commit_after>//! Compile-time generated maps and sets.
//!
//! The `phf::Map` and `phf::Set` types have roughly comparable performance to
//! a standard hash table, but can be generated as compile-time static values.
//!
//! # Usage
//!
//! If the `macros` Cargo feature is enabled, the `phf_map`, `phf_set`,
//! `phf_ordered_map`, and `phf_ordered_set` macros can be used to construct
//! the PHF type. This method can be used with a stable compiler
//! (`rustc` version 1.30+)
//!
//! ```toml
//! [dependencies]
//! phf = { version = "0.7.24", features = ["macros"] }
//! ```
//!
//! ```
//! use phf::{phf_map, phf_set};
//!
//! static MY_MAP: phf::Map<&'static str, u32> = phf_map! {
//!     "hello" => 1,
//!     "world" => 2,
//! };
//!
//! static MY_SET: phf::Set<&'static str> = phf_set! {
//!     "hello world",
//!     "hola mundo",
//! };
//!
//! fn main() {
//!     assert_eq!(MY_MAP["hello"], 1);
//!     assert!(MY_SET.contains("hello world"));
//! }
//! ```
//!
//! (Alternatively, you can use the phf_codegen crate to generate PHF datatypes
//! in a build script)
#![doc(html_root_url="https://docs.rs/phf/0.7")]
#![warn(missing_docs)]
#![cfg_attr(not(feature = "std"), no_std)]

#[cfg(feature = "std")]
extern crate std as core;

#[cfg(feature = "macros")]
/// Macro to create a `static` (compile-time) [`Map`].
///
/// Requires the `"macros"` feature.
///
/// # Example
///
/// ```rust,edition2018
/// use ::phf::{phf_map, Map};
///
/// static MY_MAP: Map<&'static str, u32> = phf_map! {
///     "hello" => 1,
///     "world" => 2,
/// };
///
/// fn main () {
///     assert_eq!(MY_MAP["hello"], 1);
/// }
/// ```
#[::proc_macro_hack::proc_macro_hack]
pub use phf_macros:: phf_map;

#[cfg(feature = "macros")]
/// Macro to create a `static` (compile-time) [`Set`].
///
/// Requires the `"macros"` feature.
///
/// # Example
///
/// ```rust,edition2018
/// use ::phf::{phf_set, Set};
///
/// static MY_SET: Set<&'static str> = phf_set! {
///     "hello world",
///     "hola mundo",
/// };
///
/// fn main ()
/// {
///     assert!(MY_SET.contains("hello world"));
/// }
/// ```
#[::proc_macro_hack::proc_macro_hack]
pub use phf_macros::phf_set;

use core::ops::Deref;

pub use phf_shared::PhfHash;
#[doc(inline)]
pub use self::map::Map;
#[doc(inline)]
pub use self::set::Set;

pub mod map;
pub mod set;

// WARNING: this is not considered part of phf's public API and is subject to
// change at any time.
//
// Basically Cow, but with the Owned version conditionally compiled
#[doc(hidden)]
pub enum Slice<T: 'static> {
    Static(&'static [T]),
    #[cfg(feature = "std")]
    Dynamic(Vec<T>),
}

impl<T> Deref for Slice<T> {
    type Target = [T];

    fn deref(&self) -> &[T] {
        match *self {
            Slice::Static(t) => t,
            #[cfg(feature = "std")]
            Slice::Dynamic(ref t) => t,
        }
    }
}
<|endoftext|>


<commit_before>//! To make attribute macros work reliably when typing, we need to take care to
//! fix up syntax errors in the code we're passing to them.
use std::mem;

use mbe::{SyntheticToken, SyntheticTokenId, TokenMap};
use rustc_hash::FxHashMap;
use syntax::{
    ast::{self, AstNode},
    match_ast, SyntaxKind, SyntaxNode, TextRange,
};
use tt::Subtree;

/// The result of calculating fixes for a syntax node -- a bunch of changes
/// (appending to and replacing nodes), the information that is needed to
/// reverse those changes afterwards, and a token map.
#[derive(Debug)]
pub(crate) struct SyntaxFixups {
    pub(crate) append: FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
    pub(crate) replace: FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
    pub(crate) undo_info: SyntaxFixupUndoInfo,
    pub(crate) token_map: TokenMap,
    pub(crate) next_id: u32,
}

/// This is the information needed to reverse the fixups.
#[derive(Debug, PartialEq, Eq)]
pub struct SyntaxFixupUndoInfo {
    original: Vec<Subtree>,
}

const EMPTY_ID: SyntheticTokenId = SyntheticTokenId(!0);

pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups {
    let mut append = FxHashMap::default();
    let mut replace = FxHashMap::default();
    let mut preorder = node.preorder();
    let mut original = Vec::new();
    let mut token_map = TokenMap::default();
    let mut next_id = 0;
    while let Some(event) = preorder.next() {
        let node = match event {
            syntax::WalkEvent::Enter(node) => node,
            syntax::WalkEvent::Leave(_) => continue,
        };

        if can_handle_error(&node) && has_error_to_handle(&node) {
            // the node contains an error node, we have to completely replace it by something valid
            let (original_tree, new_tmap, new_next_id) =
                mbe::syntax_node_to_token_tree_with_modifications(
                    &node,
                    mem::take(&mut token_map),
                    next_id,
                    Default::default(),
                    Default::default(),
                );
            token_map = new_tmap;
            next_id = new_next_id;
            let idx = original.len() as u32;
            original.push(original_tree);
            let replacement = SyntheticToken {
                kind: SyntaxKind::IDENT,
                text: "__ra_fixup".into(),
                range: node.text_range(),
                id: SyntheticTokenId(idx),
            };
            replace.insert(node.clone(), vec![replacement]);
            preorder.skip_subtree();
            continue;
        }

        // In some other situations, we can fix things by just appending some tokens.
        let end_range = TextRange::empty(node.text_range().end());
        match_ast! {
            match node {
                ast::FieldExpr(it) => {
                    if it.name_ref().is_none() {
                        // incomplete field access: some_expr.|
                        append.insert(node.clone(), vec![
                            SyntheticToken {
                                kind: SyntaxKind::IDENT,
                                text: "__ra_fixup".into(),
                                range: end_range,
                                id: EMPTY_ID,
                            },
                        ]);
                    }
                },
                ast::ExprStmt(it) => {
                    if it.semicolon_token().is_none() {
                        append.insert(node.clone(), vec![
                            SyntheticToken {
                                kind: SyntaxKind::SEMICOLON,
                                text: ";".into(),
                                range: end_range,
                                id: EMPTY_ID,
                            },
                        ]);
                    }
                },
                _ => (),
            }
        }
    }
    SyntaxFixups {
        append,
        replace,
        token_map,
        next_id,
        undo_info: SyntaxFixupUndoInfo { original },
    }
}

fn has_error(node: &SyntaxNode) -> bool {
    node.children().any(|c| c.kind() == SyntaxKind::ERROR)
}

fn can_handle_error(node: &SyntaxNode) -> bool {
    ast::Expr::can_cast(node.kind())
}

fn has_error_to_handle(node: &SyntaxNode) -> bool {
    has_error(node) || node.children().any(|c| !can_handle_error(&c) && has_error_to_handle(&c))
}

pub(crate) fn reverse_fixups(
    tt: &mut Subtree,
    token_map: &TokenMap,
    undo_info: &SyntaxFixupUndoInfo,
) {
    tt.token_trees.retain(|tt| match tt {
        tt::TokenTree::Leaf(leaf) => {
            token_map.synthetic_token_id(leaf.id()).is_none()
                || token_map.synthetic_token_id(leaf.id()) != Some(EMPTY_ID)
        }
        _ => true,
    });
    tt.token_trees.iter_mut().for_each(|tt| match tt {
        tt::TokenTree::Subtree(tt) => reverse_fixups(tt, token_map, undo_info),
        tt::TokenTree::Leaf(leaf) => {
            if let Some(id) = token_map.synthetic_token_id(leaf.id()) {
                let original = &undo_info.original[id.0 as usize];
                *tt = tt::TokenTree::Subtree(original.clone());
            }
        }
    });
}

#[cfg(test)]
mod tests {
    use expect_test::{expect, Expect};

    use super::reverse_fixups;

    #[track_caller]
    fn check(ra_fixture: &str, mut expect: Expect) {
        let parsed = syntax::SourceFile::parse(ra_fixture);
        let fixups = super::fixup_syntax(&parsed.syntax_node());
        let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications(
            &parsed.syntax_node(),
            fixups.token_map,
            fixups.next_id,
            fixups.replace,
            fixups.append,
        );

        let mut actual = tt.to_string();
        actual.push_str("\n");

        expect.indent(false);
        expect.assert_eq(&actual);

        // the fixed-up tree should be syntactically valid
        let (parse, _) = mbe::token_tree_to_syntax_node(&tt, ::mbe::TopEntryPoint::MacroItems);
        assert_eq!(
            parse.errors(),
            &[],
            "parse has syntax errors. parse tree:\n{:#?}",
            parse.syntax_node()
        );

        reverse_fixups(&mut tt, &tmap, &fixups.undo_info);

        // the fixed-up + reversed version should be equivalent to the original input
        // (but token IDs don't matter)
        let (original_as_tt, _) = mbe::syntax_node_to_token_tree(&parsed.syntax_node());
        assert_eq!(tt.to_string(), original_as_tt.to_string());
    }

    #[test]
    fn incomplete_field_expr_1() {
        check(
            r#"
fn foo() {
    a.
}
"#,
            expect![[r#"
fn foo () {a . __ra_fixup}
"#]],
        )
    }

    #[test]
    fn incomplete_field_expr_2() {
        check(
            r#"
fn foo() {
    a. ;
}
"#,
            expect![[r#"
fn foo () {a . __ra_fixup ;}
"#]],
        )
    }

    #[test]
    fn incomplete_field_expr_3() {
        check(
            r#"
fn foo() {
    a. ;
    bar();
}
"#,
            expect![[r#"
fn foo () {a . __ra_fixup ; bar () ;}
"#]],
        )
    }

    #[test]
    fn field_expr_before_call() {
        // another case that easily happens while typing
        check(
            r#"
fn foo() {
    a.b
    bar();
}
"#,
            expect![[r#"
fn foo () {a . b ; bar () ;}
"#]],
        )
    }

    #[test]
    fn extraneous_comma() {
        check(
            r#"
fn foo() {
    bar(,);
}
"#,
            expect![[r#"
fn foo () {__ra_fixup ;}
"#]],
        )
    }
}
<commit_msg>Add another case to the syntax fixup code<commit_after>//! To make attribute macros work reliably when typing, we need to take care to
//! fix up syntax errors in the code we're passing to them.
use std::mem;

use mbe::{SyntheticToken, SyntheticTokenId, TokenMap};
use rustc_hash::FxHashMap;
use syntax::{
    ast::{self, AstNode},
    match_ast, SyntaxKind, SyntaxNode, TextRange,
};
use tt::Subtree;

/// The result of calculating fixes for a syntax node -- a bunch of changes
/// (appending to and replacing nodes), the information that is needed to
/// reverse those changes afterwards, and a token map.
#[derive(Debug)]
pub(crate) struct SyntaxFixups {
    pub(crate) append: FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
    pub(crate) replace: FxHashMap<SyntaxNode, Vec<SyntheticToken>>,
    pub(crate) undo_info: SyntaxFixupUndoInfo,
    pub(crate) token_map: TokenMap,
    pub(crate) next_id: u32,
}

/// This is the information needed to reverse the fixups.
#[derive(Debug, PartialEq, Eq)]
pub struct SyntaxFixupUndoInfo {
    original: Vec<Subtree>,
}

const EMPTY_ID: SyntheticTokenId = SyntheticTokenId(!0);

pub(crate) fn fixup_syntax(node: &SyntaxNode) -> SyntaxFixups {
    let mut append = FxHashMap::default();
    let mut replace = FxHashMap::default();
    let mut preorder = node.preorder();
    let mut original = Vec::new();
    let mut token_map = TokenMap::default();
    let mut next_id = 0;
    while let Some(event) = preorder.next() {
        let node = match event {
            syntax::WalkEvent::Enter(node) => node,
            syntax::WalkEvent::Leave(_) => continue,
        };

        if can_handle_error(&node) && has_error_to_handle(&node) {
            // the node contains an error node, we have to completely replace it by something valid
            let (original_tree, new_tmap, new_next_id) =
                mbe::syntax_node_to_token_tree_with_modifications(
                    &node,
                    mem::take(&mut token_map),
                    next_id,
                    Default::default(),
                    Default::default(),
                );
            token_map = new_tmap;
            next_id = new_next_id;
            let idx = original.len() as u32;
            original.push(original_tree);
            let replacement = SyntheticToken {
                kind: SyntaxKind::IDENT,
                text: "__ra_fixup".into(),
                range: node.text_range(),
                id: SyntheticTokenId(idx),
            };
            replace.insert(node.clone(), vec![replacement]);
            preorder.skip_subtree();
            continue;
        }

        // In some other situations, we can fix things by just appending some tokens.
        let end_range = TextRange::empty(node.text_range().end());
        match_ast! {
            match node {
                ast::FieldExpr(it) => {
                    if it.name_ref().is_none() {
                        // incomplete field access: some_expr.|
                        append.insert(node.clone(), vec![
                            SyntheticToken {
                                kind: SyntaxKind::IDENT,
                                text: "__ra_fixup".into(),
                                range: end_range,
                                id: EMPTY_ID,
                            },
                        ]);
                    }
                },
                ast::ExprStmt(it) => {
                    if it.semicolon_token().is_none() {
                        append.insert(node.clone(), vec![
                            SyntheticToken {
                                kind: SyntaxKind::SEMICOLON,
                                text: ";".into(),
                                range: end_range,
                                id: EMPTY_ID,
                            },
                        ]);
                    }
                },
                ast::LetStmt(it) => {
                    if it.semicolon_token().is_none() {
                        append.insert(node.clone(), vec![
                            SyntheticToken {
                                kind: SyntaxKind::SEMICOLON,
                                text: ";".into(),
                                range: end_range,
                                id: EMPTY_ID,
                            },
                        ]);
                    }
                },
                _ => (),
            }
        }
    }
    SyntaxFixups {
        append,
        replace,
        token_map,
        next_id,
        undo_info: SyntaxFixupUndoInfo { original },
    }
}

fn has_error(node: &SyntaxNode) -> bool {
    node.children().any(|c| c.kind() == SyntaxKind::ERROR)
}

fn can_handle_error(node: &SyntaxNode) -> bool {
    ast::Expr::can_cast(node.kind())
}

fn has_error_to_handle(node: &SyntaxNode) -> bool {
    has_error(node) || node.children().any(|c| !can_handle_error(&c) && has_error_to_handle(&c))
}

pub(crate) fn reverse_fixups(
    tt: &mut Subtree,
    token_map: &TokenMap,
    undo_info: &SyntaxFixupUndoInfo,
) {
    tt.token_trees.retain(|tt| match tt {
        tt::TokenTree::Leaf(leaf) => {
            token_map.synthetic_token_id(leaf.id()).is_none()
                || token_map.synthetic_token_id(leaf.id()) != Some(EMPTY_ID)
        }
        _ => true,
    });
    tt.token_trees.iter_mut().for_each(|tt| match tt {
        tt::TokenTree::Subtree(tt) => reverse_fixups(tt, token_map, undo_info),
        tt::TokenTree::Leaf(leaf) => {
            if let Some(id) = token_map.synthetic_token_id(leaf.id()) {
                let original = &undo_info.original[id.0 as usize];
                *tt = tt::TokenTree::Subtree(original.clone());
            }
        }
    });
}

#[cfg(test)]
mod tests {
    use expect_test::{expect, Expect};

    use super::reverse_fixups;

    #[track_caller]
    fn check(ra_fixture: &str, mut expect: Expect) {
        let parsed = syntax::SourceFile::parse(ra_fixture);
        let fixups = super::fixup_syntax(&parsed.syntax_node());
        let (mut tt, tmap, _) = mbe::syntax_node_to_token_tree_with_modifications(
            &parsed.syntax_node(),
            fixups.token_map,
            fixups.next_id,
            fixups.replace,
            fixups.append,
        );

        let mut actual = tt.to_string();
        actual.push_str("\n");

        expect.indent(false);
        expect.assert_eq(&actual);

        // the fixed-up tree should be syntactically valid
        let (parse, _) = mbe::token_tree_to_syntax_node(&tt, ::mbe::TopEntryPoint::MacroItems);
        assert_eq!(
            parse.errors(),
            &[],
            "parse has syntax errors. parse tree:\n{:#?}",
            parse.syntax_node()
        );

        reverse_fixups(&mut tt, &tmap, &fixups.undo_info);

        // the fixed-up + reversed version should be equivalent to the original input
        // (but token IDs don't matter)
        let (original_as_tt, _) = mbe::syntax_node_to_token_tree(&parsed.syntax_node());
        assert_eq!(tt.to_string(), original_as_tt.to_string());
    }

    #[test]
    fn incomplete_field_expr_1() {
        check(
            r#"
fn foo() {
    a.
}
"#,
            expect![[r#"
fn foo () {a . __ra_fixup}
"#]],
        )
    }

    #[test]
    fn incomplete_field_expr_2() {
        check(
            r#"
fn foo() {
    a. ;
}
"#,
            expect![[r#"
fn foo () {a . __ra_fixup ;}
"#]],
        )
    }

    #[test]
    fn incomplete_field_expr_3() {
        check(
            r#"
fn foo() {
    a. ;
    bar();
}
"#,
            expect![[r#"
fn foo () {a . __ra_fixup ; bar () ;}
"#]],
        )
    }

    #[test]
    fn incomplete_let() {
        check(
            r#"
fn foo() {
    let x = a
}
"#,
            expect![[r#"
fn foo () {let x = a ;}
"#]],
        )
    }

    #[test]
    fn incomplete_field_expr_in_let() {
        check(
            r#"
fn foo() {
    let x = a.
}
"#,
            expect![[r#"
fn foo () {let x = a . __ra_fixup ;}
"#]],
        )
    }

    #[test]
    fn field_expr_before_call() {
        // another case that easily happens while typing
        check(
            r#"
fn foo() {
    a.b
    bar();
}
"#,
            expect![[r#"
fn foo () {a . b ; bar () ;}
"#]],
        )
    }

    #[test]
    fn extraneous_comma() {
        check(
            r#"
fn foo() {
    bar(,);
}
"#,
            expect![[r#"
fn foo () {__ra_fixup ;}
"#]],
        )
    }
}
<|endoftext|>


<commit_before>//! rust-analyzer is lazy and doesn't compute anything unless asked. This
//! sometimes is counter productive when, for example, the first goto definition
//! request takes longer to compute. This modules implemented prepopulation of
//! various caches, it's not really advanced at the moment.
mod topologic_sort;

use hir::db::DefDatabase;
use ide_db::base_db::{
    salsa::{Database, ParallelDatabase, Snapshot},
    CrateGraph, CrateId, SourceDatabase, SourceDatabaseExt,
};
use rustc_hash::{FxHashMap, FxHashSet};

use crate::RootDatabase;

/// We started indexing a crate.
#[derive(Debug)]
pub struct PrimeCachesProgress {
    pub on_crate: String,
    pub n_done: usize,
    pub n_total: usize,
}

pub(crate) fn prime_caches(db: &RootDatabase, cb: &(dyn Fn(PrimeCachesProgress) + Sync)) {
    let _p = profile::span("prime_caches");
    let graph = db.crate_graph();
    let to_prime = compute_crates_to_prime(db, &graph);

    let n_total = to_prime.len();
    for (n_done, &crate_id) in to_prime.iter().enumerate() {
        let crate_name = graph[crate_id].display_name.as_deref().unwrap_or_default().to_string();

        cb(PrimeCachesProgress { on_crate: crate_name, n_done, n_total });
        // This also computes the DefMap
        db.import_map(crate_id);
    }
}

/// We're indexing many crates.
#[derive(Debug)]
pub struct ParallelPrimeCachesProgress {
    /// the crates that we are currently priming.
    pub crates_currently_indexing: Vec<String>,
    /// the total number of crates we want to prime.
    pub crates_total: usize,
    /// the total number of crates that have finished priming
    pub crates_done: usize,
}

pub(crate) fn parallel_prime_caches(
    db: &RootDatabase,
    num_worker_threads: u8,
    cb: &(dyn Fn(ParallelPrimeCachesProgress) + Sync),
) {
    let _p = profile::span("prime_caches");

    let graph = db.crate_graph();
    let mut crates_to_prime = {
        let crate_ids = compute_crates_to_prime(db, &graph);

        let mut builder = topologic_sort::TopologicalSortIter::builder();

        for &crate_id in &crate_ids {
            let crate_data = &graph[crate_id];
            let dependencies = crate_data
                .dependencies
                .iter()
                .map(|d| d.crate_id)
                .filter(|i| crate_ids.contains(i));

            builder.add(crate_id, dependencies);
        }

        builder.build()
    };

    crossbeam_utils::thread::scope(move |s| {
        let (work_sender, work_receiver) = crossbeam_channel::unbounded();
        let (progress_sender, progress_receiver) = crossbeam_channel::unbounded();

        enum ParallelPrimeCacheWorkerProgress {
            BeginCrate { crate_id: CrateId, crate_name: String },
            EndCrate { crate_id: CrateId },
        }

        let prime_caches_worker = move |db: Snapshot<RootDatabase>| {
            while let Ok((crate_id, crate_name)) = work_receiver.recv() {
                progress_sender
                    .send(ParallelPrimeCacheWorkerProgress::BeginCrate { crate_id, crate_name })?;

                // This also computes the DefMap
                db.import_map(crate_id);

                progress_sender.send(ParallelPrimeCacheWorkerProgress::EndCrate { crate_id })?;
            }

            Ok::<_, crossbeam_channel::SendError<_>>(())
        };

        for _ in 0..num_worker_threads {
            let worker = prime_caches_worker.clone();
            let db = db.snapshot();
            s.spawn(move |_| worker(db));
        }

        let crates_total = crates_to_prime.len();
        let mut crates_done = 0;
        let mut crates_currently_indexing =
            FxHashMap::with_capacity_and_hasher(num_worker_threads as _, Default::default());

        while !crates_to_prime.is_empty() {
            db.unwind_if_cancelled();

            for crate_id in &mut crates_to_prime {
                work_sender
                    .send((
                        crate_id,
                        graph[crate_id].display_name.as_deref().unwrap_or_default().to_string(),
                    ))
                    .ok();
            }

            let worker_progress = match progress_receiver.recv() {
                Ok(p) => p,
                Err(_) => {
                    // our workers may have died from a cancelled task, so we'll check and re-raise here.
                    db.unwind_if_cancelled();
                    break;
                }
            };
            match worker_progress {
                ParallelPrimeCacheWorkerProgress::BeginCrate { crate_id, crate_name } => {
                    crates_currently_indexing.insert(crate_id, crate_name);
                }
                ParallelPrimeCacheWorkerProgress::EndCrate { crate_id } => {
                    crates_currently_indexing.remove(&crate_id);
                    crates_to_prime.mark_done(crate_id);
                    crates_done += 1;
                }
            };

            let progress = ParallelPrimeCachesProgress {
                crates_currently_indexing: crates_currently_indexing.values().cloned().collect(),
                crates_done,
                crates_total,
            };

            cb(progress);
        }
    })
    .unwrap();
}

fn compute_crates_to_prime(db: &RootDatabase, graph: &CrateGraph) -> FxHashSet<CrateId> {
    // We're only interested in the workspace crates and the `ImportMap`s of their direct
    // dependencies, though in practice the latter also compute the `DefMap`s.
    // We don't prime transitive dependencies because they're generally not visible in
    // the current workspace.
    graph
        .iter()
        .filter(|&id| {
            let file_id = graph[id].root_file_id;
            let root_id = db.file_source_root(file_id);
            !db.source_root(root_id).is_library
        })
        .flat_map(|id| graph[id].dependencies.iter().map(|krate| krate.crate_id))
        .collect()
}
<commit_msg>use index map<commit_after>//! rust-analyzer is lazy and doesn't compute anything unless asked. This
//! sometimes is counter productive when, for example, the first goto definition
//! request takes longer to compute. This modules implemented prepopulation of
//! various caches, it's not really advanced at the moment.
mod topologic_sort;

use hir::db::DefDatabase;
use ide_db::{
    base_db::{
        salsa::{Database, ParallelDatabase, Snapshot},
        CrateGraph, CrateId, SourceDatabase, SourceDatabaseExt,
    },
    FxIndexMap,
};
use rustc_hash::{FxHashMap, FxHashSet};

use crate::RootDatabase;

/// We started indexing a crate.
#[derive(Debug)]
pub struct PrimeCachesProgress {
    pub on_crate: String,
    pub n_done: usize,
    pub n_total: usize,
}

pub(crate) fn prime_caches(db: &RootDatabase, cb: &(dyn Fn(PrimeCachesProgress) + Sync)) {
    let _p = profile::span("prime_caches");
    let graph = db.crate_graph();
    let to_prime = compute_crates_to_prime(db, &graph);

    let n_total = to_prime.len();
    for (n_done, &crate_id) in to_prime.iter().enumerate() {
        let crate_name = graph[crate_id].display_name.as_deref().unwrap_or_default().to_string();

        cb(PrimeCachesProgress { on_crate: crate_name, n_done, n_total });
        // This also computes the DefMap
        db.import_map(crate_id);
    }
}

/// We're indexing many crates.
#[derive(Debug)]
pub struct ParallelPrimeCachesProgress {
    /// the crates that we are currently priming.
    pub crates_currently_indexing: Vec<String>,
    /// the total number of crates we want to prime.
    pub crates_total: usize,
    /// the total number of crates that have finished priming
    pub crates_done: usize,
}

pub(crate) fn parallel_prime_caches(
    db: &RootDatabase,
    num_worker_threads: u8,
    cb: &(dyn Fn(ParallelPrimeCachesProgress) + Sync),
) {
    let _p = profile::span("prime_caches");

    let graph = db.crate_graph();
    let mut crates_to_prime = {
        let crate_ids = compute_crates_to_prime(db, &graph);

        let mut builder = topologic_sort::TopologicalSortIter::builder();

        for &crate_id in &crate_ids {
            let crate_data = &graph[crate_id];
            let dependencies = crate_data
                .dependencies
                .iter()
                .map(|d| d.crate_id)
                .filter(|i| crate_ids.contains(i));

            builder.add(crate_id, dependencies);
        }

        builder.build()
    };

    crossbeam_utils::thread::scope(move |s| {
        let (work_sender, work_receiver) = crossbeam_channel::unbounded();
        let (progress_sender, progress_receiver) = crossbeam_channel::unbounded();

        enum ParallelPrimeCacheWorkerProgress {
            BeginCrate { crate_id: CrateId, crate_name: String },
            EndCrate { crate_id: CrateId },
        }

        let prime_caches_worker = move |db: Snapshot<RootDatabase>| {
            while let Ok((crate_id, crate_name)) = work_receiver.recv() {
                progress_sender
                    .send(ParallelPrimeCacheWorkerProgress::BeginCrate { crate_id, crate_name })?;

                // This also computes the DefMap
                db.import_map(crate_id);

                progress_sender.send(ParallelPrimeCacheWorkerProgress::EndCrate { crate_id })?;
            }

            Ok::<_, crossbeam_channel::SendError<_>>(())
        };

        for _ in 0..num_worker_threads {
            let worker = prime_caches_worker.clone();
            let db = db.snapshot();
            s.spawn(move |_| worker(db));
        }

        let crates_total = crates_to_prime.len();
        let mut crates_done = 0;

        // an index map is used to preserve ordering so we can sort the progress report in order of
        // "longest crate to index" first
        let mut crates_currently_indexing =
            FxIndexMap::with_capacity_and_hasher(num_worker_threads as _, Default::default());

        while !crates_to_prime.is_empty() {
            db.unwind_if_cancelled();

            for crate_id in &mut crates_to_prime {
                work_sender
                    .send((
                        crate_id,
                        graph[crate_id].display_name.as_deref().unwrap_or_default().to_string(),
                    ))
                    .ok();
            }

            let worker_progress = match progress_receiver.recv() {
                Ok(p) => p,
                Err(_) => {
                    // our workers may have died from a cancelled task, so we'll check and re-raise here.
                    db.unwind_if_cancelled();
                    break;
                }
            };
            match worker_progress {
                ParallelPrimeCacheWorkerProgress::BeginCrate { crate_id, crate_name } => {
                    crates_currently_indexing.insert(crate_id, crate_name);
                }
                ParallelPrimeCacheWorkerProgress::EndCrate { crate_id } => {
                    crates_currently_indexing.remove(&crate_id);
                    crates_to_prime.mark_done(crate_id);
                    crates_done += 1;
                }
            };

            let progress = ParallelPrimeCachesProgress {
                crates_currently_indexing: crates_currently_indexing.values().cloned().collect(),
                crates_done,
                crates_total,
            };

            cb(progress);
        }
    })
    .unwrap();
}

fn compute_crates_to_prime(db: &RootDatabase, graph: &CrateGraph) -> FxHashSet<CrateId> {
    // We're only interested in the workspace crates and the `ImportMap`s of their direct
    // dependencies, though in practice the latter also compute the `DefMap`s.
    // We don't prime transitive dependencies because they're generally not visible in
    // the current workspace.
    graph
        .iter()
        .filter(|&id| {
            let file_id = graph[id].root_file_id;
            let root_id = db.file_source_root(file_id);
            !db.source_root(root_id).is_library
        })
        .flat_map(|id| graph[id].dependencies.iter().map(|krate| krate.crate_id))
        .collect()
}
<|endoftext|>


<commit_before>#![recursion_limit = "1024"]

extern crate syn;
#[macro_use]
extern crate quote;
extern crate proc_macro;
extern crate edit_distance as ed;

use proc_macro::TokenStream;
use syn::Ident;

use std::iter::once;

fn get_op_arity(tra1t: &str) -> Option<usize> {
    match tra1t {
        "Quasigroup" | "Monoid" | "Semigroup" | "Loop" | "Group" | "GroupAbelian" => Some(1),
        "Ring" | "RingCommutative" | "Field" => Some(2),
        _ => None,
    }
}

fn get_closest_trait(tra1t: &str) -> &str {
    ["Quasigroup", "Monoid", "Semigroup", "Group", "GroupAbelian", "Ring", "RingCommutative", "Field"].iter()
        .map(|t| (ed::edit_distance(t, tra1t), t))
        .min().expect("Hardcoded array which we are iterating should never be empty. Programming error.").1
}

fn get_dependencies(tra1t: &str, op: usize) -> Vec<String> {
    match tra1t {
        "Quasigroup" => vec![],
        "Monoid" => vec!["Quasigroup"],
        "Semigroup" => vec![],
        "Loop" => vec!["Semigroup"],
        "Group" => vec!["Monoid", "Quasigroup", "Loop", "Semigroup"],
        "GroupAbelian" => vec!["Group", "Monoid", "Quasigroup", "Loop", "Semigroup"],
        _ => match tra1t {
            "Ring" => if op == 0 {
                vec!["GroupAbelian", "Group", "Monoid", "Quasigroup", "Loop", "Semigroup"]
            } else {
                vec!["Monoid", "Quasigroup", "Loop", "Semigroup"]
            },
            "RingCommutative" => if op == 0 {
                vec!["Ring", "GroupAbelian", "Group", "Monoid", "Quasigroup", "Loop", "Semigroup"]
            } else {
                vec!["Ring", "Monoid", "Quasigroup", "Loop", "Semigroup"]
            },
            "Field" => if op == 0 {
                vec!["RingCommutative", "Ring", "GroupAbelian", "Group", "Monoid", "Quasigroup", "Loop", "Semigroup"]
            } else {
                vec!["GroupAbelian", "Group", "Monoid", "Quasigroup", "Loop", "Semigroup"]
            },
            _ => panic!("Unknown Alga trait `{}`. Programming error.", tra1t),
        }
    }.into_iter().map(String::from).collect()
}

#[proc_macro_derive(Alga, attributes(alga_traits))]
pub fn derive_alga(input: TokenStream) -> TokenStream {
    use syn::MetaItem::*;
    use syn::NestedMetaItem::*;
    let item = syn::parse_derive_input(&input.to_string()).unwrap();
    let name = &item.ident;
    let (tra1t, op): (Vec<_>, Vec<_>) = item.attrs.iter()
        .filter_map(|a| {
                if let List(ref ident, ref traits) = a.value {
                    Some((ident, traits))
                } else {
                    None
                }
            })
        .filter(|&(i, _)| i == "alga_traits")
        .flat_map(|(_, v)| v)
        .map(|t| {
            if let &MetaItem(ref n) = t {
                n
            } else {
                panic!("Instead of a literal, trait with operator expected:\n         #[alga_traits(Trait = Operator, ...)]");
            }
        }).map(|n| {
            if let &List(ref name, ref value) = n {
                (name, value)
            } else {
                panic!("Operator has to be provided via #[alga_traits(Trait(Op1, ..))]");
            }
        }).flat_map(|(name, value)| {
            let name = name.as_ref();
            let arity = get_op_arity(name)
                .expect(&format!("Invalid Alga trait provided. Did you mean `{}`?", get_closest_trait(name)));
            assert!(value.len() == arity, "Exactly {} operators need to be specified.", arity);
            let iter = once(name.into()).chain(
                get_dependencies(name, 0)
                    .into_iter()
            ).map(|n| {
                let arity = get_op_arity(&n)
                    .expect(&format!("Invalid Alga trait `{}`. Programming error.", n));
                (Ident::from(format!("Abstract{}", n)), if arity == 1 {
                    vec![value[0].clone()]
                } else {
                    value.clone()
                })
            });
            if arity == 1 {
                iter.collect::<Vec<_>>()
            } else {
                iter.chain(
                    get_dependencies(name, 1)
                        .into_iter()
                        .map(|n| {
                            let arity = get_op_arity(&n)
                                .expect(&format!("Invalid Alga trait `{}`. Programming error.", n));
                            (Ident::from(format!("Abstract{}", n)), if arity == 1 {
                                vec![value[1].clone()]
                            } else {
                                value.clone()
                            })
                        }
                )).collect()
            }
        })
        .unzip();
    assert!(!tra1t.is_empty(),
    "Atleast one trait is required to be implemented.\n         Trait can be specified with `#[alga_traits(Trait = Operator, ...)]` attribute.");

    let dummy_const = Ident::new(format!("_ALGA_DERIVE_{}", name));
    let name = once(&name).cycle();
    let (i, t, w) = item.generics.split_for_impl();
    let (impl_generics, ty_generics, where_clause) = (once(&i).cycle(), once(&t).cycle(), once(&w).cycle());
    (quote!{
        #[allow(non_upper_case_globals, unused_attributes, unused_qualifications)]
        const #dummy_const: () = {
            extern crate alga as _alga;
            #(#[automatically_derived]
            impl #impl_generics _alga::general::#tra1t<#(#op,)*> for #name #ty_generics #where_clause {})*
        };
    }).parse().unwrap()
}
<commit_msg>Refactoring.<commit_after>#![recursion_limit = "1024"]

extern crate syn;
#[macro_use]
extern crate quote;
extern crate proc_macro;
extern crate edit_distance as ed;

use proc_macro::TokenStream;
use syn::Ident;

use std::iter::once;

fn get_op_arity(tra1t: &str) -> usize {
    match tra1t {
        "Quasigroup" | "Monoid" | "Semigroup" | "Loop" | "Group" | "GroupAbelian" => 1,
        "Ring" | "RingCommutative" | "Field" => 2,
        _ => panic!("Invalid Alga trait provided. Did you mean `{}`?", get_closest_trait(tra1t)),
    }
}

fn get_closest_trait(tra1t: &str) -> &str {
    ["Quasigroup", "Monoid", "Semigroup", "Group", "GroupAbelian", "Ring", "RingCommutative", "Field"].iter()
        .map(|t| (ed::edit_distance(t, tra1t), t))
        .min().expect("Hardcoded array which we are iterating should never be empty. Programming error.").1
}

fn get_dependencies(tra1t: &str, op: usize) -> Vec<String> {
    match tra1t {
        "Quasigroup" => vec![],
        "Monoid" => vec!["Quasigroup"],
        "Semigroup" => vec![],
        "Loop" => vec!["Semigroup"],
        "Group" => vec!["Monoid", "Quasigroup", "Loop", "Semigroup"],
        "GroupAbelian" => vec!["Group", "Monoid", "Quasigroup", "Loop", "Semigroup"],
        _ => match tra1t {
            "Ring" => if op == 0 {
                vec!["GroupAbelian", "Group", "Monoid", "Quasigroup", "Loop", "Semigroup"]
            } else {
                vec!["Monoid", "Quasigroup", "Loop", "Semigroup"]
            },
            "RingCommutative" => if op == 0 {
                vec!["Ring", "GroupAbelian", "Group", "Monoid", "Quasigroup", "Loop", "Semigroup"]
            } else {
                vec!["Ring", "Monoid", "Quasigroup", "Loop", "Semigroup"]
            },
            "Field" => if op == 0 {
                vec!["RingCommutative", "Ring", "GroupAbelian", "Group", "Monoid", "Quasigroup", "Loop", "Semigroup"]
            } else {
                vec!["GroupAbelian", "Group", "Monoid", "Quasigroup", "Loop", "Semigroup"]
            },
            _ => panic!("Unknown Alga trait `{}`. Programming error.", tra1t),
        }
    }.into_iter().map(String::from).collect()
}

#[proc_macro_derive(Alga, attributes(alga_traits))]
pub fn derive_alga(input: TokenStream) -> TokenStream {
    use syn::MetaItem::*;
    use syn::NestedMetaItem::*;
    let item = syn::parse_derive_input(&input.to_string()).unwrap();
    let name = &item.ident;
    let (tra1t, op): (Vec<_>, Vec<_>) = item.attrs.iter()
        .filter_map(|a|
            if let List(ref ident, ref traits) = a.value {
                Some((ident, traits))
            } else {
                None
            })
        .filter(|&(i, _)| i == "alga_traits")
        .flat_map(|(_, v)| v)
        .map(|t|
            if let &MetaItem(List(ref name, ref value)) = t {
                (name.as_ref(), value)
            } else {
                panic!("Operator has to be provided via #[alga_traits(Trait(Operators))]");
            })
        .flat_map(|(name, value)| {
            let arity = get_op_arity(name);
            assert!(value.len() == arity, "Exactly {} operators need to be specified.", arity);
            let create_tuple = |n: &str, i: usize| {
                let value = if get_op_arity(n) == 1 {
                    vec![value[i].clone()]
                } else {
                    value.clone()
                };
                (Ident::from(format!("Abstract{}", n)), value)
            };
            let iter = once(name.into())
                .chain(get_dependencies(name, 0))
                .map(|n| create_tuple(&n, 0));
            if arity == 1 {
                iter.collect::<Vec<_>>()
            } else {
                iter.chain(
                    get_dependencies(name, 1)
                        .into_iter()
                        .map(|n| create_tuple(&n, 1))
                ).collect()
            }
        })
        .unzip();
    assert!(!tra1t.is_empty(),
    "Atleast one trait is required to be implemented.\n         Trait can be specified with `#[alga_traits(Trait(Operators))]` attribute.");

    let dummy_const = Ident::new(format!("_ALGA_DERIVE_{}", name));
    let name = once(&name).cycle();
    let (i, t, w) = item.generics.split_for_impl();
    let (impl_generics, ty_generics, where_clause) = (once(&i).cycle(), once(&t).cycle(), once(&w).cycle());
    (quote!{
        #[allow(non_upper_case_globals, unused_attributes, unused_qualifications)]
        const #dummy_const: () = {
            extern crate alga as _alga;
            #(
                #[automatically_derived]
                impl #impl_generics _alga::general::#tra1t<#(#op,)*> for #name #ty_generics #where_clause {}
            )*
        };
    }).parse().unwrap()
}
<|endoftext|>


<commit_before>use std::path::{Path, PathBuf};
use std::fs::File;

use libmount::BindMount;

use quire::validate as V;
use config::read_config;
use config::containers::Container as Cont;
use version::short_version;
use container::mount::{remount_ro};
use container::util::{copy_dir};
use file_util::{create_dir, shallow_copy, copy};
use build_step::{BuildStep, VersionError, StepError, Digest, Config, Guard};

use builder::error::StepError as E;

// Build Steps
#[derive(Debug)]
pub struct Container(String);
tuple_struct_decode!(Container);

impl Container {
    pub fn config() -> V::Scalar {
        V::Scalar::new()
    }
}

#[derive(RustcDecodable, Debug)]
pub struct Build {
    pub container: String,
    pub source: PathBuf,
    pub path: Option<PathBuf>,
    pub temporary_mount: Option<PathBuf>,
}

impl Build {
    pub fn config() -> V::Structure<'static> {
        V::Structure::new()
        .member("container", V::Scalar::new())
        .member("source".to_string(),
            V::Directory::new().is_absolute(true).default("/"))
        .member("path".to_string(),
            V::Directory::new().is_absolute(true).optional())
        .member("temporary_mount".to_string(),
            V::Directory::new().is_absolute(true).optional())
    }
}

#[derive(RustcDecodable, Debug)]
pub struct GitSource {
    pub url: String,
    pub revision: Option<String>,
    pub branch: Option<String>,
}

#[derive(RustcDecodable, Debug)]
pub enum Source {
    Git(GitSource),
    Container(String),
    Directory,
}

#[derive(RustcDecodable, Debug)]
pub struct SubConfig {
    pub source: Source,
    pub path: PathBuf,
    pub container: String,
    pub cache: Option<bool>,
}

impl SubConfig {
    pub fn config() -> V::Structure<'static> {
        V::Structure::new()
        .member("source", V::Enum::new()
            .option("Directory", V::Nothing)
            .option("Container", V::Scalar::new())
            .option("Git", V::Structure::new()
                .member("url", V::Scalar::new())
                .member("revision", V::Scalar::new().optional())
                .member("branch", V::Scalar::new().optional()))
            .optional()
            .default_tag("Directory"))
        .member("path".to_string(), V::Directory::new()
            .is_absolute(false)
            .default("vagga.yaml"))
        .member("container", V::Scalar::new())
        .member("cache", V::Scalar::new().optional())
    }
}

pub fn build(binfo: &Build, guard: &mut Guard, build: bool)
    -> Result<(), StepError>
{
    let ref name = binfo.container;
    let cont = guard.ctx.config.containers.get(name)
        .expect("Subcontainer not found");  // TODO
    if build {
        let version = try!(short_version(&cont, &guard.ctx.config)
            .map_err(|(s, e)| format!("step {}: {}", s, e)));
        let container = Path::new("/vagga/base/.roots")
            .join(format!("{}.{}", name, version));
        let path = container.join("root")
            .join(binfo.source.strip_prefix("/").unwrap());

        // Update container use when using it as subcontainer (fixes #267)
        File::create(Path::new(&container).join("last_use"))
            .map_err(|e| warn!("Can't write image usage info: {}", e)).ok();

        if let Some(ref dest_rel) = binfo.path {
            let dest = Path::new("/vagga/root")
                .join(dest_rel.strip_prefix("/").unwrap());
            if path.is_dir() {
                try_msg!(copy_dir(&path, &dest, None, None),
                    "Error copying dir {p:?}: {err}", p=path);
            } else {
                let path_stat = try!(path.symlink_metadata()
                    .map_err(|e| StepError::Read(path.clone(), e)));
                try_msg!(shallow_copy(&path, &path_stat, &dest,
                        None, None, None),
                    "Error copying file {p:?}: {err}", p=path);
            }
        } else if let Some(ref dest_rel) = binfo.temporary_mount {
            let dest = Path::new("/vagga/root")
                .join(dest_rel.strip_prefix("/").unwrap());
            try_msg!(create_dir(&dest, false),
                "Error creating destination dir: {err}");
            try!(BindMount::new(&path, &dest).mount());
            try!(remount_ro(&dest));
            guard.ctx.mounted.push(dest);
        }
        // Revert resolv.conf back
        try!(copy(&Path::new("/etc/resolv.conf"),
                  &Path::new("/vagga/root/etc/resolv.conf"))
            .map_err(|e| format!("Error copying /etc/resolv.conf: {}", e)));
    }
    Ok(())
}

fn real_copy(name: &String, cont: &Cont, guard: &mut Guard)
    -> Result<(), StepError>
{
    let version = try!(short_version(&cont, &guard.ctx.config)
        .map_err(|(s, e)| format!("step {}: {}", s, e)));
    let container = format!("/vagga/base/.roots/{}.{}", name, version);

    // Update container use when using it as subcontainer (fixes #267)
    File::create(Path::new(&container).join("last_use"))
        .map_err(|e| warn!("Can't write image usage info: {}", e)).ok();

    let root = Path::new(&container).join("root");
    try_msg!(copy_dir(&root, &Path::new("/vagga/root"),
                      None, None),
        "Error copying dir {p:?}: {err}", p=root);
    Ok(())
}

pub fn clone(name: &String, guard: &mut Guard, build: bool)
    -> Result<(), StepError>
{
    let cont = guard.ctx.config.containers.get(name)
        .expect("Subcontainer not found");  // TODO
    for b in cont.setup.iter() {
        try!(b.build(guard, false)
            .map_err(|e| E::SubStep(b.0.clone(), Box::new(e))));
    }
    if build {
        try!(real_copy(name, cont, guard));
    }
    Ok(())
}

fn find_config(cfg: &SubConfig, guard: &mut Guard)
    -> Result<Config, StepError>
{
    let path = match cfg.source {
        Source::Container(ref container) => {
            let cont = guard.ctx.config.containers.get(container)
                .expect("Subcontainer not found");  // TODO
            let version = try!(short_version(&cont, &guard.ctx.config)
                .map_err(|(s, e)| format!("step {}: {}", s, e)));
            let container = Path::new("/vagga/base/.roots")
                .join(format!("{}.{}", container, version));

            // Update container use when using it as subcontainer (fixes #267)
            File::create(Path::new(&container).join("last_use"))
                .map_err(|e| warn!("Can't write image usage info: {}", e))
                .ok();

            container.join("root").join(&cfg.path)
        }
        Source::Git(ref _git) => {
            unimplemented!();
        }
        Source::Directory => {
            Path::new("/work").join(&cfg.path)
        }
    };
    Ok(try!(read_config(&path)))
}

pub fn subconfig(cfg: &SubConfig, guard: &mut Guard, build: bool)
    -> Result<(), StepError>
{
    let subcfg = try!(find_config(cfg, guard));
    let cont = subcfg.containers.get(&cfg.container)
        .expect("Subcontainer not found");  // TODO
    for b in cont.setup.iter() {
        try!(b.build(guard, build)
            .map_err(|e| E::SubStep(b.0.clone(), Box::new(e))));
    }
    Ok(())
}

impl BuildStep for Container {
    fn hash(&self, cfg: &Config, hash: &mut Digest)
        -> Result<(), VersionError>
    {
        let cont = try!(cfg.containers.get(&self.0)
            .ok_or(VersionError::ContainerNotFound(self.0.to_string())));
        for b in cont.setup.iter() {
            debug!("Versioning setup: {:?}", b);
            try!(b.hash(cfg, hash));
        }
        Ok(())
    }
    fn build(&self, guard: &mut Guard, build: bool)
        -> Result<(), StepError>
    {
        clone(&self.0, guard, build)
    }
    fn is_dependent_on(&self) -> Option<&str> {
        Some(&self.0)
    }
}
impl BuildStep for Build {
    fn hash(&self, cfg: &Config, hash: &mut Digest)
        -> Result<(), VersionError>
    {
        let cont = try!(cfg.containers.get(&self.container)
            .ok_or(VersionError::ContainerNotFound(self.container.to_string())));
        for b in cont.setup.iter() {
            debug!("Versioning setup: {:?}", b);
            try!(b.hash(cfg, hash));
        }
        Ok(())
    }
    fn build(&self, guard: &mut Guard, do_build: bool)
        -> Result<(), StepError>
    {
        build(&self, guard, do_build)
    }
    fn is_dependent_on(&self) -> Option<&str> {
        Some(&self.container)
    }
}
impl BuildStep for SubConfig {
    fn hash(&self, cfg: &Config, hash: &mut Digest)
        -> Result<(), VersionError>
    {
        let path = match self.source {
            Source::Container(ref container) => {
                let cinfo = try!(cfg.containers.get(container)
                    .ok_or(VersionError::ContainerNotFound(container.clone())));
                let version = try!(short_version(&cinfo, cfg));
                Path::new("/vagga/base/.roots")
                    .join(format!("{}.{}", container, version))
                    .join("root").join(&self.path)
            }
            Source::Git(ref _git) => {
                unimplemented!();
            }
            Source::Directory => {
                Path::new("/work").join(&self.path)
            }
        };
        if !path.exists() {
            return Err(VersionError::New);
        }
        let subcfg = try!(read_config(&path));
        let cont = try!(subcfg.containers.get(&self.container)
            .ok_or(VersionError::ContainerNotFound(self.container.to_string())));
        for b in cont.setup.iter() {
            debug!("Versioning setup: {:?}", b);
            try!(b.hash(cfg, hash));
        }
        Ok(())
    }
    fn build(&self, guard: &mut Guard, build: bool)
        -> Result<(), StepError>
    {
        subconfig(self, guard, build)
    }
    fn is_dependent_on(&self) -> Option<&str> {
        match self.source {
            Source::Directory => None,
            Source::Container(ref name) => Some(name),
            Source::Git(ref _git) => None,
        }
    }
}
<commit_msg>Revert "Update `resolv.conf` when using subcontainer"<commit_after>use std::path::{Path, PathBuf};
use std::fs::File;

use libmount::BindMount;

use quire::validate as V;
use config::read_config;
use config::containers::Container as Cont;
use version::short_version;
use container::mount::{remount_ro};
use container::util::{copy_dir};
use file_util::{create_dir, shallow_copy};
use build_step::{BuildStep, VersionError, StepError, Digest, Config, Guard};

use builder::error::StepError as E;

// Build Steps
#[derive(Debug)]
pub struct Container(String);
tuple_struct_decode!(Container);

impl Container {
    pub fn config() -> V::Scalar {
        V::Scalar::new()
    }
}

#[derive(RustcDecodable, Debug)]
pub struct Build {
    pub container: String,
    pub source: PathBuf,
    pub path: Option<PathBuf>,
    pub temporary_mount: Option<PathBuf>,
}

impl Build {
    pub fn config() -> V::Structure<'static> {
        V::Structure::new()
        .member("container", V::Scalar::new())
        .member("source".to_string(),
            V::Directory::new().is_absolute(true).default("/"))
        .member("path".to_string(),
            V::Directory::new().is_absolute(true).optional())
        .member("temporary_mount".to_string(),
            V::Directory::new().is_absolute(true).optional())
    }
}

#[derive(RustcDecodable, Debug)]
pub struct GitSource {
    pub url: String,
    pub revision: Option<String>,
    pub branch: Option<String>,
}

#[derive(RustcDecodable, Debug)]
pub enum Source {
    Git(GitSource),
    Container(String),
    Directory,
}

#[derive(RustcDecodable, Debug)]
pub struct SubConfig {
    pub source: Source,
    pub path: PathBuf,
    pub container: String,
    pub cache: Option<bool>,
}

impl SubConfig {
    pub fn config() -> V::Structure<'static> {
        V::Structure::new()
        .member("source", V::Enum::new()
            .option("Directory", V::Nothing)
            .option("Container", V::Scalar::new())
            .option("Git", V::Structure::new()
                .member("url", V::Scalar::new())
                .member("revision", V::Scalar::new().optional())
                .member("branch", V::Scalar::new().optional()))
            .optional()
            .default_tag("Directory"))
        .member("path".to_string(), V::Directory::new()
            .is_absolute(false)
            .default("vagga.yaml"))
        .member("container", V::Scalar::new())
        .member("cache", V::Scalar::new().optional())
    }
}

pub fn build(binfo: &Build, guard: &mut Guard, build: bool)
    -> Result<(), StepError>
{
    let ref name = binfo.container;
    let cont = guard.ctx.config.containers.get(name)
        .expect("Subcontainer not found");  // TODO
    if build {
        let version = try!(short_version(&cont, &guard.ctx.config)
            .map_err(|(s, e)| format!("step {}: {}", s, e)));
        let container = Path::new("/vagga/base/.roots")
            .join(format!("{}.{}", name, version));
        let path = container.join("root")
            .join(binfo.source.strip_prefix("/").unwrap());

        // Update container use when using it as subcontainer (fixes #267)
        File::create(Path::new(&container).join("last_use"))
            .map_err(|e| warn!("Can't write image usage info: {}", e)).ok();

        if let Some(ref dest_rel) = binfo.path {
            let dest = Path::new("/vagga/root")
                .join(dest_rel.strip_prefix("/").unwrap());
            if path.is_dir() {
                try_msg!(copy_dir(&path, &dest, None, None),
                    "Error copying dir {p:?}: {err}", p=path);
            } else {
                let path_stat = try!(path.symlink_metadata()
                    .map_err(|e| StepError::Read(path.clone(), e)));
                try_msg!(shallow_copy(&path, &path_stat, &dest,
                        None, None, None),
                    "Error copying file {p:?}: {err}", p=path);
            }
        } else if let Some(ref dest_rel) = binfo.temporary_mount {
            let dest = Path::new("/vagga/root")
                .join(dest_rel.strip_prefix("/").unwrap());
            try_msg!(create_dir(&dest, false),
                "Error creating destination dir: {err}");
            try!(BindMount::new(&path, &dest).mount());
            try!(remount_ro(&dest));
            guard.ctx.mounted.push(dest);
        }
    }
    Ok(())
}

fn real_copy(name: &String, cont: &Cont, guard: &mut Guard)
    -> Result<(), StepError>
{
    let version = try!(short_version(&cont, &guard.ctx.config)
        .map_err(|(s, e)| format!("step {}: {}", s, e)));
    let container = format!("/vagga/base/.roots/{}.{}", name, version);

    // Update container use when using it as subcontainer (fixes #267)
    File::create(Path::new(&container).join("last_use"))
        .map_err(|e| warn!("Can't write image usage info: {}", e)).ok();

    let root = Path::new(&container).join("root");
    try_msg!(copy_dir(&root, &Path::new("/vagga/root"),
                      None, None),
        "Error copying dir {p:?}: {err}", p=root);
    Ok(())
}

pub fn clone(name: &String, guard: &mut Guard, build: bool)
    -> Result<(), StepError>
{
    let cont = guard.ctx.config.containers.get(name)
        .expect("Subcontainer not found");  // TODO
    for b in cont.setup.iter() {
        try!(b.build(guard, false)
            .map_err(|e| E::SubStep(b.0.clone(), Box::new(e))));
    }
    if build {
        try!(real_copy(name, cont, guard));
    }
    Ok(())
}

fn find_config(cfg: &SubConfig, guard: &mut Guard)
    -> Result<Config, StepError>
{
    let path = match cfg.source {
        Source::Container(ref container) => {
            let cont = guard.ctx.config.containers.get(container)
                .expect("Subcontainer not found");  // TODO
            let version = try!(short_version(&cont, &guard.ctx.config)
                .map_err(|(s, e)| format!("step {}: {}", s, e)));
            let container = Path::new("/vagga/base/.roots")
                .join(format!("{}.{}", container, version));

            // Update container use when using it as subcontainer (fixes #267)
            File::create(Path::new(&container).join("last_use"))
                .map_err(|e| warn!("Can't write image usage info: {}", e))
                .ok();

            container.join("root").join(&cfg.path)
        }
        Source::Git(ref _git) => {
            unimplemented!();
        }
        Source::Directory => {
            Path::new("/work").join(&cfg.path)
        }
    };
    Ok(try!(read_config(&path)))
}

pub fn subconfig(cfg: &SubConfig, guard: &mut Guard, build: bool)
    -> Result<(), StepError>
{
    let subcfg = try!(find_config(cfg, guard));
    let cont = subcfg.containers.get(&cfg.container)
        .expect("Subcontainer not found");  // TODO
    for b in cont.setup.iter() {
        try!(b.build(guard, build)
            .map_err(|e| E::SubStep(b.0.clone(), Box::new(e))));
    }
    Ok(())
}

impl BuildStep for Container {
    fn hash(&self, cfg: &Config, hash: &mut Digest)
        -> Result<(), VersionError>
    {
        let cont = try!(cfg.containers.get(&self.0)
            .ok_or(VersionError::ContainerNotFound(self.0.to_string())));
        for b in cont.setup.iter() {
            debug!("Versioning setup: {:?}", b);
            try!(b.hash(cfg, hash));
        }
        Ok(())
    }
    fn build(&self, guard: &mut Guard, build: bool)
        -> Result<(), StepError>
    {
        clone(&self.0, guard, build)
    }
    fn is_dependent_on(&self) -> Option<&str> {
        Some(&self.0)
    }
}
impl BuildStep for Build {
    fn hash(&self, cfg: &Config, hash: &mut Digest)
        -> Result<(), VersionError>
    {
        let cont = try!(cfg.containers.get(&self.container)
            .ok_or(VersionError::ContainerNotFound(self.container.to_string())));
        for b in cont.setup.iter() {
            debug!("Versioning setup: {:?}", b);
            try!(b.hash(cfg, hash));
        }
        Ok(())
    }
    fn build(&self, guard: &mut Guard, do_build: bool)
        -> Result<(), StepError>
    {
        build(&self, guard, do_build)
    }
    fn is_dependent_on(&self) -> Option<&str> {
        Some(&self.container)
    }
}
impl BuildStep for SubConfig {
    fn hash(&self, cfg: &Config, hash: &mut Digest)
        -> Result<(), VersionError>
    {
        let path = match self.source {
            Source::Container(ref container) => {
                let cinfo = try!(cfg.containers.get(container)
                    .ok_or(VersionError::ContainerNotFound(container.clone())));
                let version = try!(short_version(&cinfo, cfg));
                Path::new("/vagga/base/.roots")
                    .join(format!("{}.{}", container, version))
                    .join("root").join(&self.path)
            }
            Source::Git(ref _git) => {
                unimplemented!();
            }
            Source::Directory => {
                Path::new("/work").join(&self.path)
            }
        };
        if !path.exists() {
            return Err(VersionError::New);
        }
        let subcfg = try!(read_config(&path));
        let cont = try!(subcfg.containers.get(&self.container)
            .ok_or(VersionError::ContainerNotFound(self.container.to_string())));
        for b in cont.setup.iter() {
            debug!("Versioning setup: {:?}", b);
            try!(b.hash(cfg, hash));
        }
        Ok(())
    }
    fn build(&self, guard: &mut Guard, build: bool)
        -> Result<(), StepError>
    {
        subconfig(self, guard, build)
    }
    fn is_dependent_on(&self) -> Option<&str> {
        match self.source {
            Source::Directory => None,
            Source::Container(ref name) => Some(name),
            Source::Git(ref _git) => None,
        }
    }
}
<|endoftext|>


<commit_before>#![crate_name="base32"]
#![crate_type="rlib"]

#![feature(phase)]

#[cfg(test)]
extern crate quickcheck;
#[cfg(test)]
#[phase(plugin)]
extern crate quickcheck_macros;

use std::iter::range_inclusive;
use std::cmp::min;

pub enum Base32Type {
    // Not sure if it's better to just have an additional `padding: bool`
    // argument to the encode function
    RFC4648Base32, CrockfordBase32, UnpaddedRFC4648Base32
}

static rfc4648_alphabet: &'static str   = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
static crockford_alphabet: &'static str = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";

pub fn encode(base32_type: Base32Type, data: &[u8]) -> Vec<Ascii> {
    let alphabet = match base32_type {
        RFC4648Base32 | UnpaddedRFC4648Base32 => rfc4648_alphabet,
        CrockfordBase32 => crockford_alphabet
    }.to_ascii();
    let mut ret = Vec::with_capacity((data.len()+3)/4*5);

    for chunk in data.chunks(5) {
        let buf = {
            let mut buf = [0u8, ..5];
            buf.clone_from_slice(chunk);
            buf
        };
        ret.push(alphabet[((buf[0] & 0xF8) >> 3) as uint]);
        ret.push(alphabet[(((buf[0] & 0x07) << 2) | ((buf[1] & 0xC0) >> 6)) as uint]);
        ret.push(alphabet[((buf[1] & 0x3E) >> 1) as uint]);
        ret.push(alphabet[(((buf[1] & 0x01) << 4) | ((buf[2] & 0xF0) >> 4)) as uint]);
        ret.push(alphabet[(((buf[2] & 0x0F) << 1) | (buf[3] >> 7)) as uint]);
        ret.push(alphabet[((buf[3] & 0x7C) >> 2) as uint]);
        ret.push(alphabet[(((buf[3] & 0x03) << 3) | ((buf[4] & 0xE0) >> 5)) as uint]);
        ret.push(alphabet[(buf[4] & 0x1F) as uint]);
    }

    if data.len() % 5 != 0 {
        let len = ret.len();
        let num_extra = 8-(data.len()%5*8+4)/5;
        match base32_type {
            UnpaddedRFC4648Base32 | CrockfordBase32 => {
                ret.truncate(len-num_extra);
            }
            RFC4648Base32 => {
                for i in range_inclusive(1, num_extra) {
                    *ret.get_mut(len-i) = '='.to_ascii();
                }
            }
        }
    }

    ret
}

static rfc4648_inv_alphabet: [u8, ..43] = [-1, -1, 26, 27, 28, 29, 30, 31, -1, -1, -1, -1, -1, 0, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25];

static crockford_inv_alphabet: [u8, ..43] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, 10, 11, 12, 13, 14, 15, 16, 17, 1, 18, 19, 1, 20, 21, 0, 22, 23, 24, 25, 26, -1, 27, 28, 29, 30, 31];

pub fn decode(base32_type: Base32Type, data: &[Ascii]) -> Option<Vec<u8>> {
    let alphabet = match base32_type {
        RFC4648Base32 | UnpaddedRFC4648Base32 => rfc4648_inv_alphabet,
        CrockfordBase32 => crockford_inv_alphabet
    };
    let mut unpadded_data_length = data.len();
    for i in range_inclusive(1u, min(6, data.len())) {
        if data[data.len() - i] != '='.to_ascii() {
            break;
        }
        unpadded_data_length -= 1;
    }
    let output_length = unpadded_data_length*5/8;
    let mut ret = Vec::with_capacity((output_length+4)/5*5);
    for chunk in data.chunks(8) {
        let buf = {
            let mut buf = [0u8, ..8];
            for (i, &c) in chunk.iter().enumerate() {
                match alphabet.get((c.to_uppercase().to_byte()-('0' as u8)) as uint) {
                    Some(&-1) | None => return None,
                    Some(&value) => buf[i] = value,
                };
            }
            buf
        };
        ret.push((buf[0] << 3) | (buf[1] >> 2));
        ret.push((buf[1] << 6) | (buf[2] << 1) | (buf[3] >> 4));
        ret.push((buf[3] << 4) | (buf[4] >> 1));
        ret.push((buf[4] << 7) | (buf[5] << 2) | (buf[6] >> 3));
        ret.push((buf[6] << 5) | buf[7]);
    }
    ret.truncate(output_length);
    Some(ret)
}

#[cfg(test)]
mod test {
    extern crate test;
    use super::{encode, decode, CrockfordBase32, RFC4648Base32, UnpaddedRFC4648Base32};
    use quickcheck;
    use std;
    use std::rand::distributions::IndependentSample;

    #[deriving(Clone)]
    struct B32 {
        c: Ascii
    }

    impl quickcheck::Arbitrary for B32 {
        fn arbitrary<G: quickcheck::Gen>(g: &mut G) -> B32 {
            let alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ".to_ascii();
            B32 {
                c: alphabet[std::rand::distributions::Range::new(0, alphabet.len()).ind_sample(g)]
            }
        }
    }

    impl std::fmt::Show for B32 {
        fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::FormatError> {
            self.c.fmt(f)
        }
    }

    #[test]
    fn masks_crockford() {
        assert_eq!(encode(CrockfordBase32, [0xF8, 0x3E, 0x0F, 0x83, 0xE0]), Vec::from_slice("Z0Z0Z0Z0".to_ascii()));
        assert_eq!(encode(CrockfordBase32, [0x07, 0xC1, 0xF0, 0x7C, 0x1F]), Vec::from_slice("0Z0Z0Z0Z".to_ascii()));
        assert_eq!(decode(CrockfordBase32, "Z0Z0Z0Z0".to_ascii()).unwrap(), vec![0xF8, 0x3E, 0x0F, 0x83, 0xE0]);
        assert_eq!(decode(CrockfordBase32, "0Z0Z0Z0Z".to_ascii()).unwrap(), vec![0x07, 0xC1, 0xF0, 0x7C, 0x1F]);
    }

    #[test]
    fn masks_rfc4648() {
        assert_eq!(encode(RFC4648Base32, [0xF8, 0x3E, 0x7F, 0x83, 0xE7]), Vec::from_slice("7A7H7A7H".to_ascii()));
        assert_eq!(encode(RFC4648Base32, [0x77, 0xC1, 0xF7, 0x7C, 0x1F]), Vec::from_slice("O7A7O7A7".to_ascii()));
        assert_eq!(decode(RFC4648Base32, "7A7H7A7H".to_ascii()).unwrap(), vec![0xF8, 0x3E, 0x7F, 0x83, 0xE7]);
        assert_eq!(decode(RFC4648Base32, "O7A7O7A7".to_ascii()).unwrap(), vec![0x77, 0xC1, 0xF7, 0x7C, 0x1F]);
        assert_eq!(encode(RFC4648Base32, [0xF8, 0x3E, 0x7F, 0x83]), Vec::from_slice("7A7H7AY=".to_ascii()));
    }

    #[test]
    fn masks_unpadded_rfc4648() {
        assert_eq!(encode(UnpaddedRFC4648Base32, [0xF8, 0x3E, 0x7F, 0x83, 0xE7]), Vec::from_slice("7A7H7A7H".to_ascii()));
        assert_eq!(encode(UnpaddedRFC4648Base32, [0x77, 0xC1, 0xF7, 0x7C, 0x1F]), Vec::from_slice("O7A7O7A7".to_ascii()));
        assert_eq!(decode(UnpaddedRFC4648Base32, "7A7H7A7H".to_ascii()).unwrap(), vec![0xF8, 0x3E, 0x7F, 0x83, 0xE7]);
        assert_eq!(decode(UnpaddedRFC4648Base32, "O7A7O7A7".to_ascii()).unwrap(), vec![0x77, 0xC1, 0xF7, 0x7C, 0x1F]);
        assert_eq!(encode(UnpaddedRFC4648Base32, [0xF8, 0x3E, 0x7F, 0x83]), Vec::from_slice("7A7H7AY".to_ascii()));
    }

    #[test]
    fn padding() {
        let num_padding = [0u, 6, 4, 3, 1];
        for i in range(1u, 6) {
            println!("Checking padding for length == {}", i);
            let encoded = encode(RFC4648Base32, range(0u8, i as u8).collect::<Vec<u8>>().as_slice());
            assert_eq!(encoded.len(), 8u);
            for j in range(0u, num_padding[i % 5]) {
                println!("Making sure index {} is padding", encoded.len()-j-1);
                assert_eq!(encoded[encoded.len()-j-1], '='.to_ascii());
            }
            for j in range(0u, 8 - num_padding[i % 5]) {
                println!("Making sure index {} is not padding", j);
                assert!(encoded[j] != '='.to_ascii());
            }
        }
    }

    #[quickcheck]
    fn invertible_crockford(data: Vec<u8>) -> bool {
        decode(CrockfordBase32, encode(CrockfordBase32, data.as_slice()).as_slice()).unwrap() == data
    }

    #[quickcheck]
    fn invertible_rfc4648(data: Vec<u8>) -> bool {
        decode(RFC4648Base32, encode(RFC4648Base32, data.as_slice()).as_slice()).unwrap() == data
    }
    #[quickcheck]
    fn invertible_unpadded_rfc4648(data: Vec<u8>) -> bool {
        decode(UnpaddedRFC4648Base32, encode(UnpaddedRFC4648Base32, data.as_slice()).as_slice()).unwrap() == data
    }

    #[quickcheck]
    fn lower_case(data: Vec<B32>) -> bool {
        let data: Vec<Ascii> = data.iter().map(|e| e.c).collect();
        decode(CrockfordBase32, data.as_slice()) == decode(CrockfordBase32, data.as_slice().to_lower().as_slice())
    }

    #[test]
    #[allow(non_snake_case)]
    fn iIlL1_oO0() {
        assert_eq!(decode(CrockfordBase32, "IiLlOo".to_ascii()), decode(CrockfordBase32, "111100".to_ascii()));
    }

    #[test]
    fn invalid_chars_crockford() {
        assert_eq!(decode(CrockfordBase32, ",".to_ascii()), None)
    }

    #[test]
    fn invalid_chars_rfc4648() {
        assert_eq!(decode(RFC4648Base32, ",".to_ascii()), None)
    }

    #[test]
    fn invalid_chars_unpadded_rfc4648() {
        assert_eq!(decode(UnpaddedRFC4648Base32, ",".to_ascii()), None)
    }

    #[bench]
    fn bench_encode(b: &mut test::Bencher) {
        let data = [0, 0, 0, 0, 0];
        b.iter(|| encode(CrockfordBase32, data.as_slice()));
        b.bytes = data.len() as u64;
    }

    #[bench]
    fn bench_decode(b: &mut test::Bencher) {
        let data = "00000000".to_ascii();
        b.iter(|| decode(CrockfordBase32, data));
        b.bytes = data.len() as u64;
    }
}
<commit_msg>Use IndexMut instead of get_mut() now that IndexMut works properly.<commit_after>#![crate_name="base32"]
#![crate_type="rlib"]

#![feature(phase)]

#[cfg(test)]
extern crate quickcheck;
#[cfg(test)]
#[phase(plugin)]
extern crate quickcheck_macros;

use std::iter::range_inclusive;
use std::cmp::min;

pub enum Base32Type {
    // Not sure if it's better to just have an additional `padding: bool`
    // argument to the encode function
    RFC4648Base32, CrockfordBase32, UnpaddedRFC4648Base32
}

static rfc4648_alphabet: &'static str   = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
static crockford_alphabet: &'static str = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";

pub fn encode(base32_type: Base32Type, data: &[u8]) -> Vec<Ascii> {
    let alphabet = match base32_type {
        RFC4648Base32 | UnpaddedRFC4648Base32 => rfc4648_alphabet,
        CrockfordBase32 => crockford_alphabet
    }.to_ascii();
    let mut ret = Vec::with_capacity((data.len()+3)/4*5);

    for chunk in data.chunks(5) {
        let buf = {
            let mut buf = [0u8, ..5];
            buf.clone_from_slice(chunk);
            buf
        };
        ret.push(alphabet[((buf[0] & 0xF8) >> 3) as uint]);
        ret.push(alphabet[(((buf[0] & 0x07) << 2) | ((buf[1] & 0xC0) >> 6)) as uint]);
        ret.push(alphabet[((buf[1] & 0x3E) >> 1) as uint]);
        ret.push(alphabet[(((buf[1] & 0x01) << 4) | ((buf[2] & 0xF0) >> 4)) as uint]);
        ret.push(alphabet[(((buf[2] & 0x0F) << 1) | (buf[3] >> 7)) as uint]);
        ret.push(alphabet[((buf[3] & 0x7C) >> 2) as uint]);
        ret.push(alphabet[(((buf[3] & 0x03) << 3) | ((buf[4] & 0xE0) >> 5)) as uint]);
        ret.push(alphabet[(buf[4] & 0x1F) as uint]);
    }

    if data.len() % 5 != 0 {
        let len = ret.len();
        let num_extra = 8-(data.len()%5*8+4)/5;
        match base32_type {
            UnpaddedRFC4648Base32 | CrockfordBase32 => {
                ret.truncate(len-num_extra);
            }
            RFC4648Base32 => {
                for i in range_inclusive(1, num_extra) {
                    ret[len-i] = '='.to_ascii();
                }
            }
        }
    }

    ret
}

static rfc4648_inv_alphabet: [u8, ..43] = [-1, -1, 26, 27, 28, 29, 30, 31, -1, -1, -1, -1, -1, 0, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25];

static crockford_inv_alphabet: [u8, ..43] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, -1, 10, 11, 12, 13, 14, 15, 16, 17, 1, 18, 19, 1, 20, 21, 0, 22, 23, 24, 25, 26, -1, 27, 28, 29, 30, 31];

pub fn decode(base32_type: Base32Type, data: &[Ascii]) -> Option<Vec<u8>> {
    let alphabet = match base32_type {
        RFC4648Base32 | UnpaddedRFC4648Base32 => rfc4648_inv_alphabet,
        CrockfordBase32 => crockford_inv_alphabet
    };
    let mut unpadded_data_length = data.len();
    for i in range_inclusive(1u, min(6, data.len())) {
        if data[data.len() - i] != '='.to_ascii() {
            break;
        }
        unpadded_data_length -= 1;
    }
    let output_length = unpadded_data_length*5/8;
    let mut ret = Vec::with_capacity((output_length+4)/5*5);
    for chunk in data.chunks(8) {
        let buf = {
            let mut buf = [0u8, ..8];
            for (i, &c) in chunk.iter().enumerate() {
                match alphabet.get((c.to_uppercase().to_byte()-('0' as u8)) as uint) {
                    Some(&-1) | None => return None,
                    Some(&value) => buf[i] = value,
                };
            }
            buf
        };
        ret.push((buf[0] << 3) | (buf[1] >> 2));
        ret.push((buf[1] << 6) | (buf[2] << 1) | (buf[3] >> 4));
        ret.push((buf[3] << 4) | (buf[4] >> 1));
        ret.push((buf[4] << 7) | (buf[5] << 2) | (buf[6] >> 3));
        ret.push((buf[6] << 5) | buf[7]);
    }
    ret.truncate(output_length);
    Some(ret)
}

#[cfg(test)]
mod test {
    extern crate test;
    use super::{encode, decode, CrockfordBase32, RFC4648Base32, UnpaddedRFC4648Base32};
    use quickcheck;
    use std;
    use std::rand::distributions::IndependentSample;

    #[deriving(Clone)]
    struct B32 {
        c: Ascii
    }

    impl quickcheck::Arbitrary for B32 {
        fn arbitrary<G: quickcheck::Gen>(g: &mut G) -> B32 {
            let alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ".to_ascii();
            B32 {
                c: alphabet[std::rand::distributions::Range::new(0, alphabet.len()).ind_sample(g)]
            }
        }
    }

    impl std::fmt::Show for B32 {
        fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::FormatError> {
            self.c.fmt(f)
        }
    }

    #[test]
    fn masks_crockford() {
        assert_eq!(encode(CrockfordBase32, [0xF8, 0x3E, 0x0F, 0x83, 0xE0]), Vec::from_slice("Z0Z0Z0Z0".to_ascii()));
        assert_eq!(encode(CrockfordBase32, [0x07, 0xC1, 0xF0, 0x7C, 0x1F]), Vec::from_slice("0Z0Z0Z0Z".to_ascii()));
        assert_eq!(decode(CrockfordBase32, "Z0Z0Z0Z0".to_ascii()).unwrap(), vec![0xF8, 0x3E, 0x0F, 0x83, 0xE0]);
        assert_eq!(decode(CrockfordBase32, "0Z0Z0Z0Z".to_ascii()).unwrap(), vec![0x07, 0xC1, 0xF0, 0x7C, 0x1F]);
    }

    #[test]
    fn masks_rfc4648() {
        assert_eq!(encode(RFC4648Base32, [0xF8, 0x3E, 0x7F, 0x83, 0xE7]), Vec::from_slice("7A7H7A7H".to_ascii()));
        assert_eq!(encode(RFC4648Base32, [0x77, 0xC1, 0xF7, 0x7C, 0x1F]), Vec::from_slice("O7A7O7A7".to_ascii()));
        assert_eq!(decode(RFC4648Base32, "7A7H7A7H".to_ascii()).unwrap(), vec![0xF8, 0x3E, 0x7F, 0x83, 0xE7]);
        assert_eq!(decode(RFC4648Base32, "O7A7O7A7".to_ascii()).unwrap(), vec![0x77, 0xC1, 0xF7, 0x7C, 0x1F]);
        assert_eq!(encode(RFC4648Base32, [0xF8, 0x3E, 0x7F, 0x83]), Vec::from_slice("7A7H7AY=".to_ascii()));
    }

    #[test]
    fn masks_unpadded_rfc4648() {
        assert_eq!(encode(UnpaddedRFC4648Base32, [0xF8, 0x3E, 0x7F, 0x83, 0xE7]), Vec::from_slice("7A7H7A7H".to_ascii()));
        assert_eq!(encode(UnpaddedRFC4648Base32, [0x77, 0xC1, 0xF7, 0x7C, 0x1F]), Vec::from_slice("O7A7O7A7".to_ascii()));
        assert_eq!(decode(UnpaddedRFC4648Base32, "7A7H7A7H".to_ascii()).unwrap(), vec![0xF8, 0x3E, 0x7F, 0x83, 0xE7]);
        assert_eq!(decode(UnpaddedRFC4648Base32, "O7A7O7A7".to_ascii()).unwrap(), vec![0x77, 0xC1, 0xF7, 0x7C, 0x1F]);
        assert_eq!(encode(UnpaddedRFC4648Base32, [0xF8, 0x3E, 0x7F, 0x83]), Vec::from_slice("7A7H7AY".to_ascii()));
    }

    #[test]
    fn padding() {
        let num_padding = [0u, 6, 4, 3, 1];
        for i in range(1u, 6) {
            println!("Checking padding for length == {}", i);
            let encoded = encode(RFC4648Base32, range(0u8, i as u8).collect::<Vec<u8>>().as_slice());
            assert_eq!(encoded.len(), 8u);
            for j in range(0u, num_padding[i % 5]) {
                println!("Making sure index {} is padding", encoded.len()-j-1);
                assert_eq!(encoded[encoded.len()-j-1], '='.to_ascii());
            }
            for j in range(0u, 8 - num_padding[i % 5]) {
                println!("Making sure index {} is not padding", j);
                assert!(encoded[j] != '='.to_ascii());
            }
        }
    }

    #[quickcheck]
    fn invertible_crockford(data: Vec<u8>) -> bool {
        decode(CrockfordBase32, encode(CrockfordBase32, data.as_slice()).as_slice()).unwrap() == data
    }

    #[quickcheck]
    fn invertible_rfc4648(data: Vec<u8>) -> bool {
        decode(RFC4648Base32, encode(RFC4648Base32, data.as_slice()).as_slice()).unwrap() == data
    }
    #[quickcheck]
    fn invertible_unpadded_rfc4648(data: Vec<u8>) -> bool {
        decode(UnpaddedRFC4648Base32, encode(UnpaddedRFC4648Base32, data.as_slice()).as_slice()).unwrap() == data
    }

    #[quickcheck]
    fn lower_case(data: Vec<B32>) -> bool {
        let data: Vec<Ascii> = data.iter().map(|e| e.c).collect();
        decode(CrockfordBase32, data.as_slice()) == decode(CrockfordBase32, data.as_slice().to_lower().as_slice())
    }

    #[test]
    #[allow(non_snake_case)]
    fn iIlL1_oO0() {
        assert_eq!(decode(CrockfordBase32, "IiLlOo".to_ascii()), decode(CrockfordBase32, "111100".to_ascii()));
    }

    #[test]
    fn invalid_chars_crockford() {
        assert_eq!(decode(CrockfordBase32, ",".to_ascii()), None)
    }

    #[test]
    fn invalid_chars_rfc4648() {
        assert_eq!(decode(RFC4648Base32, ",".to_ascii()), None)
    }

    #[test]
    fn invalid_chars_unpadded_rfc4648() {
        assert_eq!(decode(UnpaddedRFC4648Base32, ",".to_ascii()), None)
    }

    #[bench]
    fn bench_encode(b: &mut test::Bencher) {
        let data = [0, 0, 0, 0, 0];
        b.iter(|| encode(CrockfordBase32, data.as_slice()));
        b.bytes = data.len() as u64;
    }

    #[bench]
    fn bench_decode(b: &mut test::Bencher) {
        let data = "00000000".to_ascii();
        b.iter(|| decode(CrockfordBase32, data));
        b.bytes = data.len() as u64;
    }
}
<|endoftext|>