<commit_before>use std::os::raw::{c_char, c_void, c_int};
use std::mem;
use std::default::Default;
use std::fmt;

use ffi;

pub type PmDeviceId = i32;
pub type PortMidiStream = c_void;
pub type PmMessage = i32;
pub type PmTimestamp = u32;
pub const PM_NO_DEVICE: PmDeviceId = -1;

#[derive(Copy, Clone, Debug)]
#[repr(C)]
pub struct PmEvent {
    pub message: PmMessage,
    pub timestamp: PmTimestamp,
}
impl Default for PmEvent {
    fn default() -> Self {
        PmEvent {
            message: 0,
            timestamp: 0,
        }
    }
}

#[repr(C)]
pub struct PmDeviceInfo {
    /// this internal structure version
    pub struct_version: i32,
    /// underlying MIDI API, e.g. MMSystem or DirectX
    pub interf: *const c_char,
    /// device name, e.g. USB MidiSport 1x1
    pub name: *const c_char,
    /// true iff input is available
    pub input: c_int,
    /// true iff output is available
    pub output: c_int,
    /// used by generic PortMidi code to do error checking on arguments
    pub opened: i32,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(C)]
pub enum PmError {
    /// "no error" return that also indicates data available
    PmNoError = 0,
    /// "no error" return that also indicates data available
    PmGotData = 1,
    PmHostError = -10000,
    /// out of range or
    /// output device when input is requested or
    /// input device when output is requested or
    /// device is already opened
    PmInvalidDeviceId = -9999,
    PmInsufficientMemory = -9998,
    PmBufferTooSmall = -9997,
    PmBufferOverflow = -9996,
    /// PortMidiStream parameter is NULL or
    /// stream is not opened or
    /// stream is output when input is required or
    /// stream is input when output is required
    PmBadPtr = -9995,
    /// illegal midi data, e.g. missing EOX
    PmBadData = -9994,
    PmInternalError = -9993,
    /// buffer is already as large as it can be
    PmBufferMaxSize = -9992,
}
impl fmt::Display for PmError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let str_ptr = unsafe { ffi::Pm_GetErrorText(*self) };
        write!(f, "{}", ffi::ptr_to_string(str_ptr).unwrap())
    }
}
pub trait MaybeError<T> {
    fn try_from(err_code: T) -> Result<T, PmError>;
}
impl MaybeError<i32> for PmError {
    fn try_from(err_code: i32) -> Result<i32, PmError> {
        match err_code {
            -10_000...-9992 | 0 | 1 => unsafe { Err(mem::transmute(err_code)) },
            _ => Ok(err_code),
        }
    }
}
<commit_msg>Replace `i32` with `c_int`<commit_after>use std::os::raw::{c_char, c_void, c_int};
use std::mem;
use std::default::Default;
use std::fmt;

use ffi;

pub type PmDeviceId = c_int;
pub type PortMidiStream = c_void;
pub type PmMessage = c_int;
pub type PmTimestamp = u32;
pub const PM_NO_DEVICE: PmDeviceId = -1;

#[derive(Copy, Clone, Debug)]
#[repr(C)]
pub struct PmEvent {
    pub message: PmMessage,
    pub timestamp: PmTimestamp,
}
impl Default for PmEvent {
    fn default() -> Self {
        PmEvent {
            message: 0,
            timestamp: 0,
        }
    }
}

#[repr(C)]
pub struct PmDeviceInfo {
    /// this internal structure version
    pub struct_version: c_int,
    /// underlying MIDI API, e.g. MMSystem or DirectX
    pub interf: *const c_char,
    /// device name, e.g. USB MidiSport 1x1
    pub name: *const c_char,
    /// true iff input is available
    pub input: c_int,
    /// true iff output is available
    pub output: c_int,
    /// used by generic PortMidi code to do error checking on arguments
    pub opened: c_int,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(C)]
pub enum PmError {
    /// "no error" return that also indicates data available
    PmNoError = 0,
    /// "no error" return that also indicates data available
    PmGotData = 1,
    PmHostError = -10000,
    /// out of range or
    /// output device when input is requested or
    /// input device when output is requested or
    /// device is already opened
    PmInvalidDeviceId = -9999,
    PmInsufficientMemory = -9998,
    PmBufferTooSmall = -9997,
    PmBufferOverflow = -9996,
    /// PortMidiStream parameter is NULL or
    /// stream is not opened or
    /// stream is output when input is required or
    /// stream is input when output is required
    PmBadPtr = -9995,
    /// illegal midi data, e.g. missing EOX
    PmBadData = -9994,
    PmInternalError = -9993,
    /// buffer is already as large as it can be
    PmBufferMaxSize = -9992,
}
impl fmt::Display for PmError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let str_ptr = unsafe { ffi::Pm_GetErrorText(*self) };
        write!(f, "{}", ffi::ptr_to_string(str_ptr).unwrap())
    }
}
pub trait MaybeError<T> {
    fn try_from(err_code: T) -> Result<T, PmError>;
}
impl MaybeError<c_int> for PmError {
    fn try_from(err_code: c_int) -> Result<c_int, PmError> {
        match err_code {
            -10_000...-9992 | 0 | 1 => unsafe { Err(mem::transmute(err_code)) },
            _ => Ok(err_code),
        }
    }
}
<|endoftext|>


<commit_before>//! Tests for when multiple artifacts have the same output filename.
//! See https://github.com/rust-lang/cargo/issues/6313 for more details.
//! Ideally these should never happen, but I don't think we'll ever be able to
//! prevent all collisions.

use cargo_test_support::basic_manifest;
use cargo_test_support::project;
use std::env;

#[cargo_test]
fn collision_dylib() {
    // Path dependencies don't include metadata hash in filename for dylibs.
    let p = project()
        .file(
            "Cargo.toml",
            r#"
            [workspace]
            members = ["a", "b"]
            "#,
        )
        .file(
            "a/Cargo.toml",
            r#"
            [package]
            name = "a"
            version = "1.0.0"

            [lib]
            crate-type = ["dylib"]
            "#,
        )
        .file("a/src/lib.rs", "")
        .file(
            "b/Cargo.toml",
            r#"
            [package]
            name = "b"
            version = "1.0.0"

            [lib]
            crate-type = ["dylib"]
            name = "a"
            "#,
        )
        .file("b/src/lib.rs", "")
        .build();

    // `j=1` is required because on Windows you'll get an error due to
    // two processes writing to the file at the same time.
    p.cargo("build -j=1")
        .with_stderr_contains(&format!("\
[WARNING] output filename collision.
The lib target `a` in package `b v1.0.0 ([..]/foo/b)` has the same output filename as the lib target `a` in package `a v1.0.0 ([..]/foo/a)`.
Colliding filename is: [..]/foo/target/debug/deps/{}a{}
The targets should have unique names.
Consider changing their names to be unique or compiling them separately.
This may become a hard error in the future; see <https://github.com/rust-lang/cargo/issues/6313>.
", env::consts::DLL_PREFIX, env::consts::DLL_SUFFIX))
        .run();
}

#[cargo_test]
fn collision_example() {
    // Examples in a workspace can easily collide.
    let p = project()
        .file(
            "Cargo.toml",
            r#"
            [workspace]
            members = ["a", "b"]
            "#,
        )
        .file("a/Cargo.toml", &basic_manifest("a", "1.0.0"))
        .file("a/examples/ex1.rs", "fn main() {}")
        .file("b/Cargo.toml", &basic_manifest("b", "1.0.0"))
        .file("b/examples/ex1.rs", "fn main() {}")
        .build();

    // `j=1` is required because on Windows you'll get an error due to
    // two processes writing to the file at the same time.
    p.cargo("build --examples -j=1")
        .with_stderr_contains("\
[WARNING] output filename collision.
The example target `ex1` in package `b v1.0.0 ([..]/foo/b)` has the same output filename as the example target `ex1` in package `a v1.0.0 ([..]/foo/a)`.
Colliding filename is: [..]/foo/target/debug/examples/ex1[EXE]
The targets should have unique names.
Consider changing their names to be unique or compiling them separately.
This may become a hard error in the future; see <https://github.com/rust-lang/cargo/issues/6313>.
")
        .run();
}

#[cargo_test]
#[cfg(not(target_env = "msvc"))]
fn collision_export() {
    // `--out-dir` combines some things which can cause conflicts.
    let p = project()
        .file("Cargo.toml", &basic_manifest("foo", "1.0.0"))
        .file("examples/foo.rs", "fn main() {}")
        .file("src/main.rs", "fn main() {}")
        .build();

    p.cargo("build --out-dir=out -Z unstable-options --bins --examples")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains("\
[WARNING] `--out-dir` filename collision.
The example target `foo` in package `foo v1.0.0 ([..]/foo)` has the same output filename as the bin target `foo` in package `foo v1.0.0 ([..]/foo)`.
Colliding filename is: [..]/foo/out/foo[EXE]
The exported filenames should be unique.
Consider changing their names to be unique or compiling them separately.
This may become a hard error in the future; see <https://github.com/rust-lang/cargo/issues/6313>.
")
        .run();
}

#[cargo_test]
fn collision_doc() {
    let p = project()
        .file(
            "Cargo.toml",
            r#"
            [package]
            name = "foo"
            version = "0.1.0"

            [dependencies]
            foo2 = { path = "foo2" }
            "#,
        )
        .file("src/lib.rs", "")
        .file(
            "foo2/Cargo.toml",
            r#"
            [package]
            name = "foo2"
            version = "0.1.0"

            [lib]
            name = "foo"
            "#,
        )
        .file("foo2/src/lib.rs", "")
        .build();

    p.cargo("doc")
        .with_stderr_contains(
            "\
[WARNING] output filename collision.
The lib target `foo` in package `foo2 v0.1.0 ([..]/foo/foo2)` has the same output \
filename as the lib target `foo` in package `foo v0.1.0 ([..]/foo)`.
Colliding filename is: [..]/foo/target/doc/foo/index.html
The targets should have unique names.
This is a known bug where multiple crates with the same name use
the same path; see <https://github.com/rust-lang/cargo/issues/6313>.
",
        )
        .run();
}
<commit_msg>Fix rare failure in collision_export test.<commit_after>//! Tests for when multiple artifacts have the same output filename.
//! See https://github.com/rust-lang/cargo/issues/6313 for more details.
//! Ideally these should never happen, but I don't think we'll ever be able to
//! prevent all collisions.

use cargo_test_support::basic_manifest;
use cargo_test_support::project;
use std::env;

#[cargo_test]
fn collision_dylib() {
    // Path dependencies don't include metadata hash in filename for dylibs.
    let p = project()
        .file(
            "Cargo.toml",
            r#"
            [workspace]
            members = ["a", "b"]
            "#,
        )
        .file(
            "a/Cargo.toml",
            r#"
            [package]
            name = "a"
            version = "1.0.0"

            [lib]
            crate-type = ["dylib"]
            "#,
        )
        .file("a/src/lib.rs", "")
        .file(
            "b/Cargo.toml",
            r#"
            [package]
            name = "b"
            version = "1.0.0"

            [lib]
            crate-type = ["dylib"]
            name = "a"
            "#,
        )
        .file("b/src/lib.rs", "")
        .build();

    // `j=1` is required because on Windows you'll get an error due to
    // two processes writing to the file at the same time.
    p.cargo("build -j=1")
        .with_stderr_contains(&format!("\
[WARNING] output filename collision.
The lib target `a` in package `b v1.0.0 ([..]/foo/b)` has the same output filename as the lib target `a` in package `a v1.0.0 ([..]/foo/a)`.
Colliding filename is: [..]/foo/target/debug/deps/{}a{}
The targets should have unique names.
Consider changing their names to be unique or compiling them separately.
This may become a hard error in the future; see <https://github.com/rust-lang/cargo/issues/6313>.
", env::consts::DLL_PREFIX, env::consts::DLL_SUFFIX))
        .run();
}

#[cargo_test]
fn collision_example() {
    // Examples in a workspace can easily collide.
    let p = project()
        .file(
            "Cargo.toml",
            r#"
            [workspace]
            members = ["a", "b"]
            "#,
        )
        .file("a/Cargo.toml", &basic_manifest("a", "1.0.0"))
        .file("a/examples/ex1.rs", "fn main() {}")
        .file("b/Cargo.toml", &basic_manifest("b", "1.0.0"))
        .file("b/examples/ex1.rs", "fn main() {}")
        .build();

    // `j=1` is required because on Windows you'll get an error due to
    // two processes writing to the file at the same time.
    p.cargo("build --examples -j=1")
        .with_stderr_contains("\
[WARNING] output filename collision.
The example target `ex1` in package `b v1.0.0 ([..]/foo/b)` has the same output filename as the example target `ex1` in package `a v1.0.0 ([..]/foo/a)`.
Colliding filename is: [..]/foo/target/debug/examples/ex1[EXE]
The targets should have unique names.
Consider changing their names to be unique or compiling them separately.
This may become a hard error in the future; see <https://github.com/rust-lang/cargo/issues/6313>.
")
        .run();
}

#[cargo_test]
// --out-dir and examples are currently broken on MSVC.
// See https://github.com/rust-lang/cargo/issues/7493
#[cfg(not(target_env = "msvc"))]
fn collision_export() {
    // `--out-dir` combines some things which can cause conflicts.
    let p = project()
        .file("Cargo.toml", &basic_manifest("foo", "1.0.0"))
        .file("examples/foo.rs", "fn main() {}")
        .file("src/main.rs", "fn main() {}")
        .build();

    // -j1 to avoid issues with two processes writing to the same file at the
    // same time.
    p.cargo("build -j1 --out-dir=out -Z unstable-options --bins --examples")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains("\
[WARNING] `--out-dir` filename collision.
The example target `foo` in package `foo v1.0.0 ([..]/foo)` has the same output filename as the bin target `foo` in package `foo v1.0.0 ([..]/foo)`.
Colliding filename is: [..]/foo/out/foo[EXE]
The exported filenames should be unique.
Consider changing their names to be unique or compiling them separately.
This may become a hard error in the future; see <https://github.com/rust-lang/cargo/issues/6313>.
")
        .run();
}

#[cargo_test]
fn collision_doc() {
    let p = project()
        .file(
            "Cargo.toml",
            r#"
            [package]
            name = "foo"
            version = "0.1.0"

            [dependencies]
            foo2 = { path = "foo2" }
            "#,
        )
        .file("src/lib.rs", "")
        .file(
            "foo2/Cargo.toml",
            r#"
            [package]
            name = "foo2"
            version = "0.1.0"

            [lib]
            name = "foo"
            "#,
        )
        .file("foo2/src/lib.rs", "")
        .build();

    p.cargo("doc")
        .with_stderr_contains(
            "\
[WARNING] output filename collision.
The lib target `foo` in package `foo2 v0.1.0 ([..]/foo/foo2)` has the same output \
filename as the lib target `foo` in package `foo v0.1.0 ([..]/foo)`.
Colliding filename is: [..]/foo/target/doc/foo/index.html
The targets should have unique names.
This is a known bug where multiple crates with the same name use
the same path; see <https://github.com/rust-lang/cargo/issues/6313>.
",
        )
        .run();
}
<|endoftext|>


<commit_before>use super::node::write_node;
use super::node::NodeWriteGuard;
use super::search::MutSearchResult;
use super::BPlusTree;
use super::LevelTree;
use super::NodeCellRef;
use super::*;
use itertools::Itertools;
use std::fmt::Debug;
use std::sync::atomic::Ordering;

pub const LEVEL_PAGE_DIFF_MULTIPLIER: usize = 4;
pub const LEVEL_TREE_DEPTH: u32 = 2;

pub const LEVEL_M: usize = 16; // Smaller can be faster but more fragmented
pub const LEVEL_1: usize = LEVEL_M * LEVEL_PAGE_DIFF_MULTIPLIER;
pub const LEVEL_2: usize = LEVEL_1 * LEVEL_PAGE_DIFF_MULTIPLIER;

pub const NUM_LEVELS: usize = 3;

// NEVER MAKE THIS ASYNC
fn merge_prune<KS, PS>(
    level: usize,
    node: &NodeCellRef,
    src_tree: &BPlusTree<KS, PS>,
    dest_tree: &dyn LevelTree,
    boundary: &EntryKey,
) -> usize
where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    let search = mut_first::<KS, PS>(node);
    match search {
        MutSearchResult::External => {
            let (nodes, mut new_first) = select_nodes_in_boundary::<KS, PS>(node, boundary, level);
            let new_first_ref = new_first.node_ref().clone();
            let head_id = nodes[0].ext_id();
            new_first.extnode_mut(src_tree).id = head_id;
            new_first.left_ref_mut().map(|lr| *lr = Default::default());
            drop(new_first);
            debug_assert!(
                !nodes.is_empty(),
                "Cannot find any keys in range in external with boundary {:?}",
                boundary
            );
            let mut num_keys_merged = 0;
            debug!("Selected {} pages to merge", nodes.len());
            for mut node in nodes {
                let node_id = node.ext_id();
                let node_len = node.len();
                let deleted_keys = node
                    .keys()
                    .iter()
                    .filter(|k| src_tree.deleted.contains(k))
                    .cloned()
                    .collect_vec();
                let merging_keys = node
                    .extnode_mut_no_persist()
                    .keys
                    .as_slice()[..node_len]
                    .iter_mut()
                    .filter(|k| !src_tree.deleted.contains(k))
                    .map(|k| mem::take(k))
                    .collect_vec();
                let num_merging_keys = merging_keys.len();
                num_keys_merged += num_merging_keys;
                debug!("Collected {} keys, merging to destination tree", num_merging_keys);
                dest_tree.merge_with_keys(box merging_keys);
                debug!("Merge completed, cleanup");
                // Update delete set in source
                for dk in deleted_keys {
                    src_tree.deleted.remove(&dk);
                }
                clear_node(node, &new_first_ref);
                if node_id != head_id {
                    external::make_deleted::<KS, PS>(&node_id);
                }
                src_tree.len.fetch_sub(num_merging_keys, Ordering::Relaxed);
            }
            debug!("Merge prune completed in external level");
            return num_keys_merged;
        }
        MutSearchResult::Internal(sub_node) => {
            let num_keys_merged = merge_prune(level + 1, &sub_node, src_tree, dest_tree, boundary);
            let (nodes, right_node) = select_nodes_in_boundary::<KS, PS>(node, boundary, level);
            if !nodes.is_empty() {
                for node in nodes {
                    clear_node(node, right_node.node_ref());
                }
            } else {
                // The boundary does nott covered the first node in this level
                // Need to partially remove the node keys and ptrs
                let mut node = right_node;
                // Find the boundary in the node and remove all keys and ptrs to there
                match node.keys().binary_search(boundary) {
                    Ok(bound_id) => {
                        let mut new_keys = KS::init();
                        let mut new_ptrs = PS::init();
                        let mut num_keys = 0;
                        for (i, k) in node.keys()[bound_id + 1..].iter().enumerate() {
                            new_keys.as_slice()[i] = k.clone();
                            num_keys += 1;
                        }
                        for (i, p) in node.ptrs()[bound_id + 1..].iter().enumerate() {
                            new_ptrs.as_slice()[i] = p.clone();
                        }
                        let root_innode = node.innode_mut();
                        root_innode.keys = new_keys;
                        root_innode.ptrs = new_ptrs;
                        root_innode.len = num_keys;
                    }
                    Err(id) => {
                        assert_eq!(id, 0);
                    }
                }
            }
            return num_keys_merged;
        }
    }
}

fn clear_node<KS, PS>(mut node: NodeWriteGuard<KS, PS>, right_ref: &NodeCellRef)
where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    node.make_empty_node(false);
    *node.right_ref_mut().unwrap() = right_ref.clone();
    node.left_ref_mut().map(|r| *r = right_ref.clone());
}

fn select_nodes_in_boundary<KS, PS>(
    first_node: &NodeCellRef,
    right_boundary: &EntryKey,
    level: usize,
) -> (Vec<NodeWriteGuard<KS, PS>>, NodeWriteGuard<KS, PS>)
where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    let first_node = write_node::<KS, PS>(first_node);
    if first_node.right_bound() > right_boundary {
        return (vec![], first_node);
    }
    let mut next_node = write_node(first_node.right_ref().unwrap());
    let mut collected = vec![first_node];
    loop {
        let first_key = next_node.first_key();
        let node_right = next_node.right_bound();
        debug_assert!(first_key < right_boundary, "First key {:?} out of bound {:?}, collected {}", first_key, right_boundary, collected.len());
        debug_assert!(node_right <= right_boundary, "Node right {:?} out of bound {:?}, collected {}", node_right, right_boundary, collected.len());
        let next_right = write_node(next_node.right_ref().unwrap());
        collected.push(next_node);
        next_node = next_right;
        if next_node.first_key() >= right_boundary {
            trace!("Collected {} nodes at level {}", collected.len(), level);
            return (collected, next_node);
        }
    }
}

fn select_boundary<KS, PS>(node: &NodeCellRef) -> EntryKey
where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    read_node(node, |node: &NodeReadHandler<KS, PS>| {
        let node_keys = node.keys();
        let node_len = node_keys.len();
        if node.len() < 2 {
            return select_boundary::<KS, PS>(&node.ptrs()[0]);
        }
        // Pick half of the keys in the root
        // Genreally, higher level sub tree in LSM tree will select more keys to merged
        // into next level
        let mid_idx = node_len / 2;
        // Return the mid key as the boundary for selection (cut)
        node_keys[mid_idx].clone()
    })
}

pub async fn level_merge<KS, PS>(
    level: usize,
    src_tree: &BPlusTree<KS, PS>,
    dest_tree: &dyn LevelTree,
) -> usize
where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    debug!("Merging LSM tree level {}", level);
    debug_assert!(verification::tree_has_no_empty_node(&src_tree));
    debug_assert!(verification::is_tree_in_order(&src_tree, level));
    let root = src_tree.get_root();
    let key_boundary = select_boundary::<KS, PS>(&root);
    debug!(
        "Level merge level {} with boundary {:?}",
        level, key_boundary
    );
    let num_keys = merge_prune(0, &src_tree.get_root(), src_tree, dest_tree, &key_boundary);
    debug_assert!(verification::tree_has_no_empty_node(&src_tree));
    debug_assert!(verification::is_tree_in_order(&src_tree, level));
    debug!("Merge and pruned level {}, waiting for storage", level);
    storage::wait_until_updated().await;
    debug!("MERGE LEVEL {} COMPLETED", level);
    return num_keys;
}
<commit_msg>alternative select and prune pattern<commit_after>use super::node::write_node;
use super::node::NodeWriteGuard;
use super::search::MutSearchResult;
use super::BPlusTree;
use super::LevelTree;
use super::NodeCellRef;
use super::*;
use itertools::Itertools;
use std::fmt::Debug;
use std::sync::atomic::Ordering;

pub const LEVEL_PAGE_DIFF_MULTIPLIER: usize = 4;
pub const LEVEL_TREE_DEPTH: u32 = 2;

pub const LEVEL_M: usize = 16; // Smaller can be faster but more fragmented
pub const LEVEL_1: usize = LEVEL_M * LEVEL_PAGE_DIFF_MULTIPLIER;
pub const LEVEL_2: usize = LEVEL_1 * LEVEL_PAGE_DIFF_MULTIPLIER;

pub const NUM_LEVELS: usize = 3;

enum NodeSelection<KS, PS>
  where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    WholePage(Vec<NodeWriteGuard<KS, PS>>, NodeWriteGuard<KS, PS>),
    PartialPage(Vec<NodeWriteGuard<KS, PS>>, NodeWriteGuard<KS, PS>)
}

// NEVER MAKE THIS ASYNC
fn merge_prune<KS, PS>(
    level: usize,
    node: &NodeCellRef,
    src_tree: &BPlusTree<KS, PS>,
    dest_tree: &dyn LevelTree,
    boundary: &EntryKey,
) -> usize
where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    let search = mut_first::<KS, PS>(node);
    match search {
        MutSearchResult::External => {
            if let NodeSelection::WholePage(nodes, mut new_first) = select_nodes_in_boundary::<KS, PS>(node, boundary, level) {
                let new_first_ref = new_first.node_ref().clone();
                let head_id = nodes[0].ext_id();
                new_first.extnode_mut(src_tree).id = head_id;
                new_first.left_ref_mut().map(|lr| *lr = Default::default());
                drop(new_first);
                debug_assert!(
                    !nodes.is_empty(),
                    "Cannot find any keys in range in external with boundary {:?}",
                    boundary
                );
                let mut num_keys_merged = 0;
                debug!("Selected {} pages to merge", nodes.len());
                for mut node in nodes {
                    let node_id = node.ext_id();
                    let node_len = node.len();
                    let deleted_keys = node
                        .keys()
                        .iter()
                        .filter(|k| src_tree.deleted.contains(k))
                        .cloned()
                        .collect_vec();
                    let merging_keys = node
                        .extnode_mut_no_persist()
                        .keys
                        .as_slice()[..node_len]
                        .iter_mut()
                        .filter(|k| !src_tree.deleted.contains(k))
                        .map(|k| mem::take(k))
                        .collect_vec();
                    let num_merging_keys = merging_keys.len();
                    num_keys_merged += num_merging_keys;
                    debug!("Collected {} keys, merging to destination tree", num_merging_keys);
                    dest_tree.merge_with_keys(box merging_keys);
                    debug!("Merge completed, cleanup");
                    // Update delete set in source
                    for dk in deleted_keys {
                        src_tree.deleted.remove(&dk);
                    }
                    clear_node(node, &new_first_ref);
                    if node_id != head_id {
                        external::make_deleted::<KS, PS>(&node_id);
                    }
                    src_tree.len.fetch_sub(num_merging_keys, Ordering::Relaxed);
                }
                debug!("Merge prune completed in external level");
                return num_keys_merged;
            } else {
                unreachable!();
            }
        }
        MutSearchResult::Internal(sub_node) => {
            let num_keys_merged = merge_prune(level + 1, &sub_node, src_tree, dest_tree, boundary);
            match select_nodes_in_boundary::<KS, PS>(node, boundary, level) {
                NodeSelection::WholePage(nodes, right_node) => {
                    let right_ref = right_node.node_ref().clone();
                    drop(right_node);
                    clear_nodes(nodes, &right_ref);
                },
                NodeSelection::PartialPage(nodes, mut terminal_node) => {
                    clear_nodes(nodes, terminal_node.node_ref());
                    match terminal_node.keys().binary_search(boundary) {
                        Ok(bound_id) => {
                            let mut new_keys = KS::init();
                            let mut new_ptrs = PS::init();
                            let mut num_keys = 0;
                            for (i, k) in terminal_node.keys()[bound_id + 1..].iter().enumerate() {
                                new_keys.as_slice()[i] = k.clone();
                                num_keys += 1;
                            }
                            for (i, p) in terminal_node.ptrs()[bound_id + 1..].iter().enumerate() {
                                new_ptrs.as_slice()[i] = p.clone();
                            }
                            let root_innode = terminal_node.innode_mut();
                            root_innode.keys = new_keys;
                            root_innode.ptrs = new_ptrs;
                            root_innode.len = num_keys;
                        }
                        Err(id) => {
                            assert_eq!(id, 0);
                        }
                    }
                }
            }
            return num_keys_merged;
        }
    }
}

fn clear_node<KS, PS>(mut node: NodeWriteGuard<KS, PS>, right_ref: &NodeCellRef)
where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    node.make_empty_node(false);
    *node.right_ref_mut().unwrap() = right_ref.clone();
    node.left_ref_mut().map(|r| *r = right_ref.clone());
}

fn clear_nodes<KS, PS>(nodes: Vec<NodeWriteGuard<KS, PS>>, right_ref: &NodeCellRef)
where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    for node in nodes {
        clear_node(node, right_ref);
    }
}

fn select_nodes_in_boundary<KS, PS>(
    first_node: &NodeCellRef,
    right_boundary: &EntryKey,
    level: usize,
) -> NodeSelection<KS, PS>
where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    let first_node = write_node::<KS, PS>(first_node);
    debug_assert!(first_node.is_internal());
    if first_node.right_bound() > right_boundary {
        return NodeSelection::PartialPage(vec![], first_node);
    }
    let mut next_node = write_node(first_node.right_ref().unwrap());
    let mut collected = vec![first_node];
    loop {
        let first_key = next_node.first_key();
        let node_right = next_node.right_bound();
        debug_assert!(first_key < right_boundary, "First key {:?} out of bound {:?}, collected {}", first_key, right_boundary, collected.len());
        if node_right <= right_boundary {
            let next_right = write_node(next_node.right_ref().unwrap());
            collected.push(next_node);
            next_node = next_right;
            if next_node.first_key() >= right_boundary {
                trace!("Collected {} nodes at level {}", collected.len(), level);
                return NodeSelection::WholePage(collected, next_node);
            }
        } else {
            return NodeSelection::PartialPage(collected, next_node);
        }
    }
}

fn select_boundary<KS, PS>(node: &NodeCellRef) -> EntryKey
where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    read_node(node, |node: &NodeReadHandler<KS, PS>| {
        let node_keys = node.keys();
        let node_len = node_keys.len();
        if node.len() < 2 {
            return select_boundary::<KS, PS>(&node.ptrs()[0]);
        }
        // Pick half of the keys in the root
        // Genreally, higher level sub tree in LSM tree will select more keys to merged
        // into next level
        let mid_idx = node_len / 2;
        // Return the mid key as the boundary for selection (cut)
        node_keys[mid_idx].clone()
    })
}

pub async fn level_merge<KS, PS>(
    level: usize,
    src_tree: &BPlusTree<KS, PS>,
    dest_tree: &dyn LevelTree,
) -> usize
where
    KS: Slice<EntryKey> + Debug + 'static,
    PS: Slice<NodeCellRef> + 'static,
{
    debug!("Merging LSM tree level {}", level);
    debug_assert!(verification::tree_has_no_empty_node(&src_tree));
    debug_assert!(verification::is_tree_in_order(&src_tree, level));
    let root = src_tree.get_root();
    let key_boundary = select_boundary::<KS, PS>(&root);
    debug!(
        "Level merge level {} with boundary {:?}",
        level, key_boundary
    );
    let num_keys = merge_prune(0, &src_tree.get_root(), src_tree, dest_tree, &key_boundary);
    debug_assert!(verification::tree_has_no_empty_node(&src_tree));
    debug_assert!(verification::is_tree_in_order(&src_tree, level));
    debug!("Merge and pruned level {}, waiting for storage", level);
    storage::wait_until_updated().await;
    debug!("MERGE LEVEL {} COMPLETED", level);
    return num_keys;
}
<|endoftext|>


<commit_before>use super::diagnostics::Error;
use super::item::ItemInfo;
use super::Parser;

use crate::{new_sub_parser_from_file, DirectoryOwnership};

use rustc_ast::ast::{self, Attribute, Crate, Ident, ItemKind, Mod};
use rustc_ast::attr;
use rustc_ast::token::{self, TokenKind};
use rustc_errors::PResult;
use rustc_span::source_map::{FileName, SourceMap, Span, DUMMY_SP};
use rustc_span::symbol::sym;

use std::path::{self, Path, PathBuf};

/// Information about the path to a module.
// Public for rustfmt usage.
pub struct ModulePath {
    name: String,
    path_exists: bool,
    pub result: Result<ModulePathSuccess, Error>,
}

// Public for rustfmt usage.
pub struct ModulePathSuccess {
    pub path: PathBuf,
    pub directory_ownership: DirectoryOwnership,
}

impl<'a> Parser<'a> {
    /// Parses a source module as a crate. This is the main entry point for the parser.
    pub fn parse_crate_mod(&mut self) -> PResult<'a, Crate> {
        let lo = self.token.span;
        let krate = Ok(ast::Crate {
            attrs: self.parse_inner_attributes()?,
            module: self.parse_mod_items(&token::Eof, lo)?,
            span: lo.to(self.token.span),
            // Filled in by proc_macro_harness::inject()
            proc_macros: Vec::new(),
        });
        krate
    }

    /// Parses a `mod <foo> { ... }` or `mod <foo>;` item.
    pub(super) fn parse_item_mod(&mut self, attrs: &mut Vec<Attribute>) -> PResult<'a, ItemInfo> {
        let in_cfg = crate::config::process_configure_mod(self.sess, self.cfg_mods, attrs);

        let id_span = self.token.span;
        let id = self.parse_ident()?;
        let (module, mut inner_attrs) = if self.eat(&token::Semi) {
            if in_cfg && self.recurse_into_file_modules {
                // This mod is in an external file. Let's go get it!
                let ModulePathSuccess { path, directory_ownership } =
                    self.submod_path(id, &attrs, id_span)?;
                self.eval_src_mod(path, directory_ownership, id.to_string(), id_span)?
            } else {
                (ast::Mod { inner: DUMMY_SP, items: Vec::new(), inline: false }, Vec::new())
            }
        } else {
            let old_directory = self.directory.clone();
            self.push_directory(id, &attrs);

            self.expect(&token::OpenDelim(token::Brace))?;
            let mod_inner_lo = self.token.span;
            let inner_attrs = self.parse_inner_attributes()?;
            let module = self.parse_mod_items(&token::CloseDelim(token::Brace), mod_inner_lo)?;

            self.directory = old_directory;
            (module, inner_attrs)
        };
        attrs.append(&mut inner_attrs);
        Ok((id, ItemKind::Mod(module)))
    }

    /// Given a termination token, parses all of the items in a module.
    fn parse_mod_items(&mut self, term: &TokenKind, inner_lo: Span) -> PResult<'a, Mod> {
        let mut items = vec![];
        while let Some(item) = self.parse_item()? {
            items.push(item);
            self.maybe_consume_incorrect_semicolon(&items);
        }

        if !self.eat(term) {
            let token_str = super::token_descr(&self.token);
            if !self.maybe_consume_incorrect_semicolon(&items) {
                let msg = &format!("expected item, found {}", token_str);
                let mut err = self.struct_span_err(self.token.span, msg);
                err.span_label(self.token.span, "expected item");
                return Err(err);
            }
        }

        let hi = if self.token.span.is_dummy() { inner_lo } else { self.prev_token.span };

        Ok(Mod { inner: inner_lo.to(hi), items, inline: true })
    }

    fn submod_path(
        &mut self,
        id: ast::Ident,
        outer_attrs: &[Attribute],
        id_sp: Span,
    ) -> PResult<'a, ModulePathSuccess> {
        if let Some(path) = Parser::submod_path_from_attr(outer_attrs, &self.directory.path) {
            return Ok(ModulePathSuccess {
                directory_ownership: match path.file_name().and_then(|s| s.to_str()) {
                    // All `#[path]` files are treated as though they are a `mod.rs` file.
                    // This means that `mod foo;` declarations inside `#[path]`-included
                    // files are siblings,
                    //
                    // Note that this will produce weirdness when a file named `foo.rs` is
                    // `#[path]` included and contains a `mod foo;` declaration.
                    // If you encounter this, it's your own darn fault :P
                    Some(_) => DirectoryOwnership::Owned { relative: None },
                    _ => DirectoryOwnership::UnownedViaMod,
                },
                path,
            });
        }

        let relative = match self.directory.ownership {
            DirectoryOwnership::Owned { relative } => relative,
            DirectoryOwnership::UnownedViaBlock | DirectoryOwnership::UnownedViaMod => None,
        };
        let paths =
            Parser::default_submod_path(id, relative, &self.directory.path, self.sess.source_map());

        match self.directory.ownership {
            DirectoryOwnership::Owned { .. } => {
                paths.result.map_err(|err| self.span_fatal_err(id_sp, err))
            }
            DirectoryOwnership::UnownedViaBlock => {
                let msg = "Cannot declare a non-inline module inside a block \
                    unless it has a path attribute";
                let mut err = self.struct_span_err(id_sp, msg);
                if paths.path_exists {
                    let msg = format!(
                        "Maybe `use` the module `{}` instead of redeclaring it",
                        paths.name
                    );
                    err.span_note(id_sp, &msg);
                }
                Err(err)
            }
            DirectoryOwnership::UnownedViaMod => {
                let mut err =
                    self.struct_span_err(id_sp, "cannot declare a new module at this location");
                if !id_sp.is_dummy() {
                    let src_path = self.sess.source_map().span_to_filename(id_sp);
                    if let FileName::Real(src_path) = src_path {
                        if let Some(stem) = src_path.file_stem() {
                            let mut dest_path = src_path.clone();
                            dest_path.set_file_name(stem);
                            dest_path.push("mod.rs");
                            err.span_note(
                                id_sp,
                                &format!(
                                    "maybe move this module `{}` to its own \
                                                directory via `{}`",
                                    src_path.display(),
                                    dest_path.display()
                                ),
                            );
                        }
                    }
                }
                if paths.path_exists {
                    err.span_note(
                        id_sp,
                        &format!(
                            "... or maybe `use` the module `{}` instead \
                                            of possibly redeclaring it",
                            paths.name
                        ),
                    );
                }
                Err(err)
            }
        }
    }

    /// Derive a submodule path from the first found `#[path = "path_string"]`.
    /// The provided `dir_path` is joined with the `path_string`.
    // Public for rustfmt usage.
    pub fn submod_path_from_attr(attrs: &[Attribute], dir_path: &Path) -> Option<PathBuf> {
        // Extract path string from first `#[path = "path_string"]` attribute.
        let path_string = attr::first_attr_value_str_by_name(attrs, sym::path)?;
        let path_string = path_string.as_str();

        // On windows, the base path might have the form
        // `\\?\foo\bar` in which case it does not tolerate
        // mixed `/` and `\` separators, so canonicalize
        // `/` to `\`.
        #[cfg(windows)]
        let path_string = path_string.replace("/", "\\");

        Some(dir_path.join(&*path_string))
    }

    /// Returns a path to a module.
    // Public for rustfmt usage.
    pub fn default_submod_path(
        id: ast::Ident,
        relative: Option<ast::Ident>,
        dir_path: &Path,
        source_map: &SourceMap,
    ) -> ModulePath {
        // If we're in a foo.rs file instead of a mod.rs file,
        // we need to look for submodules in
        // `./foo/<id>.rs` and `./foo/<id>/mod.rs` rather than
        // `./<id>.rs` and `./<id>/mod.rs`.
        let relative_prefix_string;
        let relative_prefix = if let Some(ident) = relative {
            relative_prefix_string = format!("{}{}", ident.name, path::MAIN_SEPARATOR);
            &relative_prefix_string
        } else {
            ""
        };

        let mod_name = id.name.to_string();
        let default_path_str = format!("{}{}.rs", relative_prefix, mod_name);
        let secondary_path_str =
            format!("{}{}{}mod.rs", relative_prefix, mod_name, path::MAIN_SEPARATOR);
        let default_path = dir_path.join(&default_path_str);
        let secondary_path = dir_path.join(&secondary_path_str);
        let default_exists = source_map.file_exists(&default_path);
        let secondary_exists = source_map.file_exists(&secondary_path);

        let result = match (default_exists, secondary_exists) {
            (true, false) => Ok(ModulePathSuccess {
                path: default_path,
                directory_ownership: DirectoryOwnership::Owned { relative: Some(id) },
            }),
            (false, true) => Ok(ModulePathSuccess {
                path: secondary_path,
                directory_ownership: DirectoryOwnership::Owned { relative: None },
            }),
            (false, false) => {
                Err(Error::FileNotFoundForModule { mod_name: mod_name.clone(), default_path })
            }
            (true, true) => Err(Error::DuplicatePaths {
                mod_name: mod_name.clone(),
                default_path: default_path_str,
                secondary_path: secondary_path_str,
            }),
        };

        ModulePath { name: mod_name, path_exists: default_exists || secondary_exists, result }
    }

    /// Reads a module from a source file.
    fn eval_src_mod(
        &mut self,
        path: PathBuf,
        directory_ownership: DirectoryOwnership,
        name: String,
        id_sp: Span,
    ) -> PResult<'a, (Mod, Vec<Attribute>)> {
        let mut included_mod_stack = self.sess.included_mod_stack.borrow_mut();
        if let Some(i) = included_mod_stack.iter().position(|p| *p == path) {
            let mut err = String::from("circular modules: ");
            let len = included_mod_stack.len();
            for p in &included_mod_stack[i..len] {
                err.push_str(&p.to_string_lossy());
                err.push_str(" -> ");
            }
            err.push_str(&path.to_string_lossy());
            return Err(self.struct_span_err(id_sp, &err[..]));
        }
        included_mod_stack.push(path.clone());
        drop(included_mod_stack);

        let mut p0 =
            new_sub_parser_from_file(self.sess, &path, directory_ownership, Some(name), id_sp);
        p0.cfg_mods = self.cfg_mods;
        let mod_inner_lo = p0.token.span;
        let mod_attrs = p0.parse_inner_attributes()?;
        let mut m0 = p0.parse_mod_items(&token::Eof, mod_inner_lo)?;
        m0.inline = false;
        self.sess.included_mod_stack.borrow_mut().pop();
        Ok((m0, mod_attrs))
    }

    fn push_directory(&mut self, id: Ident, attrs: &[Attribute]) {
        if let Some(path) = attr::first_attr_value_str_by_name(attrs, sym::path) {
            self.directory.path.push(&*path.as_str());
            self.directory.ownership = DirectoryOwnership::Owned { relative: None };
        } else {
            // We have to push on the current module name in the case of relative
            // paths in order to ensure that any additional module paths from inline
            // `mod x { ... }` come after the relative extension.
            //
            // For example, a `mod z { ... }` inside `x/y.rs` should set the current
            // directory path to `/x/y/z`, not `/x/z` with a relative offset of `y`.
            if let DirectoryOwnership::Owned { relative } = &mut self.directory.ownership {
                if let Some(ident) = relative.take() {
                    // remove the relative offset
                    self.directory.path.push(&*ident.as_str());
                }
            }
            self.directory.path.push(&*id.as_str());
        }
    }
}
<commit_msg>extract error_cannot_declare_mod_here<commit_after>use super::diagnostics::Error;
use super::item::ItemInfo;
use super::Parser;

use crate::{new_sub_parser_from_file, DirectoryOwnership};

use rustc_ast::ast::{self, Attribute, Crate, Ident, ItemKind, Mod};
use rustc_ast::attr;
use rustc_ast::token::{self, TokenKind};
use rustc_errors::PResult;
use rustc_span::source_map::{FileName, SourceMap, Span, DUMMY_SP};
use rustc_span::symbol::sym;

use std::path::{self, Path, PathBuf};

/// Information about the path to a module.
// Public for rustfmt usage.
pub struct ModulePath {
    name: String,
    path_exists: bool,
    pub result: Result<ModulePathSuccess, Error>,
}

// Public for rustfmt usage.
pub struct ModulePathSuccess {
    pub path: PathBuf,
    pub directory_ownership: DirectoryOwnership,
}

impl<'a> Parser<'a> {
    /// Parses a source module as a crate. This is the main entry point for the parser.
    pub fn parse_crate_mod(&mut self) -> PResult<'a, Crate> {
        let lo = self.token.span;
        let krate = Ok(ast::Crate {
            attrs: self.parse_inner_attributes()?,
            module: self.parse_mod_items(&token::Eof, lo)?,
            span: lo.to(self.token.span),
            // Filled in by proc_macro_harness::inject()
            proc_macros: Vec::new(),
        });
        krate
    }

    /// Parses a `mod <foo> { ... }` or `mod <foo>;` item.
    pub(super) fn parse_item_mod(&mut self, attrs: &mut Vec<Attribute>) -> PResult<'a, ItemInfo> {
        let in_cfg = crate::config::process_configure_mod(self.sess, self.cfg_mods, attrs);

        let id_span = self.token.span;
        let id = self.parse_ident()?;
        let (module, mut inner_attrs) = if self.eat(&token::Semi) {
            if in_cfg && self.recurse_into_file_modules {
                // This mod is in an external file. Let's go get it!
                let ModulePathSuccess { path, directory_ownership } =
                    self.submod_path(id, &attrs, id_span)?;
                self.eval_src_mod(path, directory_ownership, id.to_string(), id_span)?
            } else {
                (ast::Mod { inner: DUMMY_SP, items: Vec::new(), inline: false }, Vec::new())
            }
        } else {
            let old_directory = self.directory.clone();
            self.push_directory(id, &attrs);

            self.expect(&token::OpenDelim(token::Brace))?;
            let mod_inner_lo = self.token.span;
            let inner_attrs = self.parse_inner_attributes()?;
            let module = self.parse_mod_items(&token::CloseDelim(token::Brace), mod_inner_lo)?;

            self.directory = old_directory;
            (module, inner_attrs)
        };
        attrs.append(&mut inner_attrs);
        Ok((id, ItemKind::Mod(module)))
    }

    /// Given a termination token, parses all of the items in a module.
    fn parse_mod_items(&mut self, term: &TokenKind, inner_lo: Span) -> PResult<'a, Mod> {
        let mut items = vec![];
        while let Some(item) = self.parse_item()? {
            items.push(item);
            self.maybe_consume_incorrect_semicolon(&items);
        }

        if !self.eat(term) {
            let token_str = super::token_descr(&self.token);
            if !self.maybe_consume_incorrect_semicolon(&items) {
                let msg = &format!("expected item, found {}", token_str);
                let mut err = self.struct_span_err(self.token.span, msg);
                err.span_label(self.token.span, "expected item");
                return Err(err);
            }
        }

        let hi = if self.token.span.is_dummy() { inner_lo } else { self.prev_token.span };

        Ok(Mod { inner: inner_lo.to(hi), items, inline: true })
    }

    fn submod_path(
        &mut self,
        id: ast::Ident,
        outer_attrs: &[Attribute],
        id_sp: Span,
    ) -> PResult<'a, ModulePathSuccess> {
        if let Some(path) = Parser::submod_path_from_attr(outer_attrs, &self.directory.path) {
            return Ok(ModulePathSuccess {
                directory_ownership: match path.file_name().and_then(|s| s.to_str()) {
                    // All `#[path]` files are treated as though they are a `mod.rs` file.
                    // This means that `mod foo;` declarations inside `#[path]`-included
                    // files are siblings,
                    //
                    // Note that this will produce weirdness when a file named `foo.rs` is
                    // `#[path]` included and contains a `mod foo;` declaration.
                    // If you encounter this, it's your own darn fault :P
                    Some(_) => DirectoryOwnership::Owned { relative: None },
                    _ => DirectoryOwnership::UnownedViaMod,
                },
                path,
            });
        }

        let relative = match self.directory.ownership {
            DirectoryOwnership::Owned { relative } => relative,
            DirectoryOwnership::UnownedViaBlock | DirectoryOwnership::UnownedViaMod => None,
        };
        let paths =
            Parser::default_submod_path(id, relative, &self.directory.path, self.sess.source_map());

        match self.directory.ownership {
            DirectoryOwnership::Owned { .. } => {
                paths.result.map_err(|err| self.span_fatal_err(id_sp, err))
            }
            DirectoryOwnership::UnownedViaBlock => {
                let msg = "Cannot declare a non-inline module inside a block \
                    unless it has a path attribute";
                let mut err = self.struct_span_err(id_sp, msg);
                if paths.path_exists {
                    let msg = format!(
                        "Maybe `use` the module `{}` instead of redeclaring it",
                        paths.name
                    );
                    err.span_note(id_sp, &msg);
                }
                Err(err)
            }
            DirectoryOwnership::UnownedViaMod => self.error_cannot_declare_mod_here(id_sp, paths),
        }
    }

    fn error_cannot_declare_mod_here<T>(&self, id_sp: Span, paths: ModulePath) -> PResult<'a, T> {
        let mut err = self.struct_span_err(id_sp, "cannot declare a new module at this location");
        if !id_sp.is_dummy() {
            if let FileName::Real(src_path) = self.sess.source_map().span_to_filename(id_sp) {
                if let Some(stem) = src_path.file_stem() {
                    let mut dest_path = src_path.clone();
                    dest_path.set_file_name(stem);
                    dest_path.push("mod.rs");
                    err.span_note(
                        id_sp,
                        &format!(
                            "maybe move this module `{}` to its own \
                                    directory via `{}`",
                            src_path.display(),
                            dest_path.display()
                        ),
                    );
                }
            }
        }
        if paths.path_exists {
            err.span_note(
                id_sp,
                &format!(
                    "... or maybe `use` the module `{}` instead \
                                of possibly redeclaring it",
                    paths.name
                ),
            );
        }
        Err(err)
    }

    /// Derive a submodule path from the first found `#[path = "path_string"]`.
    /// The provided `dir_path` is joined with the `path_string`.
    // Public for rustfmt usage.
    pub fn submod_path_from_attr(attrs: &[Attribute], dir_path: &Path) -> Option<PathBuf> {
        // Extract path string from first `#[path = "path_string"]` attribute.
        let path_string = attr::first_attr_value_str_by_name(attrs, sym::path)?;
        let path_string = path_string.as_str();

        // On windows, the base path might have the form
        // `\\?\foo\bar` in which case it does not tolerate
        // mixed `/` and `\` separators, so canonicalize
        // `/` to `\`.
        #[cfg(windows)]
        let path_string = path_string.replace("/", "\\");

        Some(dir_path.join(&*path_string))
    }

    /// Returns a path to a module.
    // Public for rustfmt usage.
    pub fn default_submod_path(
        id: ast::Ident,
        relative: Option<ast::Ident>,
        dir_path: &Path,
        source_map: &SourceMap,
    ) -> ModulePath {
        // If we're in a foo.rs file instead of a mod.rs file,
        // we need to look for submodules in
        // `./foo/<id>.rs` and `./foo/<id>/mod.rs` rather than
        // `./<id>.rs` and `./<id>/mod.rs`.
        let relative_prefix_string;
        let relative_prefix = if let Some(ident) = relative {
            relative_prefix_string = format!("{}{}", ident.name, path::MAIN_SEPARATOR);
            &relative_prefix_string
        } else {
            ""
        };

        let mod_name = id.name.to_string();
        let default_path_str = format!("{}{}.rs", relative_prefix, mod_name);
        let secondary_path_str =
            format!("{}{}{}mod.rs", relative_prefix, mod_name, path::MAIN_SEPARATOR);
        let default_path = dir_path.join(&default_path_str);
        let secondary_path = dir_path.join(&secondary_path_str);
        let default_exists = source_map.file_exists(&default_path);
        let secondary_exists = source_map.file_exists(&secondary_path);

        let result = match (default_exists, secondary_exists) {
            (true, false) => Ok(ModulePathSuccess {
                path: default_path,
                directory_ownership: DirectoryOwnership::Owned { relative: Some(id) },
            }),
            (false, true) => Ok(ModulePathSuccess {
                path: secondary_path,
                directory_ownership: DirectoryOwnership::Owned { relative: None },
            }),
            (false, false) => {
                Err(Error::FileNotFoundForModule { mod_name: mod_name.clone(), default_path })
            }
            (true, true) => Err(Error::DuplicatePaths {
                mod_name: mod_name.clone(),
                default_path: default_path_str,
                secondary_path: secondary_path_str,
            }),
        };

        ModulePath { name: mod_name, path_exists: default_exists || secondary_exists, result }
    }

    /// Reads a module from a source file.
    fn eval_src_mod(
        &mut self,
        path: PathBuf,
        directory_ownership: DirectoryOwnership,
        name: String,
        id_sp: Span,
    ) -> PResult<'a, (Mod, Vec<Attribute>)> {
        let mut included_mod_stack = self.sess.included_mod_stack.borrow_mut();
        if let Some(i) = included_mod_stack.iter().position(|p| *p == path) {
            let mut err = String::from("circular modules: ");
            let len = included_mod_stack.len();
            for p in &included_mod_stack[i..len] {
                err.push_str(&p.to_string_lossy());
                err.push_str(" -> ");
            }
            err.push_str(&path.to_string_lossy());
            return Err(self.struct_span_err(id_sp, &err[..]));
        }
        included_mod_stack.push(path.clone());
        drop(included_mod_stack);

        let mut p0 =
            new_sub_parser_from_file(self.sess, &path, directory_ownership, Some(name), id_sp);
        p0.cfg_mods = self.cfg_mods;
        let mod_inner_lo = p0.token.span;
        let mod_attrs = p0.parse_inner_attributes()?;
        let mut m0 = p0.parse_mod_items(&token::Eof, mod_inner_lo)?;
        m0.inline = false;
        self.sess.included_mod_stack.borrow_mut().pop();
        Ok((m0, mod_attrs))
    }

    fn push_directory(&mut self, id: Ident, attrs: &[Attribute]) {
        if let Some(path) = attr::first_attr_value_str_by_name(attrs, sym::path) {
            self.directory.path.push(&*path.as_str());
            self.directory.ownership = DirectoryOwnership::Owned { relative: None };
        } else {
            // We have to push on the current module name in the case of relative
            // paths in order to ensure that any additional module paths from inline
            // `mod x { ... }` come after the relative extension.
            //
            // For example, a `mod z { ... }` inside `x/y.rs` should set the current
            // directory path to `/x/y/z`, not `/x/z` with a relative offset of `y`.
            if let DirectoryOwnership::Owned { relative } = &mut self.directory.ownership {
                if let Some(ident) = relative.take() {
                    // remove the relative offset
                    self.directory.path.push(&*ident.as_str());
                }
            }
            self.directory.path.push(&*id.as_str());
        }
    }
}
<|endoftext|>


<commit_before>#![cfg(feature = "s3")]
extern crate rusoto_core;
extern crate rusoto_s3;
extern crate time;
extern crate hyper;
extern crate env_logger;
extern crate log;

use hyper::Client;
use std::fs::File;
use std::io::Read;
use time::get_time;

use rusoto_core::{DefaultCredentialsProvider, Region};
use rusoto_s3::{S3, S3Client, HeadObjectRequest, CopyObjectRequest, GetObjectRequest,
                 PutObjectRequest, DeleteObjectRequest, PutBucketCorsRequest, CORSConfiguration,
                 CORSRule, CreateBucketRequest, DeleteBucketRequest, CreateMultipartUploadRequest,
                 UploadPartRequest, CompleteMultipartUploadRequest, CompletedMultipartUpload,
                 CompletedPart, CompletedPartList, ListObjectsV2Request};
use rusoto_core::default_tls_client;

type TestClient = S3Client<DefaultCredentialsProvider, Client>;

// Rust is in bad need of an integration test harness
// This creates the S3 resources needed for a suite of tests,
// executes those tests, and then destroys the resources
#[test]
fn test_all_the_things() {
    let _ = env_logger::init();

    let client = S3Client::new(default_tls_client().unwrap(),
                               DefaultCredentialsProvider::new().unwrap(),
                               Region::UsEast1);

    let test_bucket = format!("rusoto_test_bucket_{}", get_time().sec);
    let filename = format!("test_file_{}", get_time().sec);
    let binary_filename = format!("test_file_b{}", get_time().sec);
    let multipart_filename = format!("test_multipart_file_{}", get_time().sec);

    // get a list of list_buckets
    test_list_buckets(&client);

    // create a bucket for these tests
    test_create_bucket(&client, &test_bucket);

    // list items v2
    list_items_in_bucket(&client, &test_bucket);

    // do a multipart upload
    test_multipart_upload(&client, &test_bucket, &multipart_filename);

    // modify the bucket's CORS properties
    test_put_bucket_cors(&client, &test_bucket);

    // PUT an object (no_credentials is an arbitrary choice)
    test_put_object_with_filename(&client,
                                  &test_bucket,
                                  &filename,
                                  &"tests/sample-data/no_credentials");

    // HEAD the object that was PUT
    test_head_object(&client, &test_bucket, &filename);

    // GET the object
    test_get_object(&client, &test_bucket, &filename);
    test_get_object_range(&client, &test_bucket, &filename);
    // copy the object to change its settings
    test_copy_object(&client, &test_bucket, &filename);

    // Binary objects:
    test_put_object_with_filename(&client,
                                  &test_bucket,
                                  &binary_filename,
                                  &"tests/sample-data/binary-file");
    test_get_object(&client, &test_bucket, &binary_filename);

    // paging test requires three items in the bucket, put another item there:
    // PUT an object (no_credentials is an arbitrary choice)
    let another_filename = format!("foo{}", filename);
    test_put_object_with_filename(&client,
                                  &test_bucket,
                                  &another_filename,
                                  &"tests/sample-data/no_credentials");

    // list items with paging
    list_items_in_bucket_paged(&client, &test_bucket);

    test_delete_object(&client, &test_bucket, &binary_filename);
    test_delete_object(&client, &test_bucket, &another_filename);

    // DELETE the object
    test_delete_object(&client, &test_bucket, &filename);

    // delete the test bucket
    test_delete_bucket(&client, &test_bucket);
}

fn test_multipart_upload(client: &TestClient, bucket: &str, filename: &str) {
    let create_multipart_req = CreateMultipartUploadRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        ..Default::default()
    };

    // start the multipart upload and note the upload_id generated
    let response = client.create_multipart_upload(&create_multipart_req).unwrap();
    println!("{:#?}", response);
    let upload_id = response.upload_id.unwrap();

    // create 2 upload parts
    let create_upload_part = |body: Vec<u8>, part_number: i64| -> UploadPartRequest {
        UploadPartRequest {
            body: Some(body),
            bucket: bucket.to_owned(),
            key: filename.to_owned(),
            upload_id: upload_id.to_owned(),
            part_number: part_number,
            ..Default::default()
        }
    };

    // minimum size for a non-final multipart upload part is 5MB
    let part_req1 = create_upload_part(vec!['a' as u8; 1024 * 1024 * 5], 1);
    let part_req2 = create_upload_part("foo".as_bytes().to_vec(), 2);

    // upload 2 parts and note the etags generated for them
    let mut completed_parts = CompletedPartList::new();
    for req in [part_req1, part_req2].into_iter() {
        let response = client.upload_part(&req).unwrap();
        println!("{:#?}", response);
        completed_parts.push(CompletedPart {
            e_tag: response.e_tag.clone(),
            part_number: Some(req.part_number),
        });
    }

    // complete the multipart upload with the etags of the parts
    let completed_upload = CompletedMultipartUpload { parts: Some(completed_parts) };

    let complete_req = CompleteMultipartUploadRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        upload_id: upload_id.to_owned(),
        multipart_upload: Some(completed_upload),
        ..Default::default()
    };

    let response = client.complete_multipart_upload(&complete_req).unwrap();
    println!("{:#?}", response);

    // delete the completed file
    test_delete_object(client, bucket, filename);
}

fn test_create_bucket(client: &TestClient, bucket: &str) {
    let create_bucket_req = CreateBucketRequest { bucket: bucket.to_owned(), ..Default::default() };

    let result = client.create_bucket(&create_bucket_req).unwrap();
    println!("{:#?}", result);
}

fn test_delete_bucket(client: &TestClient, bucket: &str) {
    let delete_bucket_req = DeleteBucketRequest { bucket: bucket.to_owned(), ..Default::default() };

    let result = client.delete_bucket(&delete_bucket_req).unwrap();
    println!("{:#?}", result);
}

fn test_put_object_with_filename(client: &TestClient,
                                 bucket: &str,
                                 dest_filename: &str,
                                 local_filename: &str) {
    let mut f = File::open(local_filename).unwrap();
    let mut contents: Vec<u8> = Vec::new();
    match f.read_to_end(&mut contents) {
        Err(why) => panic!("Error opening file to send to S3: {}", why),
        Ok(_) => {
            let req = PutObjectRequest {
                bucket: bucket.to_owned(),
                key: dest_filename.to_owned(),
                body: Some(contents),
                ..Default::default()
            };
            let result = client.put_object(&req);
            println!("{:#?}", result);
        }
    }
}

fn test_head_object(client: &TestClient, bucket: &str, filename: &str) {
    let head_req = HeadObjectRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        ..Default::default()
    };

    let result = client.head_object(&head_req).unwrap();
    println!("{:#?}", result);
}

fn test_get_object(client: &TestClient, bucket: &str, filename: &str) {
    let get_req = GetObjectRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        ..Default::default()
    };

    let result = client.get_object(&get_req).unwrap();
    println!("get object result: {:#?}", result);
    assert!(result.body.unwrap().len() > 0);
}

fn test_get_object_range(client: &TestClient, bucket: &str, filename: &str) {
    let get_req = GetObjectRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        range: Some("bytes=0-1".to_owned()),
        ..Default::default()
    };

    let result = client.get_object(&get_req).unwrap();
    println!("\nget object range result: {:#?}", result);
    assert_eq!(result.content_length.unwrap(), 2);
}

fn test_copy_object(client: &TestClient, bucket: &str, filename: &str) {
    let req = CopyObjectRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        copy_source: format!("{}/{}", bucket, filename),
        cache_control: Some("max-age=123".to_owned()),
        content_type: Some("application/json".to_owned()),
        metadata_directive: Some("REPLACE".to_owned()),
        ..Default::default()
    };

    let result = client.copy_object(&req).unwrap();
    println!("{:#?}", result);
}

fn test_delete_object(client: &TestClient, bucket: &str, filename: &str) {
    let del_req = DeleteObjectRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        ..Default::default()
    };

    let result = client.delete_object(&del_req).unwrap();
    println!("{:#?}", result);
}

fn test_list_buckets(client: &TestClient) {
    let result = client.list_buckets().unwrap();
    println!("\nbuckets available: {:#?}", result);
}

fn list_items_in_bucket(client: &TestClient, bucket: &str) {
    let list_obj_req = ListObjectsV2Request {
        bucket: bucket.to_owned(),
        start_after: Some("foo".to_owned()),
        ..Default::default()
    };
    let result = client.list_objects_v2(&list_obj_req).unwrap();
    println!("Items in bucket: {:#?}", result);
}

// Assuming there's already more than three item in our test bucket:
fn list_items_in_bucket_paged(client: &TestClient, bucket: &str) {
    let mut list_obj_req = ListObjectsV2Request {
        bucket: bucket.to_owned(),
        max_keys: Some(1),
        ..Default::default()
    };
    let result = client.list_objects_v2(&list_obj_req).unwrap();
    println!("Items in bucket, page 1: {:#?}", result);
    assert!(result.next_continuation_token.is_some());

    list_obj_req.continuation_token = result.next_continuation_token;
    let result = client.list_objects_v2(&list_obj_req).unwrap();
    println!("Items in bucket, page 2: {:#?}", result);
    // For the second call it the token is in `continuation_token` not `next_continuation_token`
    assert!(result.continuation_token.is_some());
}

fn test_put_bucket_cors(client: &TestClient, bucket: &str) {
    let cors_rules =
        vec![CORSRule {
                 allowed_methods: vec!["PUT".to_owned(), "POST".to_owned(), "DELETE".to_owned()],
                 allowed_origins: vec!["http://www.example.com".to_owned()],
                 allowed_headers: Some(vec!["*".to_owned()]),
                 max_age_seconds: Some(3000),
                 expose_headers: Some(vec!["x-amz-server-side-encryption".to_owned()]),
                 ..Default::default()
             }];

    let cors_configuration = CORSConfiguration { cors_rules: cors_rules };

    let req = PutBucketCorsRequest {
        bucket: bucket.to_owned(),
        cors_configuration: cors_configuration,
        ..Default::default()
    };

    let result = client.put_bucket_cors(&req).unwrap();
    println!("{:#?}", result);
}
<commit_msg>Add integration test that uses encode_key util<commit_after>#![cfg(feature = "s3")]
extern crate rusoto_core;
extern crate rusoto_s3;
extern crate time;
extern crate hyper;
extern crate env_logger;
extern crate log;

use hyper::Client;
use std::fs::File;
use std::io::Read;
use time::get_time;

use rusoto_core::{DefaultCredentialsProvider, Region};
use rusoto_s3::{S3, S3Client, HeadObjectRequest, CopyObjectRequest, GetObjectRequest,
                 PutObjectRequest, DeleteObjectRequest, PutBucketCorsRequest, CORSConfiguration,
                 CORSRule, CreateBucketRequest, DeleteBucketRequest, CreateMultipartUploadRequest,
                 UploadPartRequest, CompleteMultipartUploadRequest, CompletedMultipartUpload,
                 CompletedPart, CompletedPartList, ListObjectsV2Request};
use rusoto_core::default_tls_client;

type TestClient = S3Client<DefaultCredentialsProvider, Client>;

// Rust is in bad need of an integration test harness
// This creates the S3 resources needed for a suite of tests,
// executes those tests, and then destroys the resources
#[test]
fn test_all_the_things() {
    let _ = env_logger::init();

    let client = S3Client::new(default_tls_client().unwrap(),
                               DefaultCredentialsProvider::new().unwrap(),
                               Region::UsEast1);

    let test_bucket = format!("rusoto_test_bucket_{}", get_time().sec);
    let filename = format!("test_file_{}", get_time().sec);
    let utf8_filename = format!("test[über]file@{}", get_time().sec);
    let binary_filename = format!("test_file_b{}", get_time().sec);
    let multipart_filename = format!("test_multipart_file_{}", get_time().sec);

    // get a list of list_buckets
    test_list_buckets(&client);

    // create a bucket for these tests
    test_create_bucket(&client, &test_bucket);

    // list items v2
    list_items_in_bucket(&client, &test_bucket);

    // do a multipart upload
    test_multipart_upload(&client, &test_bucket, &multipart_filename);

    // modify the bucket's CORS properties
    test_put_bucket_cors(&client, &test_bucket);

    // PUT an object (no_credentials is an arbitrary choice)
    test_put_object_with_filename(&client,
                                  &test_bucket,
                                  &filename,
                                  &"tests/sample-data/no_credentials");

    // HEAD the object that was PUT
    test_head_object(&client, &test_bucket, &filename);

    // GET the object
    test_get_object(&client, &test_bucket, &filename);
    test_get_object_range(&client, &test_bucket, &filename);
    // copy the object to change its settings
    test_copy_object(&client, &test_bucket, &filename);

    // UTF8 filenames
    test_put_object_with_filename(&client,
                                  &test_bucket,
                                  &utf8_filename,
                                  &"tests/sample-data/no_credentials");

    test_copy_object_utf8(&client, &test_bucket, &utf8_filename);

    test_delete_object(&client, &test_bucket, &utf8_filename);

    // Binary objects:
    test_put_object_with_filename(&client,
                                  &test_bucket,
                                  &binary_filename,
                                  &"tests/sample-data/binary-file");
    test_get_object(&client, &test_bucket, &binary_filename);

    // paging test requires three items in the bucket, put another item there:
    // PUT an object (no_credentials is an arbitrary choice)
    let another_filename = format!("foo{}", filename);
    test_put_object_with_filename(&client,
                                  &test_bucket,
                                  &another_filename,
                                  &"tests/sample-data/no_credentials");

    // list items with paging
    list_items_in_bucket_paged(&client, &test_bucket);

    test_delete_object(&client, &test_bucket, &binary_filename);
    test_delete_object(&client, &test_bucket, &another_filename);

    // DELETE the object
    test_delete_object(&client, &test_bucket, &filename);

    // delete the test bucket
    test_delete_bucket(&client, &test_bucket);
}

fn test_multipart_upload(client: &TestClient, bucket: &str, filename: &str) {
    let create_multipart_req = CreateMultipartUploadRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        ..Default::default()
    };

    // start the multipart upload and note the upload_id generated
    let response = client.create_multipart_upload(&create_multipart_req).unwrap();
    println!("{:#?}", response);
    let upload_id = response.upload_id.unwrap();

    // create 2 upload parts
    let create_upload_part = |body: Vec<u8>, part_number: i64| -> UploadPartRequest {
        UploadPartRequest {
            body: Some(body),
            bucket: bucket.to_owned(),
            key: filename.to_owned(),
            upload_id: upload_id.to_owned(),
            part_number: part_number,
            ..Default::default()
        }
    };

    // minimum size for a non-final multipart upload part is 5MB
    let part_req1 = create_upload_part(vec!['a' as u8; 1024 * 1024 * 5], 1);
    let part_req2 = create_upload_part("foo".as_bytes().to_vec(), 2);

    // upload 2 parts and note the etags generated for them
    let mut completed_parts = CompletedPartList::new();
    for req in [part_req1, part_req2].into_iter() {
        let response = client.upload_part(&req).unwrap();
        println!("{:#?}", response);
        completed_parts.push(CompletedPart {
            e_tag: response.e_tag.clone(),
            part_number: Some(req.part_number),
        });
    }

    // complete the multipart upload with the etags of the parts
    let completed_upload = CompletedMultipartUpload { parts: Some(completed_parts) };

    let complete_req = CompleteMultipartUploadRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        upload_id: upload_id.to_owned(),
        multipart_upload: Some(completed_upload),
        ..Default::default()
    };

    let response = client.complete_multipart_upload(&complete_req).unwrap();
    println!("{:#?}", response);

    // delete the completed file
    test_delete_object(client, bucket, filename);
}

fn test_create_bucket(client: &TestClient, bucket: &str) {
    let create_bucket_req = CreateBucketRequest { bucket: bucket.to_owned(), ..Default::default() };

    let result = client.create_bucket(&create_bucket_req).unwrap();
    println!("{:#?}", result);
}

fn test_delete_bucket(client: &TestClient, bucket: &str) {
    let delete_bucket_req = DeleteBucketRequest { bucket: bucket.to_owned(), ..Default::default() };

    let result = client.delete_bucket(&delete_bucket_req).unwrap();
    println!("{:#?}", result);
}

fn test_put_object_with_filename(client: &TestClient,
                                 bucket: &str,
                                 dest_filename: &str,
                                 local_filename: &str) {
    let mut f = File::open(local_filename).unwrap();
    let mut contents: Vec<u8> = Vec::new();
    match f.read_to_end(&mut contents) {
        Err(why) => panic!("Error opening file to send to S3: {}", why),
        Ok(_) => {
            let req = PutObjectRequest {
                bucket: bucket.to_owned(),
                key: dest_filename.to_owned(),
                body: Some(contents),
                ..Default::default()
            };
            let result = client.put_object(&req);
            println!("{:#?}", result);
        }
    }
}

fn test_head_object(client: &TestClient, bucket: &str, filename: &str) {
    let head_req = HeadObjectRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        ..Default::default()
    };

    let result = client.head_object(&head_req).unwrap();
    println!("{:#?}", result);
}

fn test_get_object(client: &TestClient, bucket: &str, filename: &str) {
    let get_req = GetObjectRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        ..Default::default()
    };

    let result = client.get_object(&get_req).unwrap();
    println!("get object result: {:#?}", result);
    assert!(result.body.unwrap().len() > 0);
}

fn test_get_object_range(client: &TestClient, bucket: &str, filename: &str) {
    let get_req = GetObjectRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        range: Some("bytes=0-1".to_owned()),
        ..Default::default()
    };

    let result = client.get_object(&get_req).unwrap();
    println!("\nget object range result: {:#?}", result);
    assert_eq!(result.content_length.unwrap(), 2);
}

fn test_copy_object(client: &TestClient, bucket: &str, filename: &str) {
    let req = CopyObjectRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        copy_source: format!("{}/{}", bucket, filename),
        cache_control: Some("max-age=123".to_owned()),
        content_type: Some("application/json".to_owned()),
        metadata_directive: Some("REPLACE".to_owned()),
        ..Default::default()
    };

    let result = client.copy_object(&req).unwrap();
    println!("{:#?}", result);
}

fn test_copy_object_utf8(client: &TestClient, bucket: &str, filename: &str) {
    let req = CopyObjectRequest {
        bucket: bucket.to_owned(),
        key: format!("{}", filename.to_owned()),
        copy_source: rusoto_s3::util::encode_key(format!("{}/{}", bucket, filename)),
        cache_control: Some("max-age=123".to_owned()),
        content_type: Some("application/json".to_owned()),
        metadata_directive: Some("REPLACE".to_owned()),
        ..Default::default()
    };

    let result = client.copy_object(&req).unwrap();
    println!("{:#?}", result);
}

fn test_delete_object(client: &TestClient, bucket: &str, filename: &str) {
    let del_req = DeleteObjectRequest {
        bucket: bucket.to_owned(),
        key: filename.to_owned(),
        ..Default::default()
    };

    let result = client.delete_object(&del_req).unwrap();
    println!("{:#?}", result);
}

fn test_list_buckets(client: &TestClient) {
    let result = client.list_buckets().unwrap();
    println!("\nbuckets available: {:#?}", result);
}

fn list_items_in_bucket(client: &TestClient, bucket: &str) {
    let list_obj_req = ListObjectsV2Request {
        bucket: bucket.to_owned(),
        start_after: Some("foo".to_owned()),
        ..Default::default()
    };
    let result = client.list_objects_v2(&list_obj_req).unwrap();
    println!("Items in bucket: {:#?}", result);
}

// Assuming there's already more than three item in our test bucket:
fn list_items_in_bucket_paged(client: &TestClient, bucket: &str) {
    let mut list_obj_req = ListObjectsV2Request {
        bucket: bucket.to_owned(),
        max_keys: Some(1),
        ..Default::default()
    };
    let result = client.list_objects_v2(&list_obj_req).unwrap();
    println!("Items in bucket, page 1: {:#?}", result);
    assert!(result.next_continuation_token.is_some());

    list_obj_req.continuation_token = result.next_continuation_token;
    let result = client.list_objects_v2(&list_obj_req).unwrap();
    println!("Items in bucket, page 2: {:#?}", result);
    // For the second call it the token is in `continuation_token` not `next_continuation_token`
    assert!(result.continuation_token.is_some());
}

fn test_put_bucket_cors(client: &TestClient, bucket: &str) {
    let cors_rules =
        vec![CORSRule {
                 allowed_methods: vec!["PUT".to_owned(), "POST".to_owned(), "DELETE".to_owned()],
                 allowed_origins: vec!["http://www.example.com".to_owned()],
                 allowed_headers: Some(vec!["*".to_owned()]),
                 max_age_seconds: Some(3000),
                 expose_headers: Some(vec!["x-amz-server-side-encryption".to_owned()]),
                 ..Default::default()
             }];

    let cors_configuration = CORSConfiguration { cors_rules: cors_rules };

    let req = PutBucketCorsRequest {
        bucket: bucket.to_owned(),
        cors_configuration: cors_configuration,
        ..Default::default()
    };

    let result = client.put_bucket_cors(&req).unwrap();
    println!("{:#?}", result);
}
<|endoftext|>


<commit_before>extern crate bindgen;
extern crate pkg_config;

use bindgen::Builder;
use pkg_config::Config;
use std::env;
use std::path::PathBuf;

fn main() {
    let mut builder = Builder::default();

    let gdal = Config::new().probe("gdal").unwrap();
    for path in &gdal.libs {
        println!("cargo:rustc-link-lib={}", path);
    }
    for path in &gdal.include_paths {
        builder = builder.clang_arg("-I");
        builder = builder.clang_arg(path.to_str().unwrap());
    }

    let bindings = builder
        .header("wrapper.h")
        .prepend_enum_name(false)
        .constified_enum_module("CPLErr")
        .constified_enum_module("OGRwkbGeometryType")
        .constified_enum_module("OGRErr")
        .constified_enum_module("OGRFieldType")
        .ctypes_prefix("libc")
        .whitelist_function("CPL.*")
        .whitelist_function("GDAL.*")
        .whitelist_function("OGR.*")
        .whitelist_function("OSR.*")
        .whitelist_function("OCT.*")
        .whitelist_function("VSI.*")
        .whitelist_var("OGR.*")
        .whitelist_var("FALSE")
        .whitelist_var("TRUE")
        .generate()
        .expect("Unable to generate bindings");

    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
    bindings
        .write_to_file(out_path.join("bindings.rs"))
        .expect("Couldn't write bindings!");
}
<commit_msg>Avoid duplicate linked library warning ‽<commit_after>extern crate bindgen;
extern crate pkg_config;

use bindgen::Builder;
use pkg_config::Config;
use std::env;
use std::path::PathBuf;

fn main() {
    let mut builder = Builder::default();

    let gdal = Config::new().probe("gdal").unwrap();
    for path in &gdal.include_paths {
        builder = builder.clang_arg("-I");
        builder = builder.clang_arg(path.to_str().unwrap());
    }

    let bindings = builder
        .header("wrapper.h")
        .prepend_enum_name(false)
        .constified_enum_module("CPLErr")
        .constified_enum_module("OGRwkbGeometryType")
        .constified_enum_module("OGRErr")
        .constified_enum_module("OGRFieldType")
        .ctypes_prefix("libc")
        .whitelist_function("CPL.*")
        .whitelist_function("GDAL.*")
        .whitelist_function("OGR.*")
        .whitelist_function("OSR.*")
        .whitelist_function("OCT.*")
        .whitelist_function("VSI.*")
        .whitelist_var("OGR.*")
        .whitelist_var("FALSE")
        .whitelist_var("TRUE")
        .generate()
        .expect("Unable to generate bindings");

    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
    bindings
        .write_to_file(out_path.join("bindings.rs"))
        .expect("Couldn't write bindings!");
}
<|endoftext|>


<commit_before>extern mod std;

use std::*;
use io::{ReaderUtil,WriterUtil};

enum Result {
  Nil,
  Int(int),
  Data(~str),
  List(~[Result]),
  Error(~str),
  Status(~str)
}

priv fn parse_data(len: uint, sb: net::tcp::TcpSocketBuf) -> Result {
  let res =
    if (len > 0) {
      let bytes = sb.read_bytes(len as uint);
      assert bytes.len() == len;
      Data(str::from_bytes(bytes))
    } else {
      Data(~"")
    };
  assert sb.read_char() == '\r';
  assert sb.read_char() == '\n';
  return res;
}

fn parse_list(len: uint, sb: net::tcp::TcpSocketBuf) -> Result {
  let mut list: ~[Result] = ~[];
  for len.times {
    if sb.read_char() != '$' { fail }
    list.push(parse_bulk(sb));
  }
  return List(list);
}

priv fn chop(s: ~str) -> ~str {
  s.slice(0, s.len() - 1)
}

priv fn parse_bulk(sb: net::tcp::TcpSocketBuf) -> Result {
  match int::from_str(chop(sb.read_line())) {
    None => fail,
    Some(-1) => Nil,
    Some(len) if len >= 0 => parse_data(len as uint, sb),
    Some(_) => fail
  }
}

priv fn parse_multi(sb: net::tcp::TcpSocketBuf) -> Result {
  match int::from_str(chop(sb.read_line())) {
    None => fail,
    Some(-1) => Nil,
    Some(0) => List(~[]),
    Some(len) if len >= 0 => parse_list(len as uint, sb),
    Some(_) => fail
  }
}

fn parse_response(sb: net::tcp::TcpSocketBuf) -> Result {
  match sb.read_char() {
    '$' => parse_bulk(sb),
    '*' => parse_multi(sb),
    '+' => Status(chop(sb.read_line())),
    '-' => Error(chop(sb.read_line())),
    ':' => match int::from_str(chop(sb.read_line())) {
             None => fail,
             Some(i) => Int(i)
           },
    _   => fail
  }
}

fn cmd_to_str(cmd: ~[~str]) -> ~str {
  let mut res = ~"*";
  str::push_str(&mut res, cmd.len().to_str());
  str::push_str(&mut res, "\r\n");
  for cmd.each |s| {
    str::push_str(&mut res, str::concat(~[~"$", s.len().to_str(), ~"\r\n", *s, ~"\r\n"]));
  }
  res
}

fn query(cmd: ~[~str], sb: net::tcp::TcpSocketBuf) -> Result {
  let cmd = cmd_to_str(cmd);
  io::println(cmd);
  sb.write_str(cmd);
  let res = parse_response(sb);
  io::println(fmt!("%?", res));
  res
}

fn main() {
  let server_ip_addr = net::ip::v4::parse_addr("127.0.0.1");
  let server_port = 6379;
  let iotask = uv::global_loop::get();
  let connect_result = net::tcp::connect(server_ip_addr, server_port, iotask);
  let sock = result::unwrap(connect_result);
  let sb = net::tcp::socket_buf(sock);

  query(~[~"SET", ~"abc", ~"XXX"], sb);
  query(~[~"SET", ~"def", ~"123"], sb);
  query(~[~"GET", ~"abc"], sb);
  query(~[~"MGET", ~"abc", ~"def", ~"non"], sb);
}
<commit_msg>Complete protocol parsing<commit_after>extern mod std;

use std::*;
use io::{ReaderUtil,WriterUtil};

enum Result {
  Nil,
  Int(int),
  Data(~str),
  List(~[Result]),
  Error(~str),
  Status(~str)
}

priv fn parse_data(len: uint, sb: net::tcp::TcpSocketBuf) -> Result {
  let res =
    if (len > 0) {
      let bytes = sb.read_bytes(len as uint);
      assert bytes.len() == len;
      Data(str::from_bytes(bytes))
    } else {
      Data(~"")
    };
  assert sb.read_char() == '\r';
  assert sb.read_char() == '\n';
  return res;
}

fn parse_list(len: uint, sb: net::tcp::TcpSocketBuf) -> Result {
  let mut list: ~[Result] = ~[];
  for len.times {
    let v =
      match sb.read_char() {
        '$' => parse_bulk(sb),
        ':' => parse_int(sb),
         _  => fail
      };
    list.push(v);
  }
  return List(list);
}

priv fn chop(s: ~str) -> ~str {
  s.slice(0, s.len() - 1)
}

priv fn parse_bulk(sb: net::tcp::TcpSocketBuf) -> Result {
  match int::from_str(chop(sb.read_line())) {
    None => fail,
    Some(-1) => Nil,
    Some(len) if len >= 0 => parse_data(len as uint, sb),
    Some(_) => fail
  }
}

priv fn parse_multi(sb: net::tcp::TcpSocketBuf) -> Result {
  match int::from_str(chop(sb.read_line())) {
    None => fail,
    Some(-1) => Nil,
    Some(0) => List(~[]),
    Some(len) if len >= 0 => parse_list(len as uint, sb),
    Some(_) => fail
  }
}

priv fn parse_int(sb: net::tcp::TcpSocketBuf) -> Result {
  match int::from_str(chop(sb.read_line())) {
    None => fail,
    Some(i) => Int(i)
  }
}

fn parse_response(sb: net::tcp::TcpSocketBuf) -> Result {
  match sb.read_char() {
    '$' => parse_bulk(sb),
    '*' => parse_multi(sb),
    '+' => Status(chop(sb.read_line())),
    '-' => Error(chop(sb.read_line())),
    ':' => parse_int(sb),
    _   => fail
  }
}

fn cmd_to_str(cmd: ~[~str]) -> ~str {
  let mut res = ~"*";
  str::push_str(&mut res, cmd.len().to_str());
  str::push_str(&mut res, "\r\n");
  for cmd.each |s| {
    str::push_str(&mut res, str::concat(~[~"$", s.len().to_str(), ~"\r\n", *s, ~"\r\n"]));
  }
  res
}

fn query(cmd: ~[~str], sb: net::tcp::TcpSocketBuf) -> Result {
  let cmd = cmd_to_str(cmd);
  //io::println(cmd);
  sb.write_str(cmd);
  let res = parse_response(sb);
  io::println(fmt!("%?", res));
  res
}

fn main() {
  let server_ip_addr = net::ip::v4::parse_addr("127.0.0.1");
  let server_port = 6379;
  let iotask = uv::global_loop::get();
  let connect_result = net::tcp::connect(server_ip_addr, server_port, iotask);
  let sock = result::unwrap(connect_result);
  let sb = net::tcp::socket_buf(sock);

/*  for 10_000.times {
    query(~[~"GET", ~"abc"], sb);
  }
*/

  query(~[~"MULTI"], sb);
  query(~[~"GET", ~"def"], sb);
  query(~[~"INCR", ~"def"], sb);
  query(~[~"INCR", ~"def"], sb);
  query(~[~"GET", ~"def"], sb);
  query(~[~"EXEC"], sb);

  query(~[~"SET", ~"abc", ~"XXX"], sb);
  query(~[~"SET", ~"def", ~"123"], sb);
  query(~[~"GET", ~"abc"], sb);
  query(~[~"MGET", ~"abc", ~"def", ~"non"], sb);
}
<|endoftext|>