<commit_before>// Copyright 2015 Philipp Oppermann. See the README.md
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

pub use self::entry::*;
use memory::{PAGE_SIZE, Frame, FrameAllocator};
use self::table::{Table, Level4};
use self::temporary_page::TemporaryPage;
pub use self::mapper::Mapper;
use core::ops::{Deref, DerefMut};
use core::ptr::Unique;
use multiboot2::BootInformation;

mod entry;
mod table;
mod temporary_page;
mod mapper;

const ENTRY_COUNT: usize = 512;

pub type PhysicalAddress = usize;
pub type VirtualAddress = usize;

#[derive(Debug, Clone, Copy)]
pub struct Page {
    number: usize,
}

impl Page {
    pub fn containing_address(address: VirtualAddress) -> Page {
        assert!(address < 0x0000_8000_0000_0000 || address >= 0xffff_8000_0000_0000,
                "invalid address: 0x{:x}",
                address);
        Page { number: address / PAGE_SIZE }
    }

    fn start_address(&self) -> usize {
        self.number * PAGE_SIZE
    }

    fn p4_index(&self) -> usize {
        (self.number >> 27) & 0o777
    }
    fn p3_index(&self) -> usize {
        (self.number >> 18) & 0o777
    }
    fn p2_index(&self) -> usize {
        (self.number >> 9) & 0o777
    }
    fn p1_index(&self) -> usize {
        (self.number >> 0) & 0o777
    }
}

pub struct ActivePageTable {
    mapper: Mapper,
}

impl Deref for ActivePageTable {
    type Target = Mapper;

    fn deref(&self) -> &Mapper {
        &self.mapper
    }
}

impl DerefMut for ActivePageTable {
    fn deref_mut(&mut self) -> &mut Mapper {
        &mut self.mapper
    }
}

impl ActivePageTable {
    unsafe fn new() -> ActivePageTable {
        ActivePageTable { mapper: Mapper::new() }
    }

    pub fn with<F>(&mut self,
                   table: &mut InactivePageTable,
                   temporary_page: &mut temporary_page::TemporaryPage, // new
                   f: F)
        where F: FnOnce(&mut Mapper)
    {
        use x86::{controlregs, tlb};
        let flush_tlb = || unsafe { tlb::flush_all() };

        {
            let backup = Frame::containing_address(unsafe { controlregs::cr3() } as usize);

            // map temporary_page to current p4 table
            let p4_table = temporary_page.map_table_frame(backup.clone(), self);

            // overwrite recursive mapping
            self.p4_mut()[511].set(table.p4_frame.clone(), PRESENT | WRITABLE);
            flush_tlb();

            // execute f in the new context
            f(self);

            // restore recursive mapping to original p4 table
            p4_table[511].set(backup, PRESENT | WRITABLE);
            flush_tlb();
        }

        temporary_page.unmap(self);
    }
}

pub struct InactivePageTable {
    p4_frame: Frame,
}

impl InactivePageTable {
    pub fn new(frame: Frame,
               active_table: &mut ActivePageTable,
               temporary_page: &mut TemporaryPage)
               -> InactivePageTable {
        {
            let table = temporary_page.map_table_frame(frame.clone(), active_table);
            table.zero();
            table[511].set(frame.clone(), PRESENT | WRITABLE);
        }
        temporary_page.unmap(active_table);

        InactivePageTable { p4_frame: frame }
    }
}

pub fn test_paging<A>(allocator: &mut A)
    where A: FrameAllocator
{
    let mut page_table = unsafe { ActivePageTable::new() };

    // test translate
    println!("Some = {:?}", page_table.translate(0));
    println!("Some = {:?}", page_table.translate(4096)); // second P1 entry
    println!("Some = {:?}", page_table.translate(512 * 4096)); // second P2 entry
    println!("Some = {:?}", page_table.translate(300 * 512 * 4096)); // 300th P2 entry
    println!("None = {:?}", page_table.translate(512 * 512 * 4096)); // second P3 entry
    println!("Some = {:?}", page_table.translate(512 * 512 * 4096 - 1)); // last mapped byte

    // test map_to
    let addr = 42 * 512 * 512 * 4096; // 42th P3 entry
    let page = Page::containing_address(addr);
    let frame = allocator.allocate_frame().expect("no more frames");
    println!("None = {:?}, map to {:?}",
             page_table.translate(addr),
             frame);
    page_table.map_to(page, frame, EntryFlags::empty(), allocator);
    println!("Some = {:?}", page_table.translate(addr));
    println!("next free frame: {:?}", allocator.allocate_frame());

    // test unmap
    println!("{:#x}",
             unsafe { *(Page::containing_address(addr).start_address() as *const u64) });
    page_table.unmap(Page::containing_address(addr), allocator);
    println!("None = {:?}", page_table.translate(addr));
}

pub fn remap_the_kernel<A>(allocator: &mut A, boot_info: &BootInformation)
    where A: FrameAllocator
{
    use core::ops::Range;

    let mut temporary_page = TemporaryPage::new(Page { number: 0xcafebabe }, allocator);

    let mut active_table = unsafe { ActivePageTable::new() };
    let mut new_table = {
        let frame = allocator.allocate_frame().expect("no more frames");
        InactivePageTable::new(frame, &mut active_table, &mut temporary_page)
    };

    active_table.with(&mut new_table, &mut temporary_page, |mapper| {
        let elf_sections_tag = boot_info.elf_sections_tag()
                                        .expect("Memory map tag required");

        for section in elf_sections_tag.sections() {
            use multiboot2::ELF_SECTION_ALLOCATED;
            use self::entry::WRITABLE;

            if !section.flags().contains(ELF_SECTION_ALLOCATED) {
                // section is not loaded to memory
                continue;
            }

            println!("mapping section at addr: {:#x}, size: {:#x}",
                     section.addr,
                     section.size);

            let flags = WRITABLE; // TODO use real section flags

            let range = Range {
                start: section.addr as usize,
                end: (section.addr + section.size) as usize,
            };
            for address in range.step_by(PAGE_SIZE) {
                assert!(address % PAGE_SIZE == 0, "sections need to be page aligned");
                let frame = Frame::containing_address(address);
                mapper.identity_map(frame, flags, allocator);
            }
        }
    });
}
<commit_msg>Try to switch tables, causes boot loop<commit_after>// Copyright 2015 Philipp Oppermann. See the README.md
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

pub use self::entry::*;
use memory::{PAGE_SIZE, Frame, FrameAllocator};
use self::table::{Table, Level4};
use self::temporary_page::TemporaryPage;
pub use self::mapper::Mapper;
use core::ops::{Deref, DerefMut};
use core::ptr::Unique;
use multiboot2::BootInformation;

mod entry;
mod table;
mod temporary_page;
mod mapper;

const ENTRY_COUNT: usize = 512;

pub type PhysicalAddress = usize;
pub type VirtualAddress = usize;

#[derive(Debug, Clone, Copy)]
pub struct Page {
    number: usize,
}

impl Page {
    pub fn containing_address(address: VirtualAddress) -> Page {
        assert!(address < 0x0000_8000_0000_0000 || address >= 0xffff_8000_0000_0000,
                "invalid address: 0x{:x}",
                address);
        Page { number: address / PAGE_SIZE }
    }

    fn start_address(&self) -> usize {
        self.number * PAGE_SIZE
    }

    fn p4_index(&self) -> usize {
        (self.number >> 27) & 0o777
    }
    fn p3_index(&self) -> usize {
        (self.number >> 18) & 0o777
    }
    fn p2_index(&self) -> usize {
        (self.number >> 9) & 0o777
    }
    fn p1_index(&self) -> usize {
        (self.number >> 0) & 0o777
    }
}

pub struct ActivePageTable {
    mapper: Mapper,
}

impl Deref for ActivePageTable {
    type Target = Mapper;

    fn deref(&self) -> &Mapper {
        &self.mapper
    }
}

impl DerefMut for ActivePageTable {
    fn deref_mut(&mut self) -> &mut Mapper {
        &mut self.mapper
    }
}

impl ActivePageTable {
    unsafe fn new() -> ActivePageTable {
        ActivePageTable { mapper: Mapper::new() }
    }

    pub fn with<F>(&mut self,
                   table: &mut InactivePageTable,
                   temporary_page: &mut temporary_page::TemporaryPage, // new
                   f: F)
        where F: FnOnce(&mut Mapper)
    {
        use x86::{controlregs, tlb};
        let flush_tlb = || unsafe { tlb::flush_all() };

        {
            let backup = Frame::containing_address(unsafe { controlregs::cr3() } as usize);

            // map temporary_page to current p4 table
            let p4_table = temporary_page.map_table_frame(backup.clone(), self);

            // overwrite recursive mapping
            self.p4_mut()[511].set(table.p4_frame.clone(), PRESENT | WRITABLE);
            flush_tlb();

            // execute f in the new context
            f(self);

            // restore recursive mapping to original p4 table
            p4_table[511].set(backup, PRESENT | WRITABLE);
            flush_tlb();
        }

        temporary_page.unmap(self);
    }

    pub fn switch(&mut self, new_table: InactivePageTable) -> InactivePageTable {
        use x86::controlregs;

        let old_table = InactivePageTable {
            p4_frame: Frame::containing_address(unsafe { controlregs::cr3() } as usize),
        };
        unsafe {
            controlregs::cr3_write(new_table.p4_frame.start_address() as u64);
        }
        old_table
    }
}

pub struct InactivePageTable {
    p4_frame: Frame,
}

impl InactivePageTable {
    pub fn new(frame: Frame,
               active_table: &mut ActivePageTable,
               temporary_page: &mut TemporaryPage)
               -> InactivePageTable {
        {
            let table = temporary_page.map_table_frame(frame.clone(), active_table);
            table.zero();
            table[511].set(frame.clone(), PRESENT | WRITABLE);
        }
        temporary_page.unmap(active_table);

        InactivePageTable { p4_frame: frame }
    }
}

pub fn test_paging<A>(allocator: &mut A)
    where A: FrameAllocator
{
    let mut page_table = unsafe { ActivePageTable::new() };

    // test translate
    println!("Some = {:?}", page_table.translate(0));
    println!("Some = {:?}", page_table.translate(4096)); // second P1 entry
    println!("Some = {:?}", page_table.translate(512 * 4096)); // second P2 entry
    println!("Some = {:?}", page_table.translate(300 * 512 * 4096)); // 300th P2 entry
    println!("None = {:?}", page_table.translate(512 * 512 * 4096)); // second P3 entry
    println!("Some = {:?}", page_table.translate(512 * 512 * 4096 - 1)); // last mapped byte

    // test map_to
    let addr = 42 * 512 * 512 * 4096; // 42th P3 entry
    let page = Page::containing_address(addr);
    let frame = allocator.allocate_frame().expect("no more frames");
    println!("None = {:?}, map to {:?}",
             page_table.translate(addr),
             frame);
    page_table.map_to(page, frame, EntryFlags::empty(), allocator);
    println!("Some = {:?}", page_table.translate(addr));
    println!("next free frame: {:?}", allocator.allocate_frame());

    // test unmap
    println!("{:#x}",
             unsafe { *(Page::containing_address(addr).start_address() as *const u64) });
    page_table.unmap(Page::containing_address(addr), allocator);
    println!("None = {:?}", page_table.translate(addr));
}

pub fn remap_the_kernel<A>(allocator: &mut A, boot_info: &BootInformation)
    where A: FrameAllocator
{
    use core::ops::Range;

    let mut temporary_page = TemporaryPage::new(Page { number: 0xcafebabe }, allocator);

    let mut active_table = unsafe { ActivePageTable::new() };
    let mut new_table = {
        let frame = allocator.allocate_frame().expect("no more frames");
        InactivePageTable::new(frame, &mut active_table, &mut temporary_page)
    };

    active_table.with(&mut new_table, &mut temporary_page, |mapper| {
        let elf_sections_tag = boot_info.elf_sections_tag()
                                        .expect("Memory map tag required");

        for section in elf_sections_tag.sections() {
            use multiboot2::ELF_SECTION_ALLOCATED;
            use self::entry::WRITABLE;

            if !section.flags().contains(ELF_SECTION_ALLOCATED) {
                // section is not loaded to memory
                continue;
            }

            println!("mapping section at addr: {:#x}, size: {:#x}",
                     section.addr,
                     section.size);

            let flags = WRITABLE; // TODO use real section flags

            let range = Range {
                start: section.addr as usize,
                end: (section.addr + section.size) as usize,
            };
            for address in range.step_by(PAGE_SIZE) {
                assert!(address % PAGE_SIZE == 0, "sections need to be page aligned");
                let frame = Frame::containing_address(address);
                mapper.identity_map(frame, flags, allocator);
            }
        }
    });

    let old_table = active_table.switch(new_table);
    println!("NEW TABLE!!!");
}
<|endoftext|>


<commit_before>//! Models about OAuth2 applications.

use super::{
    id::UserId,
    user::User,
    utils::default_true
};

/// Information about a user's application. An application does not necessarily
/// have an associated bot user.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct ApplicationInfo {
    /// The bot user associated with the application. See [`BotApplication`] for
    /// more information.
    ///
    /// [`BotApplication`]: struct.BotApplication.html
    pub bot: Option<BotApplication>,
    /// Indicator of whether the bot is public.
    ///
    /// If a bot is public, anyone may invite it to their [`Guild`]. While a bot
    /// is private, only the owner may add it to a guild.
    ///
    /// [`Guild`]: ../guild/struct.Guild.html
    #[serde(default = "default_true")]
    pub bot_public: bool,
    /// Indicator of whether the bot requires an OAuth2 code grant.
    pub bot_require_code_grant: bool,
    /// A description of the application, assigned by the application owner.
    pub description: String,
    /// A set of bitflags assigned to the application, which represent gated
    /// feature flags that have been enabled for the application.
    pub flags: Option<u64>,
    /// A hash pointing to the application's icon.
    ///
    /// This is not necessarily equivalent to the bot user's avatar.
    pub icon: Option<String>,
    /// The unique numeric Id of the application.
    pub id: UserId,
    /// The name assigned to the application by the application owner.
    pub name: String,
    /// A list of redirect URIs assigned to the application.
    pub redirect_uris: Vec<String>,
    /// A list of RPC Origins assigned to the application.
    pub rpc_origins: Vec<String>,
    /// The given secret to the application.
    ///
    /// This is not equivalent to the application's bot user's token.
    pub secret: String,
}

/// Information about an application with an application's bot user.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct BotApplication {
    /// The unique Id of the bot user.
    pub id: UserId,
    /// A hash of the avatar, if one is assigned.
    ///
    /// Can be used to generate a full URL to the avatar.
    pub avatar: Option<String>,
    /// Indicator of whether it is a bot.
    #[serde(default)]
    pub bot: bool,
    /// The discriminator assigned to the bot user.
    ///
    /// While discriminators are not unique, the `username#discriminator` pair
    /// is.
    pub discriminator: u16,
    /// The bot user's username.
    pub name: String,
    /// The token used to authenticate as the bot user.
    ///
    /// **Note**: Keep this information private, as untrusted sources can use it
    /// to perform any action with a bot user.
    pub token: String,
}

/// Information about the current application and its owner.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct CurrentApplicationInfo {
    pub description: String,
    pub icon: Option<String>,
    pub id: UserId,
    pub name: String,
    pub owner: User,
    #[serde(default)] pub rpc_origins: Vec<String>,
}
<commit_msg>Fix current_application_info (#517)<commit_after>//! Models about OAuth2 applications.

use super::{
    id::UserId,
    user::User,
    utils::default_true
};

/// Information about a user's application. An application does not necessarily
/// have an associated bot user.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct ApplicationInfo {
    /// The bot user associated with the application. See [`BotApplication`] for
    /// more information.
    ///
    /// [`BotApplication`]: struct.BotApplication.html
    pub bot: Option<BotApplication>,
    /// Indicator of whether the bot is public.
    ///
    /// If a bot is public, anyone may invite it to their [`Guild`]. While a bot
    /// is private, only the owner may add it to a guild.
    ///
    /// [`Guild`]: ../guild/struct.Guild.html
    #[serde(default = "default_true")]
    pub bot_public: bool,
    /// Indicator of whether the bot requires an OAuth2 code grant.
    pub bot_require_code_grant: bool,
    /// A description of the application, assigned by the application owner.
    pub description: String,
    /// A set of bitflags assigned to the application, which represent gated
    /// feature flags that have been enabled for the application.
    pub flags: Option<u64>,
    /// A hash pointing to the application's icon.
    ///
    /// This is not necessarily equivalent to the bot user's avatar.
    pub icon: Option<String>,
    /// The unique numeric Id of the application.
    pub id: UserId,
    /// The name assigned to the application by the application owner.
    pub name: String,
    /// A list of redirect URIs assigned to the application.
    pub redirect_uris: Vec<String>,
    /// A list of RPC Origins assigned to the application.
    pub rpc_origins: Vec<String>,
    /// The given secret to the application.
    ///
    /// This is not equivalent to the application's bot user's token.
    pub secret: String,
}

/// Information about an application with an application's bot user.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct BotApplication {
    /// The unique Id of the bot user.
    pub id: UserId,
    /// A hash of the avatar, if one is assigned.
    ///
    /// Can be used to generate a full URL to the avatar.
    pub avatar: Option<String>,
    /// Indicator of whether it is a bot.
    #[serde(default)]
    pub bot: bool,
    /// The discriminator assigned to the bot user.
    ///
    /// While discriminators are not unique, the `username#discriminator` pair
    /// is.
    pub discriminator: u16,
    /// The bot user's username.
    pub name: String,
    /// The token used to authenticate as the bot user.
    ///
    /// **Note**: Keep this information private, as untrusted sources can use it
    /// to perform any action with a bot user.
    pub token: String,
}

/// Information about the current application and its owner.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct CurrentApplicationInfo {
    pub description: String,
    pub icon: Option<String>,
    pub id: UserId,
    pub name: String,
    pub owner: User,
    #[serde(default)] pub rpc_origins: Vec<String>,
    pub bot_public: bool,
    pub bot_require_code_grant: bool,
}
<|endoftext|>


<commit_before>extern crate handlebars;
extern crate rustc_serialize;

use std::path::Path;
use std::collections::BTreeMap;

use self::rustc_serialize::json;
use self::handlebars::{Handlebars, HelperDef, RenderError, RenderContext, Helper, Context};

// Handlebars helper to construct TOC
#[derive(Clone, Copy)]
pub struct RenderToc;

impl HelperDef for RenderToc {
  fn call(&self, c: &Context, _h: &Helper, _: &Handlebars, rc: &mut RenderContext) -> Result<(), RenderError> {

    // get value from context data
    // rc.get_path() is current json parent path, you should always use it like this
    // param is the key of value you want to display
    let chapters = c.navigate(rc.get_path(), "chapters");
    let current = c.navigate(rc.get_path(), "path").to_string().replace("\"", "");
    let path_to_root = c.navigate(rc.get_path(), "path_to_root").to_string().replace("\"", "");
    try!(rc.writer.write("
".as_bytes()));

    // Decode json format
    let decoded: Vec<BTreeMap<String,String>> = json::decode(&chapters.to_string()).unwrap();

    let mut current_level = 1;

    for item in decoded {

        let level = item.get("section").expect("Error: section should be Some(_)").len() / 2;
        if level > current_level {
            try!(rc.writer.write("
".as_bytes()));
            try!(rc.writer.write("
".as_bytes()));
            try!(rc.writer.write("
".as_bytes()));
        } else if level < current_level {
            try!(rc.writer.write("
".as_bytes()));
            try!(rc.writer.write("
".as_bytes()));
        }
        else {
            try!(rc.writer.write("
".as_bytes()));
        }

        // Link
        let path_exists = if let Some(path) = item.get("path") {
            if path.len() > 0 {
                try!(rc.writer.write("<a href=\"".as_bytes()));

                // Prefix with path to root
                try!(rc.writer.write(path_to_root.as_bytes()));

                // Add link
                try!(rc.writer.write(
                    Path::new(
                        item.get("path")
                            .expect("Error: path should be Some(_)")
                        ).with_extension("html")
                        .to_str().unwrap().as_bytes()
                    ));

                try!(rc.writer.write("\"".as_bytes()));

                if path == &current {
                    try!(rc.writer.write("class=\"active\"".as_bytes()));
                }

                try!(rc.writer.write(">".as_bytes()));
                true
            } else {
                false
            }
        }else {
            false
        };

        try!(rc.writer.write("".as_bytes()));
        try!(rc.writer.write(item.get("section").expect("Error: section should be Some(_)").as_bytes()));
        try!(rc.writer.write(" ".as_bytes()));
        try!(rc.writer.write(item.get("name").expect("Error: name should be Some(_)").as_bytes()));

        if path_exists {
            try!(rc.writer.write("".as_bytes()));
        }

        try!(rc.writer.write("
".as_bytes()));

        current_level = level;
    }

    try!(rc.writer.write("
".as_bytes()));
    Ok(())
  }
}
<commit_msg>Fix #34<commit_after>extern crate handlebars;
extern crate rustc_serialize;

use std::path::Path;
use std::collections::BTreeMap;

use self::rustc_serialize::json;
use self::handlebars::{Handlebars, HelperDef, RenderError, RenderContext, Helper, Context};

// Handlebars helper to construct TOC
#[derive(Clone, Copy)]
pub struct RenderToc;

impl HelperDef for RenderToc {
  fn call(&self, c: &Context, _h: &Helper, _: &Handlebars, rc: &mut RenderContext) -> Result<(), RenderError> {

    // get value from context data
    // rc.get_path() is current json parent path, you should always use it like this
    // param is the key of value you want to display
    let chapters = c.navigate(rc.get_path(), "chapters");
    let current = c.navigate(rc.get_path(), "path").to_string().replace("\"", "");
    let path_to_root = c.navigate(rc.get_path(), "path_to_root").to_string().replace("\"", "");
    try!(rc.writer.write("
".as_bytes()));

    // Decode json format
    let decoded: Vec<BTreeMap<String,String>> = json::decode(&chapters.to_string()).unwrap();

    let mut current_level = 1;

    for item in decoded {

        let level = item.get("section").expect("Error: section should be Some(_)").len() / 2;
        if level > current_level {
            try!(rc.writer.write("
".as_bytes()));
            try!(rc.writer.write("
".as_bytes()));
            try!(rc.writer.write("
".as_bytes()));
        } else if level < current_level {
            while level < current_level {
                try!(rc.writer.write("
".as_bytes()));
                try!(rc.writer.write("
".as_bytes()));
                current_level = current_level - 1;
            }
            try!(rc.writer.write("
".as_bytes()));
        }
        else {
            try!(rc.writer.write("
".as_bytes()));
        }

        // Link
        let path_exists = if let Some(path) = item.get("path") {
            if path.len() > 0 {
                try!(rc.writer.write("<a href=\"".as_bytes()));

                // Prefix with path to root
                try!(rc.writer.write(path_to_root.as_bytes()));

                // Add link
                try!(rc.writer.write(
                    Path::new(
                        item.get("path")
                            .expect("Error: path should be Some(_)")
                        ).with_extension("html")
                        .to_str().unwrap().as_bytes()
                    ));

                try!(rc.writer.write("\"".as_bytes()));

                if path == &current {
                    try!(rc.writer.write("class=\"active\"".as_bytes()));
                }

                try!(rc.writer.write(">".as_bytes()));
                true
            } else {
                false
            }
        }else {
            false
        };

        try!(rc.writer.write("".as_bytes()));
        try!(rc.writer.write(item.get("section").expect("Error: section should be Some(_)").as_bytes()));
        try!(rc.writer.write(" ".as_bytes()));
        try!(rc.writer.write(item.get("name").expect("Error: name should be Some(_)").as_bytes()));

        if path_exists {
            try!(rc.writer.write("".as_bytes()));
        }

        try!(rc.writer.write("
".as_bytes()));

        current_level = level;
    }

    try!(rc.writer.write("
".as_bytes()));
    Ok(())
  }
}
<|endoftext|>


<commit_before>//! View bits in memory with turtles
//!
//! This example uses [@myrrlyn]'s [`bitvec`] crate to turn data into strings of
//! bits, and then draws them on the screen.
//!
//! You are encouraged to change both the data used to seed the turtle, and the
//! `bitvec` calls that control how the turtle acts, to see what changes.
//!
//! [@myrrlyn]: //github.com/myrrlyn
//! [`bitvec`]: //crates.io/crates/bitvec

// This imports the things we need from `bitvec`, including the `Bits` trait for
// the `.bits::<_>()` method we use to view memory.
use bitvec::prelude::*;
use turtle::Turtle;

// Modify these constants to change the behavior of the example.

/// This text will be inspected as individual bytes, and drawn on the screen.
/// You can change it to see what different text looks like when viewed as bits.
///
/// The example program will print more information about the parts of the text
/// to the console while the turtle draws, so that you can see how each glyph
/// corresponds to parts of the rendered memory.
static TEXT: &str = "¡Hola, mundo! 🌍🌏🌎";

/// This number will have its bit pattern printed on screen. Rust provides some
/// interesting numbers in its standard library; you can replace this with other
/// numbers to see what they look like. Pi is provided as the default solely
/// because it is well-known, and has an interesting pattern.
const NUMBER: f64 = std::f64::consts::PI;

/// This controls the width of the drawn line for each bit.
const BIT_WIDTH: f64 = 20.0;

/// This controls the vertical spacing between rows of bit lines.
const BIT_HEIGHT: f64 = 10.0;

/// Set the horizontal spacing between successive bits in a row
const BIT_MARGIN: f64 = BIT_WIDTH / 2.0;

/// Compute the total width of a bit plus its spacing
const BIT_BOX: f64 = BIT_WIDTH + BIT_MARGIN;

fn main() {
    // This block sets up the turtle to draw bits more or less centered in the
    // screen. The turtle works by walking horizontally for each bit in a byte,
    // then backtracking and walking vertically to the next byte.
    let mut turtle = Turtle::new();
    // The turtle starts in the center of the screen, but we want to move it
    // around before drawing.
    turtle.pen_up();

    // Compute the boundaries of the part of the screen where the turtle will
    // draw. We expect to be drawing eight bits, with half to the right of
    // center and half to the left.
    let right_edge = BIT_BOX * 8.0 / 2.0;
    // We also expect to be drawing a row for each byte in the text, with an
    // additional separator row for each *character*, half above and half below
    // the center of the screen. This computes how many rows of text we will
    // draw, then moves the turtle appropriately.
    let byte_rows = TEXT.len();
    let char_gaps = TEXT.chars().count();
    let top_edge = BIT_HEIGHT * ((byte_rows + char_gaps) as f64 / 2.0);
    // The turtle starts from the top right of the region,
    turtle.forward(top_edge);
    turtle.right(90.0);
    turtle.forward(right_edge);
    // and walks left
    turtle.left(180.0);

    draw_text(&mut turtle, TEXT);

    // The `draw_number` function reads bits from left to right, so the turtle
    // should also walk from left to right. The `draw_number` function expects
    // that it will be drawing rows sixteen bits long, so it needs to move
    // forward another four bits' worth of space in order to be in the correct
    // spot.
    turtle.forward(8.0 * BIT_BOX / 2.0);
    turtle.forward(16.0 * BIT_BOX / 2.0);
    // Then, it needs to turn around, to walk in the other direction.
    turtle.right(180.0);

    draw_number(&mut turtle, NUMBER);

    // Reader exercise:
    //
    // The IEEE-754 format for `f64` numbers separates them into three parts:
    //
    // 1. The sign marks whether the number is positive or negative: 1 bit
    // 2. The exponent marks how far from zero the number is: 11 bits
    // 3. The fraction describes the number: 52 bits.
    //
    // Using these widths (1 bit, 11 bits, 52 bits), the knowledge that
    // `&BitSlice` is a normal Rust slice, and the API documentation for
    // `std::iter::Iterator`, see if you can display each portion of an `f64`
    // as its own row.
    //
    // Hints:
    //
    // - The variable `bits` is set up to view the entire number, from most
    // significant bit to least.
    // - You can get access to a structure that performs iteration by calling
    //   `bits.iter()`.
    // - You can use the `Iterator::by_ref` method to prevent `Iterator` adapter
    //   functions from destroying the source iterator.
    // - `&BitSlice` is an ordinary Rust slice, so you can use `[start .. end]`
    //   range indexing to get smaller pieces of it.
}

/// Draws the bits of a text span on the screen.
fn draw_text(turtle: &mut Turtle, text: &str) {
    // Rust strings can iterate over their individual characters. This block
    // loops over characters, collecting their start point in the text so that
    // we can grab the encoded bytes of each one.
    let mut row_num = 0;
    for (char_num, (start, codepoint)) in text.char_indices().enumerate() {
        println!("Character {}: {}", char_num, codepoint);
        // Each character has a variable width, so we need to find that.
        let byte_count = codepoint.len_utf8();
        // And then collect the bytes of the string that make up the character.
        // `start` gives us the starting position in the text sequence, and
        // `byte_count` gives us the length in bytes of the character, so we
        // need to select the range beginning at `start`, running for
        // `byte_count`. Another style of writing this that you might see in
        // Rust libraries is `[start ..][.. length]`.
        let row: &[u8] = &text.as_bytes()[start .. start + byte_count];

        // For each byte (`u8`), we use `bitvec` to make a view into its bits.
        // `bitvec` provides the `.bits::<_>()` method on Rust integers for easy
        // access to its view types.
        //
        // The `LittleEndian` means that the view moves from least significant
        // bit to most significant. Since we want to display on screen the most
        // significant bit on the left, and the least on the right, the turtle
        // will have to move from right to left to match.
        //
        // The `LittleEndian` and `BigEndian` types describe different ways to
        // view the same data. You can read more about them in the `bitvec`
        // docs, and at Wikipedia:
        // https://docs.rs/bitvec/0.16.1/bitvec/cursor/index.html
        // https://en.wikipedia.org/wiki/Endianness#Bit_endianness
        for byte in row {
            println!("  Byte {:02}:\n    Value: 0x{:02X}\n    Bits: {:08b}", row_num, byte, byte);

            let bits: &BitSlice<_, _> = byte.bits::<LittleEndian>();

            // Then we draw the byte's bits as a row
            draw_row(turtle, bits);
            //  And go to the next row
            next_row(turtle, 90.0);

            row_num += 1;
        }
        // This puts a dividing line between each *character* in the text.
        // Some characters may have more than one byte, and those bytes will be
        // grouped together.
        delimit(turtle, 8.0 * BIT_BOX - BIT_MARGIN);
    }
}

/// Draws the bits of a number on screen.
fn draw_number(turtle: &mut Turtle, number: f64) {
    // `bitvec` can look at more than just `u8`. Let's try looking at the bits
    // that represent a number!
    //
    // Some numbers, like `f64`, have special rules for their representation in
    // bits. `bitvec` only knows about raw bits, so it does not provide direct
    // support for `f64`. Rust lets us get the bit representation from an `f64`
    // with the method `to_bits(f64) -> u64`, which forgets about the `f64`
    // rules and uses the number's storage as ordinary bits.
    //
    // You can read more about the rules for `f64`'s storage in memory, and
    // behavior in programs, here:
    // https://en.wikipedia.org/wiki/Double-precision_floating-point_format
    let raw_number: u64 = number.to_bits();

    // `bitvec` can also view bits from left to right, with `BigEndian`.
    let bits: &BitSlice<_, _> = raw_number.bits::<BigEndian>();

    // The `&BitSlice` type acts just like `&[bool]`, so it comes with a
    // `.chunks` method which divides it into smaller pieces. `bitvec` can take
    // any number, not just multiples of 8, but 16 is a convenient number to
    // look at. Try changing it to a different number, like 10, to see what
    // happens!
    for (num, row) in bits.chunks(16).enumerate() {
        println!("Row {} bits: {:b}", num, row);
        // Each chunk produced is a smaller `&BitSlice`, just like
        // `&[bool].chunks` produces smaller `&[bool]`s, so we can draw it.
        draw_row(turtle, row);

        next_row(turtle, -90.0);
    }
}

/// Draw a row of bits on the screen.
///
/// This takes a reference to a turtle, which draws, and a reference to a slice
/// of bits, which provides the data to draw.
///
/// Note that this works whether we're going through the bits left to right
/// (`BigEndian`) or right to left (`LittleEndian`), because we assume that the
/// turtle is going to start on the correct side and be facing the correct way
/// for this drawing to work.
fn draw_row<C, T>(turtle: &mut Turtle, row: &BitSlice<C, T>)
where C: Cursor, T: BitStore {
    // `&BitSlice` can iterate over bits. It is just like `&[bool]`, and so it
    // produces `&bool` for each loop.
    for &bit in row {
        // This checks if the bit produced by the row is `1` or `0`, and sets
        // the pen color to black (`1`) or light grey (`0`)
        if bit {
            turtle.set_pen_color("black");
        }
        else {
            turtle.set_pen_color("light grey");
        }

        // For each bit, the loop puts down the pen to draw a line of the bit's
        // color, then picks up the pen to add some horizontal spacing between
        // them.
        turtle.pen_down();
        turtle.forward(BIT_WIDTH);
        turtle.pen_up();
        turtle.forward(BIT_MARGIN);
    }
    //  Rewind the turtle
    for _ in 0 .. row.len() {
        turtle.backward(BIT_BOX);
    }
}

/// Produces a separator line to demark different sections of memory.
fn delimit(turtle: &mut Turtle, width: f64) {
    turtle.set_pen_color("grey");
    turtle.pen_down();
    turtle.forward(width);
    turtle.backward(width);

    next_row(turtle, 90.0);
}

/// Moves the turtle down a row
fn next_row(turtle: &mut Turtle, angle: f64) {
    turtle.pen_up();
    turtle.left(angle);
    turtle.forward(BIT_HEIGHT);
    turtle.right(angle);
}
<commit_msg>Move the exercise comment into `draw_number`<commit_after>//! View bits in memory with turtles
//!
//! This example uses [@myrrlyn]'s [`bitvec`] crate to turn data into strings of
//! bits, and then draws them on the screen.
//!
//! You are encouraged to change both the data used to seed the turtle, and the
//! `bitvec` calls that control how the turtle acts, to see what changes.
//!
//! [@myrrlyn]: //github.com/myrrlyn
//! [`bitvec`]: //crates.io/crates/bitvec

// This imports the things we need from `bitvec`, including the `Bits` trait for
// the `.bits::<_>()` method we use to view memory.
use bitvec::prelude::*;
use turtle::Turtle;

// Modify these constants to change the behavior of the example.

/// This text will be inspected as individual bytes, and drawn on the screen.
/// You can change it to see what different text looks like when viewed as bits.
///
/// The example program will print more information about the parts of the text
/// to the console while the turtle draws, so that you can see how each glyph
/// corresponds to parts of the rendered memory.
static TEXT: &str = "¡Hola, mundo! 🌍🌏🌎";

/// This number will have its bit pattern printed on screen. Rust provides some
/// interesting numbers in its standard library; you can replace this with other
/// numbers to see what they look like. Pi is provided as the default solely
/// because it is well-known, and has an interesting pattern.
const NUMBER: f64 = std::f64::consts::PI;

/// This controls the width of the drawn line for each bit.
const BIT_WIDTH: f64 = 20.0;

/// This controls the vertical spacing between rows of bit lines.
const BIT_HEIGHT: f64 = 10.0;

/// Set the horizontal spacing between successive bits in a row
const BIT_MARGIN: f64 = BIT_WIDTH / 2.0;

/// Compute the total width of a bit plus its spacing
const BIT_BOX: f64 = BIT_WIDTH + BIT_MARGIN;

fn main() {
    // This block sets up the turtle to draw bits more or less centered in the
    // screen. The turtle works by walking horizontally for each bit in a byte,
    // then backtracking and walking vertically to the next byte.
    let mut turtle = Turtle::new();
    // The turtle starts in the center of the screen, but we want to move it
    // around before drawing.
    turtle.pen_up();

    // Compute the boundaries of the part of the screen where the turtle will
    // draw. We expect to be drawing eight bits, with half to the right of
    // center and half to the left.
    let right_edge = BIT_BOX * 8.0 / 2.0;
    // We also expect to be drawing a row for each byte in the text, with an
    // additional separator row for each *character*, half above and half below
    // the center of the screen. This computes how many rows of text we will
    // draw, then moves the turtle appropriately.
    let byte_rows = TEXT.len();
    let char_gaps = TEXT.chars().count();
    let top_edge = BIT_HEIGHT * ((byte_rows + char_gaps) as f64 / 2.0);
    // The turtle starts from the top right of the region,
    turtle.forward(top_edge);
    turtle.right(90.0);
    turtle.forward(right_edge);
    // and walks left
    turtle.left(180.0);

    draw_text(&mut turtle, TEXT);

    // The `draw_number` function reads bits from left to right, so the turtle
    // should also walk from left to right. The `draw_number` function expects
    // that it will be drawing rows sixteen bits long, so it needs to move
    // forward another four bits' worth of space in order to be in the correct
    // spot.
    turtle.forward(8.0 * BIT_BOX / 2.0);
    turtle.forward(16.0 * BIT_BOX / 2.0);
    // Then, it needs to turn around, to walk in the other direction.
    turtle.right(180.0);

    draw_number(&mut turtle, NUMBER);
}

/// Draws the bits of a text span on the screen.
fn draw_text(turtle: &mut Turtle, text: &str) {
    // Rust strings can iterate over their individual characters. This block
    // loops over characters, collecting their start point in the text so that
    // we can grab the encoded bytes of each one.
    let mut row_num = 0;
    for (char_num, (start, codepoint)) in text.char_indices().enumerate() {
        println!("Character {}: {}", char_num, codepoint);
        // Each character has a variable width, so we need to find that.
        let byte_count = codepoint.len_utf8();
        // And then collect the bytes of the string that make up the character.
        // `start` gives us the starting position in the text sequence, and
        // `byte_count` gives us the length in bytes of the character, so we
        // need to select the range beginning at `start`, running for
        // `byte_count`. Another style of writing this that you might see in
        // Rust libraries is `[start ..][.. length]`.
        let row: &[u8] = &text.as_bytes()[start .. start + byte_count];

        // For each byte (`u8`), we use `bitvec` to make a view into its bits.
        // `bitvec` provides the `.bits::<_>()` method on Rust integers for easy
        // access to its view types.
        //
        // The `LittleEndian` means that the view moves from least significant
        // bit to most significant. Since we want to display on screen the most
        // significant bit on the left, and the least on the right, the turtle
        // will have to move from right to left to match.
        //
        // The `LittleEndian` and `BigEndian` types describe different ways to
        // view the same data. You can read more about them in the `bitvec`
        // docs, and at Wikipedia:
        // https://docs.rs/bitvec/0.16.1/bitvec/cursor/index.html
        // https://en.wikipedia.org/wiki/Endianness#Bit_endianness
        for byte in row {
            println!("  Byte {:02}:\n    Value: 0x{:02X}\n    Bits: {:08b}", row_num, byte, byte);

            let bits: &BitSlice<_, _> = byte.bits::<LittleEndian>();

            // Then we draw the byte's bits as a row
            draw_row(turtle, bits);
            //  And go to the next row
            next_row(turtle, 90.0);

            row_num += 1;
        }
        // This puts a dividing line between each *character* in the text.
        // Some characters may have more than one byte, and those bytes will be
        // grouped together.
        delimit(turtle, 8.0 * BIT_BOX - BIT_MARGIN);
    }
}

/// Draws the bits of a number on screen.
fn draw_number(turtle: &mut Turtle, number: f64) {
    // `bitvec` can look at more than just `u8`. Let's try looking at the bits
    // that represent a number!
    //
    // Some numbers, like `f64`, have special rules for their representation in
    // bits. `bitvec` only knows about raw bits, so it does not provide direct
    // support for `f64`. Rust lets us get the bit representation from an `f64`
    // with the method `to_bits(f64) -> u64`, which forgets about the `f64`
    // rules and uses the number's storage as ordinary bits.
    //
    // You can read more about the rules for `f64`'s storage in memory, and
    // behavior in programs, here:
    // https://en.wikipedia.org/wiki/Double-precision_floating-point_format
    let raw_number: u64 = number.to_bits();

    // `bitvec` can also view bits from left to right, with `BigEndian`.
    let bits: &BitSlice<_, _> = raw_number.bits::<BigEndian>();

    // The `&BitSlice` type acts just like `&[bool]`, so it comes with a
    // `.chunks` method which divides it into smaller pieces. `bitvec` can take
    // any number, not just multiples of 8, but 16 is a convenient number to
    // look at. Try changing it to a different number, like 10, to see what
    // happens!
    for (num, row) in bits.chunks(16).enumerate() {
        println!("Row {} bits: {:b}", num, row);
        // Each chunk produced is a smaller `&BitSlice`, just like
        // `&[bool].chunks` produces smaller `&[bool]`s, so we can draw it.
        draw_row(turtle, row);

        next_row(turtle, -90.0);
    }

    // Reader exercise:
    //
    // The IEEE-754 format for `f64` numbers separates them into three parts:
    //
    // 1. The sign marks whether the number is positive or negative: 1 bit
    // 2. The exponent marks how far from zero the number is: 11 bits
    // 3. The fraction describes the number: 52 bits.
    //
    // Using these widths (1 bit, 11 bits, 52 bits), the knowledge that
    // `&BitSlice` is a normal Rust slice, and the API documentation for
    // `std::iter::Iterator`, see if you can display each portion of an `f64`
    // as its own row.
    //
    // Hints:
    //
    // - The variable `bits` is set up to view the entire number, from most
    // significant bit to least.
    // - You can get access to a structure that performs iteration by calling
    //   `bits.iter()`.
    // - You can use the `Iterator::by_ref` method to prevent `Iterator` adapter
    //   functions from destroying the source iterator.
    // - `&BitSlice` is an ordinary Rust slice, so you can use `[start .. end]`
    //   range indexing to get smaller pieces of it.
}

/// Draw a row of bits on the screen.
///
/// This takes a reference to a turtle, which draws, and a reference to a slice
/// of bits, which provides the data to draw.
///
/// Note that this works whether we're going through the bits left to right
/// (`BigEndian`) or right to left (`LittleEndian`), because we assume that the
/// turtle is going to start on the correct side and be facing the correct way
/// for this drawing to work.
fn draw_row<C, T>(turtle: &mut Turtle, row: &BitSlice<C, T>)
where C: Cursor, T: BitStore {
    // `&BitSlice` can iterate over bits. It is just like `&[bool]`, and so it
    // produces `&bool` for each loop.
    for &bit in row {
        // This checks if the bit produced by the row is `1` or `0`, and sets
        // the pen color to black (`1`) or light grey (`0`)
        if bit {
            turtle.set_pen_color("black");
        }
        else {
            turtle.set_pen_color("light grey");
        }

        // For each bit, the loop puts down the pen to draw a line of the bit's
        // color, then picks up the pen to add some horizontal spacing between
        // them.
        turtle.pen_down();
        turtle.forward(BIT_WIDTH);
        turtle.pen_up();
        turtle.forward(BIT_MARGIN);
    }
    //  Rewind the turtle
    for _ in 0 .. row.len() {
        turtle.backward(BIT_BOX);
    }
}

/// Produces a separator line to demark different sections of memory.
fn delimit(turtle: &mut Turtle, width: f64) {
    turtle.set_pen_color("grey");
    turtle.pen_down();
    turtle.forward(width);
    turtle.backward(width);

    next_row(turtle, 90.0);
}

/// Moves the turtle down a row
fn next_row(turtle: &mut Turtle, angle: f64) {
    turtle.pen_up();
    turtle.left(angle);
    turtle.forward(BIT_HEIGHT);
    turtle.right(angle);
}
<|endoftext|>


<commit_before>// Cargo.toml
//
// [dependencies]
// rand = "*"

extern crate rand;

use rand::Rng;

fn main() {
    println!("{}", rand::thread_rng().next_u32());
}
<commit_msg>fix rand test<commit_after>// Cargo.toml
//
// rand = "*"

extern crate rand;

use rand::Rng;

fn main() {
    println!("{}", rand::thread_rng().next_u32());
}
<|endoftext|>


<commit_before>/*
 * Copyright 2015 Ben Ashford
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//! Implementations of both Search-by-URI and Search-by-Query operations

use std::collections::BTreeMap;

use hyper::status::StatusCode;

use rustc_serialize::Decodable;
use rustc_serialize::json::{Json, ToJson};

use ::Client;
use ::error::EsError;
use ::query::Query;
use ::units::Duration;
use ::util::StrJoin;
use super::common::Options;
use super::decode_json;
use super::format_indexes_and_types;
use super::format_query_string;
use super::ShardCountResult;

pub struct SearchURIOperation<'a, 'b> {
    /// The HTTP client
    client: &'a mut Client,

    /// The indexes to which this query applies
    indexes: &'b [&'b str],

    /// The types to which this query applies
    doc_types: &'b [&'b str],

    /// Optional options
    options: Options<'b>
}

/// Options for the various search_type parameters
pub enum SearchType {
    DFSQueryThenFetch,
    DFSQueryAndFetch,
    QueryThenFetch,
    QueryAndFetch
}

impl ToString for SearchType {
    fn to_string(&self) -> String {
        match self {
            &SearchType::DFSQueryThenFetch => "dfs_query_then_fetch",
            &SearchType::DFSQueryAndFetch  => "dfs_query_and_fetch",
            &SearchType::QueryThenFetch    => "query_then_fetch",
            &SearchType::QueryAndFetch     => "query_and_fetch"
        }.to_string()
    }
}

impl<'a, 'b> SearchURIOperation<'a, 'b> {
    pub fn new(client: &'a mut Client) -> SearchURIOperation<'a, 'b> {
        SearchURIOperation {
            client:    client,
            indexes:   &[],
            doc_types: &[],
            options:   Options::new()
        }
    }

    pub fn with_indexes(&'b mut self, indexes: &'b [&'b str]) -> &'b mut Self {
        self.indexes = indexes;
        self
    }

    pub fn with_types(&'b mut self, doc_types: &'b [&'b str]) -> &'b mut Self {
        self.doc_types = doc_types;
        self
    }

    pub fn with_query<S: Into<String>>(&'b mut self, qs: S) -> &'b mut Self {
        self.options.push(("q", qs.into()));
        self
    }

    add_option!(with_df, "df");
    add_option!(with_analyzer, "analyzer");
    add_option!(with_lowercase_expanded_terms, "lowercase_expanded_terms");
    add_option!(with_analyze_wildcard, "analyze_wildcard");
    add_option!(with_default_operator, "default_operator");
    add_option!(with_lenient, "lenient");
    add_option!(with_explain, "explain");
    add_option!(with_source, "_source");
    add_option!(with_sort, "sort");
    add_option!(with_routing, "routing");
    add_option!(with_track_scores, "track_scores");
    add_option!(with_timeout, "timeout");
    add_option!(with_terminate_after, "terminate_after");
    add_option!(with_from, "from");
    add_option!(with_size, "size");
    add_option!(with_search_type, "search_type");

    pub fn with_fields(&'b mut self, fields: &[&str]) -> &'b mut Self {
        self.options.push(("fields", fields.iter().join(",")));
        self
    }

    pub fn send(&'b mut self) -> Result<SearchResult, EsError> {
        let url = format!("/{}/_search{}",
                          format_indexes_and_types(&self.indexes, &self.doc_types),
                          format_query_string(&self.options));
        info!("Searching with: {}", url);
        let (status_code, result) = try!(self.client.get_op(&url));
        info!("Search result (status: {}, result: {:?})", status_code, result);
        match status_code {
            StatusCode::Ok => Ok(SearchResult::from(&result.unwrap())),
            _              => Err(EsError::EsError(format!("Unexpected status: {}", status_code)))
        }
    }
}

struct SearchQueryOperationBody<'b> {
    /// The query
    query: Option<&'b Query>,

    /// Timeout
    timeout: Option<&'b str>,

    /// From
    from: i64,

    /// Size
    size: i64,

    /// Terminate early (marked as experimental in the ES docs)
    terminate_after: Option<i64>,

    /// Stats groups to which the query belongs
    stats: Option<Vec<String>>,

    /// Minimum score to use
    min_score: Option<f64>
}

impl<'a> ToJson for SearchQueryOperationBody<'a> {
    fn to_json(&self) -> Json {
        let mut d = BTreeMap::new();
        d.insert("from".to_string(), self.from.to_json());
        d.insert("size".to_string(), self.size.to_json());
        optional_add!(d, self.query, "query");
        optional_add!(d, self.timeout, "timeout");
        optional_add!(d, self.terminate_after, "terminate_after");
        optional_add!(d, self.stats, "stats");
        optional_add!(d, self.min_score, "min_score");
        Json::Object(d)
    }
}

pub struct SearchQueryOperation<'a, 'b> {
    /// The HTTP client
    client: &'a mut Client,

    /// The indexes to which this query applies
    indexes: &'b [&'b str],

    /// The types to which the query applies
    doc_types: &'b [&'b str],

    /// Optionals
    options: Options<'b>,

    /// The query body
    body: SearchQueryOperationBody<'b>
}

impl <'a, 'b> SearchQueryOperation<'a, 'b> {
    pub fn new(client: &'a mut Client) -> SearchQueryOperation<'a, 'b> {
        SearchQueryOperation {
            client:    client,
            indexes:   &[],
            doc_types: &[],
            options:   Options::new(),
            body:      SearchQueryOperationBody {
                query:           None,
                timeout:         None,
                from:            0,
                size:            10,
                terminate_after: None,
                stats:           None,
                min_score:       None
            }
        }
    }

    pub fn with_indexes(&'b mut self, indexes: &'b [&'b str]) -> &'b mut Self {
        self.indexes = indexes;
        self
    }

    pub fn with_types(&'b mut self, doc_types: &'b [&'b str]) -> &'b mut Self {
        self.doc_types = doc_types;
        self
    }

    pub fn with_query(&'b mut self, query: &'b Query) -> &'b mut Self {
        self.body.query = Some(query);
        self
    }

    pub fn with_timeout(&'b mut self, timeout: &'b str) -> &'b mut Self {
        self.body.timeout = Some(timeout);
        self
    }

    pub fn with_from(&'b mut self, from: i64) -> &'b mut Self {
        self.body.from = from;
        self
    }

    pub fn with_size(&'b mut self, size: i64) -> &'b mut Self {
        self.body.size = size;
        self
    }

    pub fn with_terminate_after(&'b mut self, terminate_after: i64) -> &'b mut Self {
        self.body.terminate_after = Some(terminate_after);
        self
    }

    pub fn with_stats<S>(&'b mut self, stats: &[S]) -> &'b mut Self
        where S: ToString
    {
        self.body.stats = Some(stats.iter().map(|s| s.to_string()).collect());
        self
    }

    pub fn with_min_score(&'b mut self, min_score: f64) -> &'b mut Self {
        self.body.min_score = Some(min_score);
        self
    }

    add_option!(with_routing, "routing");
    add_option!(with_search_type, "search_type");
    add_option!(with_query_cache, "query_cache");

    pub fn send(&'b mut self) -> Result<SearchResult, EsError> {
        let url = format!("/{}/_search{}",
                          format_indexes_and_types(&self.indexes, &self.doc_types),
                          format_query_string(&self.options));
        let (status_code, result) = try!(self.client.post_body_op(&url, &self.body.to_json()));
        match status_code {
            StatusCode::Ok => Ok(SearchResult::from(&result.unwrap())),
            _              => Err(EsError::EsError(format!("Unexpected status: {}", status_code)))
        }
    }

    pub fn scan(&'b mut self, scroll: Duration) -> Result<ScanResult, EsError> {
        self.options.push(("search_type", "scan".to_string()));
        self.options.push(("scroll", scroll.to_string()));
        let url = format!("/{}/_search{}",
                          format_indexes_and_types(&self.indexes, &self.doc_types),
                          format_query_string(&self.options));
        let (status_code, result) = try!(self.client.post_body_op(&url, &self.body.to_json()));
        match status_code {
            StatusCode::Ok => Ok(ScanResult::from(scroll, &result.unwrap())),
            _              => Err(EsError::EsError(format!("Unexpected status: {}", status_code)))
        }
    }
}

#[derive(Debug)]
pub struct SearchHitsHitsResult {
    pub index:    String,
    pub doc_type: String,
    pub id:       String,
    pub score:    f64,
    pub source:   Option<Json>,
    pub fields:   Option<Json>
}

impl SearchHitsHitsResult {
    /// Get the source document as a struct, the raw JSON version is available
    /// directly from the source field
    pub fn source<T: Decodable>(self) -> Result<T, EsError> {
        match self.source {
            Some(source) => decode_json(source),
            None         => Err(EsError::EsError("No source field".to_string()))
        }
    }
}

impl<'a> From<&'a Json> for SearchHitsHitsResult {
    fn from(r: &'a Json) -> SearchHitsHitsResult {
        SearchHitsHitsResult {
            index:    get_json_string!(r, "_index"),
            doc_type: get_json_string!(r, "_type"),
            id:       get_json_string!(r, "_id"),
            score:    get_json_f64!(r, "_score"),
            source:   r.find("_source").map(|s| s.clone()),
            fields:   r.find("fields").map(|s| s.clone())
        }
    }
}

#[derive(Debug)]
pub struct SearchHitsResult {
    pub total: i64,
    pub hits:  Vec<SearchHitsHitsResult>
}

impl<'a> From<&'a Json> for SearchHitsResult {
    fn from(r: &'a Json) -> SearchHitsResult {
        SearchHitsResult {
            total: get_json_i64!(r, "total"),
            hits:  r.find("hits")
                .unwrap()
                .as_array()
                .unwrap()
                .iter()
                .map(|j| SearchHitsHitsResult::from(j))
                .collect()
        }
    }
}

pub struct SearchResult {
    pub took:      i64,
    pub timed_out: bool,
    pub shards:    ShardCountResult,
    pub hits:      SearchHitsResult
}

impl<'a> From<&'a Json> for SearchResult {
    fn from(r: &'a Json) -> SearchResult {
        SearchResult {
            took:      get_json_i64!(r, "took"),
            timed_out: get_json_bool!(r, "timed_out"),
            shards:    decode_json(r.find("_shards")
                                   .unwrap()
                                   .clone()).unwrap(),
            hits:      SearchHitsResult::from(r.find("hits")
                                              .unwrap())
        }
    }
}

pub struct ScanResult {
    scroll_id:     String,
    scroll:        Duration,
    pub took:      i64,
    pub timed_out: bool,
    pub shards:    ShardCountResult,
    pub hits:      SearchHitsResult
}

impl ScanResult {
    fn from<'b>(scroll: Duration, r: &'b Json) -> ScanResult {
        ScanResult {
            scroll:    scroll,
            scroll_id: get_json_string!(r, "_scroll_id"),
            took:      get_json_i64!(r, "took"),
            timed_out: get_json_bool!(r, "timed_out"),
            shards:    decode_json(r.find("_shards")
                                   .unwrap()
                                   .clone()).unwrap(),
            hits:      SearchHitsResult::from(r.find("hits")
                                              .unwrap())
        }
    }

    pub fn scroll(&mut self, client: &mut Client) -> Result<SearchResult, EsError> {
        let url = format!("/_search/scroll?scroll={}&scroll_id={}",
                          self.scroll.to_string(),
                          self.scroll_id);
        let (status_code, result) = try!(client.get_op(&url));
        match status_code {
            StatusCode::Ok => {
                let r = result.unwrap();
                self.scroll_id = get_json_string!(r, "_scroll_id");
                Ok(SearchResult::from(&r))
            },
            _              => Err(EsError::EsError(format!("Unexpected status: {}", status_code)))
        }
    }
}

#[cfg(test)]
mod tests {
    use ::Client;
    use ::tests::TestDocument;

    use ::operations::bulk::Action;
    use ::units::{Duration, DurationUnit};

    fn make_document(idx: i64) -> TestDocument {
        TestDocument::new()
            .with_str_field(&format!("BulkDoc: {}", idx))
            .with_int_field(idx)
    }

    fn setup_scan_data(client: &mut Client, index_name: &str) {
        let actions:Vec<Action> = (0..1000).map(|idx| {
            Action::index(make_document(idx))
        }).collect();

        client.bulk(&actions)
            .with_index(index_name)
            .with_doc_type("doc_type")
            .send()
            .unwrap();

        client.refresh().with_indexes(&[index_name]).send().unwrap();
    }

    #[test]
    fn test_scan_and_scroll() {
        let mut client = ::tests::make_client();
        let index_name = "tests_test_scan_and_scroll";
        ::tests::clean_db(&mut client, index_name);
        setup_scan_data(&mut client, index_name);

        let indexes = [index_name];

        let mut scan_result = client.search_query()
            .with_indexes(&indexes)
            .with_size(100)
            .scan(Duration::new(1, DurationUnit::Minute))
            .unwrap();

        assert_eq!(1000, scan_result.hits.total);
        let mut total = 0;

        loop {
            let page = scan_result.scroll(&mut client).unwrap();
            let page_total = page.hits.hits.len();
            total += page_total;
            if page_total == 0 && total == 1000 {
                break;
            }
            assert!(total <= 1000);
        }
    }
}
<commit_msg>Close a scan/scroll token<commit_after>/*
 * Copyright 2015 Ben Ashford
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//! Implementations of both Search-by-URI and Search-by-Query operations

use std::collections::BTreeMap;

use hyper::status::StatusCode;

use rustc_serialize::Decodable;
use rustc_serialize::json::{Json, ToJson};

use ::Client;
use ::error::EsError;
use ::query::Query;
use ::units::Duration;
use ::util::StrJoin;
use super::common::Options;
use super::decode_json;
use super::format_indexes_and_types;
use super::format_query_string;
use super::ShardCountResult;

pub struct SearchURIOperation<'a, 'b> {
    /// The HTTP client
    client: &'a mut Client,

    /// The indexes to which this query applies
    indexes: &'b [&'b str],

    /// The types to which this query applies
    doc_types: &'b [&'b str],

    /// Optional options
    options: Options<'b>
}

/// Options for the various search_type parameters
pub enum SearchType {
    DFSQueryThenFetch,
    DFSQueryAndFetch,
    QueryThenFetch,
    QueryAndFetch
}

impl ToString for SearchType {
    fn to_string(&self) -> String {
        match self {
            &SearchType::DFSQueryThenFetch => "dfs_query_then_fetch",
            &SearchType::DFSQueryAndFetch  => "dfs_query_and_fetch",
            &SearchType::QueryThenFetch    => "query_then_fetch",
            &SearchType::QueryAndFetch     => "query_and_fetch"
        }.to_string()
    }
}

impl<'a, 'b> SearchURIOperation<'a, 'b> {
    pub fn new(client: &'a mut Client) -> SearchURIOperation<'a, 'b> {
        SearchURIOperation {
            client:    client,
            indexes:   &[],
            doc_types: &[],
            options:   Options::new()
        }
    }

    pub fn with_indexes(&'b mut self, indexes: &'b [&'b str]) -> &'b mut Self {
        self.indexes = indexes;
        self
    }

    pub fn with_types(&'b mut self, doc_types: &'b [&'b str]) -> &'b mut Self {
        self.doc_types = doc_types;
        self
    }

    pub fn with_query<S: Into<String>>(&'b mut self, qs: S) -> &'b mut Self {
        self.options.push(("q", qs.into()));
        self
    }

    add_option!(with_df, "df");
    add_option!(with_analyzer, "analyzer");
    add_option!(with_lowercase_expanded_terms, "lowercase_expanded_terms");
    add_option!(with_analyze_wildcard, "analyze_wildcard");
    add_option!(with_default_operator, "default_operator");
    add_option!(with_lenient, "lenient");
    add_option!(with_explain, "explain");
    add_option!(with_source, "_source");
    add_option!(with_sort, "sort");
    add_option!(with_routing, "routing");
    add_option!(with_track_scores, "track_scores");
    add_option!(with_timeout, "timeout");
    add_option!(with_terminate_after, "terminate_after");
    add_option!(with_from, "from");
    add_option!(with_size, "size");
    add_option!(with_search_type, "search_type");

    pub fn with_fields(&'b mut self, fields: &[&str]) -> &'b mut Self {
        self.options.push(("fields", fields.iter().join(",")));
        self
    }

    pub fn send(&'b mut self) -> Result<SearchResult, EsError> {
        let url = format!("/{}/_search{}",
                          format_indexes_and_types(&self.indexes, &self.doc_types),
                          format_query_string(&self.options));
        info!("Searching with: {}", url);
        let (status_code, result) = try!(self.client.get_op(&url));
        info!("Search result (status: {}, result: {:?})", status_code, result);
        match status_code {
            StatusCode::Ok => Ok(SearchResult::from(&result.unwrap())),
            _              => Err(EsError::EsError(format!("Unexpected status: {}", status_code)))
        }
    }
}

struct SearchQueryOperationBody<'b> {
    /// The query
    query: Option<&'b Query>,

    /// Timeout
    timeout: Option<&'b str>,

    /// From
    from: i64,

    /// Size
    size: i64,

    /// Terminate early (marked as experimental in the ES docs)
    terminate_after: Option<i64>,

    /// Stats groups to which the query belongs
    stats: Option<Vec<String>>,

    /// Minimum score to use
    min_score: Option<f64>
}

impl<'a> ToJson for SearchQueryOperationBody<'a> {
    fn to_json(&self) -> Json {
        let mut d = BTreeMap::new();
        d.insert("from".to_string(), self.from.to_json());
        d.insert("size".to_string(), self.size.to_json());
        optional_add!(d, self.query, "query");
        optional_add!(d, self.timeout, "timeout");
        optional_add!(d, self.terminate_after, "terminate_after");
        optional_add!(d, self.stats, "stats");
        optional_add!(d, self.min_score, "min_score");
        Json::Object(d)
    }
}

pub struct SearchQueryOperation<'a, 'b> {
    /// The HTTP client
    client: &'a mut Client,

    /// The indexes to which this query applies
    indexes: &'b [&'b str],

    /// The types to which the query applies
    doc_types: &'b [&'b str],

    /// Optionals
    options: Options<'b>,

    /// The query body
    body: SearchQueryOperationBody<'b>
}

impl <'a, 'b> SearchQueryOperation<'a, 'b> {
    pub fn new(client: &'a mut Client) -> SearchQueryOperation<'a, 'b> {
        SearchQueryOperation {
            client:    client,
            indexes:   &[],
            doc_types: &[],
            options:   Options::new(),
            body:      SearchQueryOperationBody {
                query:           None,
                timeout:         None,
                from:            0,
                size:            10,
                terminate_after: None,
                stats:           None,
                min_score:       None
            }
        }
    }

    pub fn with_indexes(&'b mut self, indexes: &'b [&'b str]) -> &'b mut Self {
        self.indexes = indexes;
        self
    }

    pub fn with_types(&'b mut self, doc_types: &'b [&'b str]) -> &'b mut Self {
        self.doc_types = doc_types;
        self
    }

    pub fn with_query(&'b mut self, query: &'b Query) -> &'b mut Self {
        self.body.query = Some(query);
        self
    }

    pub fn with_timeout(&'b mut self, timeout: &'b str) -> &'b mut Self {
        self.body.timeout = Some(timeout);
        self
    }

    pub fn with_from(&'b mut self, from: i64) -> &'b mut Self {
        self.body.from = from;
        self
    }

    pub fn with_size(&'b mut self, size: i64) -> &'b mut Self {
        self.body.size = size;
        self
    }

    pub fn with_terminate_after(&'b mut self, terminate_after: i64) -> &'b mut Self {
        self.body.terminate_after = Some(terminate_after);
        self
    }

    pub fn with_stats<S>(&'b mut self, stats: &[S]) -> &'b mut Self
        where S: ToString
    {
        self.body.stats = Some(stats.iter().map(|s| s.to_string()).collect());
        self
    }

    pub fn with_min_score(&'b mut self, min_score: f64) -> &'b mut Self {
        self.body.min_score = Some(min_score);
        self
    }

    add_option!(with_routing, "routing");
    add_option!(with_search_type, "search_type");
    add_option!(with_query_cache, "query_cache");

    pub fn send(&'b mut self) -> Result<SearchResult, EsError> {
        let url = format!("/{}/_search{}",
                          format_indexes_and_types(&self.indexes, &self.doc_types),
                          format_query_string(&self.options));
        let (status_code, result) = try!(self.client.post_body_op(&url, &self.body.to_json()));
        match status_code {
            StatusCode::Ok => Ok(SearchResult::from(&result.unwrap())),
            _              => Err(EsError::EsError(format!("Unexpected status: {}", status_code)))
        }
    }

    pub fn scan(&'b mut self, scroll: Duration) -> Result<ScanResult, EsError> {
        self.options.push(("search_type", "scan".to_string()));
        self.options.push(("scroll", scroll.to_string()));
        let url = format!("/{}/_search{}",
                          format_indexes_and_types(&self.indexes, &self.doc_types),
                          format_query_string(&self.options));
        let (status_code, result) = try!(self.client.post_body_op(&url, &self.body.to_json()));
        match status_code {
            StatusCode::Ok => Ok(ScanResult::from(scroll, &result.unwrap())),
            _              => Err(EsError::EsError(format!("Unexpected status: {}", status_code)))
        }
    }
}

#[derive(Debug)]
pub struct SearchHitsHitsResult {
    pub index:    String,
    pub doc_type: String,
    pub id:       String,
    pub score:    f64,
    pub source:   Option<Json>,
    pub fields:   Option<Json>
}

impl SearchHitsHitsResult {
    /// Get the source document as a struct, the raw JSON version is available
    /// directly from the source field
    pub fn source<T: Decodable>(self) -> Result<T, EsError> {
        match self.source {
            Some(source) => decode_json(source),
            None         => Err(EsError::EsError("No source field".to_string()))
        }
    }
}

impl<'a> From<&'a Json> for SearchHitsHitsResult {
    fn from(r: &'a Json) -> SearchHitsHitsResult {
        SearchHitsHitsResult {
            index:    get_json_string!(r, "_index"),
            doc_type: get_json_string!(r, "_type"),
            id:       get_json_string!(r, "_id"),
            score:    get_json_f64!(r, "_score"),
            source:   r.find("_source").map(|s| s.clone()),
            fields:   r.find("fields").map(|s| s.clone())
        }
    }
}

#[derive(Debug)]
pub struct SearchHitsResult {
    pub total: i64,
    pub hits:  Vec<SearchHitsHitsResult>
}

impl<'a> From<&'a Json> for SearchHitsResult {
    fn from(r: &'a Json) -> SearchHitsResult {
        SearchHitsResult {
            total: get_json_i64!(r, "total"),
            hits:  r.find("hits")
                .unwrap()
                .as_array()
                .unwrap()
                .iter()
                .map(|j| SearchHitsHitsResult::from(j))
                .collect()
        }
    }
}

pub struct SearchResult {
    pub took:      i64,
    pub timed_out: bool,
    pub shards:    ShardCountResult,
    pub hits:      SearchHitsResult
}

impl<'a> From<&'a Json> for SearchResult {
    fn from(r: &'a Json) -> SearchResult {
        SearchResult {
            took:      get_json_i64!(r, "took"),
            timed_out: get_json_bool!(r, "timed_out"),
            shards:    decode_json(r.find("_shards")
                                   .unwrap()
                                   .clone()).unwrap(),
            hits:      SearchHitsResult::from(r.find("hits")
                                              .unwrap())
        }
    }
}

pub struct ScanResult {
    scroll_id:     String,
    scroll:        Duration,
    pub took:      i64,
    pub timed_out: bool,
    pub shards:    ShardCountResult,
    pub hits:      SearchHitsResult
}

impl ScanResult {
    fn from<'b>(scroll: Duration, r: &'b Json) -> ScanResult {
        ScanResult {
            scroll:    scroll,
            scroll_id: get_json_string!(r, "_scroll_id"),
            took:      get_json_i64!(r, "took"),
            timed_out: get_json_bool!(r, "timed_out"),
            shards:    decode_json(r.find("_shards")
                                   .unwrap()
                                   .clone()).unwrap(),
            hits:      SearchHitsResult::from(r.find("hits")
                                              .unwrap())
        }
    }

    pub fn scroll(&mut self, client: &mut Client) -> Result<SearchResult, EsError> {
        let url = format!("/_search/scroll?scroll={}&scroll_id={}",
                          self.scroll.to_string(),
                          self.scroll_id);
        let (status_code, result) = try!(client.get_op(&url));
        match status_code {
            StatusCode::Ok => {
                let r = result.unwrap();
                self.scroll_id = get_json_string!(r, "_scroll_id");
                Ok(SearchResult::from(&r))
            },
            _              => Err(EsError::EsError(format!("Unexpected status: {}", status_code)))
        }
    }

    pub fn close(&self, client: &mut Client) -> Result<(), EsError> {
        let url = format!("/_search/scroll?scroll_id={}", self.scroll_id);
        let (status_code, result) = try!(client.delete_op(&url));
        match status_code {
            StatusCode::Ok       => Ok(()),
            StatusCode::NotFound => Ok(()),
            _                    => Err(EsError::EsError(format!("Unexpected status: {}, {}",
                                                                 status_code,
                                                                 result.unwrap())))
        }
    }
}

#[cfg(test)]
mod tests {
    use ::Client;
    use ::tests::TestDocument;

    use ::operations::bulk::Action;
    use ::units::{Duration, DurationUnit};

    fn make_document(idx: i64) -> TestDocument {
        TestDocument::new()
            .with_str_field(&format!("BulkDoc: {}", idx))
            .with_int_field(idx)
    }

    fn setup_scan_data(client: &mut Client, index_name: &str) {
        let actions:Vec<Action> = (0..1000).map(|idx| {
            Action::index(make_document(idx))
        }).collect();

        client.bulk(&actions)
            .with_index(index_name)
            .with_doc_type("doc_type")
            .send()
            .unwrap();

        client.refresh().with_indexes(&[index_name]).send().unwrap();
    }

    #[test]
    fn test_close() {
        let mut client = ::tests::make_client();
        let index_name = "tests_test_close";
        ::tests::clean_db(&mut client, index_name);
        setup_scan_data(&mut client, index_name);

        let indexes = [index_name];

        let mut scan_result = client.search_query()
            .with_indexes(&indexes)
            .with_size(100)
            .scan(Duration::new(1, DurationUnit::Minute))
            .unwrap();

        scan_result.scroll(&mut client).unwrap();

        scan_result.close(&mut client).unwrap();
    }

    #[test]
    fn test_scan_and_scroll() {
        let mut client = ::tests::make_client();
        let index_name = "tests_test_scan_and_scroll";
        ::tests::clean_db(&mut client, index_name);
        setup_scan_data(&mut client, index_name);

        let indexes = [index_name];

        let mut scan_result = client.search_query()
            .with_indexes(&indexes)
            .with_size(100)
            .scan(Duration::new(1, DurationUnit::Minute))
            .unwrap();

        assert_eq!(1000, scan_result.hits.total);
        let mut total = 0;

        loop {
            let page = scan_result.scroll(&mut client).unwrap();
            let page_total = page.hits.hits.len();
            total += page_total;
            if page_total == 0 && total == 1000 {
                break;
            }
            assert!(total <= 1000);
        }

        scan_result.close(&mut client).unwrap();
    }
}
<|endoftext|>