<commit_before>// Copyright (C) 2015, Alberto Corona <alberto@0x1a.us>
// All rights reserved. This file is part of rpf, distributed under the
// BSD 3-Clause license. For full terms please see the LICENSE file.

use std::path::{PathBuf,Path};
use std::fs::{PathExt};

/// Adds some useful functions for manipulating and retrieving information from
/// paths
pub trait PathMod {
    /// Returns a `PathBuf` of `&self`'s last component
    ///
    /// # Example
    ///
    /// ```
    /// use rpf::PathMod;
    /// use std::path::Path;
    ///
    /// let path = Path::new("/tmp/test/mod");
    /// let last = path.last_component();
    /// ```
    fn last_component(&self) -> PathBuf;

    /// Returns a `PathBuf` of `&self`'s first component
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("/tmp/test/mod");
    /// let first = path.first_component();
    /// assert_eq!(first, PathBuf::from("/"));
    /// ```
    fn first_component(&self) -> PathBuf;

    /// Returns a `PathBuf` of `&self` relative from `rel_from`
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("mod");
    /// let parent = PathBuf::from("/tmp/test/mod");
    /// let rel = path.rel_to(&parent);
    /// ```
    fn rel_to<T: AsRef<Path>>(&self, rel_from: &T) -> PathBuf;

    /// Returns a `&str` for a path, returns a blank string if unable to
    /// get a string for the path
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("/usr/share");
    /// let path_str = &path.as_str();
    /// assert_eq!(path_str, &"/usr/share");
    /// ```
    fn as_str(&self) -> &str;

    /// Returns a `String` for a path, returns an empty string if unable to get
    /// a string for a path
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path_string = PathBuf::from("/var/log/test").as_string();
    /// assert_eq!(path_string, "/var/log/test".to_string());
    /// ```
    fn as_string(&self) -> String;

    /// Returns true if a given path is a symbolic link, returns false if the
    /// path is a regular file, a directory, or does not exist
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("test-symlink");
    /// assert_eq!(path.is_symlink(), false);
    /// ```
    fn is_symlink(&self) -> bool;
}

impl PathMod for PathBuf {
    fn last_component(&self) -> PathBuf {
        match self.components().last() {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn first_component(&self) -> PathBuf {
        match self.components().nth(0) {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn rel_to<T: AsRef<Path>>(&self, rel_from: &T) -> PathBuf {
        self.relative_from(&rel_from).unwrap_or(&PathBuf::new()).to_path_buf()
    }

    fn as_str(&self) -> &str {
        match self.to_str() {
            Some(s) => { s },
            None => { "" },
        }
    }

    fn as_string(&self) -> String {
        self.as_str().to_string()
    }

    fn is_symlink(&self) -> bool {
        if self.exists() {
            if self.is_dir() || self.is_file() {
                return false
            }
            return true
        } else { return false }
    }
}

impl PathMod for Path {
    fn last_component(&self) -> PathBuf {
        match self.components().last() {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn first_component(&self) -> PathBuf {
        match self.components().nth(0) {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn rel_to<T: AsRef<Path>>(&self, rel_from: &T) -> PathBuf {
        self.relative_from(&rel_from).unwrap_or(&PathBuf::new()).to_path_buf()
    }

    fn as_str(&self) -> &str {
        match self.to_str() {
            Some(s) => { s },
            None => { "" },
        }
    }

    fn as_string(&self) -> String {
        self.as_str().to_string()
    }

    fn is_symlink(&self) -> bool {
        if self.exists() {
            if self.is_dir() || self.is_file() {
                return false
            }
            return true
        } else { return false }
    }
}

#[test]
fn test_pathmod_first_comp() {
    let comp = Path::new("/etc/test").first_component();
    assert_eq!(comp, PathBuf::from("/"));
}

#[test]
fn test_pathmod_last_comp() {
    let comp = Path::new("/etc/test").last_component();
    assert_eq!(comp, PathBuf::from("test"));
}

#[test]
fn test_pathmod_as_str() {
    let string = Path::new("/var/log/test").as_str();
    assert_eq!(string, "/var/log/test");
}

#[test]
fn test_pathmod_as_string() {
    let string = Path::new("/var/log/test").as_string();
    assert_eq!(string, "/var/log/test".to_string());
}

#[test]
fn test_pathmod_is_symlink() {
    let path = Path::new("/var/log/test");
    assert_eq!(path.is_symlink(), false);
}
<commit_msg>pathmod: remove `is_symlink`, available in std<commit_after>// Copyright (C) 2015, Alberto Corona <alberto@0x1a.us>
// All rights reserved. This file is part of rpf, distributed under the
// BSD 3-Clause license. For full terms please see the LICENSE file.

use std::path::{PathBuf,Path};
use std::fs::{PathExt};

/// Adds some useful functions for manipulating and retrieving information from
/// paths
pub trait PathMod {
    /// Returns a `PathBuf` of `&self`'s last component
    ///
    /// # Example
    ///
    /// ```
    /// use rpf::PathMod;
    /// use std::path::Path;
    ///
    /// let path = Path::new("/tmp/test/mod");
    /// let last = path.last_component();
    /// ```
    fn last_component(&self) -> PathBuf;

    /// Returns a `PathBuf` of `&self`'s first component
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("/tmp/test/mod");
    /// let first = path.first_component();
    /// assert_eq!(first, PathBuf::from("/"));
    /// ```
    fn first_component(&self) -> PathBuf;

    /// Returns a `PathBuf` of `&self` relative from `rel_from`
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("mod");
    /// let parent = PathBuf::from("/tmp/test/mod");
    /// let rel = path.rel_to(&parent);
    /// ```
    fn rel_to<T: AsRef<Path>>(&self, rel_from: &T) -> PathBuf;

    /// Returns a `&str` for a path, returns a blank string if unable to
    /// get a string for the path
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path = PathBuf::from("/usr/share");
    /// let path_str = &path.as_str();
    /// assert_eq!(path_str, &"/usr/share");
    /// ```
    fn as_str(&self) -> &str;

    /// Returns a `String` for a path, returns an empty string if unable to get
    /// a string for a path
    ///
    /// # Example
    /// ```
    /// use rpf::PathMod;
    /// use std::path::PathBuf;
    ///
    /// let path_string = PathBuf::from("/var/log/test").as_string();
    /// assert_eq!(path_string, "/var/log/test".to_string());
    /// ```
    fn as_string(&self) -> String;
}

impl PathMod for PathBuf {
    fn last_component(&self) -> PathBuf {
        match self.components().last() {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn first_component(&self) -> PathBuf {
        match self.components().nth(0) {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn rel_to<T: AsRef<Path>>(&self, rel_from: &T) -> PathBuf {
        self.relative_from(&rel_from).unwrap_or(&PathBuf::new()).to_path_buf()
    }

    fn as_str(&self) -> &str {
        match self.to_str() {
            Some(s) => { s },
            None => { "" },
        }
    }

    fn as_string(&self) -> String {
        self.as_str().to_string()
    }
}

impl PathMod for Path {
    fn last_component(&self) -> PathBuf {
        match self.components().last() {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn first_component(&self) -> PathBuf {
        match self.components().nth(0) {
            Some(s) => { PathBuf::from(s.as_os_str()) },
            None => { PathBuf::new() },
        }
    }

    fn rel_to<T: AsRef<Path>>(&self, rel_from: &T) -> PathBuf {
        self.relative_from(&rel_from).unwrap_or(&PathBuf::new()).to_path_buf()
    }

    fn as_str(&self) -> &str {
        match self.to_str() {
            Some(s) => { s },
            None => { "" },
        }
    }

    fn as_string(&self) -> String {
        self.as_str().to_string()
    }
}

#[test]
fn test_pathmod_first_comp() {
    let comp = Path::new("/etc/test").first_component();
    assert_eq!(comp, PathBuf::from("/"));
}

#[test]
fn test_pathmod_last_comp() {
    let comp = Path::new("/etc/test").last_component();
    assert_eq!(comp, PathBuf::from("test"));
}

#[test]
fn test_pathmod_as_str() {
    let string = Path::new("/var/log/test").as_str();
    assert_eq!(string, "/var/log/test");
}

#[test]
fn test_pathmod_as_string() {
    let string = Path::new("/var/log/test").as_string();
    assert_eq!(string, "/var/log/test".to_string());
}
<|endoftext|>


<commit_before>use core::intrinsics::transmute;

pub unsafe trait VSpaceWindow<'a> {
    fn base(&self) -> usize;
    fn size(&self) -> usize;
    unsafe fn make<T>(&self) -> &'a T {
        unimplemented!()
    }
    /* Passing in a window to describe the window you want to create
     * is a little backwards, but it provides the nicest API as far
     * as I can tell */
    unsafe fn subwindow<'i, I, O: ?Sized>(&self, window: I) -> &'a O where O: VSpaceWindow<'a> + Default, I: VSpaceWindow<'i> {
        /* transmate to extend the lifetime. This is safe as
         * it is a zero sized object */
        transmute(&O::default())
    }
}
<commit_msg>Check the ranges on subwindows<commit_after>use core::intrinsics::transmute;

pub unsafe trait VSpaceWindow<'a> {
    fn base(&self) -> usize;
    fn size(&self) -> usize;
    unsafe fn make<T>(&self) -> &'a T {
        unimplemented!()
    }
    /* Passing in a window to describe the window you want to create
     * is a little backwards, but it provides the nicest API as far
     * as I can tell */
    unsafe fn subwindow<'i, I, O: ?Sized>(&self, window: I) -> &'a O where O: VSpaceWindow<'a> + Default, I: VSpaceWindow<'i> {
        /* Validate the range for this window */
        if !self.range_valid(window.base(), window.size()) {
            panic!("Cannot construct window with range {} {}, from {} {}");
        }
        /* transmate to extend the lifetime. This is safe as
         * it is a zero sized object */
        transmute(&O::default())
    }
    fn range_valid(&self, b: usize, s: usize) -> bool {
        /* We have moved the `- s` to the other side of the equation
         * so that if `self.base() + self.size()` overflowed
         * then subtracting `s` will either correct if the range is
         * valid, or not correct in which case the range is invalid
         * and the comparison will fail */
        b >= self.base() && b <= self.base() + self.size() - s
    }
}
<|endoftext|>


<commit_before>use hir::{self, HasCrate, HirDisplay};
use stdx::format_to;
use syntax::ast::{self, AstNode, HasName, HasVisibility};

use crate::{
    utils::{find_impl_block_end, find_struct_impl, generate_impl_text},
    AssistContext, AssistId, AssistKind, Assists, GroupLabel,
};

// Assist: generate_setter
//
// Generate a setter method.
//
// ```
// struct Person {
//     nam$0e: String,
// }
// ```
// ->
// ```
// struct Person {
//     name: String,
// }
//
// impl Person {
//     /// Set the person's name.
//     fn set_name(&mut self, name: String) {
//         self.name = name;
//     }
// }
// ```
pub(crate) fn generate_delegate(acc: &mut Assists, ctx: &AssistContext) -> Option<()> {
    let strukt = ctx.find_node_at_offset::<ast::Struct>()?;
    let field = ctx.find_node_at_offset::<ast::RecordField>()?;

    let field_name = field.name()?;
    let field_ty = field.ty()?;

    let sema_field_ty = ctx.sema.resolve_type(&field_ty)?;
    let krate = sema_field_ty.krate(ctx.db());
    let mut methods = vec![];
    sema_field_ty.iterate_assoc_items(ctx.db(), krate, |item| {
        if let hir::AssocItem::Function(f) = item {
            if f.self_param(ctx.db()).is_some() {
                methods.push(f)
            }
        }
        Some(())
    });

    let target = field_ty.syntax().text_range();
    for method in methods {
        let impl_def = find_struct_impl(
            ctx,
            &ast::Adt::Struct(strukt.clone()),
            &method.name(ctx.db()).to_string(),
        )?;
        acc.add_group(
            &GroupLabel("Generate delegate".to_owned()),
            AssistId("generate_delegate", AssistKind::Generate),
            format!("Generate a delegate method for '{}'", method.name(ctx.db())),
            target,
            |builder| {
                let mut buf = String::with_capacity(512);

                let vis = strukt.visibility().map_or(String::new(), |v| format!("{} ", v));
                let return_type = method.ret_type(ctx.db());
                let return_type = if return_type.is_unit() || return_type.is_unknown() {
                    String::new()
                } else {
                    let module = match ctx.sema.scope(strukt.syntax()).module() {
                        Some(m) => m,
                        None => return,
                    };
                    match return_type.display_source_code(ctx.db(), module.into()) {
                        Ok(rt) => format!("-> {}", rt),
                        Err(_) => return,
                    }
                };

                format_to!(
                    buf,
                    "{}fn {}(&self) {} {{
                    self.{}.{}()
                }}",
                    vis,
                    method.name(ctx.db()),
                    return_type,
                    field_name,
                    method.name(ctx.db())
                );

                let start_offset = impl_def
                    .and_then(|impl_def| find_impl_block_end(impl_def, &mut buf))
                    .unwrap_or_else(|| {
                        buf = generate_impl_text(&ast::Adt::Struct(strukt.clone()), &buf);
                        strukt.syntax().text_range().end()
                    });

                builder.insert(start_offset, buf);
            },
        )?;
    }
    Some(())
}

#[cfg(test)]
mod tests {
    use crate::tests::check_assist;

    use super::*;

    #[test]
    fn test_generate_setter_from_field() {
        check_assist(
            generate_delegate,
            r#"
struct Age(u8);
impl Age {
    fn age(&self) -> u8 {
        self.0

    }
}

struct Person {
    ag$0e: Age,
}
"#,
            r#"
struct Age(u8);
impl Age {
    fn age(&self) -> u8 {
        self.0
    }
}

struct Person {
    age: Age,
}

impl Person {
    fn age(&self) -> u8 {
        self.age.age()
    }
}"#,
        );
    }
}
<commit_msg>create function<commit_after>use hir::{self, HasCrate, HirDisplay};
use stdx::format_to;
use syntax::ast::{self, make, AstNode, HasName, HasVisibility};

use crate::{
    utils::{find_impl_block_end, find_struct_impl, generate_impl_text, render_snippet, Cursor},
    AssistContext, AssistId, AssistKind, Assists, GroupLabel,
};

// Assist: generate_setter
//
// Generate a setter method.
//
// ```
// struct Person {
//     nam$0e: String,
// }
// ```
// ->
// ```
// struct Person {
//     name: String,
// }
//
// impl Person {
//     /// Set the person's name.
//     fn set_name(&mut self, name: String) {
//         self.name = name;
//     }
// }
// ```
pub(crate) fn generate_delegate(acc: &mut Assists, ctx: &AssistContext) -> Option<()> {
    let strukt = ctx.find_node_at_offset::<ast::Struct>()?;
    let field = ctx.find_node_at_offset::<ast::RecordField>()?;

    let field_name = field.name()?;
    let field_ty = field.ty()?;

    let sema_field_ty = ctx.sema.resolve_type(&field_ty)?;
    let krate = sema_field_ty.krate(ctx.db());
    let mut methods = vec![];
    sema_field_ty.iterate_assoc_items(ctx.db(), krate, |item| {
        if let hir::AssocItem::Function(f) = item {
            if f.self_param(ctx.db()).is_some() {
                methods.push(f)
            }
        }
        Some(())
    });

    let target = field_ty.syntax().text_range();
    for method in methods {
        let impl_def = find_struct_impl(
            ctx,
            &ast::Adt::Struct(strukt.clone()),
            &method.name(ctx.db()).to_string(),
        )?;
        acc.add_group(
            &GroupLabel("Generate delegate".to_owned()),
            AssistId("generate_delegate", AssistKind::Generate),
            format!("Generate a delegate method for '{}'", method.name(ctx.db())),
            target,
            |builder| {
                let mut buf = String::with_capacity(512);

                let vis = strukt.visibility().map_or(String::new(), |v| format!("{} ", v));
                let return_type = method.ret_type(ctx.db());
                let return_type = if return_type.is_unit() || return_type.is_unknown() {
                    String::new()
                } else {
                    let module = match ctx.sema.scope(strukt.syntax()).module() {
                        Some(m) => m,
                        None => return,
                    };
                    match return_type.display_source_code(ctx.db(), module.into()) {
                        Ok(rt) => format!("-> {}", rt),
                        Err(_) => return,
                    }
                };

                // make function
                let vis = strukt.visibility();
                let name = make::name(&method.name(ctx.db()).to_string());
                let type_params = None;
                let params = make::param_list(None, []);
                let body = make::block_expr([], None);
                let ret_type = &method.ret_type(ctx.db()).display(ctx.db()).to_string();
                let ret_type = Some(make::ret_type(make::ty(ret_type)));
                let is_async = false;
                let f = make::fn_(vis, name, type_params, params, body, ret_type, is_async);

                let start_offset = impl_def
                    .and_then(|impl_def| find_impl_block_end(impl_def, &mut buf))
                    .unwrap_or_else(|| {
                        buf = generate_impl_text(&ast::Adt::Struct(strukt.clone()), &buf);
                        strukt.syntax().text_range().end()
                    });

                let cap = ctx.config.snippet_cap.unwrap(); // FIXME.
                let cursor = Cursor::Before(f.syntax());

                builder.insert_snippet(
                    cap,
                    start_offset,
                    format!("\n\n{}", render_snippet(cap, f.syntax(), cursor)),
                );
            },
        )?;
    }
    Some(())
}

#[cfg(test)]
mod tests {
    use crate::tests::check_assist;

    use super::*;

    #[test]
    fn test_generate_setter_from_field() {
        check_assist(
            generate_delegate,
            r#"
struct Age(u8);
impl Age {
    fn age(&self) -> u8 {
        self.0

    }
}

struct Person {
    ag$0e: Age,
}
"#,
            r#"
struct Age(u8);
impl Age {
    fn age(&self) -> u8 {
        self.0
    }
}

struct Person {
    age: Age,
}

impl Person {
    fn age(&self) -> u8 {
        self.age.age()
    }
}"#,
        );
    }
}
<|endoftext|>


<commit_before>#![feature(core, os, io, path)]

extern crate zip;
use std::os;
use std::old_io::File;
use zip::ZipReader;
use zip::fileinfo::FileInfo;

fn main() {
    let args = os::args();
    match args.len()
    {
        2 => list(&mut zip_file(&args[1][])),
        3 => extract(&mut zip_file(&args[1][]), &args[2][]),
        _ => usage(&args[0][])
    }
}

macro_rules! do_or_die{
    ($expr:expr) => (match $expr {
        Ok(val) => val,
        Err(err) => {println!("{}",err); panic!()}
    })
}

fn zip_file(file: &str) -> ZipReader<File>{
    do_or_die!(zip::ZipReader::open(&Path::new(file)))
}

fn output_file(file: &str)->File{
    do_or_die!(File::create(&Path::new(file)))
}

fn zipped_file_info(zip: &mut ZipReader<File>, file: &str) -> FileInfo{
    do_or_die!(zip.info(file))
}

fn list(reader: &mut ZipReader<File>)->(){
    for file in reader.files(){
        let (year, month, day, hour, minute, second) = file.last_modified_datetime;
        let mod_time = format!("{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second);
        println!("{} ({}): bytes: {:10}, compressed: {:10}",
            file.name, mod_time, file.compressed_size, file.uncompressed_size);
    }
}

fn extract(zip: &mut ZipReader<File>, file: &str)->(){
    let mut out = output_file(file);
    let info = zipped_file_info(zip, file);
    do_or_die!(zip.extract(&info, &mut out));
}

fn usage(this: &str)->(){
    println!("Usage: {} [file.zip] [file_to_extract]", this);
}
<commit_msg>Make zip-demo code close to root<commit_after>#![feature(core, os, io, path)]

extern crate zip;

use std::os;
use std::old_io::File;
use zip::ZipReader;
use zip::fileinfo::FileInfo;

fn main() {
    let args = os::args();
    match args.len(){
        2 => zip_content(&mut zip_file(&args[1][])),
        3 => extract_file(&mut zip_file(&args[1][]), &args[2][]),
        _ => usage(&args[0][])
    }
}

macro_rules! do_or_die{
    ($expr:expr) => (match $expr {
        Ok(val) => val,
        Err(err) => {println!("{}",err); panic!()}
    })
}

fn zip_file(file: &str) -> ZipReader<File>{
    do_or_die!(zip::ZipReader::open(&Path::new(file)))
}

fn output_file(file: &str)->File{
    do_or_die!(File::create(&Path::new(file)))
}

fn zipped_file_info(zip: &mut ZipReader<File>, file: &str) -> FileInfo{
    do_or_die!(zip.info(file))
}

fn zip_content(reader: &mut ZipReader<File>)->(){
    for file in reader.files(){
        let (year, month, day, hour, minute, second) = file.last_modified_datetime;
        let mod_time = format!("{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second);
        println!("{} ({}): bytes: {:10}, compressed: {:10}",
            file.name, mod_time, file.compressed_size, file.uncompressed_size);
    }
}

fn extract_file(zip: &mut ZipReader<File>, file: &str)->(){
    let mut out = output_file(file);
    let info = zipped_file_info(zip, file);
    do_or_die!(zip.extract(&info, &mut out));
}

fn usage(this: &str)->(){
    println!("Usage: {} [file.zip] [file_to_extract]", this);
}
<|endoftext|>


<commit_before>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::request_manager::RequestManager;
use diem_config::config::{PeerNetworkId, UpstreamConfig};
use netcore::transport::ConnectionOrigin;
use std::{collections::HashMap, time::Duration};
const NUM_CHUNKS_TO_PROCESS: u64 = 50;
const NUM_PICKS_TO_MAKE: u64 = 1000;

#[test]
fn test_chunk_success() {
    let num_validators = 4;
    let (mut request_manager, validators) =
        generate_request_manager_and_test_validators(0, num_validators);

    // Process empty chunk responses from all validators
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        for validator_index in 0..num_validators {
            request_manager.process_empty_chunk(&validators[validator_index as usize]);
        }
    }

    // Process successful chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_success_response(&validators[0]);
    }

    // Calculate selection counts for validators
    let pick_counts = calculate_pick_counts_for_validators(&mut request_manager, NUM_PICKS_TO_MAKE);

    // Verify validator 0 is chosen more often than the other validators
    let validator_0_count = pick_counts.get(&validators[0]).unwrap_or(&0);
    for (validator_index, validator) in validators.iter().enumerate() {
        if validator_index != 0 {
            assert!(validator_0_count > pick_counts.get(&validator).unwrap());
        }
    }
}

#[test]
fn test_chunk_timeout() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 4);

    let validator_0 = vec![validators[0].clone()];

    // Process multiple request timeouts from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.add_request(1, validator_0.clone());
        request_manager.check_timeout(1);
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_chunk_version_mismatch() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 4);

    let validator_0 = validators[0].clone();

    // Process multiple chunk version mismatches from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.add_request(100, vec![validator_0.clone()]);
        request_manager
            .process_chunk_version_mismatch(&validator_0, 100, 0)
            .unwrap_err();
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_empty_chunk() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(10, 4);

    // Process multiple empty chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_empty_chunk(&validators[0]);
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_disable_peer() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 1);

    let validator_0 = validators[0].clone();

    // Verify single validator in peers
    assert!(request_manager.is_known_upstream_peer(&validator_0));
    assert!(!request_manager.no_available_peers());

    // Disable validator 0
    request_manager.disable_peer(&validator_0, ConnectionOrigin::Outbound);

    // Verify validator 0 is still known, but no longer available
    assert!(request_manager.is_known_upstream_peer(&validator_0));
    assert!(request_manager.no_available_peers());

    // Add validator 0 and verify it's now enabled
    request_manager.enable_peer(validator_0, ConnectionOrigin::Outbound);
    assert!(!request_manager.no_available_peers());
}

#[test]
fn test_invalid_chunk() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(10, 4);

    // Process multiple invalid chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_invalid_chunk(&validators[0]);
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_remove_requests() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 2);

    let validator_0 = vec![validators[0].clone()];
    let validator_1 = vec![validators[1].clone()];

    // Add version requests to request manager
    request_manager.add_request(1, validator_0.clone());
    request_manager.add_request(3, validator_1.clone());
    request_manager.add_request(5, validator_0.clone());
    request_manager.add_request(10, validator_0);
    request_manager.add_request(12, validator_1);

    // Remove all request versions below 5
    request_manager.remove_requests(5);

    // Verify versions updated correctly
    assert!(request_manager.get_last_request_time(1).is_none());
    assert!(request_manager.get_last_request_time(3).is_none());
    assert!(request_manager.get_last_request_time(5).is_some());
    assert!(request_manager.get_last_request_time(10).is_some());
    assert!(request_manager.get_last_request_time(12).is_some());
}

#[test]
fn test_request_times() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 2);

    // Verify first request time doesn't exist for missing request
    assert!(request_manager.get_first_request_time(1).is_none());

    // Add version requests to request manager
    request_manager.add_request(1, vec![validators[0].clone()]);
    request_manager.add_request(1, vec![validators[1].clone()]);

    // Verify first request time is less than last request time
    assert!(
        request_manager.get_first_request_time(1).unwrap()
            < request_manager.get_last_request_time(1).unwrap()
    );
}

/// Verify that the bad validator is chosen less often than the other validators (due to
/// having a lower peer score internally).
fn verify_bad_validator_picked_less_often(
    request_manager: &mut RequestManager,
    validators: &[PeerNetworkId],
    bad_validator_index: usize,
) {
    // Calculate selection counts for validators
    let pick_counts = calculate_pick_counts_for_validators(request_manager, NUM_PICKS_TO_MAKE);

    // Verify bad validator is chosen less often than the other validators
    let bad_validator_count = pick_counts
        .get(&validators[bad_validator_index])
        .unwrap_or(&0);
    for (validator_index, validator) in validators.iter().enumerate() {
        if validator_index != bad_validator_index {
            assert!(bad_validator_count < pick_counts.get(&validator).unwrap());
        }
    }
}

/// Picks a peer to send a chunk request to (multiple times) and constructs a pick count
/// for each of the chosen peers.
fn calculate_pick_counts_for_validators(
    request_manager: &mut RequestManager,
    number_of_picks_to_execute: u64,
) -> HashMap<PeerNetworkId, u64> {
    let mut pick_counts = HashMap::new();

    for _ in 0..number_of_picks_to_execute {
        let picked_peers = request_manager.pick_peers();
        assert_eq!(1, picked_peers.len()); // Ensure only one validator per multicast level

        let picked_peer = picked_peers[0].clone();
        let counter = pick_counts.entry(picked_peer).or_insert(0);
        *counter += 1;
    }

    pick_counts
}

/// Generates a new request manager with a specified number of validator peers enabled.
fn generate_request_manager_and_test_validators(
    request_timeout: u64,
    num_validators: u64,
) -> (RequestManager, Vec<PeerNetworkId>) {
    let mut request_manager = RequestManager::new(
        UpstreamConfig::default(),
        Duration::from_secs(request_timeout),
        Duration::from_secs(30),
        HashMap::new(),
    );

    let mut validators = Vec::new();
    for _ in 0..num_validators {
        let validator = PeerNetworkId::random_validator();
        request_manager.enable_peer(validator.clone(), ConnectionOrigin::Outbound);
        validators.push(validator);
    }

    (request_manager, validators)
}
<commit_msg>[State Sync] Better unit test names in request_manager.rs<commit_after>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::request_manager::RequestManager;
use diem_config::config::{PeerNetworkId, UpstreamConfig};
use netcore::transport::ConnectionOrigin;
use std::{collections::HashMap, time::Duration};

const NUM_CHUNKS_TO_PROCESS: u64 = 50;
const NUM_PICKS_TO_MAKE: u64 = 1000;

#[test]
fn test_disable_peer() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 1);

    let validator_0 = validators[0].clone();

    // Verify single validator in peers
    assert!(request_manager.is_known_upstream_peer(&validator_0));
    assert!(!request_manager.no_available_peers());

    // Disable validator 0
    request_manager.disable_peer(&validator_0, ConnectionOrigin::Outbound);

    // Verify validator 0 is still known, but no longer available
    assert!(request_manager.is_known_upstream_peer(&validator_0));
    assert!(request_manager.no_available_peers());

    // Add validator 0 and verify it's now enabled
    request_manager.enable_peer(validator_0, ConnectionOrigin::Outbound);
    assert!(!request_manager.no_available_peers());
}

#[test]
fn test_score_chunk_success() {
    let num_validators = 4;
    let (mut request_manager, validators) =
        generate_request_manager_and_test_validators(0, num_validators);

    // Process empty chunk responses from all validators
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        for validator_index in 0..num_validators {
            request_manager.process_empty_chunk(&validators[validator_index as usize]);
        }
    }

    // Process successful chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_success_response(&validators[0]);
    }

    // Calculate selection counts for validators
    let pick_counts = calculate_pick_counts_for_validators(&mut request_manager, NUM_PICKS_TO_MAKE);

    // Verify validator 0 is chosen more often than the other validators
    let validator_0_count = pick_counts.get(&validators[0]).unwrap_or(&0);
    for (validator_index, validator) in validators.iter().enumerate() {
        if validator_index != 0 {
            assert!(validator_0_count > pick_counts.get(&validator).unwrap());
        }
    }
}

#[test]
fn test_score_chunk_timeout() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 4);

    let validator_0 = vec![validators[0].clone()];

    // Process multiple request timeouts from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.add_request(1, validator_0.clone());
        assert!(request_manager.check_timeout(1).unwrap());
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_score_chunk_version_mismatch() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 4);

    let validator_0 = validators[0].clone();

    // Process multiple chunk version mismatches from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.add_request(100, vec![validator_0.clone()]);
        request_manager
            .process_chunk_version_mismatch(&validator_0, 100, 0)
            .unwrap_err();
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_score_empty_chunk() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(10, 4);

    // Process multiple empty chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_empty_chunk(&validators[0]);
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_score_invalid_chunk() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(10, 4);

    // Process multiple invalid chunk responses from validator 0
    for _ in 0..NUM_CHUNKS_TO_PROCESS {
        request_manager.process_invalid_chunk(&validators[0]);
    }

    // Verify validator 0 is chosen less often than the other validators
    verify_bad_validator_picked_less_often(&mut request_manager, &validators, 0);
}

#[test]
fn test_remove_requests() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 2);

    let validator_0 = vec![validators[0].clone()];
    let validator_1 = vec![validators[1].clone()];

    // Add version requests to request manager
    request_manager.add_request(1, validator_0.clone());
    request_manager.add_request(3, validator_1.clone());
    request_manager.add_request(5, validator_0.clone());
    request_manager.add_request(10, validator_0);
    request_manager.add_request(12, validator_1);

    // Remove all request versions below 5
    request_manager.remove_requests(5);

    // Verify versions updated correctly
    assert!(request_manager.get_last_request_time(1).is_none());
    assert!(request_manager.get_last_request_time(3).is_none());
    assert!(request_manager.get_last_request_time(5).is_some());
    assert!(request_manager.get_last_request_time(10).is_some());
    assert!(request_manager.get_last_request_time(12).is_some());
}

#[test]
fn test_request_times() {
    let (mut request_manager, validators) = generate_request_manager_and_test_validators(0, 2);

    // Verify first request time doesn't exist for missing request
    assert!(request_manager.get_first_request_time(1).is_none());

    // Add version requests to request manager
    request_manager.add_request(1, vec![validators[0].clone()]);
    request_manager.add_request(1, vec![validators[1].clone()]);

    // Verify first request time is less than last request time
    assert!(
        request_manager.get_first_request_time(1).unwrap()
            < request_manager.get_last_request_time(1).unwrap()
    );
}

/// Verify that the bad validator is chosen less often than the other validators (due to
/// having a lower peer score internally).
fn verify_bad_validator_picked_less_often(
    request_manager: &mut RequestManager,
    validators: &[PeerNetworkId],
    bad_validator_index: usize,
) {
    // Calculate selection counts for validators
    let pick_counts = calculate_pick_counts_for_validators(request_manager, NUM_PICKS_TO_MAKE);

    // Verify bad validator is chosen less often than the other validators
    let bad_validator_count = pick_counts
        .get(&validators[bad_validator_index])
        .unwrap_or(&0);
    for (validator_index, validator) in validators.iter().enumerate() {
        if validator_index != bad_validator_index {
            assert!(bad_validator_count < pick_counts.get(&validator).unwrap());
        }
    }
}

/// Picks a peer to send a chunk request to (multiple times) and constructs a pick count
/// for each of the chosen peers.
fn calculate_pick_counts_for_validators(
    request_manager: &mut RequestManager,
    number_of_picks_to_execute: u64,
) -> HashMap<PeerNetworkId, u64> {
    let mut pick_counts = HashMap::new();

    for _ in 0..number_of_picks_to_execute {
        let picked_peers = request_manager.pick_peers();
        assert_eq!(1, picked_peers.len()); // Ensure only one validator per multicast level

        let picked_peer = picked_peers[0].clone();
        let counter = pick_counts.entry(picked_peer).or_insert(0);
        *counter += 1;
    }

    pick_counts
}

/// Generates a new request manager with a specified number of validator peers enabled.
fn generate_request_manager_and_test_validators(
    request_timeout: u64,
    num_validators: u64,
) -> (RequestManager, Vec<PeerNetworkId>) {
    let mut request_manager = RequestManager::new(
        UpstreamConfig::default(),
        Duration::from_secs(request_timeout),
        Duration::from_secs(30),
        HashMap::new(),
    );

    let mut validators = Vec::new();
    for _ in 0..num_validators {
        let validator = PeerNetworkId::random_validator();
        request_manager.enable_peer(validator.clone(), ConnectionOrigin::Outbound);
        validators.push(validator);
    }

    (request_manager, validators)
}
<|endoftext|>


<commit_before>use regex;
use regex::Regex;
use std::str::FromStr;
use std::str::Lines;
use std::cmp;
use std::i32;
use std::usize;
use std::iter::Enumerate;
use std::iter::Peekable;
use std::str::CharIndices;

use token::Token;
use token::TokenType;

pub struct TokenIterator<'a>  {
	text: &'a str,
	lines: Enumerate<Lines<'a>>,

	allRegex: Vec<(TokenType,Regex)>,

	current_line_number: usize,
	current_line_char_pos: usize,
	current_line: &'a str,

	slice_start: usize,

	initialized: bool
}

impl<'a>  TokenIterator<'a>  {
	//type Item = str;

	pub fn new(string: &'a str) -> TokenIterator {

		let single_char_token = Regex::new(r"\+|-|\*|/|<|=|&|!|\(|\)|;|\.|:").unwrap();
		let two_char_token = Regex::new(r"(\.{2})|(:={1})").unwrap();
		let keyword = Regex::new(r"(assert)|(string)|(print)|(bool)|(read)|(var)|(for)|(end)|(int)|(in)|(do)").unwrap();
		let integer = Regex::new(r"\b\d+\b").unwrap();
		let string_literal = Regex::new(r#"^"([^"\\]|\\.)*"$"#).unwrap();
		let identifier = Regex::new(r"^[A-Za-z][A-Za-z0-9]*|([A-Za-z]+[0-9]*)").unwrap();

		let allRegex: Vec<(TokenType, Regex)> = vec!(
		(TokenType::string_literal, string_literal),
		(TokenType::keyword, keyword),
		(TokenType::identifier, identifier),
		(TokenType::two_char, two_char_token),
		(TokenType::single_char, single_char_token),
		(TokenType::integer, integer));
		let src = string.trim_left_matches("\u{feff}"); //strip BOM

		let t = TokenIterator {
			initialized: false,
            text: src,
			allRegex: allRegex,
			lines: src.lines().enumerate(),
			current_line: "",
			slice_start: 0,
			current_line_number: 1,
			current_line_char_pos: 0,
        };
		t
    }

	fn handle_end_of_line(&mut self) -> bool {
		//println!("{}, {}", self.current_line_char_pos, self.current_line.len());
		//self.visualize_line_slice("", self.current_line_char_pos, self.current_line.len());
		if(self.current_line_char_pos >= self.current_line.len() ) { //get next line chars iterator if at the end
			match self.lines.next() {
				Some((line_number, line)) => {
					if(!self.initialized) {
						self.initialized = true;
					} else {
						self.slice_start += 1;
					}
					self.current_line_number = line_number;
					self.current_line = line;
					self.current_line_char_pos = 0;
				},
				None => {
					//return none if lines exhausted too
					return false;
				}
			}
		}
		true
	}

	fn visualize_line_pos(&self, pos: usize) {
		let line1 = self.current_line;
		let mut buffer = String::new();
		for _ in 0 .. pos+1 {
			buffer.push(' ');
		}
		buffer.push('^');

		println!("#####################");
		println!("{}", line1);
		println!("{}", buffer);
		println!("#####################");
	}

	fn visualize_line_slice(&self, slice: &str, start: usize, end: usize) {
		let line1 = self.current_line;
		let mut buffer = String::new();
		for _ in 0 .. start {
			buffer.push(' ');
		}
		buffer.push('^');

		if(end - start > 0){
			for _ in 0 .. (end - start - 1) {
				buffer.push(' ');
			}

			buffer.push('^');
		}

		println!("#####################");

		println!("slice: \t{}",slice);
		println!("line: \t{}",line1);
		println!("      \t{}",buffer);
		println!("{}-{}",start, end);
		println!("#####################");
	}
}

impl<'a> Iterator for TokenIterator<'a> {
	type Item = Token<'a>;

	fn next(&mut self) -> Option<Token<'a>> {
		let mut token: Option<Token> = None;

		if !self.handle_end_of_line() {
			return token;
		}

		let mut lexeme = "";
		let mut lexeme_candidate;
		let mut found = false;
		let mut skipped_initial_whitespace = false;
		let mut skipping_whitespace = false;
		let mut token_start_on_line = 0;
		let mut t_start = usize::MAX;
		let mut t_end = usize::MIN;
		let mut token_type = TokenType::NA;

		for column in self.current_line_char_pos .. self.current_line.len()+1 {
			//println!("{}",self.current_line.chars().nth(column).unwrap_or(' '));
			unsafe {
				lexeme_candidate = self.current_line.slice_unchecked(self.current_line_char_pos, column);
			}
			//println!("{}",lexeme_candidate);

			if ( !skipped_initial_whitespace && lexeme_candidate.chars().nth(0).unwrap_or('a').is_whitespace() ) {
				//skip rest of whitespace
				skipping_whitespace = true;
				self.current_line_char_pos += 1;
				continue;
			} else if (skipping_whitespace && lexeme_candidate != " ") {
				skipped_initial_whitespace = true;
			}

			for tuple_ref in self.allRegex.iter() {
				let (tokentype, ref rex): (TokenType, Regex) = *tuple_ref;
				if let Some((start, end)) = rex.find(lexeme_candidate) {
					let line_pos_start = self.current_line_char_pos+start;
					let line_pos_end = self.current_line_char_pos+end;
					if(line_pos_start <= t_start && line_pos_end > t_end) {

						//self.visualize_line_slice(lexeme_candidate, line_pos_start, line_pos_end);

						lexeme = lexeme_candidate;
						t_start = line_pos_start;
						t_end = column;
						token_start_on_line = line_pos_start;
						token_type = tokentype;
						found = true;
					}
				}
			}
		}

		if found {
			token = Some(Token{
				line: self.current_line_number + 1,
				column: token_start_on_line + 1,
				lexeme: lexeme,
				token_type: token_type});

			//set next slice start on line
			self.current_line_char_pos = t_end;

		} else {
			return self.next(); //not found, try again. EOL breaks recursive iteration
		}
		token
	}
}

#[test]
fn tokenize_two_consecutive() -> () {
	let code = include_str!("../sample4.txt");
	println!("{}", code);
	let mut iterator = TokenIterator::new(code);
	assert_eq!(Some(Token{line:1, column: 1, lexeme: "var", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 5, lexeme: "X", token_type: TokenType::identifier}),iterator.next());
}

#[test]
fn tokenize_two_consecutive_file_3() -> () {
	let code = include_str!("../sample3.txt");
	println!("{}", code);
	let mut iterator = TokenIterator::new(code);
	assert_eq!(Some(Token{line:1, column: 1, lexeme: "print", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 7, lexeme: "\"Give a number\"", token_type: TokenType::string_literal}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 22, lexeme: ";", token_type: TokenType::single_char}),iterator.next());
	assert_eq!(Some(Token{line:2, column: 1, lexeme: "var", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:2, column: 5, lexeme: "n", token_type: TokenType::identifier}),iterator.next());
}

#[test]
fn tokenize_full_line() -> () {
	let code = include_str!("../sample4.txt");
	println!("{}", code);
	let mut iterator = TokenIterator::new(code);
	assert_eq!(Some(Token{line:1, column: 1, lexeme: "var", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 5, lexeme: "X", token_type: TokenType::identifier}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 7, lexeme: ":", token_type: TokenType::single_char}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 9, lexeme: "int", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 12, lexeme: ";", token_type: TokenType::single_char}),iterator.next());
}

#[test]
fn tokenize_multiple_lines() -> () {
	let code = include_str!("../sample4.txt");
	println!("{}", code);
	let mut iterator = TokenIterator::new(code);
	iterator.next();
	iterator.next();
	iterator.next();
	iterator.next();
	iterator.next();
	assert_eq!(Some(Token{line:2, column: 1, lexeme: "X", token_type: TokenType::identifier}),iterator.next());
	assert_eq!(Some(Token{line:2, column: 3, lexeme: ":=", token_type: TokenType::two_char}),iterator.next());
	assert_eq!(Some(Token{line:2, column: 6, lexeme: "15", token_type: TokenType::integer}),iterator.next());
	assert_eq!(Some(Token{line:2, column: 8, lexeme: ";", token_type: TokenType::single_char}),iterator.next());
	assert_eq!(Some(Token{line:3, column: 1, lexeme: "print", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:3, column: 7, lexeme: "X", token_type: TokenType::identifier}),iterator.next());
	assert_eq!(Some(Token{line:3, column: 8, lexeme: ";", token_type: TokenType::single_char}),iterator.next());
}<commit_msg>Tokenizer no longer uses regex for matching<commit_after>use std::str::FromStr;
use std::str::Lines;
use std::cmp;
use std::i32;
use std::usize;
use std::iter::Enumerate;
use std::iter::Peekable;
use std::str::CharIndices;

use token::Token;
use token::TokenType;

const single_char_token: &'static [&'static str] = &["+","-","*","/","<","=","&","!","(",")",";",".",":"];
const two_char_token: &'static [&'static str] = &["..",":="];
const keyword: &'static [&'static str] = &["assert","string","print","bool","read","var","for","end","int","in","do"];

pub struct TokenIterator<'a>  {
	text: &'a str,
	lines: Enumerate<Lines<'a>>,

	token_matchers: Vec<(TokenType, fn(&str) -> bool)>,

	current_line_number: usize,
	current_line_char_pos: usize,
	current_line: &'a str,

	slice_start: usize,

	initialized: bool
}

impl<'a>  TokenIterator<'a>  {
	//type Item = str;

	pub fn new(string: &'a str) -> TokenIterator {

		let token_matchers: Vec<(TokenType, fn(&str) -> bool)> = vec!(
		(TokenType::string_literal, is_string_literal),
		(TokenType::keyword, is_keyword_token),
		(TokenType::identifier, is_identifier),
		(TokenType::two_char, is_two_char_token),
		(TokenType::single_char, is_single_char_token),
		(TokenType::integer, is_integer));
		let src = string.trim_left_matches("\u{feff}"); //strip BOM

		let t = TokenIterator {
			initialized: false,
            text: src,
			token_matchers: token_matchers,
			lines: src.lines().enumerate(),
			current_line: "",
			slice_start: 0,
			current_line_number: 1,
			current_line_char_pos: 0,
        };
		t
    }

	fn handle_end_of_line(&mut self) -> bool {
		//println!("{}, {}", self.current_line_char_pos, self.current_line.len());
		//self.visualize_line_slice("", self.current_line_char_pos, self.current_line.len());
		if(self.current_line_char_pos >= self.current_line.len() ) { //get next line chars iterator if at the end
			match self.lines.next() {
				Some((line_number, line)) => {
					if(!self.initialized) {
						self.initialized = true;
					} else {
						self.slice_start += 1;
					}
					self.current_line_number = line_number;
					self.current_line = line;
					self.current_line_char_pos = 0;
				},
				None => {
					//return none if lines exhausted too
					return false;
				}
			}
		}
		true
	}

	fn visualize_line_pos(&self, pos: usize) {
		let line1 = self.current_line;
		let mut buffer = String::new();
		for _ in 0 .. pos+1 {
			buffer.push(' ');
		}
		buffer.push('^');

		println!("#####################");
		println!("{}", line1);
		println!("{}", buffer);
		println!("#####################");
	}

	fn visualize_line_slice(&self, slice: &str, start: usize, end: usize) {
		let line1 = self.current_line;
		let mut buffer = String::new();
		for _ in 0 .. start {
			buffer.push(' ');
		}
		buffer.push('^');

		if(end - start > 0){
			for _ in 0 .. (end - start - 1) {
				buffer.push(' ');
			}

			buffer.push('^');
		}

		println!("#####################");

		println!("slice: \t{}",slice);
		println!("line: \t{}",line1);
		println!("      \t{}",buffer);
		println!("{}-{}",start, end);
		println!("#####################");
	}
}

impl<'a> Iterator for TokenIterator<'a> {
	type Item = Token<'a>;

	fn next(&mut self) -> Option<Token<'a>> {
		let mut token: Option<Token> = None;

		if !self.handle_end_of_line() {
			return token;
		}

		let mut lexeme = "";
		let mut lexeme_candidate;
		let mut found = false;
		let mut skipped_initial_whitespace = false;
		let mut skipping_whitespace = false;
		let mut token_start_on_line = 0;
		let mut t_start = usize::MAX;
		let mut t_end = usize::MIN;
		let mut token_type = TokenType::NA;
		let mut longest_lexeme = usize::MIN;

		for column in self.current_line_char_pos .. self.current_line.len()+1 {
			//println!("{}",self.current_line.chars().nth(column).unwrap_or(' '));
			unsafe {
				lexeme_candidate = self.current_line.slice_unchecked(self.current_line_char_pos, column);
			}
			//println!("{}",lexeme_candidate);

			if ( !skipped_initial_whitespace && lexeme_candidate.chars().nth(0).unwrap_or('a').is_whitespace() ) {
				//skip rest of whitespace
				skipping_whitespace = true;
				self.current_line_char_pos += 1;
				continue;
			} else if (skipping_whitespace && lexeme_candidate != " ") {
				skipped_initial_whitespace = true;
			}

			for tuple_ref in self.token_matchers.iter() {
				let (tokentype, matcher_fn): (TokenType, fn(&str) -> bool) = *tuple_ref;
				if(matcher_fn(lexeme_candidate)) {
					let line_pos_start = self.current_line_char_pos;
					let line_pos_end = self.current_line_char_pos + lexeme_candidate.len();
					let lexeme_length = lexeme_candidate.chars().count();
					if(lexeme_length > longest_lexeme) { //prioritize length

						//println!("New longest lexeme {:?}, {:?}, {:?}", lexeme_candidate ,tokentype, lexeme_length);
						longest_lexeme = lexeme_length;
						lexeme = lexeme_candidate;
						t_start = line_pos_start;
						t_end = column;
						token_start_on_line = line_pos_start;
						token_type = tokentype;
						found = true;
					}
				}
			}
		}

		if found {
			token = Some(Token{
				line: self.current_line_number + 1,
				column: token_start_on_line + 1,
				lexeme: lexeme,
				token_type: token_type});

			//set next slice start on line
			self.current_line_char_pos = t_end;

		} else {
			return self.next(); //not found, try again. EOL breaks recursive iteration
		}
		token
	}
}

fn is_single_char_token(lexeme: &str) -> bool {single_char_token.iter().any(|&x| x == lexeme)}

fn is_two_char_token(lexeme: &str) -> bool {two_char_token.iter().any(|&x| x == lexeme)}

fn is_keyword_token(lexeme: &str) -> bool {	keyword.iter().any(|&x| x == lexeme)}

fn is_string_literal(lexeme: &str) -> bool {
	let starts_with_quote = lexeme.starts_with("\"");
	let mut chars = lexeme.chars();
	let mut escape_next = false;
	let mut num_unescaped_quotes = 0;
	let mut last_ch = ' ';
	while let Some(ch) = chars.next() {
		last_ch = ch;
		match ch {
			'\\' => escape_next = true,
			'"'	=> {
				if(!escape_next) {
					num_unescaped_quotes += 1;
				}
			}
			_ => escape_next = false
		}
	}
	starts_with_quote && last_ch == '"' && num_unescaped_quotes % 2 == 0

}

fn is_integer(lexeme: &str) -> bool {lexeme.parse::<i32>().is_ok()}

fn is_identifier(lexeme: &str) -> bool {
	let allAreAlphaNum = lexeme.chars().all(|ch: char| {
		ch.is_alphanumeric() || ch == '_'
	});
	let firstIsAlphabetic = lexeme.chars().nth(0).unwrap_or('1').is_alphabetic();
	firstIsAlphabetic && allAreAlphaNum
}

#[test]
fn recornize_identifier() -> () {
	assert_eq!(false, is_identifier("var X : int"));
	assert_eq!(false, is_identifier("var X : i"));
	assert_eq!(true, is_identifier("X"));
	assert_eq!(true, is_identifier("awesome_var1"));
	assert_eq!(false, is_identifier("1illegalvar"));
}

#[test]
fn recognize_string_literal() -> () {
	let ok = r#""foo bar baz""#;
	let missing_quote = r#""foo bar baz"#;
	let unescaped_quote = r#""foo "bar baz""#;
	let escaped_quote = r#""foo \"bar baz""#;
	assert_eq!(true, is_string_literal(ok));
	assert_eq!(false, is_string_literal(missing_quote));
	assert_eq!(false, is_string_literal(unescaped_quote));
	assert_eq!(true, is_string_literal(escaped_quote));
}

#[test]
fn tokenize_two_consecutive() -> () {
	let code = include_str!("../sample4.txt");
	println!("{}", code);
	let mut iterator = TokenIterator::new(code);
	assert_eq!(Some(Token{line:1, column: 1, lexeme: "var", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 5, lexeme: "X", token_type: TokenType::identifier}),iterator.next());
}

#[test]
fn tokenize_two_consecutive_file_3() -> () {
	let code = include_str!("../sample3.txt");
	println!("{}", code);
	let mut iterator = TokenIterator::new(code);
	assert_eq!(Some(Token{line:1, column: 1, lexeme: "print", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 7, lexeme: "\"Give a number\"", token_type: TokenType::string_literal}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 22, lexeme: ";", token_type: TokenType::single_char}),iterator.next());
	assert_eq!(Some(Token{line:2, column: 1, lexeme: "var", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:2, column: 5, lexeme: "n", token_type: TokenType::identifier}),iterator.next());
}

#[test]
fn tokenize_full_line() -> () {
	let code = include_str!("../sample4.txt");
	println!("{}", code);
	let mut iterator = TokenIterator::new(code);
	assert_eq!(Some(Token{line:1, column: 1, lexeme: "var", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 5, lexeme: "X", token_type: TokenType::identifier}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 7, lexeme: ":", token_type: TokenType::single_char}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 9, lexeme: "int", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:1, column: 12, lexeme: ";", token_type: TokenType::single_char}),iterator.next());
}

#[test]
fn tokenize_multiple_lines() -> () {
	let code = include_str!("../sample4.txt");
	println!("{}", code);
	let mut iterator = TokenIterator::new(code);
	iterator.next();
	iterator.next();
	iterator.next();
	iterator.next();
	iterator.next();
	assert_eq!(Some(Token{line:2, column: 1, lexeme: "X", token_type: TokenType::identifier}),iterator.next());
	assert_eq!(Some(Token{line:2, column: 3, lexeme: ":=", token_type: TokenType::two_char}),iterator.next());
	assert_eq!(Some(Token{line:2, column: 6, lexeme: "15", token_type: TokenType::integer}),iterator.next());
	assert_eq!(Some(Token{line:2, column: 8, lexeme: ";", token_type: TokenType::single_char}),iterator.next());
	assert_eq!(Some(Token{line:3, column: 1, lexeme: "print", token_type: TokenType::keyword}),iterator.next());
	assert_eq!(Some(Token{line:3, column: 7, lexeme: "X", token_type: TokenType::identifier}),iterator.next());
	assert_eq!(Some(Token{line:3, column: 8, lexeme: ";", token_type: TokenType::single_char}),iterator.next());
}
<|endoftext|>


<commit_before>use std::num;

pub trait Addr: Int { }

impl Addr for u8 { }
impl Addr for u16 { }
impl Addr for u32 { }
impl Addr for u64 { }

pub trait Addressable<A: Addr> {
	fn get (&self, addr: A) -> u8;
	fn set (&mut self, addr: A, data: u8);

	fn getx (&self, addr:A, offset: int) -> u8 {
		if offset < 0 {
			self.get(addr - num::cast(-offset).unwrap())
		} else {
			self.get(addr + num::cast(offset).unwrap())
		}
	}

	fn get_be_n (&self, addr: A, nbytes: uint) -> u64 {
		assert!(nbytes > 0 && nbytes < 8);
		let mut val = 0u64;
		let mut i = nbytes;
		while i > 0 {
			i -= 1;
			val |= self.get(addr + num::cast(i).unwrap()) as u64 << (nbytes-i-1) * 8;
		}
		val
	}

	fn get_be<T: Int> (&self, addr: A) -> T {
		let none: Option<T> = None;
		num::cast(self.get_be_n(addr, num::Primitive::bytes(none))).unwrap()
	}

	fn get_le_n (&self, addr: A, nbytes: uint) -> u64 {
		assert!(nbytes > 0 && nbytes < 8);
		let mut val = 0u64;
		let mut i = nbytes;
		while i > 0 {
			i -= 1;
			val |= self.get(addr + num::cast(i).unwrap()) as u64 << i * 8;
		}
		val
	}

	fn get_le<T: Int> (&self, addr: A) -> T {
		let none: Option<T> = None;
		num::cast(self.get_le_n(addr, num::Primitive::bytes(none))).unwrap()
	}

	fn setx (&mut self, addr: A, offset: int, data: u8) {
		if offset < 0 {
			self.set(addr - num::cast(-offset).unwrap(), data);
		} else {
			self.set(addr + num::cast(offset).unwrap(), data);
		}
	}

	fn set_be_n (&mut self, addr: A, nbytes: uint, val: u64) {
		assert!(nbytes > 0 && nbytes < 8);
		let mut i = nbytes;
		while i > 0 {
			i -= 1;
			self.set(addr + num::cast(i).unwrap(), (val >> (nbytes-i-1) * 8) as u8);
		}
	}

	fn set_be<T: Int> (&mut self, addr: A, val: T) {
		let none: Option<T> = None;
		self.set_be_n(addr, num::Primitive::bytes(none), num::cast(val).unwrap());
	}

	fn set_le_n (&mut self, addr: A, nbytes: uint, val: u64) {
		assert!(nbytes > 0 && nbytes < 8);
		let mut i = nbytes;
		while i > 0 {
			i -= 1;
			self.set(addr + num::cast(i).unwrap(), (val >> i * 8) as u8);
		}
	}

	fn set_le<T: Int> (&mut self, addr: A, val: T) {
		let none: Option<T> = None;
		self.set_le_n(addr, num::Primitive::bytes(none), num::cast(val).unwrap());
	}
}

#[cfg(test)]
mod test {
	use super::Addressable;

	struct DummyData;
	impl Addressable<u16> for DummyData {
		fn get (&self, addr: u16) -> u8 {
			addr as u8
		}
		fn set (&mut self, addr: u16, data: u8) {
			assert_eq!(data, addr as u8);
		}
	}

	#[test]
	fn test_get () {
		let data = DummyData;
		assert_eq!(data.get(0x0012), 0x12);
		assert_eq!(data.get(0x1234), 0x34);
	}

	#[test]
	fn test_getx () {
		let data = DummyData;
		assert_eq!(data.getx(0x0012_u16, 5), 0x17);
		assert_eq!(data.getx(0x1234_u16, 5), 0x39);
		assert_eq!(data.getx(0x0012_u16, -3), 0x0f);
		assert_eq!(data.getx(0x1234_u16, -3), 0x31);
	}

	#[test]
	fn test_get_big_endian () {
		let data = DummyData;
		assert_eq!(      0x02_u8 , data.get_be(0x0002_u16));
		assert_eq!(      0x54_u8 , data.get_be(0x0054_u16));
		assert_eq!(    0x0203_u16, data.get_be(0x0002_u16));
		assert_eq!(    0x5455_u16, data.get_be(0x0054_u16));
		assert_eq!(0x02030405_u32, data.get_be(0x0002_u16));
		assert_eq!(0x54555657_u32, data.get_be(0x0054_u16));
	}

	#[test]
	fn test_get_signed_big_endian () {
		let data = DummyData;
		assert_eq!(       0x54_i8 , data.get_be(0x0054_u16));
		assert_eq!(      -0x5b_i8 , data.get_be(0x00a5_u16));
		assert_eq!(     0x5455_i16, data.get_be(0x0054_u16));
		assert_eq!(    -0x5a5a_i16, data.get_be(0x00a5_u16));
		assert_eq!( 0x54555657_i32, data.get_be(0x0054_u16));
		assert_eq!(-0x5a595858_i32, data.get_be(0x00a5_u16));
	}

	#[test]
	fn test_get_little_endian () {
		let data = DummyData;
		assert_eq!(      0x02_u8 , data.get_le(0x0002_u16));
		assert_eq!(      0x54_u8 , data.get_le(0x0054_u16));
		assert_eq!(    0x0302_u16, data.get_le(0x0002_u16));
		assert_eq!(    0x5554_u16, data.get_le(0x0054_u16));
		assert_eq!(0x05040302_u32, data.get_le(0x0002_u16));
		assert_eq!(0x57565554_u32, data.get_le(0x0054_u16));
	}

	#[test]
	fn test_get_signed_little_endian () {
		let data = DummyData;
		assert_eq!(       0x54_i8 , data.get_le(0x0054_u16));
		assert_eq!(      -0x5b_i8 , data.get_le(0x00a5_u16));
		assert_eq!(     0x5554_i16, data.get_le(0x0054_u16));
		assert_eq!(    -0x595b_i16, data.get_le(0x00a5_u16));
		assert_eq!( 0x57565554_i32, data.get_le(0x0054_u16));
		assert_eq!(-0x5758595b_i32, data.get_le(0x00a5_u16));
	}

	#[test]
	fn test_set () {
		let mut data = DummyData;
		data.set(0x0012, 0x12);
		data.set(0x1234, 0x34);
	}

	#[test]
	fn test_setx () {
		let mut data = DummyData;
		data.setx(0x0012_u16, 5, 0x17);
		data.setx(0x1234_u16, 5, 0x39);
		data.setx(0x0012_u16, -3, 0x0f);
		data.setx(0x1234_u16, -3, 0x31);
	}

	#[test]
	fn test_set_big_endian () {
		let mut data = DummyData;
		data.set_be(0x0002_u16,       0x02_u8 );
		data.set_be(0x0054_u16,       0x54_u8 );
		data.set_be(0x0002_u16,     0x0203_u16);
		data.set_be(0x0054_u16,     0x5455_u16);
		data.set_be(0x0002_u16, 0x02030405_u32);
		data.set_be(0x0054_u16, 0x54555657_u32);
	}

	#[test]
	fn test_set_signed_big_endian () {
		let mut data = DummyData;
		data.set_be(0x0054_u16,        0x54_i8 );
		data.set_be(0x00a5_u16,       -0x5b_i8 );
		data.set_be(0x0054_u16,      0x5455_i16);
		data.set_be(0x00a5_u16,     -0x5a5a_i16);
		data.set_be(0x0054_u16,  0x54555657_i32);
		data.set_be(0x00a5_u16, -0x5a595858_i32);
	}

	#[test]
	fn test_set_little_endian () {
		let mut data = DummyData;
		data.set_le(0x0002_u16,       0x02_u8 );
		data.set_le(0x0054_u16,       0x54_u8 );
		data.set_le(0x0002_u16,     0x0302_u16);
		data.set_le(0x0054_u16,     0x5554_u16);
		data.set_le(0x0002_u16, 0x05040302_u32);
		data.set_le(0x0054_u16, 0x57565554_u32);
	}

	#[test]
	fn test_set_signed_little_endian () {
		let mut data = DummyData;
		data.set_le(0x0054_u16,        0x54_i8 );
		data.set_le(0x00a5_u16,       -0x5b_i8 );
		data.set_le(0x0054_u16,      0x5554_i16);
		data.set_le(0x00a5_u16,     -0x595b_i16);
		data.set_le(0x0054_u16,  0x57565554_i32);
		data.set_le(0x00a5_u16, -0x5758595b_i32);
	}
}
<commit_msg>Add comments to addressable module<commit_after>use std::num;

/// A trait for all addresses
pub trait Addr: Int { }

/// Supported address sizes
impl Addr for u8 { }
impl Addr for u16 { }
impl Addr for u32 { }
impl Addr for u64 { }
// TODO: Shall we support arbitrary address sizes like u12?

/// A trait for anything that has an address bus and can get/set data. The
/// data size that can be get/set is u8 always, the address size is given
/// as a type parameter and can be of any size (typically u16 or u32).
/// TODO: Support data sizes other than u8?
pub trait Addressable<A: Addr> {
	fn get (&self, addr: A) -> u8;
	fn set (&mut self, addr: A, data: u8);

	fn getx (&self, addr:A, offset: int) -> u8 {
		if offset < 0 {
			self.get(addr - num::cast(-offset).unwrap())
		} else {
			self.get(addr + num::cast(offset).unwrap())
		}
	}

	fn get_be_n (&self, addr: A, nbytes: uint) -> u64 {
		assert!(nbytes > 0 && nbytes < 8);
		let mut val = 0u64;
		let mut i = nbytes;
		while i > 0 {
			i -= 1;
			val |= self.get(addr + num::cast(i).unwrap()) as u64 << (nbytes-i-1) * 8;
		}
		val
	}

	fn get_be<T: Int> (&self, addr: A) -> T {
		let none: Option<T> = None;
		num::cast(self.get_be_n(addr, num::Primitive::bytes(none))).unwrap()
	}

	fn get_le_n (&self, addr: A, nbytes: uint) -> u64 {
		assert!(nbytes > 0 && nbytes < 8);
		let mut val = 0u64;
		let mut i = nbytes;
		while i > 0 {
			i -= 1;
			val |= self.get(addr + num::cast(i).unwrap()) as u64 << i * 8;
		}
		val
	}

	fn get_le<T: Int> (&self, addr: A) -> T {
		let none: Option<T> = None;
		num::cast(self.get_le_n(addr, num::Primitive::bytes(none))).unwrap()
	}

	fn setx (&mut self, addr: A, offset: int, data: u8) {
		if offset < 0 {
			self.set(addr - num::cast(-offset).unwrap(), data);
		} else {
			self.set(addr + num::cast(offset).unwrap(), data);
		}
	}

	fn set_be_n (&mut self, addr: A, nbytes: uint, val: u64) {
		assert!(nbytes > 0 && nbytes < 8);
		let mut i = nbytes;
		while i > 0 {
			i -= 1;
			self.set(addr + num::cast(i).unwrap(), (val >> (nbytes-i-1) * 8) as u8);
		}
	}

	fn set_be<T: Int> (&mut self, addr: A, val: T) {
		let none: Option<T> = None;
		self.set_be_n(addr, num::Primitive::bytes(none), num::cast(val).unwrap());
	}

	fn set_le_n (&mut self, addr: A, nbytes: uint, val: u64) {
		assert!(nbytes > 0 && nbytes < 8);
		let mut i = nbytes;
		while i > 0 {
			i -= 1;
			self.set(addr + num::cast(i).unwrap(), (val >> i * 8) as u8);
		}
	}

	fn set_le<T: Int> (&mut self, addr: A, val: T) {
		let none: Option<T> = None;
		self.set_le_n(addr, num::Primitive::bytes(none), num::cast(val).unwrap());
	}
}

#[cfg(test)]
mod test {
	use super::Addressable;

	struct DummyData;
	impl Addressable<u16> for DummyData {
		fn get (&self, addr: u16) -> u8 {
			addr as u8
		}
		fn set (&mut self, addr: u16, data: u8) {
			assert_eq!(data, addr as u8);
		}
	}

	#[test]
	fn test_get () {
		let data = DummyData;
		assert_eq!(data.get(0x0012), 0x12);
		assert_eq!(data.get(0x1234), 0x34);
	}

	#[test]
	fn test_getx () {
		let data = DummyData;
		assert_eq!(data.getx(0x0012_u16, 5), 0x17);
		assert_eq!(data.getx(0x1234_u16, 5), 0x39);
		assert_eq!(data.getx(0x0012_u16, -3), 0x0f);
		assert_eq!(data.getx(0x1234_u16, -3), 0x31);
	}

	#[test]
	fn test_get_big_endian () {
		let data = DummyData;
		assert_eq!(      0x02_u8 , data.get_be(0x0002_u16));
		assert_eq!(      0x54_u8 , data.get_be(0x0054_u16));
		assert_eq!(    0x0203_u16, data.get_be(0x0002_u16));
		assert_eq!(    0x5455_u16, data.get_be(0x0054_u16));
		assert_eq!(0x02030405_u32, data.get_be(0x0002_u16));
		assert_eq!(0x54555657_u32, data.get_be(0x0054_u16));
	}

	#[test]
	fn test_get_signed_big_endian () {
		let data = DummyData;
		assert_eq!(       0x54_i8 , data.get_be(0x0054_u16));
		assert_eq!(      -0x5b_i8 , data.get_be(0x00a5_u16));
		assert_eq!(     0x5455_i16, data.get_be(0x0054_u16));
		assert_eq!(    -0x5a5a_i16, data.get_be(0x00a5_u16));
		assert_eq!( 0x54555657_i32, data.get_be(0x0054_u16));
		assert_eq!(-0x5a595858_i32, data.get_be(0x00a5_u16));
	}

	#[test]
	fn test_get_little_endian () {
		let data = DummyData;
		assert_eq!(      0x02_u8 , data.get_le(0x0002_u16));
		assert_eq!(      0x54_u8 , data.get_le(0x0054_u16));
		assert_eq!(    0x0302_u16, data.get_le(0x0002_u16));
		assert_eq!(    0x5554_u16, data.get_le(0x0054_u16));
		assert_eq!(0x05040302_u32, data.get_le(0x0002_u16));
		assert_eq!(0x57565554_u32, data.get_le(0x0054_u16));
	}

	#[test]
	fn test_get_signed_little_endian () {
		let data = DummyData;
		assert_eq!(       0x54_i8 , data.get_le(0x0054_u16));
		assert_eq!(      -0x5b_i8 , data.get_le(0x00a5_u16));
		assert_eq!(     0x5554_i16, data.get_le(0x0054_u16));
		assert_eq!(    -0x595b_i16, data.get_le(0x00a5_u16));
		assert_eq!( 0x57565554_i32, data.get_le(0x0054_u16));
		assert_eq!(-0x5758595b_i32, data.get_le(0x00a5_u16));
	}

	#[test]
	fn test_set () {
		let mut data = DummyData;
		data.set(0x0012, 0x12);
		data.set(0x1234, 0x34);
	}

	#[test]
	fn test_setx () {
		let mut data = DummyData;
		data.setx(0x0012_u16, 5, 0x17);
		data.setx(0x1234_u16, 5, 0x39);
		data.setx(0x0012_u16, -3, 0x0f);
		data.setx(0x1234_u16, -3, 0x31);
	}

	#[test]
	fn test_set_big_endian () {
		let mut data = DummyData;
		data.set_be(0x0002_u16,       0x02_u8 );
		data.set_be(0x0054_u16,       0x54_u8 );
		data.set_be(0x0002_u16,     0x0203_u16);
		data.set_be(0x0054_u16,     0x5455_u16);
		data.set_be(0x0002_u16, 0x02030405_u32);
		data.set_be(0x0054_u16, 0x54555657_u32);
	}

	#[test]
	fn test_set_signed_big_endian () {
		let mut data = DummyData;
		data.set_be(0x0054_u16,        0x54_i8 );
		data.set_be(0x00a5_u16,       -0x5b_i8 );
		data.set_be(0x0054_u16,      0x5455_i16);
		data.set_be(0x00a5_u16,     -0x5a5a_i16);
		data.set_be(0x0054_u16,  0x54555657_i32);
		data.set_be(0x00a5_u16, -0x5a595858_i32);
	}

	#[test]
	fn test_set_little_endian () {
		let mut data = DummyData;
		data.set_le(0x0002_u16,       0x02_u8 );
		data.set_le(0x0054_u16,       0x54_u8 );
		data.set_le(0x0002_u16,     0x0302_u16);
		data.set_le(0x0054_u16,     0x5554_u16);
		data.set_le(0x0002_u16, 0x05040302_u32);
		data.set_le(0x0054_u16, 0x57565554_u32);
	}

	#[test]
	fn test_set_signed_little_endian () {
		let mut data = DummyData;
		data.set_le(0x0054_u16,        0x54_i8 );
		data.set_le(0x00a5_u16,       -0x5b_i8 );
		data.set_le(0x0054_u16,      0x5554_i16);
		data.set_le(0x00a5_u16,     -0x595b_i16);
		data.set_le(0x0054_u16,  0x57565554_i32);
		data.set_le(0x00a5_u16, -0x5758595b_i32);
	}
}
<|endoftext|>


<commit_before>use nom::{
  be_u8, be_u16, be_u32, be_u64,
  le_u32,
  IResult,
  ErrorCode, Err,
  Needed,
};

use std::str::from_utf8;

use metadata::{
  Block, BlockData,
  StreamInfo, Application, VorbisComment, CueSheet, Picture,
  SeekPoint, CueSheetTrack, CueSheetTrackIndex, PictureType,
};

use utility::to_u32;

named!(stream_info <&[u8], BlockData>,
  chain!(
    min_block_size: be_u16 ~
    max_block_size: be_u16 ~
    min_frame_size: map!(take!(3), to_u32) ~
    max_frame_size: map!(take!(3), to_u32) ~
    bytes: take!(8) ~
    md5_sum: take!(16),
    || {
      let sample_rate     = ((bytes[0] as u32) << 12) +
                            ((bytes[1] as u32) << 4)  +
                            (bytes[2] as u32) >> 4;
      let channels        = (bytes[2] >> 1) & 0b0111;
      let bits_per_sample = ((bytes[2] & 0b01) << 4) +
                            bytes[3] >> 4;
      let total_samples   = (((bytes[3] as u64) & 0x0f) << 32) +
                            ((bytes[4] as u64) << 24) +
                            ((bytes[5] as u64) << 16) +
                            ((bytes[6] as u64) << 8) +
                            (bytes[7] as u64);

      BlockData::StreamInfo(StreamInfo {
        min_block_size: min_block_size,
        max_block_size: max_block_size,
        min_frame_size: min_frame_size,
        max_frame_size: max_frame_size,
        sample_rate: sample_rate,
        channels: channels + 1,
        bits_per_sample: bits_per_sample + 1,
        total_samples: total_samples,
        md5_sum: md5_sum,
      })
    }
  )
);

fn padding(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  map!(input, take!(length), |_| BlockData::Padding(0))
}

fn application(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  chain!(input,
    id: take_str!(4) ~
    data: take!(length - 4),
    || {
      BlockData::Application(Application {
        id: id,
        data: data,
      })
    }
  )
}

named!(seek_point <&[u8], SeekPoint>,
  chain!(
    sample_number: be_u64 ~
    stream_offset: be_u64 ~
    frame_samples: be_u16,
    || {
      SeekPoint {
        sample_number: sample_number,
        stream_offset: stream_offset,
        frame_samples: frame_samples,
      }
    }
  )
);

fn seek_table(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  let seek_count = (length / 18) as usize;

  map!(input, count!(seek_point, seek_count), BlockData::SeekTable)
}

named!(vorbis_comment <&[u8], BlockData>,
  chain!(
    vendor_string_length: le_u32 ~
    vendor_string: take_str!(vendor_string_length)  ~
    number_of_comments: le_u32 ~
    comments: count!(comment_field, number_of_comments as usize),
    || {
      BlockData::VorbisComment(VorbisComment {
        vendor_string: vendor_string,
        comments: comments,
      })
    }
  )
);

named!(comment_field <&[u8], &str>,
  chain!(
    comment_length: le_u32 ~
    comment: take_str!(comment_length),
    || { comment }
  )
);

named!(cue_sheet <&[u8], BlockData>,
  chain!(
    media_catalog_number: take_str!(128) ~
    lead_in: be_u64 ~
    bytes: take!(259) ~ // TODO: last (7 + 258 * 8) bits must be 0
    num_tracks: be_u8 ~
    tracks: count!(cue_sheet_track, num_tracks as usize),
    || {
      let is_cd = ((bytes[0] >> 7) & 0b01) == 1;

      BlockData::CueSheet(CueSheet {
        media_catalog_number: media_catalog_number,
        lead_in: lead_in,
        is_cd: is_cd,
        tracks: tracks,
      })
    }
  )
);

named!(cue_sheet_track <&[u8], CueSheetTrack>,
  chain!(
    offset: be_u64 ~
    number: be_u8 ~
    isrc: take_str!(12) ~
    bytes: take!(14) ~ // TODO: last (6 + 13 * 8) bits must be 0
    num_indices: be_u8 ~
    indices: count!(cue_sheet_track_index, num_indices as usize),
    || {
      let isnt_audio      = ((bytes[0] >> 7) & 0b01) == 1;
      let is_pre_emphasis = ((bytes[0] >> 6) & 0b01) == 1;

      CueSheetTrack {
        offset: offset,
        number: number,
        isrc: isrc,
        isnt_audio: isnt_audio,
        is_pre_emphasis: is_pre_emphasis,
        indices: indices,
      }
    }
  )
);

named!(cue_sheet_track_index <&[u8], CueSheetTrackIndex>,
  chain!(
    offset: be_u64 ~
    number: be_u8 ~
    take!(3), // TODO: these bytes must be 0
    || {
      CueSheetTrackIndex {
        offset: offset,
        number: number,
      }
    }
  )
);

named!(picture <&[u8], BlockData>,
  chain!(
    picture_type_num: be_u32 ~
    mime_type_length:  be_u32 ~
    mime_type: take_str!(mime_type_length) ~
    description_length: be_u32 ~
    description: take_str!(description_length) ~
    width: be_u32 ~
    height: be_u32 ~
    depth: be_u32 ~
    colors: be_u32 ~
    data_length: be_u32 ~
    data: take!(data_length),
    || {
      let picture_type = match picture_type_num {
        0  => PictureType::Other,
        1  => PictureType::FileIconStandard,
        2  => PictureType::FileIcon,
        3  => PictureType::FrontCover,
        4  => PictureType::BackCover,
        5  => PictureType::LeafletPage,
        6  => PictureType::Media,
        7  => PictureType::LeadArtist,
        8  => PictureType::Artist,
        9  => PictureType::Conductor,
        10 => PictureType::Band,
        11 => PictureType::Composer,
        12 => PictureType::Lyricist,
        13 => PictureType::RecordingLocation,
        14 => PictureType::DuringRecording,
        15 => PictureType::DuringPerformace,
        16 => PictureType::VideoScreenCapture,
        17 => PictureType::Fish,
        18 => PictureType::Illustration,
        19 => PictureType::BandLogoType,
        20 => PictureType::PublisherLogoType,
        _  => PictureType::Other,
      };

      BlockData::Picture(Picture {
        picture_type: picture_type,
        mime_type: mime_type,
        description: description,
        width: width,
        height: height,
        depth: depth,
        colors: colors,
        data: data,
      })
    }
  )
);

// As of FLAC v1.3.1, there is support for up to 127 different metadata
// `Block`s but actually 7 that are implemented. When the `Block` type isn't
// recognised, this block gets skipped over with this parser.
fn unknown(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  map!(input, take!(length), BlockData::Unknown)
}

named!(header <&[u8], (u8, bool, u32)>,
  chain!(
    block_byte: be_u8 ~
    length: map!(take!(3), to_u32),
    || {
      let is_last    = (block_byte >> 7) == 1;
      let block_type = block_byte & 0b01111111;

      (block_type, is_last, length)
    }
  )
);

fn block_data(input: &[u8], block_type: u8, length: u32)
              -> IResult<&[u8], BlockData> {
  match block_type {
    0       => stream_info(input),
    1       => padding(input, length),
    2       => application(input, length),
    3       => seek_table(input, length),
    4       => vorbis_comment(input),
    5       => cue_sheet(input),
    6       => picture(input),
    7...126 => unknown(input, length),
    _       => IResult::Error(Err::Position(ErrorCode::Alt as u32, input)),
  }
}

named!(block <&[u8], Block>,
  chain!(
    block_header: header ~
    data: apply!(block_data, block_header.0, block_header.2),
    || {
      Block {
        is_last: block_header.1,
        length: block_header.2,
        data: data
      }
    }
  )
);

/// Parses one or more metadata `Block`s
///
/// The first block should always be `StreamInfo` since that is the only
/// required `Block`. At the moment `many_blocks` doesn't check that
/// requirement.
pub fn many_blocks(input: &[u8]) -> IResult<&[u8], Vec<Block>> {
  let mut is_last   = false;
  let mut blocks    = Vec::new();
  let mut start     = 0;
  let mut remaining = input.len();

  while !is_last {
    match block(&input[start..]) {
      IResult::Done(i, block) => {
        let result_len = i.len();

        if result_len == input[start..].len() {
          break;
        }

        start    += remaining - result_len;
        remaining = result_len;
        is_last   = block.is_last;

        blocks.push(block);
      }
      _                       => break,
    }
  }

  if blocks.len() == 0 {
    IResult::Error(Err::Position(ErrorCode::Many1 as u32, input))
  } else if is_last {
    IResult::Done(&input[start..], blocks)
  } else {
    IResult::Incomplete(Needed::Unknown)
  }
}
<commit_msg>Check to see if indices_num is zero<commit_after>use nom::{
  be_u8, be_u16, be_u32, be_u64,
  le_u32,
  IResult,
  ErrorCode, Err,
  Needed,
};

use std::str::from_utf8;

use metadata::{
  Block, BlockData,
  StreamInfo, Application, VorbisComment, CueSheet, Picture,
  SeekPoint, CueSheetTrack, CueSheetTrackIndex, PictureType,
};

use utility::to_u32;

named!(stream_info <&[u8], BlockData>,
  chain!(
    min_block_size: be_u16 ~
    max_block_size: be_u16 ~
    min_frame_size: map!(take!(3), to_u32) ~
    max_frame_size: map!(take!(3), to_u32) ~
    bytes: take!(8) ~
    md5_sum: take!(16),
    || {
      let sample_rate     = ((bytes[0] as u32) << 12) +
                            ((bytes[1] as u32) << 4)  +
                            (bytes[2] as u32) >> 4;
      let channels        = (bytes[2] >> 1) & 0b0111;
      let bits_per_sample = ((bytes[2] & 0b01) << 4) +
                            bytes[3] >> 4;
      let total_samples   = (((bytes[3] as u64) & 0x0f) << 32) +
                            ((bytes[4] as u64) << 24) +
                            ((bytes[5] as u64) << 16) +
                            ((bytes[6] as u64) << 8) +
                            (bytes[7] as u64);

      BlockData::StreamInfo(StreamInfo {
        min_block_size: min_block_size,
        max_block_size: max_block_size,
        min_frame_size: min_frame_size,
        max_frame_size: max_frame_size,
        sample_rate: sample_rate,
        channels: channels + 1,
        bits_per_sample: bits_per_sample + 1,
        total_samples: total_samples,
        md5_sum: md5_sum,
      })
    }
  )
);

fn padding(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  map!(input, take!(length), |_| BlockData::Padding(0))
}

fn application(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  chain!(input,
    id: take_str!(4) ~
    data: take!(length - 4),
    || {
      BlockData::Application(Application {
        id: id,
        data: data,
      })
    }
  )
}

named!(seek_point <&[u8], SeekPoint>,
  chain!(
    sample_number: be_u64 ~
    stream_offset: be_u64 ~
    frame_samples: be_u16,
    || {
      SeekPoint {
        sample_number: sample_number,
        stream_offset: stream_offset,
        frame_samples: frame_samples,
      }
    }
  )
);

fn seek_table(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  let seek_count = (length / 18) as usize;

  map!(input, count!(seek_point, seek_count), BlockData::SeekTable)
}

named!(vorbis_comment <&[u8], BlockData>,
  chain!(
    vendor_string_length: le_u32 ~
    vendor_string: take_str!(vendor_string_length)  ~
    number_of_comments: le_u32 ~
    comments: count!(comment_field, number_of_comments as usize),
    || {
      BlockData::VorbisComment(VorbisComment {
        vendor_string: vendor_string,
        comments: comments,
      })
    }
  )
);

named!(comment_field <&[u8], &str>,
  chain!(
    comment_length: le_u32 ~
    comment: take_str!(comment_length),
    || { comment }
  )
);

named!(cue_sheet <&[u8], BlockData>,
  chain!(
    media_catalog_number: take_str!(128) ~
    lead_in: be_u64 ~
    bytes: take!(259) ~ // TODO: last (7 + 258 * 8) bits must be 0
    num_tracks: be_u8 ~
    tracks: count!(cue_sheet_track, num_tracks as usize),
    || {
      let is_cd = ((bytes[0] >> 7) & 0b01) == 1;

      BlockData::CueSheet(CueSheet {
        media_catalog_number: media_catalog_number,
        lead_in: lead_in,
        is_cd: is_cd,
        tracks: tracks,
      })
    }
  )
);

named!(cue_sheet_track <&[u8], CueSheetTrack>,
  chain!(
    offset: be_u64 ~
    number: be_u8 ~
    isrc: take_str!(12) ~
    bytes: take!(14) ~ // TODO: last (6 + 13 * 8) bits must be 0
    num_indices: be_u8 ~
    indices: cond!(
      num_indices != 0,
      count!(cue_sheet_track_index, num_indices as usize)
    ),
    || {
      let isnt_audio      = ((bytes[0] >> 7) & 0b01) == 1;
      let is_pre_emphasis = ((bytes[0] >> 6) & 0b01) == 1;

      CueSheetTrack {
        offset: offset,
        number: number,
        isrc: isrc,
        isnt_audio: isnt_audio,
        is_pre_emphasis: is_pre_emphasis,
        indices: indices.unwrap_or(Vec::new()),
      }
    }
  )
);

named!(cue_sheet_track_index <&[u8], CueSheetTrackIndex>,
  chain!(
    offset: be_u64 ~
    number: be_u8 ~
    take!(3), // TODO: these bytes must be 0
    || {
      CueSheetTrackIndex {
        offset: offset,
        number: number,
      }
    }
  )
);

named!(picture <&[u8], BlockData>,
  chain!(
    picture_type_num: be_u32 ~
    mime_type_length:  be_u32 ~
    mime_type: take_str!(mime_type_length) ~
    description_length: be_u32 ~
    description: take_str!(description_length) ~
    width: be_u32 ~
    height: be_u32 ~
    depth: be_u32 ~
    colors: be_u32 ~
    data_length: be_u32 ~
    data: take!(data_length),
    || {
      let picture_type = match picture_type_num {
        0  => PictureType::Other,
        1  => PictureType::FileIconStandard,
        2  => PictureType::FileIcon,
        3  => PictureType::FrontCover,
        4  => PictureType::BackCover,
        5  => PictureType::LeafletPage,
        6  => PictureType::Media,
        7  => PictureType::LeadArtist,
        8  => PictureType::Artist,
        9  => PictureType::Conductor,
        10 => PictureType::Band,
        11 => PictureType::Composer,
        12 => PictureType::Lyricist,
        13 => PictureType::RecordingLocation,
        14 => PictureType::DuringRecording,
        15 => PictureType::DuringPerformace,
        16 => PictureType::VideoScreenCapture,
        17 => PictureType::Fish,
        18 => PictureType::Illustration,
        19 => PictureType::BandLogoType,
        20 => PictureType::PublisherLogoType,
        _  => PictureType::Other,
      };

      BlockData::Picture(Picture {
        picture_type: picture_type,
        mime_type: mime_type,
        description: description,
        width: width,
        height: height,
        depth: depth,
        colors: colors,
        data: data,
      })
    }
  )
);

// As of FLAC v1.3.1, there is support for up to 127 different metadata
// `Block`s but actually 7 that are implemented. When the `Block` type isn't
// recognised, this block gets skipped over with this parser.
fn unknown(input: &[u8], length: u32) -> IResult<&[u8], BlockData> {
  map!(input, take!(length), BlockData::Unknown)
}

named!(header <&[u8], (u8, bool, u32)>,
  chain!(
    block_byte: be_u8 ~
    length: map!(take!(3), to_u32),
    || {
      let is_last    = (block_byte >> 7) == 1;
      let block_type = block_byte & 0b01111111;

      (block_type, is_last, length)
    }
  )
);

fn block_data(input: &[u8], block_type: u8, length: u32)
              -> IResult<&[u8], BlockData> {
  match block_type {
    0       => stream_info(input),
    1       => padding(input, length),
    2       => application(input, length),
    3       => seek_table(input, length),
    4       => vorbis_comment(input),
    5       => cue_sheet(input),
    6       => picture(input),
    7...126 => unknown(input, length),
    _       => IResult::Error(Err::Position(ErrorCode::Alt as u32, input)),
  }
}

named!(block <&[u8], Block>,
  chain!(
    block_header: header ~
    data: apply!(block_data, block_header.0, block_header.2),
    || {
      Block {
        is_last: block_header.1,
        length: block_header.2,
        data: data
      }
    }
  )
);

/// Parses one or more metadata `Block`s
///
/// The first block should always be `StreamInfo` since that is the only
/// required `Block`. At the moment `many_blocks` doesn't check that
/// requirement.
pub fn many_blocks(input: &[u8]) -> IResult<&[u8], Vec<Block>> {
  let mut is_last   = false;
  let mut blocks    = Vec::new();
  let mut start     = 0;
  let mut remaining = input.len();

  while !is_last {
    match block(&input[start..]) {
      IResult::Done(i, block) => {
        let result_len = i.len();

        if result_len == input[start..].len() {
          break;
        }

        start    += remaining - result_len;
        remaining = result_len;
        is_last   = block.is_last;

        blocks.push(block);
      }
      _                       => break,
    }
  }

  if blocks.len() == 0 {
    IResult::Error(Err::Position(ErrorCode::Many1 as u32, input))
  } else if is_last {
    IResult::Done(&input[start..], blocks)
  } else {
    IResult::Incomplete(Needed::Unknown)
  }
}
<|endoftext|>