<commit_before>//! Rust friendly bindings to the various *nix system functions.
//!
//! Modules are structured according to the C header file that they would be
//! defined in.
#![crate_name = "nix"]
#![cfg(unix)]
#![allow(non_camel_case_types)]
// latest bitflags triggers a rustc bug with cross-crate macro expansions causing dead_code
// warnings even though the macro expands into something with allow(dead_code)
#![allow(dead_code)]
#![deny(warnings)]

#[macro_use]
extern crate bitflags;

#[macro_use]
extern crate cfg_if;
extern crate void;

#[cfg(test)]
extern crate nix_test as nixtest;

#[macro_use] mod macros;

// In rust 1.8+ this should be `pub extern crate libc` but prior
// to https://github.com/rust-lang/rust/issues/26775 being resolved
// it is necessary to get a little creative.
pub mod libc {
    extern crate libc;
    pub use self::libc::*;
}

pub use libc::{c_int, c_void};
pub use errno::Errno;

pub mod errno;
pub mod features;
pub mod fcntl;

#[cfg(any(target_os = "linux", target_os = "android"))]
pub mod mount;

#[cfg(any(target_os = "linux"))]
pub mod mqueue;

#[cfg(any(target_os = "linux", target_os = "macos"))]
pub mod poll;

pub mod net;

#[cfg(any(target_os = "linux", target_os = "android"))]
pub mod sched;

pub mod sys;

// This can be implemented for other platforms as soon as libc
// provides bindings for them.
#[cfg(all(target_os = "linux",
          any(target_arch = "x86", target_arch = "x86_64")))]
pub mod ucontext;

pub mod unistd;

/*
 *
 * ===== Result / Error =====
 *
 */

use libc::c_char;
use std::{ptr, result};
use std::ffi::{CStr, OsStr};
use std::path::{Path, PathBuf};
use std::os::unix::ffi::OsStrExt;
use std::io;
use std::fmt;
use std::error;
use libc::PATH_MAX;

pub type Result<T> = result::Result<T, Error>;

#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Error {
    Sys(errno::Errno),
    InvalidPath,
}

impl Error {
    pub fn from_errno(errno: errno::Errno) -> Error {
        Error::Sys(errno)
    }

    pub fn last() -> Error {
        Error::Sys(errno::Errno::last())
    }

    pub fn invalid_argument() -> Error {
        Error::Sys(errno::EINVAL)
    }

    pub fn errno(&self) -> errno::Errno {
        match *self {
            Error::Sys(errno) => errno,
            Error::InvalidPath => errno::Errno::EINVAL,
        }
    }
}

impl From<errno::Errno> for Error {
    fn from(errno: errno::Errno) -> Error { Error::from_errno(errno) }
}

impl error::Error for Error {
    fn description(&self) -> &str {
        match self {
            &Error::InvalidPath => "Invalid path",
            &Error::Sys(ref errno) => errno.desc(),
        }
    }
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &Error::InvalidPath => write!(f, "Invalid path"),
            &Error::Sys(errno) => write!(f, "{:?}: {}", errno, errno.desc()),
        }
    }
}

impl From<Error> for io::Error {
    fn from(err: Error) -> Self {
        match err {
            Error::InvalidPath => io::Error::new(io::ErrorKind::InvalidInput, err),
            Error::Sys(errno) => io::Error::from_raw_os_error(errno as i32),
        }
    }
}

pub trait NixPath {
    fn len(&self) -> usize;

    fn with_nix_path<T, F>(&self, f: F) -> Result<T>
        where F: FnOnce(&CStr) -> T;
}

impl NixPath for str {
    fn len(&self) -> usize {
        NixPath::len(OsStr::new(self))
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T>
        where F: FnOnce(&CStr) -> T {
            OsStr::new(self).with_nix_path(f)
        }
}

impl NixPath for OsStr {
    fn len(&self) -> usize {
        self.as_bytes().len()
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T>
        where F: FnOnce(&CStr) -> T {
            self.as_bytes().with_nix_path(f)
        }
}

impl NixPath for CStr {
    fn len(&self) -> usize {
        self.to_bytes().len()
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T>
            where F: FnOnce(&CStr) -> T {
        // Equivalence with the [u8] impl.
        if self.len() >= PATH_MAX as usize {
            return Err(Error::InvalidPath);
        }

        Ok(f(self))
    }
}

impl NixPath for [u8] {
    fn len(&self) -> usize {
        self.len()
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T>
            where F: FnOnce(&CStr) -> T {
        let mut buf = [0u8; PATH_MAX as usize];

        if self.len() >= PATH_MAX as usize {
            return Err(Error::InvalidPath);
        }

        match self.iter().position(|b| *b == 0) {
            Some(_) => Err(Error::InvalidPath),
            None => {
                unsafe {
                    // TODO: Replace with bytes::copy_memory. rust-lang/rust#24028
                    ptr::copy_nonoverlapping(self.as_ptr(), buf.as_mut_ptr(), self.len());
                    Ok(f(CStr::from_ptr(buf.as_ptr() as *const c_char)))
                }

            }
        }
    }
}

impl NixPath for Path {
    fn len(&self) -> usize {
        NixPath::len(self.as_os_str())
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T> where F: FnOnce(&CStr) -> T {
        self.as_os_str().with_nix_path(f)
    }
}

impl NixPath for PathBuf {
    fn len(&self) -> usize {
        NixPath::len(self.as_os_str())
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T> where F: FnOnce(&CStr) -> T {
        self.as_os_str().with_nix_path(f)
    }
}

/// Treats `None` as an empty string.
impl<'a, NP: ?Sized + NixPath>  NixPath for Option<&'a NP> {
    fn len(&self) -> usize {
        self.map_or(0, NixPath::len)
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T> where F: FnOnce(&CStr) -> T {
        if let Some(nix_path) = *self {
            nix_path.with_nix_path(f)
        } else {
            unsafe { CStr::from_ptr("\0".as_ptr() as *const _).with_nix_path(f) }
        }
    }
}
<commit_msg>Only deny warnings in test<commit_after>//! Rust friendly bindings to the various *nix system functions.
//!
//! Modules are structured according to the C header file that they would be
//! defined in.
#![crate_name = "nix"]
#![cfg(unix)]
#![allow(non_camel_case_types)]
// latest bitflags triggers a rustc bug with cross-crate macro expansions causing dead_code
// warnings even though the macro expands into something with allow(dead_code)
#![allow(dead_code)]
#![cfg_attr(test, deny(warnings))]

#[macro_use]
extern crate bitflags;

#[macro_use]
extern crate cfg_if;
extern crate void;

#[cfg(test)]
extern crate nix_test as nixtest;

#[macro_use] mod macros;

// In rust 1.8+ this should be `pub extern crate libc` but prior
// to https://github.com/rust-lang/rust/issues/26775 being resolved
// it is necessary to get a little creative.
pub mod libc {
    extern crate libc;
    pub use self::libc::*;
}

pub use libc::{c_int, c_void};
pub use errno::Errno;

pub mod errno;
pub mod features;
pub mod fcntl;

#[cfg(any(target_os = "linux", target_os = "android"))]
pub mod mount;

#[cfg(any(target_os = "linux"))]
pub mod mqueue;

#[cfg(any(target_os = "linux", target_os = "macos"))]
pub mod poll;

pub mod net;

#[cfg(any(target_os = "linux", target_os = "android"))]
pub mod sched;

pub mod sys;

// This can be implemented for other platforms as soon as libc
// provides bindings for them.
#[cfg(all(target_os = "linux",
          any(target_arch = "x86", target_arch = "x86_64")))]
pub mod ucontext;

pub mod unistd;

/*
 *
 * ===== Result / Error =====
 *
 */

use libc::c_char;
use std::{ptr, result};
use std::ffi::{CStr, OsStr};
use std::path::{Path, PathBuf};
use std::os::unix::ffi::OsStrExt;
use std::io;
use std::fmt;
use std::error;
use libc::PATH_MAX;

pub type Result<T> = result::Result<T, Error>;

#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Error {
    Sys(errno::Errno),
    InvalidPath,
}

impl Error {
    pub fn from_errno(errno: errno::Errno) -> Error {
        Error::Sys(errno)
    }

    pub fn last() -> Error {
        Error::Sys(errno::Errno::last())
    }

    pub fn invalid_argument() -> Error {
        Error::Sys(errno::EINVAL)
    }

    pub fn errno(&self) -> errno::Errno {
        match *self {
            Error::Sys(errno) => errno,
            Error::InvalidPath => errno::Errno::EINVAL,
        }
    }
}

impl From<errno::Errno> for Error {
    fn from(errno: errno::Errno) -> Error { Error::from_errno(errno) }
}

impl error::Error for Error {
    fn description(&self) -> &str {
        match self {
            &Error::InvalidPath => "Invalid path",
            &Error::Sys(ref errno) => errno.desc(),
        }
    }
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            &Error::InvalidPath => write!(f, "Invalid path"),
            &Error::Sys(errno) => write!(f, "{:?}: {}", errno, errno.desc()),
        }
    }
}

impl From<Error> for io::Error {
    fn from(err: Error) -> Self {
        match err {
            Error::InvalidPath => io::Error::new(io::ErrorKind::InvalidInput, err),
            Error::Sys(errno) => io::Error::from_raw_os_error(errno as i32),
        }
    }
}

pub trait NixPath {
    fn len(&self) -> usize;

    fn with_nix_path<T, F>(&self, f: F) -> Result<T>
        where F: FnOnce(&CStr) -> T;
}

impl NixPath for str {
    fn len(&self) -> usize {
        NixPath::len(OsStr::new(self))
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T>
        where F: FnOnce(&CStr) -> T {
            OsStr::new(self).with_nix_path(f)
        }
}

impl NixPath for OsStr {
    fn len(&self) -> usize {
        self.as_bytes().len()
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T>
        where F: FnOnce(&CStr) -> T {
            self.as_bytes().with_nix_path(f)
        }
}

impl NixPath for CStr {
    fn len(&self) -> usize {
        self.to_bytes().len()
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T>
            where F: FnOnce(&CStr) -> T {
        // Equivalence with the [u8] impl.
        if self.len() >= PATH_MAX as usize {
            return Err(Error::InvalidPath);
        }

        Ok(f(self))
    }
}

impl NixPath for [u8] {
    fn len(&self) -> usize {
        self.len()
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T>
            where F: FnOnce(&CStr) -> T {
        let mut buf = [0u8; PATH_MAX as usize];

        if self.len() >= PATH_MAX as usize {
            return Err(Error::InvalidPath);
        }

        match self.iter().position(|b| *b == 0) {
            Some(_) => Err(Error::InvalidPath),
            None => {
                unsafe {
                    // TODO: Replace with bytes::copy_memory. rust-lang/rust#24028
                    ptr::copy_nonoverlapping(self.as_ptr(), buf.as_mut_ptr(), self.len());
                    Ok(f(CStr::from_ptr(buf.as_ptr() as *const c_char)))
                }

            }
        }
    }
}

impl NixPath for Path {
    fn len(&self) -> usize {
        NixPath::len(self.as_os_str())
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T> where F: FnOnce(&CStr) -> T {
        self.as_os_str().with_nix_path(f)
    }
}

impl NixPath for PathBuf {
    fn len(&self) -> usize {
        NixPath::len(self.as_os_str())
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T> where F: FnOnce(&CStr) -> T {
        self.as_os_str().with_nix_path(f)
    }
}

/// Treats `None` as an empty string.
impl<'a, NP: ?Sized + NixPath>  NixPath for Option<&'a NP> {
    fn len(&self) -> usize {
        self.map_or(0, NixPath::len)
    }

    fn with_nix_path<T, F>(&self, f: F) -> Result<T> where F: FnOnce(&CStr) -> T {
        if let Some(nix_path) = *self {
            nix_path.with_nix_path(f)
        } else {
            unsafe { CStr::from_ptr("\0".as_ptr() as *const _).with_nix_path(f) }
        }
    }
}
<|endoftext|>


<commit_before>#![feature(collections, unboxed_closures, core)]
extern crate itertools;
extern crate rustc_serialize;

use std::collections::{BTreeMap, HashMap, HashSet};
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use std::iter;
use std::ops;

use itertools::{GroupBy, Itertools};

/// A Document contains an id and content.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Eq, Debug, RustcEncodable)]
pub struct Document {
    id: String,
    content: String,
}

impl Drop for Document {
    fn drop(&mut self) {
        println!("Dropped doc {}", self.id);
    }
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document { id: id.into(), content: content.into(), }
    }

    pub fn id(&self) -> &str {
        &self.id
    }

    pub fn content(&self) -> &str {
        &self.content
    }
}

impl Hash for Document {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        state.write(self.id.as_bytes());
    }
}

impl PartialEq<Document> for Document {
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug)]
struct IndexedDocument {
    pub doc: Arc<Document>,
    pub highlighted: (usize, usize),
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

fn analyze_doc(doc: &str)
-> iter::FlatMap<
    iter::Filter<
        GroupBy<bool, std::str::CharIndices, fn(&(usize, char)) -> bool>,
        fn(&(bool, Vec<(usize, char)>)) -> bool>,
    iter::Map<ops::Range<usize>, Ngrams>,
    fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>
{
    doc.char_indices()
        .group_by(is_whitespace as fn(&(usize, char)) -> bool)
        .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
        .flat_map(ngrams)
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to sets of Documents.
#[derive(Debug)]
pub struct InvertedIndex {
    index: BTreeMap<String, HashSet<IndexedDocument>>,
    docs: BTreeMap<String, Arc<Document>>,
}

impl InvertedIndex {
    pub fn new() -> InvertedIndex {
        InvertedIndex {
            index: BTreeMap::new(),
            docs: BTreeMap::new(),
        }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let doc = Arc::new(doc);
        let analyzed  = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, highlights) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&IndexedDocument { doc: doc.clone(), highlighted: highlights });
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            self.index.entry(ngram).or_insert_with(|| HashSet::new())
                .insert(IndexedDocument { doc: doc.clone(), highlighted: highlighted });
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    pub fn search(&self, query: &str) -> HashSet<SearchResult> {
        let map = query.split_whitespace()
            .flat_map(|word| self.index.get(&word.to_lowercase()))
            .flat_map(|docs| docs)
            .cloned()
            .fold(HashMap::new(), |mut map, search_result| {
                map.entry(search_result.doc)
                    .or_insert(Vec::new())
                    .push(search_result.highlighted);
                map
            });
        map.into_iter()
            .map(|(doc, mut highlights)| {
                highlights.sort();
                SearchResult { doc: doc, highlights: highlights }
            })
        .collect()
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug, RustcEncodable)]
pub struct SearchResult {
    doc: Arc<Document>,
    highlights: Vec<(usize, usize)>,
}

impl SearchResult {
    #[cfg(test)]
    fn new(doc: Arc<Document>, highlights: Vec<(usize, usize)>) -> SearchResult {
        SearchResult {
            doc: doc,
            highlights: highlights,
        }
    }

    pub fn doc(&self) -> &Arc<Document> {
        &self.doc
    }

    pub fn highlights(&self) -> &Vec<(usize, usize)> {
        &self.highlights
    }

    pub fn highlighted_content(&self) -> String {
        let &SearchResult {ref doc, ref highlights} = self;
        let content = doc.content();
        let mut begin_idx = 0;
        let mut parts = vec![];
        for &(begin, end) in highlights {
            parts.push(&content[begin_idx..begin]);
            parts.push("");
            parts.push(&content[begin..end]);
            parts.push("");
            begin_idx = end;
        }
        parts.push(&content[begin_idx..]);
        parts.into_iter().join("")
    }
}

#[test]
fn test_search1() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    index.index(doc1.clone());
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc2.clone());
    let search_results = index.search("today");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(25, 30)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 18)])
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());
}

#[test]
fn test_ngrams() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(6, 8), (25, 27)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 15)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());

}

#[test]
fn test_search2() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("2", "what to do today");
    let doc2 = Document::new("3", "hey today");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(5, 7), (11, 13)]),
        SearchResult::new(Arc::new(doc2), vec![(4, 6)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!(expected[0].highlighted_content(), "what to do today");
}

#[test]
fn test_unicode() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc.clone());
    let to_search = "å";
    let search_results = index.search(to_search);
    let &SearchResult { ref doc, ref highlights } = search_results.iter().next().unwrap();
    let (begin, end) = highlights[0];
    assert_eq!(&doc.content()[begin..end], to_search);
}

#[test]
fn test_update_doc() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc);
    let doc = Document::new("0", "different");
    index.index(doc);
    let search_results = index.search("å");
    assert!(search_results.is_empty());
    assert_eq!(index.docs.len(), 1);
}
<commit_msg>Make documents RustcDecodable<commit_after>#![feature(collections, unboxed_closures, core)]
extern crate itertools;
extern crate rustc_serialize;

use std::collections::{BTreeMap, HashMap, HashSet};
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use std::iter;
use std::ops;

use itertools::{GroupBy, Itertools};

/// A Document contains an id and content.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Eq, Debug, RustcEncodable, RustcDecodable)]
pub struct Document {
    id: String,
    content: String,
}

impl Drop for Document {
    fn drop(&mut self) {
        println!("Dropped doc {}", self.id);
    }
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document { id: id.into(), content: content.into(), }
    }

    pub fn id(&self) -> &str {
        &self.id
    }

    pub fn content(&self) -> &str {
        &self.content
    }
}

impl Hash for Document {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        state.write(self.id.as_bytes());
    }
}

impl PartialEq<Document> for Document {
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug)]
struct IndexedDocument {
    pub doc: Arc<Document>,
    pub highlighted: (usize, usize),
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

fn analyze_doc(doc: &str)
-> iter::FlatMap<
    iter::Filter<
        GroupBy<bool, std::str::CharIndices, fn(&(usize, char)) -> bool>,
        fn(&(bool, Vec<(usize, char)>)) -> bool>,
    iter::Map<ops::Range<usize>, Ngrams>,
    fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>
{
    doc.char_indices()
        .group_by(is_whitespace as fn(&(usize, char)) -> bool)
        .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
        .flat_map(ngrams)
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to sets of Documents.
#[derive(Debug)]
pub struct InvertedIndex {
    index: BTreeMap<String, HashSet<IndexedDocument>>,
    docs: BTreeMap<String, Arc<Document>>,
}

impl InvertedIndex {
    pub fn new() -> InvertedIndex {
        InvertedIndex {
            index: BTreeMap::new(),
            docs: BTreeMap::new(),
        }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let doc = Arc::new(doc);
        let analyzed  = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, highlights) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&IndexedDocument { doc: doc.clone(), highlighted: highlights });
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            self.index.entry(ngram).or_insert_with(|| HashSet::new())
                .insert(IndexedDocument { doc: doc.clone(), highlighted: highlighted });
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    pub fn search(&self, query: &str) -> HashSet<SearchResult> {
        let map = query.split_whitespace()
            .flat_map(|word| self.index.get(&word.to_lowercase()))
            .flat_map(|docs| docs)
            .cloned()
            .fold(HashMap::new(), |mut map, search_result| {
                map.entry(search_result.doc)
                    .or_insert(Vec::new())
                    .push(search_result.highlighted);
                map
            });
        map.into_iter()
            .map(|(doc, mut highlights)| {
                highlights.sort();
                SearchResult { doc: doc, highlights: highlights }
            })
        .collect()
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug, RustcEncodable)]
pub struct SearchResult {
    doc: Arc<Document>,
    highlights: Vec<(usize, usize)>,
}

impl SearchResult {
    #[cfg(test)]
    fn new(doc: Arc<Document>, highlights: Vec<(usize, usize)>) -> SearchResult {
        SearchResult {
            doc: doc,
            highlights: highlights,
        }
    }

    pub fn doc(&self) -> &Arc<Document> {
        &self.doc
    }

    pub fn highlights(&self) -> &Vec<(usize, usize)> {
        &self.highlights
    }

    pub fn highlighted_content(&self) -> String {
        let &SearchResult {ref doc, ref highlights} = self;
        let content = doc.content();
        let mut begin_idx = 0;
        let mut parts = vec![];
        for &(begin, end) in highlights {
            parts.push(&content[begin_idx..begin]);
            parts.push("");
            parts.push(&content[begin..end]);
            parts.push("");
            begin_idx = end;
        }
        parts.push(&content[begin_idx..]);
        parts.into_iter().join("")
    }
}

#[test]
fn test_search1() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    index.index(doc1.clone());
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc2.clone());
    let search_results = index.search("today");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(25, 30)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 18)])
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());
}

#[test]
fn test_ngrams() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(6, 8), (25, 27)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 15)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());

}

#[test]
fn test_search2() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("2", "what to do today");
    let doc2 = Document::new("3", "hey today");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(5, 7), (11, 13)]),
        SearchResult::new(Arc::new(doc2), vec![(4, 6)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!(expected[0].highlighted_content(), "what to do today");
}

#[test]
fn test_unicode() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc.clone());
    let to_search = "å";
    let search_results = index.search(to_search);
    let &SearchResult { ref doc, ref highlights } = search_results.iter().next().unwrap();
    let (begin, end) = highlights[0];
    assert_eq!(&doc.content()[begin..end], to_search);
}

#[test]
fn test_update_doc() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc);
    let doc = Document::new("0", "different");
    index.index(doc);
    let search_results = index.search("å");
    assert!(search_results.is_empty());
    assert_eq!(index.docs.len(), 1);
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate nom;

use nom::IResult;
use nom::IResult::*;
use nom::Err::*;
use std::fmt;

mod helper;
use helper::*;

macro_rules! errprint {
    ($($arg:tt)*) => (
        std::io::stderr().write_fmt(format_args!($($arg)*))
        );
}

macro_rules! errln {
    ($fmt:expr) => (errprint!(concat!($fmt, "\n")));
    ($fmt:expr, $($arg:tt)*) => (errprint!(concat!($fmt, "\n"), $($arg)*));
}

#[allow(unused_imports)] use std::io::Write;

#[derive(Eq,PartialEq)]
struct Date {
    year: i32,
    month: u32,
    day: u32,
}

#[derive(Eq,PartialEq)]
struct Time {
    hour: u32,
    minute: u32,
    second: u32,
    tz_offset: i32,
}

#[derive(Eq,PartialEq)]
struct DateTime {
    date: Date,
    time: Time,
}

impl fmt::Debug for Date {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:0>4?}-{:0>2?}-{:0>2?}",
               self.year, self.month, self.day)
    }
}
impl fmt::Debug for Time {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:0>2?}:{:0>2?}:{:0>2?}Z{:0>4?}",
               self.hour, self.minute, self.second, self.tz_offset*100)
    }
}
impl fmt::Debug for DateTime {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:?}T{:?}", self.date, self.time)
    }
}

/// Take n bytes and ensure that they are only in the provided range of bytes
macro_rules! take_n_between(
    ($input:expr, $count:expr, $min:expr, $max:expr) => (
        {
            let new_min = $min as u8;
            let new_max = $max as u8;
            let cnt = $count as usize;
            if $input.len() < cnt {
                nom::IResult::Incomplete(nom::Needed::Size(cnt))
            } else {
                for idx in 0..$count {
                    if $input[idx] < new_min || $input[idx] > new_max {
                        return nom::IResult::Error(nom::Err::Position(42 as u32,$input));
                    }
                }

                nom::IResult::Done(&$input[$count..], &$input[0..$count])
            }
        }
        );
    );

macro_rules! char_between(
    ($input:expr, $min:expr, $max:expr) => (
        take_n_between!($input, 1, $min, $max)
    );
);

macro_rules! empty_or(
    ($i:expr, $submac:ident!( $($args:tt)* )) => (
        if $i.len() == 0 {
            nom::IResult::Done($i, None)
        } else {
            match $submac!($i, $($args)*) {
                nom::IResult::Done(i,o)     => nom::IResult::Done(i, Some(o)),
                nom::IResult::Error(e)      => nom::IResult::Error(e),
                nom::IResult::Incomplete(i) => nom::IResult::Incomplete(i)

            }
        }
    );
);

named!(year_prefix, alt!(tag!("+") | tag!("-")));

named!(pub year <&[u8], i32>, chain!(
        pref: opt!(year_prefix) ~
        year: take_n_between!(4, '0', '9')
        ,
        || {
            match pref {
                Some(b"-") => -buf_to_i32(year),
                _ => buf_to_i32(year)
            }
        }));

named!(lower_month <&[u8], u32>, chain!(tag!("0") ~ s:char_between!('1', '9') , || buf_to_u32(s)));
named!(upper_month <&[u8], u32>, chain!(tag!("1") ~ s:char_between!('0', '2') ,       || 10+buf_to_u32(s)));

named!(month <&[u8], u32>, alt!(lower_month | upper_month));

named!(day_zero <&[u8], u32>,  chain!(tag!("0") ~ s:char_between!('1', '9') ,  || buf_to_u32(s)));
named!(day_one <&[u8], u32>,   chain!(tag!("1") ~ s:char_between!('0', '9') , || 10+buf_to_u32(s)));
named!(day_two <&[u8], u32>,   chain!(tag!("2") ~ s:char_between!('0', '9') , || 20+buf_to_u32(s)));
named!(day_three <&[u8], u32>, chain!(tag!("3") ~ s:char_between!('0', '1') ,         || 30+buf_to_u32(s)));

named!(day <&[u8], u32>, alt!(day_zero | day_one | day_two | day_three));

named!(date <&[u8], Date>, chain!(
        y: year ~
        tag!("-") ~
        m: month ~
        tag!("-") ~
        d: day
        ,
        || { Date{ year: y, month: m, day: d } }
        ));

named!(lower_hour <&[u8], u32>, chain!(f:char_between!('0','1') ~ s:char_between!('0','9') ,
                                       || { buf_to_u32(f)*10 + buf_to_u32(s) } ));
named!(upper_hour <&[u8], u32>, chain!(tag!("2") ~ s:char_between!('0','4') , || 20+buf_to_u32(s)));
named!(hour <&[u8], u32>, alt!(lower_hour | upper_hour));

named!(below_sixty <&[u8], u32>, chain!(f:char_between!('0','5') ~ s:char_between!('0','9') ,
                                       || { buf_to_u32(f)*10 + buf_to_u32(s) } ));

named!(minute <&[u8], u32>, call!(below_sixty));
named!(second <&[u8], u32>, call!(below_sixty));

named!(append_seconds <&[u8], u32>, chain!(tag!(":") ~ sec:second, || sec));

named!(time <&[u8], Time>, chain!(
        h: hour ~
        tag!(":") ~
        m: minute ~
        s: empty_or!(
            chain!(tag!(":") ~ s:second , || s)
            )
        ,
        || {
            Time {
                hour: h,
                minute: m,
                second: s.unwrap_or(0),
                tz_offset: 0
            }
        }
        ));

named!(timezone <&[u8], u32>, chain!(
        tag!("+") ~
        h: hour ~
        tag!(":")? ~
        m: minute
        ,
        || { h }));

named!(tz_z, tag!("Z"));
named!(timezone_utc <&[u8], u32>, map!(tz_z, |_| 0));

named!(time_with_timezone <&[u8], Time>, chain!(
        t: time ~
        s: empty_or!(alt!(timezone_utc | timezone))
        ,
        || {
            Time {
                hour: t.hour,
                minute: t.minute,
                second: t.second,
                tz_offset: s.unwrap_or(0) as i32
            }
        }
        ));

#[test]
fn parse_year() {
    assert_eq!(Done(&[][..], 2015), year(b"2015"));
    assert_eq!(Done(&[][..], -0333), year(b"-0333"));

    assert_eq!(Done(&b"-"[..], 2015), year(b"2015-"));

    assert!(year(b"abcd").is_err());
    assert!(year(b"2a03").is_err());
}

#[test]
fn parse_month() {
    assert_eq!(Done(&[][..], 1), month(b"01"));
    assert_eq!(Done(&[][..], 6), month(b"06"));
    assert_eq!(Done(&[][..], 12), month(b"12"));
    assert_eq!(Done(&b"-"[..], 12), month(b"12-"));

    assert!(month(b"13").is_err());
    assert!(month(b"00").is_err());
}

#[test]
fn parse_day() {
    assert_eq!(Done(&[][..], 1), day(b"01"));
    assert_eq!(Done(&[][..], 12), day(b"12"));
    assert_eq!(Done(&[][..], 20), day(b"20"));
    assert_eq!(Done(&[][..], 28), day(b"28"));
    assert_eq!(Done(&[][..], 30), day(b"30"));
    assert_eq!(Done(&[][..], 31), day(b"31"));
    assert_eq!(Done(&b"-"[..], 31), day(b"31-"));

    assert!(day(b"00").is_err());
    assert!(day(b"32").is_err());
}

#[test]
fn parse_date() {
    assert_eq!(Done(&[][..], Date{ year: 2015, month: 6, day: 26 }), date(b"2015-06-26"));
    assert_eq!(Done(&[][..], Date{ year: -333, month: 7, day: 11 }), date(b"-0333-07-11"));

    assert!(date(b"201").is_incomplete());
    assert!(date(b"2015p00p00").is_err());
    assert!(date(b"pppp").is_err());
}

#[test]
fn parse_hour() {
    assert_eq!(Done(&[][..], 0), hour(b"00"));
    assert_eq!(Done(&[][..], 1), hour(b"01"));
    assert_eq!(Done(&[][..], 6), hour(b"06"));
    assert_eq!(Done(&[][..], 12), hour(b"12"));
    assert_eq!(Done(&[][..], 13), hour(b"13"));
    assert_eq!(Done(&[][..], 20), hour(b"20"));
    assert_eq!(Done(&[][..], 24), hour(b"24"));

    assert!(hour(b"25").is_err());
    assert!(hour(b"30").is_err());
    assert!(hour(b"ab").is_err());
}

#[test]
fn parse_minute() {
    assert_eq!(Done(&[][..], 0), minute(b"00"));
    assert_eq!(Done(&[][..], 1), minute(b"01"));
    assert_eq!(Done(&[][..], 30), minute(b"30"));
    assert_eq!(Done(&[][..], 59), minute(b"59"));

    assert!(minute(b"60").is_err());
    assert!(minute(b"61").is_err());
    assert!(minute(b"ab").is_err());
}

#[test]
fn parse_second() {
    assert_eq!(Done(&[][..], 0), second(b"00"));
    assert_eq!(Done(&[][..], 1), second(b"01"));
    assert_eq!(Done(&[][..], 30), second(b"30"));
    assert_eq!(Done(&[][..], 59), second(b"59"));

    assert!(second(b"60").is_err());
    assert!(second(b"61").is_err());
    assert!(second(b"ab").is_err());
}

#[test]
fn parse_time() {
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second: 16, tz_offset: 0}), time(b"16:43:16"));
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second:  0, tz_offset: 0}), time(b"16:43"));

    assert!(time(b"20:").is_incomplete());
    assert!(time(b"20p42p16").is_err());
    assert!(time(b"pppp").is_err());
}

#[test]
fn parse_time_with_timezone() {
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second: 16, tz_offset: 0}),
               time_with_timezone(b"16:43:16"));
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second: 16, tz_offset: 0}),
               time_with_timezone(b"16:43:16Z"));
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second: 16, tz_offset: 0}),
               time_with_timezone(b"16:43:16+00:00"));
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second: 16, tz_offset: 5}),
               time_with_timezone(b"16:43:16+05:00"));

    assert!(time_with_timezone(b"20:").is_incomplete());
    assert!(time_with_timezone(b"20p42p16").is_err());
    assert!(time_with_timezone(b"pppp").is_err());

    assert!(time_with_timezone(b"16:43:16+").is_err());
    assert!(time_with_timezone(b"16:43:16+0").is_err());
    assert!(time_with_timezone(b"16:43:16+05:").is_err());
}
<commit_msg>Actually return None<commit_after>#[macro_use]
extern crate nom;

use nom::IResult;
use nom::IResult::*;
use nom::Err::*;
use std::fmt;

mod helper;
use helper::*;

macro_rules! errprint {
    ($($arg:tt)*) => (
        std::io::stderr().write_fmt(format_args!($($arg)*))
        );
}

macro_rules! errln {
    ($fmt:expr) => (errprint!(concat!($fmt, "\n")));
    ($fmt:expr, $($arg:tt)*) => (errprint!(concat!($fmt, "\n"), $($arg)*));
}

#[allow(unused_imports)] use std::io::Write;

#[derive(Eq,PartialEq)]
struct Date {
    year: i32,
    month: u32,
    day: u32,
}

#[derive(Eq,PartialEq)]
struct Time {
    hour: u32,
    minute: u32,
    second: u32,
    tz_offset: i32,
}

#[derive(Eq,PartialEq)]
struct DateTime {
    date: Date,
    time: Time,
}

impl fmt::Debug for Date {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:0>4?}-{:0>2?}-{:0>2?}",
               self.year, self.month, self.day)
    }
}
impl fmt::Debug for Time {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:0>2?}:{:0>2?}:{:0>2?}Z{:0>4?}",
               self.hour, self.minute, self.second, self.tz_offset*100)
    }
}
impl fmt::Debug for DateTime {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:?}T{:?}", self.date, self.time)
    }
}

/// Take n bytes and ensure that they are only in the provided range of bytes
macro_rules! take_n_between(
    ($input:expr, $count:expr, $min:expr, $max:expr) => (
        {
            let new_min = $min as u8;
            let new_max = $max as u8;
            let cnt = $count as usize;
            if $input.len() < cnt {
                nom::IResult::Incomplete(nom::Needed::Size(cnt))
            } else {
                for idx in 0..$count {
                    if $input[idx] < new_min || $input[idx] > new_max {
                        return nom::IResult::Error(nom::Err::Position(42 as u32,$input));
                    }
                }

                nom::IResult::Done(&$input[$count..], &$input[0..$count])
            }
        }
        );
    );

macro_rules! char_between(
    ($input:expr, $min:expr, $max:expr) => (
        take_n_between!($input, 1, $min, $max)
    );
);

macro_rules! empty_or(
    ($i:expr, $submac:ident!( $($args:tt)* )) => (
        if $i.len() == 0 {
            nom::IResult::Done($i, None)
        } else {
            match $submac!($i, $($args)*) {
                nom::IResult::Done(i,o)     => nom::IResult::Done(i, Some(o)),
                nom::IResult::Error(_)      => nom::IResult::Done($i, None),
                nom::IResult::Incomplete(i) => nom::IResult::Incomplete(i)

            }
        }
    );
);

named!(year_prefix, alt!(tag!("+") | tag!("-")));

named!(pub year <&[u8], i32>, chain!(
        pref: opt!(year_prefix) ~
        year: take_n_between!(4, '0', '9')
        ,
        || {
            match pref {
                Some(b"-") => -buf_to_i32(year),
                _ => buf_to_i32(year)
            }
        }));

named!(lower_month <&[u8], u32>, chain!(tag!("0") ~ s:char_between!('1', '9') , || buf_to_u32(s)));
named!(upper_month <&[u8], u32>, chain!(tag!("1") ~ s:char_between!('0', '2') ,       || 10+buf_to_u32(s)));

named!(month <&[u8], u32>, alt!(lower_month | upper_month));

named!(day_zero <&[u8], u32>,  chain!(tag!("0") ~ s:char_between!('1', '9') ,  || buf_to_u32(s)));
named!(day_one <&[u8], u32>,   chain!(tag!("1") ~ s:char_between!('0', '9') , || 10+buf_to_u32(s)));
named!(day_two <&[u8], u32>,   chain!(tag!("2") ~ s:char_between!('0', '9') , || 20+buf_to_u32(s)));
named!(day_three <&[u8], u32>, chain!(tag!("3") ~ s:char_between!('0', '1') ,         || 30+buf_to_u32(s)));

named!(day <&[u8], u32>, alt!(day_zero | day_one | day_two | day_three));

named!(date <&[u8], Date>, chain!(
        y: year ~
        tag!("-") ~
        m: month ~
        tag!("-") ~
        d: day
        ,
        || { Date{ year: y, month: m, day: d } }
        ));

named!(lower_hour <&[u8], u32>, chain!(f:char_between!('0','1') ~ s:char_between!('0','9') ,
                                       || { buf_to_u32(f)*10 + buf_to_u32(s) } ));
named!(upper_hour <&[u8], u32>, chain!(tag!("2") ~ s:char_between!('0','4') , || 20+buf_to_u32(s)));
named!(hour <&[u8], u32>, alt!(lower_hour | upper_hour));

named!(below_sixty <&[u8], u32>, chain!(f:char_between!('0','5') ~ s:char_between!('0','9') ,
                                       || { buf_to_u32(f)*10 + buf_to_u32(s) } ));

named!(minute <&[u8], u32>, call!(below_sixty));
named!(second <&[u8], u32>, call!(below_sixty));

named!(append_seconds <&[u8], u32>, chain!(tag!(":") ~ sec:second, || sec));

named!(time <&[u8], Time>, chain!(
        h: hour ~
        tag!(":") ~
        m: minute ~
        s: empty_or!(
            chain!(tag!(":") ~ s:second , || s)
            )
        ,
        || {
            Time {
                hour: h,
                minute: m,
                second: s.unwrap_or(0),
                tz_offset: 0
            }
        }
        ));

named!(timezone <&[u8], u32>, chain!(
        tag!("+") ~
        h: hour ~
        tag!(":")? ~
        m: minute
        ,
        || { h }));

named!(tz_z, tag!("Z"));
named!(timezone_utc <&[u8], u32>, map!(tz_z, |_| 0));

named!(time_with_timezone <&[u8], Time>, chain!(
        t: time ~
        s: empty_or!(alt!(timezone_utc | timezone))
        ,
        || {
            Time {
                hour: t.hour,
                minute: t.minute,
                second: t.second,
                tz_offset: s.unwrap_or(0) as i32
            }
        }
        ));

#[test]
fn parse_year() {
    assert_eq!(Done(&[][..], 2015), year(b"2015"));
    assert_eq!(Done(&[][..], -0333), year(b"-0333"));

    assert_eq!(Done(&b"-"[..], 2015), year(b"2015-"));

    assert!(year(b"abcd").is_err());
    assert!(year(b"2a03").is_err());
}

#[test]
fn parse_month() {
    assert_eq!(Done(&[][..], 1), month(b"01"));
    assert_eq!(Done(&[][..], 6), month(b"06"));
    assert_eq!(Done(&[][..], 12), month(b"12"));
    assert_eq!(Done(&b"-"[..], 12), month(b"12-"));

    assert!(month(b"13").is_err());
    assert!(month(b"00").is_err());
}

#[test]
fn parse_day() {
    assert_eq!(Done(&[][..], 1), day(b"01"));
    assert_eq!(Done(&[][..], 12), day(b"12"));
    assert_eq!(Done(&[][..], 20), day(b"20"));
    assert_eq!(Done(&[][..], 28), day(b"28"));
    assert_eq!(Done(&[][..], 30), day(b"30"));
    assert_eq!(Done(&[][..], 31), day(b"31"));
    assert_eq!(Done(&b"-"[..], 31), day(b"31-"));

    assert!(day(b"00").is_err());
    assert!(day(b"32").is_err());
}

#[test]
fn parse_date() {
    assert_eq!(Done(&[][..], Date{ year: 2015, month: 6, day: 26 }), date(b"2015-06-26"));
    assert_eq!(Done(&[][..], Date{ year: -333, month: 7, day: 11 }), date(b"-0333-07-11"));

    assert!(date(b"201").is_incomplete());
    assert!(date(b"2015p00p00").is_err());
    assert!(date(b"pppp").is_err());
}

#[test]
fn parse_hour() {
    assert_eq!(Done(&[][..], 0), hour(b"00"));
    assert_eq!(Done(&[][..], 1), hour(b"01"));
    assert_eq!(Done(&[][..], 6), hour(b"06"));
    assert_eq!(Done(&[][..], 12), hour(b"12"));
    assert_eq!(Done(&[][..], 13), hour(b"13"));
    assert_eq!(Done(&[][..], 20), hour(b"20"));
    assert_eq!(Done(&[][..], 24), hour(b"24"));

    assert!(hour(b"25").is_err());
    assert!(hour(b"30").is_err());
    assert!(hour(b"ab").is_err());
}

#[test]
fn parse_minute() {
    assert_eq!(Done(&[][..], 0), minute(b"00"));
    assert_eq!(Done(&[][..], 1), minute(b"01"));
    assert_eq!(Done(&[][..], 30), minute(b"30"));
    assert_eq!(Done(&[][..], 59), minute(b"59"));

    assert!(minute(b"60").is_err());
    assert!(minute(b"61").is_err());
    assert!(minute(b"ab").is_err());
}

#[test]
fn parse_second() {
    assert_eq!(Done(&[][..], 0), second(b"00"));
    assert_eq!(Done(&[][..], 1), second(b"01"));
    assert_eq!(Done(&[][..], 30), second(b"30"));
    assert_eq!(Done(&[][..], 59), second(b"59"));

    assert!(second(b"60").is_err());
    assert!(second(b"61").is_err());
    assert!(second(b"ab").is_err());
}

#[test]
fn parse_time() {
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second: 16, tz_offset: 0}), time(b"16:43:16"));
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second:  0, tz_offset: 0}), time(b"16:43"));

    assert!(time(b"20:").is_incomplete());
    assert!(time(b"20p42p16").is_err());
    assert!(time(b"pppp").is_err());
}

#[test]
fn parse_time_with_timezone() {
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second: 16, tz_offset: 0}),
               time_with_timezone(b"16:43:16"));
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second: 16, tz_offset: 0}),
               time_with_timezone(b"16:43:16Z"));
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second: 16, tz_offset: 0}),
               time_with_timezone(b"16:43:16+00:00"));
    assert_eq!(Done(&[][..], Time{ hour: 16, minute: 43, second: 16, tz_offset: 5}),
               time_with_timezone(b"16:43:16+05:00"));

    assert!(time_with_timezone(b"20:").is_incomplete());
    assert!(time_with_timezone(b"20p42p16").is_err());
    assert!(time_with_timezone(b"pppp").is_err());

    assert!(time_with_timezone(b"16:43:16+").is_err());
    assert!(time_with_timezone(b"16:43:16+0").is_err());
    assert!(time_with_timezone(b"16:43:16+05:").is_err());
}
<|endoftext|>


<commit_before>#![crate_name = "binstat"]
#![deny(missing_doc)]

//! A library for modelling and reasoning about multidimensional
//! binary states.

use std::collections::Bitv;

/// Contains a binary state and the choices.
pub struct BinStateChoices {
    /// The state composed of bits.
    pub state: Bitv,
    /// The choices represented as bits that can be flipped.
    pub choices: Bitv
}

impl BinStateChoices {
    /// Gets pairs from pair of booleans.
    pub fn from_pairs(pairs: &[(bool, bool)]) -> BinStateChoices {
        BinStateChoices {
            state: pairs.iter().map(|&(a, _)| a).collect(),
            choices: pairs.iter().map(|&(_, b)| b).collect()
        }
    }
}

/// Models a multidimensional binary state.
pub struct BinGraph<TAction> {
    /// The states and the bits that can be flipped.
    pub data: Vec<BinStateChoices>,
    /// A description the action that takes when flipping bits.
    pub actions: Vec<TAction>,
}

impl<TAction> BinGraph<TAction> {
    /// Creates a new BinGraph.
    pub fn new() -> BinGraph<TAction> {
        BinGraph {
            data: Vec::new(),
            actions: Vec::new()
        }
    }

    /// Adds new information about state and choices.
    ///
    /// The first bool indicate state, the second whether it can change.
    pub fn push_pairs(&mut self, pairs: &[(bool, bool)]) {
        self.data.push(BinStateChoices::from_pairs(pairs));
    }
}

#[test]
fn test_exclusive() {
    let mut g: BinGraph<&'static str> = BinGraph::new();
    g.actions.push("Go swiming");
    g.actions.push("Read a book");

    g.push_pairs([
        (false, true), // go swimming
        (false, true), // read a book
    ]);
    g.push_pairs([
        (true, true), // is swimming, can stop.
        (false, false), // is not reading book, can't read book.
    ]);
}

<commit_msg>Added default suggestion<commit_after>#![crate_name = "binstat"]
#![deny(missing_doc)]

//! A library for modelling and reasoning about multidimensional
//! binary states.

#[cfg(test)]
extern crate debug;

use std::collections::Bitv;

/// Contains a binary state and the choices.
#[deriving(PartialEq, Eq, Show)]
pub struct BinStateChoices {
    /// The state composed of bits.
    pub state: Bitv,
    /// The choices represented as bits that can be flipped.
    pub choices: Bitv
}

impl BinStateChoices {
    /// Gets pairs from pair of booleans.
    pub fn from_pairs(pairs: &[(bool, bool)]) -> BinStateChoices {
        BinStateChoices {
            state: pairs.iter().map(|&(a, _)| a).collect(),
            choices: pairs.iter().map(|&(_, b)| b).collect()
        }
    }
}

/// Models a multidimensional binary state.
pub struct BinGraph<TAction> {
    /// The states and the bits that can be flipped.
    pub data: Vec<BinStateChoices>,
    /// A description the action that takes when flipping bits.
    pub actions: Vec<TAction>,
}

impl<TAction> BinGraph<TAction> {
    /// Creates a new BinGraph.
    pub fn new() -> BinGraph<TAction> {
        BinGraph {
            data: Vec::new(),
            actions: Vec::new()
        }
    }

    /// Adds new information about state and choices.
    ///
    /// The first bool indicate state, the second whether it can change.
    pub fn push_pairs(&mut self, pairs: &[(bool, bool)]) {
        self.data.push(BinStateChoices::from_pairs(pairs));
    }

    /// Returns the suggestion in default state that are open to all choices.
    pub fn default_suggestion(&self) -> BinStateChoices {
        let open_for_all_choices: Vec<(bool, bool)> =
            range(0, self.actions.len()).map(|_| (false, true)).collect();
        BinStateChoices::from_pairs(open_for_all_choices.as_slice())
    }

    /// Suggests a missing state with choices.
    pub fn suggestion(&self) -> Option<BinStateChoices> {
        if self.data.len() == 0 {
            Some(self.default_suggestion())
        } else {
            None
        }
    }
}

#[test]
fn test_exclusive() {
    let mut g: BinGraph<&'static str> = BinGraph::new();
    g.actions.push("Go swiming");
    g.actions.push("Read a book");

    let suggestion = g.suggestion().unwrap();
    assert_eq!(suggestion, BinStateChoices::from_pairs([
        (false, true),
        (false, true)
    ]));

    g.data.push(suggestion);

    g.push_pairs([
        (true, true), // is swimming, can stop.
        (false, false), // is not reading book, can't read book.
    ]);
}

<|endoftext|>


<commit_before>#![crate_id = "audiostream"]
#![doc(html_root_url = "http://www.rust-ci.org/tari/audiostream.rs/doc/audiostream/")]

#![experimental]
#![deny(dead_code,missing_doc)]
#![feature(macro_rules)]
#![feature(default_type_params)]

//! Audio steam pipelines and processing.
//!
//! Streams are represented as sequences of buffers, each of which contains
//! zero or more samples. Data is produced from `Source`s on demand and fed
//! into a chain of zero or more `Sink`s until it reaches the end of the
//! pipeline. The pipeline always operates in a "pull" mode, where `Source`s
//! yield buffers only as fast as requested by a `Sink`.
//!
//! Valid sample formats are represented with the `Sample` trait. In general,
//! buffers are of type `&[Sample]`, presenting a single channel of audio
//! data. However, this convention is not enforced.

extern crate libao = "ao";

use std::num::Zero;
use std::sync::atomics::{AtomicBool, AcqRel};

pub mod ao;
pub mod synth;

/// Type bound for sample formats.
///
/// Implementation assumes `f64` is sufficient to represent all other formats
/// without loss.
pub trait Sample : Num + NumCast {
    /// Maximum value of a valid sample.
    fn max() -> Self;
    /// Minimum value of a valid sample.
    fn min() -> Self;
    /// Clip a value to be in range [min, max] (inclusive).
    fn clip(&self) -> Self;

    /// Convert from `Self` to an arbitrary other sample format.
    ///
    /// The default intermediate format here is `f64`, capable of losslessly
    /// converting all formats shorter than 52 bits.
    fn convert<X: Sample, I: Sample = f64>(a: Self) -> X {
        let a_i: I = NumCast::from(a).unwrap();
        let self_max: Self = Sample::max();
        let self_max_i: I = NumCast::from(self_max).unwrap();
        let ratio: I = a_i / self_max_i;

        let x_max: X = Sample::max();
        let x_max_i: I = NumCast::from(x_max).unwrap();

        NumCast::from(ratio * x_max_i).unwrap()
    }
}

macro_rules! sample_impl(
    ($t:ty, $min:expr .. $max:expr) => (
        impl Sample for $t {
            fn max() -> $t { $min }
            fn min() -> $t { $max }
            fn clip(&self) -> $t {
                if *self < Sample::min() {
                    Sample::min()
                } else if *self > Sample::max() {
                    Sample::max()
                } else {
                    *self
                }
            }
        }
    );
    ($t:ty) => (
        sample_impl!($t, ::std::num::Bounded::min_value()
                      .. ::std::num::Bounded::max_value())
    )
)
sample_impl!(i8)
sample_impl!(i16)
// Conspicuously missing: i24
sample_impl!(i32)
sample_impl!(f32, -1.0 .. 1.0)
sample_impl!(f64, -1.0 .. 1.0)

/// A source of samples.
///
/// Generates buffers of samples of type `T` and passes them to a consumer.
pub trait Source<T> {
    /// Emit the next buffer.
    fn next<'a>(&'a mut self) -> Option<&'a mut [T]>;
}

/// A thing.
pub trait Sink {
    /// Process a single buffer.
    ///
    /// Returns `Some` if there will be more buffers to process, or `None`
    /// otherwise.
    fn run_once(&mut self) -> Option<()>;

    /// Process buffers indefinitely, until end of stream or terminated.
    ///
    /// As long as `term_cond` is `true` and there are buffers available,
    /// this will process buffers. If `term_cond` is cleared, no additional
    /// buffers will be processed and the function returns.
    ///
    /// If `term_cond` is never modified, this is equivalent to repeatedly
    /// calling `run_once` until it returns `None`.
    fn run(&mut self, term_cond: &AtomicBool) {
        loop {
            if term_cond.load(AcqRel) || self.run_once().is_none() {
                return;
            }
        }
    }
}

/// A source of uncontrolled samples.
///
/// Owns buffers that get passed down through a pipeline, providing no
/// guarantees about what's in the buffer beyond that it's safe to read
/// and write.
///
/// This struct is used internally by most synthesis sources, and is
/// generally not useful to library users. It may be useful, however,
/// for building custom sources.
pub struct UninitializedSource<F> {
    buffer: Vec<F>
}

impl<F: Sample> UninitializedSource<F> {
    /// Create a source of uncontrolled samples.
    ///
    /// The yielded buffers will have `size` items.
    pub fn new(size: uint) -> UninitializedSource<F> {
        UninitializedSource {
            buffer: Vec::from_fn(size, |_| Zero::zero())
        }
    }
}

impl<F> Source<F> for UninitializedSource<F> {
    fn next<'a>(&'a mut self) -> Option<&'a mut [F]> {
        Some(self.buffer.as_mut_slice())
    }
}
<commit_msg>crate_id => crate_name<commit_after>#![crate_name = "audiostream"]
#![doc(html_root_url = "http://www.rust-ci.org/tari/audiostream.rs/doc/audiostream/")]

#![experimental]
#![deny(dead_code,missing_doc)]
#![feature(macro_rules)]
#![feature(default_type_params)]

//! Audio steam pipelines and processing.
//!
//! Streams are represented as sequences of buffers, each of which contains
//! zero or more samples. Data is produced from `Source`s on demand and fed
//! into a chain of zero or more `Sink`s until it reaches the end of the
//! pipeline. The pipeline always operates in a "pull" mode, where `Source`s
//! yield buffers only as fast as requested by a `Sink`.
//!
//! Valid sample formats are represented with the `Sample` trait. In general,
//! buffers are of type `&[Sample]`, presenting a single channel of audio
//! data. However, this convention is not enforced.

extern crate libao = "ao";

use std::num::Zero;
use std::sync::atomics::{AtomicBool, AcqRel};

pub mod ao;
pub mod synth;

/// Type bound for sample formats.
///
/// Implementation assumes `f64` is sufficient to represent all other formats
/// without loss.
pub trait Sample : Num + NumCast {
    /// Maximum value of a valid sample.
    fn max() -> Self;
    /// Minimum value of a valid sample.
    fn min() -> Self;
    /// Clip a value to be in range [min, max] (inclusive).
    fn clip(&self) -> Self;

    /// Convert from `Self` to an arbitrary other sample format.
    ///
    /// The default intermediate format here is `f64`, capable of losslessly
    /// converting all formats shorter than 52 bits.
    fn convert<X: Sample, I: Sample = f64>(a: Self) -> X {
        let a_i: I = NumCast::from(a).unwrap();
        let self_max: Self = Sample::max();
        let self_max_i: I = NumCast::from(self_max).unwrap();
        let ratio: I = a_i / self_max_i;

        let x_max: X = Sample::max();
        let x_max_i: I = NumCast::from(x_max).unwrap();

        NumCast::from(ratio * x_max_i).unwrap()
    }
}

macro_rules! sample_impl(
    ($t:ty, $min:expr .. $max:expr) => (
        impl Sample for $t {
            fn max() -> $t { $min }
            fn min() -> $t { $max }
            fn clip(&self) -> $t {
                if *self < Sample::min() {
                    Sample::min()
                } else if *self > Sample::max() {
                    Sample::max()
                } else {
                    *self
                }
            }
        }
    );
    ($t:ty) => (
        sample_impl!($t, ::std::num::Bounded::min_value()
                      .. ::std::num::Bounded::max_value())
    )
)
sample_impl!(i8)
sample_impl!(i16)
// Conspicuously missing: i24
sample_impl!(i32)
sample_impl!(f32, -1.0 .. 1.0)
sample_impl!(f64, -1.0 .. 1.0)

/// A source of samples.
///
/// Generates buffers of samples of type `T` and passes them to a consumer.
pub trait Source<T> {
    /// Emit the next buffer.
    fn next<'a>(&'a mut self) -> Option<&'a mut [T]>;
}

/// A thing.
pub trait Sink {
    /// Process a single buffer.
    ///
    /// Returns `Some` if there will be more buffers to process, or `None`
    /// otherwise.
    fn run_once(&mut self) -> Option<()>;

    /// Process buffers indefinitely, until end of stream or terminated.
    ///
    /// As long as `term_cond` is `true` and there are buffers available,
    /// this will process buffers. If `term_cond` is cleared, no additional
    /// buffers will be processed and the function returns.
    ///
    /// If `term_cond` is never modified, this is equivalent to repeatedly
    /// calling `run_once` until it returns `None`.
    fn run(&mut self, term_cond: &AtomicBool) {
        loop {
            if term_cond.load(AcqRel) || self.run_once().is_none() {
                return;
            }
        }
    }
}

/// A source of uncontrolled samples.
///
/// Owns buffers that get passed down through a pipeline, providing no
/// guarantees about what's in the buffer beyond that it's safe to read
/// and write.
///
/// This struct is used internally by most synthesis sources, and is
/// generally not useful to library users. It may be useful, however,
/// for building custom sources.
pub struct UninitializedSource<F> {
    buffer: Vec<F>
}

impl<F: Sample> UninitializedSource<F> {
    /// Create a source of uncontrolled samples.
    ///
    /// The yielded buffers will have `size` items.
    pub fn new(size: uint) -> UninitializedSource<F> {
        UninitializedSource {
            buffer: Vec::from_fn(size, |_| Zero::zero())
        }
    }
}

impl<F> Source<F> for UninitializedSource<F> {
    fn next<'a>(&'a mut self) -> Option<&'a mut [F]> {
        Some(self.buffer.as_mut_slice())
    }
}
<|endoftext|>


<commit_before>// tag_safe
//
// A linting plugin to flag calls to methods not marked "tag_safe"
// from methods marked "tag_safe".
//
// Author: John Hodge (thePowersGang/Mutabah)
//
// TODO: Support '#[tag_unsafe(type)]' which is used when a method has no marker
// - Allows default safe fallback, with upwards propagation.
//
#![crate_name="tag_safe"]
#![crate_type="dylib"]
#![feature(plugin_registrar, rustc_private)]

#[macro_use]
extern crate log;

extern crate syntax;
#[macro_use]
extern crate rustc;

use syntax::ast;
use syntax::visit;
use syntax::codemap::Span;
use rustc::lint::LintPassObject;
use rustc::plugin::Registry;
use rustc::lint::{Context, LintPass, LintArray};
use syntax::attr::AttrMetaMethods;
use rustc::middle::{def,ty};

declare_lint!(NOT_TAGGED_SAFE, Warn, "Warn about use of non-tagged methods within tagged function");

#[derive(Copy,Clone,Debug)]
enum SafetyType
{
    Safe,
    Unsafe,
    Unknown,
}

#[derive(Default)]
struct Pass
{
    /// Cache of flag types
    flag_types: Vec<String>,
    /// Node => (Type => IsSafe)
    flag_cache: ::rustc::util::nodemap::NodeMap< ::rustc::util::nodemap::FnvHashMap<usize, SafetyType> >,

    lvl: usize,
}

struct Visitor<'a, 'tcx: 'a, F: FnMut(&Span) + 'a>
{
    pass: &'a mut Pass,
    tcx: &'a ty::ctxt<'tcx>,
    name: &'a str,
    unknown_assume: bool,
    cb: F,
}

// Hack to provide indenting in debug calls
struct Indent(usize);
impl ::std::fmt::Display for Indent {
    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
        for s in ::std::iter::repeat(" ").take(self.0) {
            try!(write!(f, "{}", s));
        }
        Ok( () )
    }
}

impl LintPass for Pass {
    fn get_lints(&self) -> LintArray {
        lint_array!(NOT_TAGGED_SAFE)
    }

    fn check_fn(&mut self, cx: &Context, _kind: visit::FnKind, _decl: &ast::FnDecl, body: &ast::Block, _: Span, id: ast::NodeId) {
        let attrs = cx.tcx.map.attrs(id);
        for ty in attrs.iter()
            .filter(|a| a.check_name("tag_safe"))
            .filter_map(|a| a.meta_item_list())
            .flat_map(|x| x.iter())
        {
            // Search body for calls to non safe methods
            let mut v = Visitor{
                    pass: self, tcx: cx.tcx, name: &ty.name(),
                    // - Assumes an untagged method is unsafe
                    unknown_assume: false,
                    cb: |span| {
                            cx.span_lint(NOT_TAGGED_SAFE, *span,
                                &format!("Calling {}-unsafe method from a #[tag_safe({})] method", ty.name(), ty.name())[..]
                                );
                        },
                    };
            debug!("Method {:?} is marked safe '{}'", id, ty.name());
            visit::walk_block(&mut v, body);
        }
    }
}

impl Pass
{
    // Searches for the relevant marker
    fn check_for_marker(tcx: &ty::ctxt, id: ast::NodeId, marker: &str, name: &str) -> bool
    {
        debug!("Checking for marker {}({}) on {:?}", marker, name, id);
        tcx.map.attrs(id).iter()
            .filter_map( |a| if a.check_name(marker) { a.meta_item_list() } else { None })
            .flat_map(|x| x.iter())
            .any(|a| a.name() == name)
    }

    /// Recursively check that the provided function is either safe or unsafe.
    // Used to avoid excessive annotating
    fn recurse_fcn_body(&mut self, tcx: &ty::ctxt, node_id: ast::NodeId, name_id: usize, name: &str, unknown_assume: bool) -> bool
    {
        // Cache this method as unknown (to prevent infinite recursion)
        self.flag_cache.entry(node_id)
            .or_insert(Default::default())
            .insert(name_id, SafetyType::Unknown)
            ;

        // and apply a visitor to all
        match tcx.map.get(node_id)
        {
        rustc::ast_map::NodeItem(i) =>
            match i.node {
            ast::ItemFn(_, _, _, _, _, ref body) => {
                // Enumerate this function's code, recursively checking for a call to an unsafe method
                let mut is_safe = true;
                {
                    let mut v = Visitor {
                        pass: self, tcx: tcx, name: name,
                        unknown_assume: true,
                        cb: |_| { is_safe = false; }
                        };
                    visit::walk_block(&mut v, body);
                }
                is_safe
                },
            _ => unknown_assume,
            },
        rustc::ast_map::NodeImplItem(i) =>
            match i.node {
            ast::MethodImplItem(_, ref body) => {
                let mut is_safe = true;
                {
                    let mut v = Visitor {
                        pass: self, tcx: tcx, name: name,
                        unknown_assume: true,
                        cb: |_| { is_safe = false; }
                        };
                    visit::walk_block(&mut v, body);
                }
                is_safe
                },
            _ => unknown_assume,
            },
        v @ _ => {
            error!("Node ID {} points to non-item {:?}", node_id, v);
            unknown_assume
            }
        }
    }

    /// Check that a method within this crate is safe with the provided tag
    fn crate_method_is_safe(&mut self, tcx: &ty::ctxt, node_id: ast::NodeId, name: &str, unknown_assume: bool) -> bool
    {
        // Obtain tag name ID (avoids storing a string in the map)
        let name_id =
            match self.flag_types.iter().position(|a| *a == name)
            {
            Some(v) => v,
            None => {
                self.flag_types.push( String::from(name) );
                self.flag_types.len() - 1
                },
            };

        // Check cache first
        if let Some(&st) = self.flag_cache.get(&node_id).and_then(|a| a.get(&name_id))
        {
            match st
            {
            SafetyType::Safe => true,
            SafetyType::Unsafe => false,
            SafetyType::Unknown => unknown_assume,
            }
        }
        else
        {
            // Search for a safety marker, possibly recursing
            let is_safe =
                if Self::check_for_marker(tcx, node_id, "tag_safe", name) {
                    true
                }
                else if Self::check_for_marker(tcx, node_id, "tag_unsafe", name) {
                    false
                }
                else {
                    self.recurse_fcn_body(tcx, node_id, name_id, name, unknown_assume)
                };
            // Save resultant value
            self.flag_cache.entry(node_id)
                .or_insert(Default::default())
                .insert(name_id, if is_safe { SafetyType::Safe } else { SafetyType::Unsafe })
                ;
            is_safe
        }
    }

    /// Locate a #[tag_safe(<name>)] attribute on the passed item
    pub fn method_is_safe(&mut self, tcx: &ty::ctxt, id: ast::DefId, name: &str, unknown_assume: bool) -> bool
    {
        debug!("{}Checking method {:?} (A {})", Indent(self.lvl), id, unknown_assume);
        self.lvl += 1;
        let rv = if id.krate == 0 {
                self.crate_method_is_safe(tcx, id.node, name, unknown_assume)
            }
            else {
                for a in tcx.get_attrs(id).iter()
                {
                    if a.check_name("tag_safe") {
                        if a.meta_item_list().iter().flat_map(|a| a.iter()).any(|a| a.name() == name) {
                            return true;
                        }
                    }
                    if a.check_name("tag_unsafe") {
                        if a.meta_item_list().iter().flat_map(|a| a.iter()).any(|a| a.name() == name) {
                            return false;
                        }
                    }
                }
                warn!("TODO: Crate ID non-zero {:?} (assuming safe)", id);
                // TODO: Check the crate import for an annotation
                true
            };
        self.lvl -= 1;
        debug!("{}Checking method {:?} = {}", Indent(self.lvl), id, rv);
        rv
    }
}

impl<'a, 'tcx: 'a, F: FnMut(&Span)> visit::Visitor<'a> for Visitor<'a,'tcx, F>
{
    // Locate function/method calls in a code block
    // - uses visit_expr_post because it doesn't _need_ to do anything
    fn visit_expr_post(&mut self, ex: &'a ast::Expr) {
        match ex.node
        {
        // Call expressions - check that it's a path call
        ast::ExprCall(ref fcn, _) =>
            match fcn.node
            {
            ast::ExprPath(ref _qs, ref _p) => {
                    if let def::DefFn(did, _) = self.tcx.resolve_expr(&fcn) {
                        // Check for a safety tag
                        if !self.pass.method_is_safe(self.tcx, did, self.name, self.unknown_assume)
                        {
                            (self.cb)(&ex.span);
                        }
                    }
                },
            _ => {},
            },

        // Method call expressions - get the relevant method
        ast::ExprMethodCall(ref _id, ref _tys, ref _exprs) =>
            {
                let tables = self.tcx.tables.borrow();
                let mm = &tables.method_map;

                let callee = mm.get( &ty::MethodCall::expr(ex.id) ).unwrap();
                let id = callee.def_id;

                //if let ty::MethodStatic(id) = callee.origin {
                        // Check for a safety tag
                        if !self.pass.method_is_safe(self.tcx, id, self.name, self.unknown_assume) {
                            (self.cb)(&ex.span);
                        }
                //}
            },

        // Ignore any other type of node
        _ => {},
        }
    }
}

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    use syntax::feature_gate::AttributeType;
    reg.register_lint_pass(Box::new(Pass::default()) as LintPassObject);

    reg.register_attribute(String::from("tag_safe"),   AttributeType::Whitelisted);
    reg.register_attribute(String::from("tag_unsafe"), AttributeType::Whitelisted);
}

// vim: ts=4 expandtab sw=4
<commit_msg>Add foreign item support (e.g. 'extern fn')<commit_after>// tag_safe
//
// A linting plugin to flag calls to methods not marked "tag_safe"
// from methods marked "tag_safe".
//
// Author: John Hodge (thePowersGang/Mutabah)
//
// TODO: Support '#[tag_unsafe(type)]' which is used when a method has no marker
// - Allows default safe fallback, with upwards propagation.
//
#![crate_name="tag_safe"]
#![crate_type="dylib"]
#![feature(plugin_registrar, rustc_private)]

#[macro_use]
extern crate log;

extern crate syntax;
#[macro_use]
extern crate rustc;

use syntax::ast;
use syntax::visit;
use syntax::codemap::Span;
use rustc::lint::LintPassObject;
use rustc::plugin::Registry;
use rustc::lint::{Context, LintPass, LintArray};
use syntax::attr::AttrMetaMethods;
use rustc::middle::{def,ty};

declare_lint!(NOT_TAGGED_SAFE, Warn, "Warn about use of non-tagged methods within tagged function");

#[derive(Copy,Clone,Debug)]
enum SafetyType
{
    Safe,
    Unsafe,
    Unknown,
}

#[derive(Default)]
struct Pass
{
    /// Cache of flag types
    flag_types: Vec<String>,
    /// Node => (Type => IsSafe)
    flag_cache: ::rustc::util::nodemap::NodeMap< ::rustc::util::nodemap::FnvHashMap<usize, SafetyType> >,

    lvl: usize,
}

struct Visitor<'a, 'tcx: 'a, F: FnMut(&Span) + 'a>
{
    pass: &'a mut Pass,
    tcx: &'a ty::ctxt<'tcx>,
    name: &'a str,
    unknown_assume: bool,
    cb: F,
}

// Hack to provide indenting in debug calls
struct Indent(usize);
impl ::std::fmt::Display for Indent {
    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
        for s in ::std::iter::repeat(" ").take(self.0) {
            try!(write!(f, "{}", s));
        }
        Ok( () )
    }
}

impl LintPass for Pass {
    fn get_lints(&self) -> LintArray {
        lint_array!(NOT_TAGGED_SAFE)
    }

    fn check_fn(&mut self, cx: &Context, _kind: visit::FnKind, _decl: &ast::FnDecl, body: &ast::Block, _: Span, id: ast::NodeId) {
        let attrs = cx.tcx.map.attrs(id);
        for ty in attrs.iter()
            .filter(|a| a.check_name("tag_safe"))
            .filter_map(|a| a.meta_item_list())
            .flat_map(|x| x.iter())
        {
            // Search body for calls to non safe methods
            let mut v = Visitor{
                    pass: self, tcx: cx.tcx, name: &ty.name(),
                    // - Assumes an untagged method is unsafe
                    unknown_assume: false,
                    cb: |span| {
                            cx.span_lint(NOT_TAGGED_SAFE, *span,
                                &format!("Calling {}-unsafe method from a #[tag_safe({})] method", ty.name(), ty.name())[..]
                                );
                        },
                    };
            debug!("Method {:?} is marked safe '{}'", id, ty.name());
            visit::walk_block(&mut v, body);
        }
    }
}

impl Pass
{
    // Searches for the relevant marker
    fn check_for_marker(tcx: &ty::ctxt, id: ast::NodeId, marker: &str, name: &str) -> bool
    {
        debug!("Checking for marker {}({}) on {:?}", marker, name, id);
        tcx.map.attrs(id).iter()
            .filter_map( |a| if a.check_name(marker) { a.meta_item_list() } else { None })
            .flat_map(|x| x.iter())
            .any(|a| a.name() == name)
    }

    /// Recursively check that the provided function is either safe or unsafe.
    // Used to avoid excessive annotating
    fn recurse_fcn_body(&mut self, tcx: &ty::ctxt, node_id: ast::NodeId, name_id: usize, name: &str, unknown_assume: bool) -> bool
    {
        // Cache this method as unknown (to prevent infinite recursion)
        self.flag_cache.entry(node_id)
            .or_insert(Default::default())
            .insert(name_id, SafetyType::Unknown)
            ;

        // and apply a visitor to all
        match tcx.map.get(node_id)
        {
        rustc::ast_map::NodeItem(i) =>
            match i.node {
            ast::ItemFn(_, _, _, _, _, ref body) => {
                // Enumerate this function's code, recursively checking for a call to an unsafe method
                let mut is_safe = true;
                {
                    let mut v = Visitor {
                        pass: self, tcx: tcx, name: name,
                        unknown_assume: true,
                        cb: |_| { is_safe = false; }
                        };
                    visit::walk_block(&mut v, body);
                }
                is_safe
                },
            _ => unknown_assume,
            },
        rustc::ast_map::NodeImplItem(i) =>
            match i.node {
            ast::MethodImplItem(_, ref body) => {
                let mut is_safe = true;
                {
                    let mut v = Visitor {
                        pass: self, tcx: tcx, name: name,
                        unknown_assume: true,
                        cb: |_| { is_safe = false; }
                        };
                    visit::walk_block(&mut v, body);
                }
                is_safe
                },
            _ => unknown_assume,
            },
        rustc::ast_map::NodeForeignItem(i) =>
            if Self::check_for_marker(tcx, i.id, "tag_safe", name) {
                true
            }
            else if Self::check_for_marker(tcx, i.id, "tag_unsafe", name) {
                false
            }
            else {
                unknown_assume
            },
        v @ _ => {
            error!("Node ID {} points to non-item {:?}", node_id, v);
            unknown_assume
            }
        }
    }

    /// Check that a method within this crate is safe with the provided tag
    fn crate_method_is_safe(&mut self, tcx: &ty::ctxt, node_id: ast::NodeId, name: &str, unknown_assume: bool) -> bool
    {
        // Obtain tag name ID (avoids storing a string in the map)
        let name_id =
            match self.flag_types.iter().position(|a| *a == name)
            {
            Some(v) => v,
            None => {
                self.flag_types.push( String::from(name) );
                self.flag_types.len() - 1
                },
            };

        // Check cache first
        if let Some(&st) = self.flag_cache.get(&node_id).and_then(|a| a.get(&name_id))
        {
            match st
            {
            SafetyType::Safe => true,
            SafetyType::Unsafe => false,
            SafetyType::Unknown => unknown_assume,
            }
        }
        else
        {
            // Search for a safety marker, possibly recursing
            let is_safe =
                if Self::check_for_marker(tcx, node_id, "tag_safe", name) {
                    true
                }
                else if Self::check_for_marker(tcx, node_id, "tag_unsafe", name) {
                    false
                }
                else {
                    self.recurse_fcn_body(tcx, node_id, name_id, name, unknown_assume)
                };
            // Save resultant value
            self.flag_cache.entry(node_id)
                .or_insert(Default::default())
                .insert(name_id, if is_safe { SafetyType::Safe } else { SafetyType::Unsafe })
                ;
            is_safe
        }
    }

    /// Locate a #[tag_safe(<name>)] attribute on the passed item
    pub fn method_is_safe(&mut self, tcx: &ty::ctxt, id: ast::DefId, name: &str, unknown_assume: bool) -> bool
    {
        debug!("{}Checking method {:?} (A {})", Indent(self.lvl), id, unknown_assume);
        self.lvl += 1;
        let rv = if id.krate == 0 {
                self.crate_method_is_safe(tcx, id.node, name, unknown_assume)
            }
            else {
                for a in tcx.get_attrs(id).iter()
                {
                    if a.check_name("tag_safe") {
                        if a.meta_item_list().iter().flat_map(|a| a.iter()).any(|a| a.name() == name) {
                            return true;
                        }
                    }
                    if a.check_name("tag_unsafe") {
                        if a.meta_item_list().iter().flat_map(|a| a.iter()).any(|a| a.name() == name) {
                            return false;
                        }
                    }
                }
                warn!("TODO: Crate ID non-zero {:?} (assuming safe)", id);
                // TODO: Check the crate import for an annotation
                true
            };
        self.lvl -= 1;
        debug!("{}Checking method {:?} = {}", Indent(self.lvl), id, rv);
        rv
    }
}

impl<'a, 'tcx: 'a, F: FnMut(&Span)> visit::Visitor<'a> for Visitor<'a,'tcx, F>
{
    // Locate function/method calls in a code block
    // - uses visit_expr_post because it doesn't _need_ to do anything
    fn visit_expr_post(&mut self, ex: &'a ast::Expr) {
        match ex.node
        {
        // Call expressions - check that it's a path call
        ast::ExprCall(ref fcn, _) =>
            match fcn.node
            {
            ast::ExprPath(ref _qs, ref _p) => {
                    if let def::DefFn(did, _) = self.tcx.resolve_expr(&fcn) {
                        // Check for a safety tag
                        if !self.pass.method_is_safe(self.tcx, did, self.name, self.unknown_assume)
                        {
                            (self.cb)(&ex.span);
                        }
                    }
                },
            _ => {},
            },

        // Method call expressions - get the relevant method
        ast::ExprMethodCall(ref _id, ref _tys, ref _exprs) =>
            {
                let tables = self.tcx.tables.borrow();
                let mm = &tables.method_map;

                let callee = mm.get( &ty::MethodCall::expr(ex.id) ).unwrap();
                let id = callee.def_id;

                //if let ty::MethodStatic(id) = callee.origin {
                        // Check for a safety tag
                        if !self.pass.method_is_safe(self.tcx, id, self.name, self.unknown_assume) {
                            (self.cb)(&ex.span);
                        }
                //}
            },

        // Ignore any other type of node
        _ => {},
        }
    }
}

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    use syntax::feature_gate::AttributeType;
    reg.register_lint_pass(Box::new(Pass::default()) as LintPassObject);

    reg.register_attribute(String::from("tag_safe"),   AttributeType::Whitelisted);
    reg.register_attribute(String::from("tag_unsafe"), AttributeType::Whitelisted);
}

// vim: ts=4 expandtab sw=4
<|endoftext|>


<commit_before>#![no_std]
#![cfg_attr(
    feature = "cargo-clippy",
    allow(
        cast_lossless,
        cyclomatic_complexity,
        many_single_char_names,
        unreadable_literal,
    )
)]

mod common;
mod d2s;
mod d2s_full_table;
mod digit_table;
mod f2s;

pub use d2s::d2s_buffered_n;
pub use f2s::f2s_buffered_n;
<commit_msg>Ignore clippy lint on passing FloatingDecimal64<commit_after>#![no_std]
#![cfg_attr(
    feature = "cargo-clippy",
    allow(
        cast_lossless,
        cyclomatic_complexity,
        many_single_char_names,
        needless_pass_by_value,
        unreadable_literal,
    )
)]

mod common;
mod d2s;
mod d2s_full_table;
mod digit_table;
mod f2s;

pub use d2s::d2s_buffered_n;
pub use f2s::f2s_buffered_n;
<|endoftext|>


<commit_before>extern crate protobuf;
extern crate byteorder;
use std::io::prelude::*;
use protobuf::{MessageStatic, Message, parse_from_bytes};
use byteorder::{ReadBytesExt, LittleEndian, WriteBytesExt};
use std::io::Cursor;

pub fn read_pbuf<M, R>(reader: &mut R) -> Vec<M>
  where M: MessageStatic, R: Read {
  let mut msg: M;
  let mut msgs: Vec<M> = Vec::new();
  let mut head = vec![0; 2];

  loop {
    match reader.read_exact(&mut head) {
      Ok(n) => n,
      Err(_) => break
    };
    let h = header(&head);
    let mut proto = vec![0; h];
    match reader.read_exact(&mut proto) {
      Ok(n) => n,
      Err(e) => panic!("{:?}", e)
    };
    msg = match parse_from_bytes::<M>(&mut proto) {
              Ok(n) => n,
              Err(e) => panic!("{:?}", e)
            };
    msgs.push(msg);
  }
  msgs
}

fn header(sz: &Vec<u8>) -> usize {
  let mut rdr = Cursor::new(sz);
  match rdr.read_i16::<LittleEndian>() {
    Ok(x) => x as usize,
    Err(e) => panic!("Something wicked this way comes: {}", e)
  }
}

pub fn write_pbuf<M, W>(msg: &M, writer: &mut W) -> Option<usize>
  where M: Message, W: Write {
  let bts = Message::write_to_bytes(msg).unwrap();
  let l = bts.len();
  match writer.write_u16::<LittleEndian>(l as u16) {
    Ok(n) => n,
    Err(e) => panic!("{:?}", e)
  };
  match writer.write(&bts[..]) {
    Ok(n) => assert_eq!(l, n),
    Err(e) => panic!("{:?}", e)
  }
  Some(l)
}
<commit_msg>Fix write error<commit_after>extern crate protobuf;
extern crate byteorder;
use std::io::prelude::*;
use protobuf::{MessageStatic, Message, parse_from_bytes};
use byteorder::{ReadBytesExt, LittleEndian, WriteBytesExt};
use std::io::Cursor;

pub mod error;
use error::PBErr;

pub fn read_many<M, R>(reader: &mut R) -> Result<Vec<M>, PBErr>
  where M: MessageStatic, R: Read {
  let mut msg: M;
  let mut msgs: Vec<M> = Vec::new();
  let mut head = vec![0; 2];

  while let Ok(_) = reader.read_exact(&mut head) {
    let h = header(&head)?;
    let mut proto = vec![0; h];
    reader.read_exact(&mut proto)?;
    msg = parse_from_bytes::<M>(&proto)?;
    msgs.push(msg);
  }
  Ok(msgs)
}

pub fn read_one<M, R>(reader: &mut R) -> Result<M, PBErr>
  where M: MessageStatic, R: Read {
    let mut head = vec![0; 2];

    reader.read_exact(&mut head)?;
    let h = header(&head)?;
    let mut proto = vec![0; h];
    reader.read_exact(&mut proto)?;
    Ok(parse_from_bytes::<M>(&proto)?)
}

fn header(sz: &[u8]) -> Result<usize, PBErr> {
  Ok(Cursor::new(sz).read_i16::<LittleEndian>()? as usize)
}

pub fn write<M, W>(msg: &M, writer: &mut W) -> Result<usize, PBErr>
  where M: Message, W: Write {
  let bts = Message::write_to_bytes(msg)?;
  let l = bts.len();
  writer.write_u16::<LittleEndian>(l as u16)?;
  writer.write_all(&bts[..])?;
  Ok(l)
}
<|endoftext|>