<commit_before>// Copyright 2014, 2015 Mikhail Zabaluev <mikhail.zabaluev@gmail.com>
// Copyright 2012 The Rust Project Developers
// See the COPYRIGHT file at the top-level directory of this distribution
// and at http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::iter::{Iterator, range};
use std::ptr;
use libc;

use super::{CStr, CStrBuf, OwnedCString};
use super::{libc_free, parse_c_multistring};

pub fn check_c_str(c_str: &CStr, expected: &[u8]) {
    let buf = c_str.as_ptr();
    let len = expected.len();
    for i in range(0, len) {
        let byte = unsafe { *buf.offset(i as isize) as u8 };
        assert_eq!(byte, expected[i]);
    }
    let term = unsafe { *buf.offset(len as isize) as u8 };
    assert_eq!(term, 0);
}

unsafe fn bytes_dup_raw(s: &[u8]) -> *const libc::c_char {
    let len = s.len();
    let dup = libc::malloc((len + 1) as libc::size_t) as *mut u8;
    ptr::copy_nonoverlapping_memory(dup, s.as_ptr(), len);
    *dup.offset(len as isize) = 0;
    dup as *const libc::c_char
}

fn bytes_dup(s: &[u8]) -> OwnedCString {
    unsafe {
        OwnedCString::new(bytes_dup_raw(s), libc_free)
    }
}

fn str_dup(s: &str) -> OwnedCString {
    bytes_dup(s.as_bytes())
}

#[test]
fn test_str_multistring_parsing() {
    unsafe {
        let input = b"zero\0one\0\0";
        let ptr = input.as_ptr();
        let expected = ["zero", "one"];
        let mut it = expected.iter();
        let result = parse_c_multistring(ptr as *const libc::c_char, None,
            |cbytes| {
                assert_eq!(cbytes, it.next().unwrap().as_bytes());
            });
        assert_eq!(result, 2);
        assert!(it.next().is_none());
    }
}

#[test]
fn test_c_str_macro() {
    let c_str = c_str!("hello");
    check_c_str(c_str, b"hello");
}

#[test]
fn test_string_deref() {
    let c_str = str_dup("hello");
    check_c_str(&*c_str, b"hello");
}

#[test]
fn test_string_as_ptr() {
    let c_str = str_dup("hello");
    let len = unsafe { libc::strlen(c_str.as_ptr()) };
    assert_eq!(len, 5);
}

#[test]
fn test_iterator() {
    let c_string = str_dup("");
    let mut iter = c_string.iter();
    assert_eq!(iter.next(), None);

    let c_string = str_dup("hello");
    let mut iter = c_string.iter();
    assert_eq!(iter.next(), Some('h' as libc::c_char));
    assert_eq!(iter.next(), Some('e' as libc::c_char));
    assert_eq!(iter.next(), Some('l' as libc::c_char));
    assert_eq!(iter.next(), Some('l' as libc::c_char));
    assert_eq!(iter.next(), Some('o' as libc::c_char));
    assert_eq!(iter.next(), None);
}

#[test]
fn test_parse_as_bytes() {
    let c_str = str_dup("hello");
    let bytes = unsafe { super::parse_as_bytes(c_str.ptr, &c_str) };
    assert_eq!(bytes, b"hello");
    let c_str = str_dup("");
    let bytes = unsafe { super::parse_as_bytes(c_str.ptr, &c_str) };
    assert_eq!(bytes, b"");
    let c_str = bytes_dup(b"foo\xFF");
    let bytes = unsafe { super::parse_as_bytes(c_str.ptr, &c_str) };
    assert_eq!(bytes, b"foo\xFF");
}

#[test]
fn test_parse_as_utf8() {
    let c_str = str_dup("hello");
    let res = unsafe { super::parse_as_utf8(c_str.ptr, &c_str) };
    assert_eq!(res, Ok("hello"));
    let c_str = str_dup("");
    let res = unsafe { super::parse_as_utf8(c_str.ptr, &c_str) };
    assert_eq!(res, Ok(""));
    let c_str = bytes_dup(b"foo\xFF");
    let res = unsafe { super::parse_as_utf8(c_str.ptr, &c_str) };
    assert!(res.is_err());
}

#[test]
fn test_c_str_buf_from_bytes() {
    let c_str = CStrBuf::from_bytes(b"").unwrap();
    check_c_str(&*c_str, b"");

    let c_str = CStrBuf::from_bytes(b"foo\xFF").unwrap();
    check_c_str(&*c_str, b"foo\xFF");

    // Owned variant
    let c_str = CStrBuf::from_bytes(b"Mary had a little \xD0\x0D, Little \xD0\x0D").unwrap();
    check_c_str(&*c_str, b"Mary had a little \xD0\x0D, Little \xD0\x0D");

    let res = CStrBuf::from_bytes(b"got\0nul");
    let err = res.err().unwrap();
    assert_eq!(err.position, 3);
}

#[test]
fn test_c_str_buf_from_str() {
    let c_str = CStrBuf::from_str("").unwrap();
    check_c_str(&*c_str, b"");

    let c_str = CStrBuf::from_str("hello").unwrap();
    check_c_str(&*c_str, b"hello");

    // Owned variant
    let c_str = CStrBuf::from_str("Mary had a little lamb, Little lamb").unwrap();
    check_c_str(&*c_str, b"Mary had a little lamb, Little lamb");

    let res = CStrBuf::from_str("got\0nul");
    let err = res.err().unwrap();
    assert_eq!(err.position, 3);
}

#[test]
fn test_c_str_buf_from_vec() {
    let c_str = CStrBuf::from_vec(b"".to_vec()).unwrap();
    check_c_str(&*c_str, b"");

    let c_str = CStrBuf::from_vec(b"hello".to_vec()).unwrap();
    check_c_str(&*c_str, b"hello");

    // Owned variant
    let c_str = CStrBuf::from_vec(b"Mary had a little lamb, Little lamb".to_vec()).unwrap();
    check_c_str(&*c_str, b"Mary had a little lamb, Little lamb");

    let res = CStrBuf::from_vec(b"got\0nul".to_vec());
    let err = res.err().unwrap();
    assert_eq!(err.nul_error().position, 3);
    let vec = err.into_bytes();
    assert_eq!(vec.as_slice(), b"got\0nul");
}

#[test]
fn test_c_str_buf_into_vec() {
    let c_str = CStrBuf::from_str("").unwrap();
    let vec = c_str.into_vec();
    assert_eq!(vec.as_slice(), b"");
    let c_str = CStrBuf::from_str("hello").unwrap();
    let vec = c_str.into_vec();
    assert_eq!(vec.as_slice(), b"hello");
    let c_str = CStrBuf::from_bytes(b"foo\xFF").unwrap();
    let vec = c_str.into_vec();
    assert_eq!(vec.as_slice(), b"foo\xFF");

    // Owned variant
    let c_str = CStrBuf::from_str("Mary had a little lamb, Little lamb").unwrap();
    let vec = c_str.into_vec();
    assert_eq!(vec.as_slice(), b"Mary had a little lamb, Little lamb");
    let c_str = CStrBuf::from_bytes(b"Mary had a little \xD0\x0D, Little \xD0\x0D").unwrap();
    let vec = c_str.into_vec();
    assert_eq!(vec.as_slice(), b"Mary had a little \xD0\x0D, Little \xD0\x0D");
}

#[test]
fn test_c_str_is_empty() {
    let c_str = CStrBuf::from_str("").unwrap();
    assert!(c_str.is_empty());
}

#[test]
fn test_c_string_parse_as_bytes() {
    let c_str = str_dup("hello");
    assert_eq!(c_str.parse_as_bytes(), b"hello");
    let c_str = str_dup("");
    assert_eq!(c_str.parse_as_bytes(), b"");
    let c_str = bytes_dup(b"foo\xFF");
    assert_eq!(c_str.parse_as_bytes(), b"foo\xFF");
}

#[test]
fn test_c_string_parse_as_utf8() {
    let c_str = str_dup("hello");
    assert_eq!(c_str.parse_as_utf8(), Ok("hello"));
    let c_str = str_dup("");
    assert_eq!(c_str.parse_as_utf8(), Ok(""));
    let c_str = bytes_dup(b"foo\xFF");
    assert!(c_str.parse_as_utf8().is_err());
}

#[test]
#[should_fail]
fn test_parse_null_as_bytes_fail() {
    unsafe {
        let p: *const libc::c_char = ptr::null();
        let _ = super::parse_as_bytes(p, &p);
    };
}

#[test]
#[should_fail]
fn test_parse_null_as_utf8_fail() {
    unsafe {
        let p: *const libc::c_char = ptr::null();
        let _ = super::parse_as_utf8(p, &p);
    };
}

#[test]
#[should_fail]
fn test_c_string_new_fail() {
    let _c_str: OwnedCString = unsafe {
        OwnedCString::new(ptr::null(), libc_free)
    };
}

#[test]
#[should_fail]
fn test_from_static_bytes_fail() {
    let _c_str = super::from_static_bytes(b"no nul");
}

#[test]
#[should_fail]
fn test_from_static_str_fail() {
    let _c_str = super::from_static_str("no nul");
}

#[test]
#[should_fail]
fn test_from_raw_ptr_fail() {
    let p: *const libc::c_char = ptr::null();
    let _c_str = unsafe { super::from_raw_ptr(p, &p) };
}

// Do you love me, Travis?
<commit_msg>Renamed some tests<commit_after>// Copyright 2014, 2015 Mikhail Zabaluev <mikhail.zabaluev@gmail.com>
// Copyright 2012 The Rust Project Developers
// See the COPYRIGHT file at the top-level directory of this distribution
// and at http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::iter::{Iterator, range};
use std::ptr;
use libc;

use super::{CStr, CStrBuf, OwnedCString};
use super::{libc_free, parse_c_multistring};

pub fn check_c_str(c_str: &CStr, expected: &[u8]) {
    let buf = c_str.as_ptr();
    let len = expected.len();
    for i in range(0, len) {
        let byte = unsafe { *buf.offset(i as isize) as u8 };
        assert_eq!(byte, expected[i]);
    }
    let term = unsafe { *buf.offset(len as isize) as u8 };
    assert_eq!(term, 0);
}

unsafe fn bytes_dup_raw(s: &[u8]) -> *const libc::c_char {
    let len = s.len();
    let dup = libc::malloc((len + 1) as libc::size_t) as *mut u8;
    ptr::copy_nonoverlapping_memory(dup, s.as_ptr(), len);
    *dup.offset(len as isize) = 0;
    dup as *const libc::c_char
}

fn bytes_dup(s: &[u8]) -> OwnedCString {
    unsafe {
        OwnedCString::new(bytes_dup_raw(s), libc_free)
    }
}

fn str_dup(s: &str) -> OwnedCString {
    bytes_dup(s.as_bytes())
}

#[test]
fn test_parse_c_multistring() {
    unsafe {
        let input = b"zero\0one\0\0";
        let ptr = input.as_ptr();
        let expected = ["zero", "one"];
        let mut it = expected.iter();
        let result = parse_c_multistring(ptr as *const libc::c_char, None,
            |cbytes| {
                assert_eq!(cbytes, it.next().unwrap().as_bytes());
            });
        assert_eq!(result, 2);
        assert!(it.next().is_none());
    }
}

#[test]
fn test_c_str_macro() {
    let c_str = c_str!("hello");
    check_c_str(c_str, b"hello");
}

#[test]
fn test_owned_c_string_deref() {
    let c_str = str_dup("hello");
    check_c_str(&*c_str, b"hello");
}

#[test]
fn test_owned_c_string_as_ptr() {
    let c_str = str_dup("hello");
    let len = unsafe { libc::strlen(c_str.as_ptr()) };
    assert_eq!(len, 5);
}

#[test]
fn test_iterator() {
    let c_string = str_dup("");
    let mut iter = c_string.iter();
    assert_eq!(iter.next(), None);

    let c_string = str_dup("hello");
    let mut iter = c_string.iter();
    assert_eq!(iter.next(), Some('h' as libc::c_char));
    assert_eq!(iter.next(), Some('e' as libc::c_char));
    assert_eq!(iter.next(), Some('l' as libc::c_char));
    assert_eq!(iter.next(), Some('l' as libc::c_char));
    assert_eq!(iter.next(), Some('o' as libc::c_char));
    assert_eq!(iter.next(), None);
}

#[test]
fn test_parse_as_bytes() {
    let c_str = str_dup("hello");
    let bytes = unsafe { super::parse_as_bytes(c_str.ptr, &c_str) };
    assert_eq!(bytes, b"hello");
    let c_str = str_dup("");
    let bytes = unsafe { super::parse_as_bytes(c_str.ptr, &c_str) };
    assert_eq!(bytes, b"");
    let c_str = bytes_dup(b"foo\xFF");
    let bytes = unsafe { super::parse_as_bytes(c_str.ptr, &c_str) };
    assert_eq!(bytes, b"foo\xFF");
}

#[test]
fn test_parse_as_utf8() {
    let c_str = str_dup("hello");
    let res = unsafe { super::parse_as_utf8(c_str.ptr, &c_str) };
    assert_eq!(res, Ok("hello"));
    let c_str = str_dup("");
    let res = unsafe { super::parse_as_utf8(c_str.ptr, &c_str) };
    assert_eq!(res, Ok(""));
    let c_str = bytes_dup(b"foo\xFF");
    let res = unsafe { super::parse_as_utf8(c_str.ptr, &c_str) };
    assert!(res.is_err());
}

#[test]
fn test_c_str_buf_from_bytes() {
    let c_str = CStrBuf::from_bytes(b"").unwrap();
    check_c_str(&*c_str, b"");

    let c_str = CStrBuf::from_bytes(b"foo\xFF").unwrap();
    check_c_str(&*c_str, b"foo\xFF");

    // Owned variant
    let c_str = CStrBuf::from_bytes(b"Mary had a little \xD0\x0D, Little \xD0\x0D").unwrap();
    check_c_str(&*c_str, b"Mary had a little \xD0\x0D, Little \xD0\x0D");

    let res = CStrBuf::from_bytes(b"got\0nul");
    let err = res.err().unwrap();
    assert_eq!(err.position, 3);
}

#[test]
fn test_c_str_buf_from_str() {
    let c_str = CStrBuf::from_str("").unwrap();
    check_c_str(&*c_str, b"");

    let c_str = CStrBuf::from_str("hello").unwrap();
    check_c_str(&*c_str, b"hello");

    // Owned variant
    let c_str = CStrBuf::from_str("Mary had a little lamb, Little lamb").unwrap();
    check_c_str(&*c_str, b"Mary had a little lamb, Little lamb");

    let res = CStrBuf::from_str("got\0nul");
    let err = res.err().unwrap();
    assert_eq!(err.position, 3);
}

#[test]
fn test_c_str_buf_from_vec() {
    let c_str = CStrBuf::from_vec(b"".to_vec()).unwrap();
    check_c_str(&*c_str, b"");

    let c_str = CStrBuf::from_vec(b"hello".to_vec()).unwrap();
    check_c_str(&*c_str, b"hello");

    // Owned variant
    let c_str = CStrBuf::from_vec(b"Mary had a little lamb, Little lamb".to_vec()).unwrap();
    check_c_str(&*c_str, b"Mary had a little lamb, Little lamb");

    let res = CStrBuf::from_vec(b"got\0nul".to_vec());
    let err = res.err().unwrap();
    assert_eq!(err.nul_error().position, 3);
    let vec = err.into_bytes();
    assert_eq!(vec.as_slice(), b"got\0nul");
}

#[test]
fn test_c_str_buf_into_vec() {
    let c_str = CStrBuf::from_str("").unwrap();
    let vec = c_str.into_vec();
    assert_eq!(vec.as_slice(), b"");
    let c_str = CStrBuf::from_str("hello").unwrap();
    let vec = c_str.into_vec();
    assert_eq!(vec.as_slice(), b"hello");
    let c_str = CStrBuf::from_bytes(b"foo\xFF").unwrap();
    let vec = c_str.into_vec();
    assert_eq!(vec.as_slice(), b"foo\xFF");

    // Owned variant
    let c_str = CStrBuf::from_str("Mary had a little lamb, Little lamb").unwrap();
    let vec = c_str.into_vec();
    assert_eq!(vec.as_slice(), b"Mary had a little lamb, Little lamb");
    let c_str = CStrBuf::from_bytes(b"Mary had a little \xD0\x0D, Little \xD0\x0D").unwrap();
    let vec = c_str.into_vec();
    assert_eq!(vec.as_slice(), b"Mary had a little \xD0\x0D, Little \xD0\x0D");
}

#[test]
fn test_c_str_is_empty() {
    let c_str = CStrBuf::from_str("").unwrap();
    assert!(c_str.is_empty());
}

#[test]
fn test_owned_c_string_parse_as_bytes() {
    let c_str = str_dup("hello");
    assert_eq!(c_str.parse_as_bytes(), b"hello");
    let c_str = str_dup("");
    assert_eq!(c_str.parse_as_bytes(), b"");
    let c_str = bytes_dup(b"foo\xFF");
    assert_eq!(c_str.parse_as_bytes(), b"foo\xFF");
}

#[test]
fn test_owned_c_string_parse_as_utf8() {
    let c_str = str_dup("hello");
    assert_eq!(c_str.parse_as_utf8(), Ok("hello"));
    let c_str = str_dup("");
    assert_eq!(c_str.parse_as_utf8(), Ok(""));
    let c_str = bytes_dup(b"foo\xFF");
    assert!(c_str.parse_as_utf8().is_err());
}

#[test]
#[should_fail]
fn test_parse_null_as_bytes_fail() {
    unsafe {
        let p: *const libc::c_char = ptr::null();
        let _ = super::parse_as_bytes(p, &p);
    };
}

#[test]
#[should_fail]
fn test_parse_null_as_utf8_fail() {
    unsafe {
        let p: *const libc::c_char = ptr::null();
        let _ = super::parse_as_utf8(p, &p);
    };
}

#[test]
#[should_fail]
fn test_c_string_new_fail() {
    let _c_str: OwnedCString = unsafe {
        OwnedCString::new(ptr::null(), libc_free)
    };
}

#[test]
#[should_fail]
fn test_from_static_bytes_fail() {
    let _c_str = super::from_static_bytes(b"no nul");
}

#[test]
#[should_fail]
fn test_from_static_str_fail() {
    let _c_str = super::from_static_str("no nul");
}

#[test]
#[should_fail]
fn test_from_raw_ptr_fail() {
    let p: *const libc::c_char = ptr::null();
    let _c_str = unsafe { super::from_raw_ptr(p, &p) };
}

// Do you love me, Travis?
<|endoftext|>


<commit_before>use super::{Logger, Discard};

#[test]
fn logger_fmt_debug_sanity() {

    let root = Logger::root(Discard, o!("a" => "aa"));
    let log = root.new(o!("b" => "bb", "c" => "cc"));

    assert_eq!(format!("{:?}", log), "Logger(b, c, a)");
}

#[cfg(test)]
mod tests {
    use {Logger, Discard};
    /// ensure o! macro expands without error inside a module
    #[test]
    fn test_o_macro_expansion() {
        let _ = Logger::root(Discard, o!("a" => "aa"));
    }
}
<commit_msg>Fix tests after owned values ordering change<commit_after>use super::{Logger, Discard};

#[test]
fn logger_fmt_debug_sanity() {

    let root = Logger::root(Discard, o!("a" => "aa"));
    let log = root.new(o!("b" => "bb", "c" => "cc"));

    assert_eq!(format!("{:?}", log), "Logger(c, b, a)");
}

#[cfg(test)]
mod tests {
    use {Logger, Discard};
    /// ensure o! macro expands without error inside a module
    #[test]
    fn test_o_macro_expansion() {
        let _ = Logger::root(Discard, o!("a" => "aa"));
    }
}
<|endoftext|>


<commit_before>use std::collections;
use std::fmt;
use std::ops;
use std::slice;
use std::vec;

/// A sentence consists of zero or more `Token`s. It is a small wrapper
/// around `Vec<Token>` that provides some extra convenience. For example,
/// the implementation of the `Display` trait outputs the sentence in
/// CoNLL-X format.
#[derive(Debug,PartialEq)]
pub struct Sentence {
    tokens: Vec<Token>,
}

impl Sentence {
    /// Create a new `Sentence` from a vector of tokens.
    pub fn new(tokens: Vec<Token>) -> Sentence {
        Sentence { tokens: tokens }
    }

    /// Get the underlying vector of tokens.
    pub fn as_tokens(&self) -> &Vec<Token> {
        &self.tokens
    }

    /// Get an iterator over the sentence tokens.
    pub fn iter(&self) -> slice::Iter<Token> {
        self.tokens.iter()
    }

    /// Get a mutable iterator over the sentence tokens.
    pub fn iter_mut(&mut self) -> slice::IterMut<Token> {
        self.tokens.iter_mut()
    }
}

impl fmt::Display for Sentence {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let last_id = self.tokens.len() - 1;

        for (id, token) in self.iter().enumerate() {
            if id == last_id {
                try!(write!(f, "{}\t{}", id + 1, token))
            } else {
                try!(write!(f, "{}\t{}\n", id + 1, token))
            }
        }

        Ok(())
    }
}

impl ops::Index<usize> for Sentence {
    type Output = Token;
    fn index(&self, index: usize) -> &Token {
        &self.tokens[index]
    }
}

impl ops::IndexMut<usize> for Sentence {
    fn index_mut(&mut self, index: usize) -> &mut Token {
        &mut self.tokens[index]
    }
}

impl IntoIterator for Sentence {
    type Item = Token;
    type IntoIter = vec::IntoIter<Token>;

    fn into_iter(self) -> vec::IntoIter<Token> {
        self.tokens.into_iter()
    }
}

impl<'a> IntoIterator for &'a Sentence {
    type Item = &'a Token;
    type IntoIter = slice::Iter<'a, Token>;

    fn into_iter(self) -> slice::Iter<'a, Token> {
        self.iter()
    }
}

impl<'a> IntoIterator for &'a mut Sentence {
    type Item = &'a mut Token;
    type IntoIter = slice::IterMut<'a, Token>;

    fn into_iter(mut self) -> slice::IterMut<'a, Token> {
        self.iter_mut()
    }
}

/// A builder for `Token`s.
///
/// The `Token` type stores a CoNLL-X token. However, since this format
/// permits a large number of fields, construction of a token can get
/// tedious. This builder provides a fluent interface for creating `Token`s.
pub struct TokenBuilder {
    token: Token,
}

impl TokenBuilder {
    /// Create a `Token` builder with all fields set to absent.
    pub fn new() -> TokenBuilder {
        TokenBuilder { token: Token::new() }
    }

    /// Set the word form or punctuation symbol.
    pub fn form<S>(mut self, form: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_form(Some(form));
        self
    }

    /// Set the lemma or stem of the word form.
    pub fn lemma<S>(mut self, lemma: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_lemma(Some(lemma));
        self
    }

    /// Set the coarse-grained part-of-speech tag.
    pub fn cpos<S>(mut self, cpos: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_cpos(Some(cpos));
        self
    }

    /// Set the fine-grained part-of-speech tag.
    pub fn pos<S>(mut self, pos: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_pos(Some(pos));
        self
    }

    /// Set the syntactic and/or morphological features of the token.
    pub fn features<S>(mut self, features: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_features(Some(features));
        self
    }

    /// Set the head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree.
    pub fn head(mut self, head: usize) -> TokenBuilder {
        self.token.set_head(Some(head));
        self
    }

    /// Set the dependency relation to the head of this token.
    pub fn head_rel<S>(mut self, head_rel: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_head_rel(Some(head_rel));
        self
    }

    /// Set the projective head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree. The dependency structure resulting from the
    /// projective heads must be projective.
    pub fn p_head(mut self, p_head: usize) -> TokenBuilder {
        self.token.set_p_head(Some(p_head));
        self
    }

    /// Set the dependency relation to the projective head of this token.
    pub fn p_head_rel<S>(mut self, p_head_rel: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_p_head_rel(Some(p_head_rel));
        self
    }

    pub fn token(self) -> Token {
        self.token
    }
}

/// A token in the CoNLL-X dependency format.
///
/// The fields of CoNLLX tokens are described at:
///
/// http://ilk.uvt.nl/conll/
///
/// This type provides exactly the same fields, except for the ID field
/// (since it can be derived from the sentence position of the token).
/// If a particular field is absent (*_* in the CoNLL-X format), its
/// value is `None`.
#[derive(Debug,PartialEq)]
pub struct Token {
    form: Option<String>,
    lemma: Option<String>,
    cpos: Option<String>,
    pos: Option<String>,
    features: Option<Features>,
    head: Option<usize>,
    head_rel: Option<String>,
    p_head: Option<usize>,
    p_head_rel: Option<String>,
}

impl Token {
    /// Create a new token where all the fields are absent.
    pub fn new() -> Token {
        Token {
            form: None,
            lemma: None,
            cpos: None,
            pos: None,
            features: None,
            head: None,
            head_rel: None,
            p_head: None,
            p_head_rel: None,
        }
    }

    /// Get the word form or punctuation symbol.
    pub fn form(&self) -> Option<&str> {
        self.form.as_ref().map(String::as_ref)
    }

    /// Get the lemma or stem of the word form.
    pub fn lemma(&self) -> Option<&str> {
        self.lemma.as_ref().map(String::as_ref)
    }

    /// Get the coarse-grained part-of-speech tag.
    pub fn cpos(&self) -> Option<&str> {
        self.cpos.as_ref().map(String::as_ref)
    }

    /// Get the fine-grained part-of-speech tag.
    pub fn pos(&self) -> Option<&str> {
        self.pos.as_ref().map(String::as_ref)
    }

    /// Get the syntactic and/or morphological features of the token.
    pub fn features(&self) -> Option<&Features> {
        self.features.as_ref()
    }

    /// Get the head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree.
    pub fn head(&self) -> Option<usize> {
        self.head
    }

    /// Get the dependency relation to the head of this token.
    pub fn head_rel(&self) -> Option<&str> {
        self.head_rel.as_ref().map(String::as_ref)
    }

    /// Get the projective head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree. The dependency structure resulting from the
    /// projective heads must be projective.
    pub fn p_head(&self) -> Option<usize> {
        self.p_head
    }

    /// Get the dependency relation to the projective head of this token.
    pub fn p_head_rel(&self) -> Option<&str> {
        self.p_head_rel.as_ref().map(String::as_ref)
    }

    /// Set the word form or punctuation symbol.
    pub fn set_form<S>(&mut self, form: Option<S>)
        where S: Into<String>
    {
        self.form = form.map(|i| i.into())
    }

    /// Set the lemma or stem of the word form.
    pub fn set_lemma<S>(&mut self, lemma: Option<S>)
        where S: Into<String>
    {
        self.lemma = lemma.map(|i| i.into())
    }

    /// Set the coarse-grained part-of-speech tag.
    pub fn set_cpos<S>(&mut self, cpos: Option<S>)
        where S: Into<String>
    {
        self.cpos = cpos.map(|i| i.into())
    }

    /// Set the fine-grained part-of-speech tag.
    pub fn set_pos<S>(&mut self, pos: Option<S>)
        where S: Into<String>
    {
        self.pos = pos.map(|i| i.into())
    }

    /// Set the syntactic and/or morphological features of the token.
    pub fn set_features<S>(&mut self, features: Option<S>)
        where S: Into<String>
    {
        self.features = features.map(|s| Features { features: s.into() })
    }

    /// Set the head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree.
    pub fn set_head(&mut self, head: Option<usize>) {
        self.head = head
    }

    /// Set the dependency relation to the head of this token.
    pub fn set_head_rel<S>(&mut self, head_rel: Option<S>)
        where S: Into<String>
    {
        self.head_rel = head_rel.map(|i| i.into())
    }

    /// Set the projective head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree. The dependency structure resulting from the
    /// projective heads must be projective.
    pub fn set_p_head(&mut self, p_head: Option<usize>) {
        self.p_head = p_head
    }

    /// Set the dependency relation to the projective head of this token.
    pub fn set_p_head_rel<S>(&mut self, p_head_rel: Option<S>)
        where S: Into<String>
    {
        self.p_head_rel = p_head_rel.map(|i| i.into())
    }
}

/// This data type is used to represent features on a `Token`. In the CoNLL-X
/// specification, these are morphological features of the token. However,
/// they are used as a free form list or mapping of features.
#[derive(Debug,PartialEq)]
pub struct Features {
    features: String,
}

impl Features {
    /// Get the features field as a key-value mapping. This assumes that
    /// the key-value pairs are separed using a vertical bar (`|`) and keys
    /// and values using a colon (`:`). If the value is absent, corresponding
    /// value in the mapping is `None`.
    pub fn as_map(&self) -> collections::BTreeMap<String, Option<String>> {
        let mut features = collections::BTreeMap::new();

        for fv in self.features.split('|') {
            let mut iter = fv.split(':');
            if let Some(k) = iter.next() {
                let v = iter.next().map(|s| s.to_owned());
                features.insert(k.to_owned(), v.to_owned());
            }
        }

        features
    }

    /// Get the features field.
    pub fn as_str(&self) -> &str {
        self.features.as_ref()
    }
}

impl fmt::Display for Features {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.write_str(self.features.as_ref())
    }
}

pub const EMPTY_TOKEN: &'static str = "_";

impl fmt::Display for Token {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let head_str = self.head.as_ref().map(|n| n.to_string());
        let p_head_str = self.p_head.as_ref().map(|n| n.to_string());

        write!(f,
               "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
               self.form.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.lemma.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.cpos.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.pos.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.features.as_ref().map(|s| s.as_str()).unwrap_or(EMPTY_TOKEN),
               head_str.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.head_rel.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.p_head.clone().map(|n| n.to_string()).unwrap_or(EMPTY_TOKEN.to_string()),
               p_head_str.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN))
    }
}
<commit_msg>Derive Clone on Token and Sentence.<commit_after>use std::collections;
use std::fmt;
use std::ops;
use std::slice;
use std::vec;

/// A sentence consists of zero or more `Token`s. It is a small wrapper
/// around `Vec<Token>` that provides some extra convenience. For example,
/// the implementation of the `Display` trait outputs the sentence in
/// CoNLL-X format.
#[derive(Clone, Debug,PartialEq)]
pub struct Sentence {
    tokens: Vec<Token>,
}

impl Sentence {
    /// Create a new `Sentence` from a vector of tokens.
    pub fn new(tokens: Vec<Token>) -> Sentence {
        Sentence { tokens: tokens }
    }

    /// Get the underlying vector of tokens.
    pub fn as_tokens(&self) -> &Vec<Token> {
        &self.tokens
    }

    /// Get an iterator over the sentence tokens.
    pub fn iter(&self) -> slice::Iter<Token> {
        self.tokens.iter()
    }

    /// Get a mutable iterator over the sentence tokens.
    pub fn iter_mut(&mut self) -> slice::IterMut<Token> {
        self.tokens.iter_mut()
    }
}

impl fmt::Display for Sentence {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let last_id = self.tokens.len() - 1;

        for (id, token) in self.iter().enumerate() {
            if id == last_id {
                try!(write!(f, "{}\t{}", id + 1, token))
            } else {
                try!(write!(f, "{}\t{}\n", id + 1, token))
            }
        }

        Ok(())
    }
}

impl ops::Index<usize> for Sentence {
    type Output = Token;
    fn index(&self, index: usize) -> &Token {
        &self.tokens[index]
    }
}

impl ops::IndexMut<usize> for Sentence {
    fn index_mut(&mut self, index: usize) -> &mut Token {
        &mut self.tokens[index]
    }
}

impl IntoIterator for Sentence {
    type Item = Token;
    type IntoIter = vec::IntoIter<Token>;

    fn into_iter(self) -> vec::IntoIter<Token> {
        self.tokens.into_iter()
    }
}

impl<'a> IntoIterator for &'a Sentence {
    type Item = &'a Token;
    type IntoIter = slice::Iter<'a, Token>;

    fn into_iter(self) -> slice::Iter<'a, Token> {
        self.iter()
    }
}

impl<'a> IntoIterator for &'a mut Sentence {
    type Item = &'a mut Token;
    type IntoIter = slice::IterMut<'a, Token>;

    fn into_iter(mut self) -> slice::IterMut<'a, Token> {
        self.iter_mut()
    }
}

/// A builder for `Token`s.
///
/// The `Token` type stores a CoNLL-X token. However, since this format
/// permits a large number of fields, construction of a token can get
/// tedious. This builder provides a fluent interface for creating `Token`s.
pub struct TokenBuilder {
    token: Token,
}

impl TokenBuilder {
    /// Create a `Token` builder with all fields set to absent.
    pub fn new() -> TokenBuilder {
        TokenBuilder { token: Token::new() }
    }

    /// Set the word form or punctuation symbol.
    pub fn form<S>(mut self, form: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_form(Some(form));
        self
    }

    /// Set the lemma or stem of the word form.
    pub fn lemma<S>(mut self, lemma: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_lemma(Some(lemma));
        self
    }

    /// Set the coarse-grained part-of-speech tag.
    pub fn cpos<S>(mut self, cpos: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_cpos(Some(cpos));
        self
    }

    /// Set the fine-grained part-of-speech tag.
    pub fn pos<S>(mut self, pos: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_pos(Some(pos));
        self
    }

    /// Set the syntactic and/or morphological features of the token.
    pub fn features<S>(mut self, features: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_features(Some(features));
        self
    }

    /// Set the head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree.
    pub fn head(mut self, head: usize) -> TokenBuilder {
        self.token.set_head(Some(head));
        self
    }

    /// Set the dependency relation to the head of this token.
    pub fn head_rel<S>(mut self, head_rel: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_head_rel(Some(head_rel));
        self
    }

    /// Set the projective head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree. The dependency structure resulting from the
    /// projective heads must be projective.
    pub fn p_head(mut self, p_head: usize) -> TokenBuilder {
        self.token.set_p_head(Some(p_head));
        self
    }

    /// Set the dependency relation to the projective head of this token.
    pub fn p_head_rel<S>(mut self, p_head_rel: S) -> TokenBuilder
        where S: Into<String>
    {
        self.token.set_p_head_rel(Some(p_head_rel));
        self
    }

    pub fn token(self) -> Token {
        self.token
    }
}

/// A token in the CoNLL-X dependency format.
///
/// The fields of CoNLLX tokens are described at:
///
/// http://ilk.uvt.nl/conll/
///
/// This type provides exactly the same fields, except for the ID field
/// (since it can be derived from the sentence position of the token).
/// If a particular field is absent (*_* in the CoNLL-X format), its
/// value is `None`.
#[derive(Clone,Debug,PartialEq)]
pub struct Token {
    form: Option<String>,
    lemma: Option<String>,
    cpos: Option<String>,
    pos: Option<String>,
    features: Option<Features>,
    head: Option<usize>,
    head_rel: Option<String>,
    p_head: Option<usize>,
    p_head_rel: Option<String>,
}

impl Token {
    /// Create a new token where all the fields are absent.
    pub fn new() -> Token {
        Token {
            form: None,
            lemma: None,
            cpos: None,
            pos: None,
            features: None,
            head: None,
            head_rel: None,
            p_head: None,
            p_head_rel: None,
        }
    }

    /// Get the word form or punctuation symbol.
    pub fn form(&self) -> Option<&str> {
        self.form.as_ref().map(String::as_ref)
    }

    /// Get the lemma or stem of the word form.
    pub fn lemma(&self) -> Option<&str> {
        self.lemma.as_ref().map(String::as_ref)
    }

    /// Get the coarse-grained part-of-speech tag.
    pub fn cpos(&self) -> Option<&str> {
        self.cpos.as_ref().map(String::as_ref)
    }

    /// Get the fine-grained part-of-speech tag.
    pub fn pos(&self) -> Option<&str> {
        self.pos.as_ref().map(String::as_ref)
    }

    /// Get the syntactic and/or morphological features of the token.
    pub fn features(&self) -> Option<&Features> {
        self.features.as_ref()
    }

    /// Get the head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree.
    pub fn head(&self) -> Option<usize> {
        self.head
    }

    /// Get the dependency relation to the head of this token.
    pub fn head_rel(&self) -> Option<&str> {
        self.head_rel.as_ref().map(String::as_ref)
    }

    /// Get the projective head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree. The dependency structure resulting from the
    /// projective heads must be projective.
    pub fn p_head(&self) -> Option<usize> {
        self.p_head
    }

    /// Get the dependency relation to the projective head of this token.
    pub fn p_head_rel(&self) -> Option<&str> {
        self.p_head_rel.as_ref().map(String::as_ref)
    }

    /// Set the word form or punctuation symbol.
    pub fn set_form<S>(&mut self, form: Option<S>)
        where S: Into<String>
    {
        self.form = form.map(|i| i.into())
    }

    /// Set the lemma or stem of the word form.
    pub fn set_lemma<S>(&mut self, lemma: Option<S>)
        where S: Into<String>
    {
        self.lemma = lemma.map(|i| i.into())
    }

    /// Set the coarse-grained part-of-speech tag.
    pub fn set_cpos<S>(&mut self, cpos: Option<S>)
        where S: Into<String>
    {
        self.cpos = cpos.map(|i| i.into())
    }

    /// Set the fine-grained part-of-speech tag.
    pub fn set_pos<S>(&mut self, pos: Option<S>)
        where S: Into<String>
    {
        self.pos = pos.map(|i| i.into())
    }

    /// Set the syntactic and/or morphological features of the token.
    pub fn set_features<S>(&mut self, features: Option<S>)
        where S: Into<String>
    {
        self.features = features.map(|s| Features { features: s.into() })
    }

    /// Set the head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree.
    pub fn set_head(&mut self, head: Option<usize>) {
        self.head = head
    }

    /// Set the dependency relation to the head of this token.
    pub fn set_head_rel<S>(&mut self, head_rel: Option<S>)
        where S: Into<String>
    {
        self.head_rel = head_rel.map(|i| i.into())
    }

    /// Set the projective head of the token. This is the sentence position
    /// of the head **plus one**. If the head is 0, the token the root
    /// of the dependency tree. The dependency structure resulting from the
    /// projective heads must be projective.
    pub fn set_p_head(&mut self, p_head: Option<usize>) {
        self.p_head = p_head
    }

    /// Set the dependency relation to the projective head of this token.
    pub fn set_p_head_rel<S>(&mut self, p_head_rel: Option<S>)
        where S: Into<String>
    {
        self.p_head_rel = p_head_rel.map(|i| i.into())
    }
}

/// This data type is used to represent features on a `Token`. In the CoNLL-X
/// specification, these are morphological features of the token. However,
/// they are used as a free form list or mapping of features.
#[derive(Clone,Debug,PartialEq)]
pub struct Features {
    features: String,
}

impl Features {
    /// Get the features field as a key-value mapping. This assumes that
    /// the key-value pairs are separed using a vertical bar (`|`) and keys
    /// and values using a colon (`:`). If the value is absent, corresponding
    /// value in the mapping is `None`.
    pub fn as_map(&self) -> collections::BTreeMap<String, Option<String>> {
        let mut features = collections::BTreeMap::new();

        for fv in self.features.split('|') {
            let mut iter = fv.split(':');
            if let Some(k) = iter.next() {
                let v = iter.next().map(|s| s.to_owned());
                features.insert(k.to_owned(), v.to_owned());
            }
        }

        features
    }

    /// Get the features field.
    pub fn as_str(&self) -> &str {
        self.features.as_ref()
    }
}

impl fmt::Display for Features {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.write_str(self.features.as_ref())
    }
}

pub const EMPTY_TOKEN: &'static str = "_";

impl fmt::Display for Token {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let head_str = self.head.as_ref().map(|n| n.to_string());
        let p_head_str = self.p_head.as_ref().map(|n| n.to_string());

        write!(f,
               "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
               self.form.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.lemma.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.cpos.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.pos.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.features.as_ref().map(|s| s.as_str()).unwrap_or(EMPTY_TOKEN),
               head_str.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.head_rel.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN),
               self.p_head.clone().map(|n| n.to_string()).unwrap_or(EMPTY_TOKEN.to_string()),
               p_head_str.as_ref().map(|s| s.as_ref()).unwrap_or(EMPTY_TOKEN))
    }
}
<|endoftext|>


<commit_before>//! Tokens in the dependency graph.

use std::collections::BTreeMap;
use std::fmt;
use std::fmt::{Debug, Display};
use std::iter::FromIterator;
use std::mem;

use itertools::Itertools;
use lazy_init::Lazy;

pub const EMPTY_TOKEN: &'static str = "_";

/// A builder for `Token`s.
///
/// The `Token` type stores a CoNLL-X token. However, since this format
/// permits a large number of fields, construction of a token can get
/// tedious. This builder provides a fluent interface for creating `Token`s.
pub struct TokenBuilder {
    token: Token,
}

impl TokenBuilder {
    /// Create a `Token` builder with all non-form fields set to absent.
    pub fn new(form: impl Into<String>) -> TokenBuilder {
        TokenBuilder {
            token: Token::new(form),
        }
    }

    /// Set the word form or punctuation symbol.
    pub fn form(mut self, form: impl Into<String>) -> TokenBuilder {
        self.token.set_form(form);
        self
    }

    /// Set the lemma or stem of the word form.
    pub fn lemma(mut self, lemma: impl Into<String>) -> TokenBuilder {
        self.token.set_lemma(Some(lemma));
        self
    }

    /// Set the coarse-grained part-of-speech tag.
    pub fn cpos(mut self, cpos: impl Into<String>) -> TokenBuilder {
        self.token.set_cpos(Some(cpos));
        self
    }

    /// Set the fine-grained part-of-speech tag.
    pub fn pos(mut self, pos: impl Into<String>) -> TokenBuilder {
        self.token.set_pos(Some(pos));
        self
    }

    /// Set the syntactic and/or morphological features of the token.
    pub fn features(mut self, features: Features) -> TokenBuilder {
        self.token.set_features(Some(features));
        self
    }
}

impl From<Token> for TokenBuilder {
    fn from(token: Token) -> Self {
        TokenBuilder { token }
    }
}

impl From<TokenBuilder> for Token {
    fn from(builder: TokenBuilder) -> Self {
        builder.token
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Token {
    form: String,
    lemma: Option<String>,
    cpos: Option<String>,
    pos: Option<String>,
    features: Option<Features>,
}

impl Token {
    /// Create a new token where all the non-form fields are absent.
    pub fn new(form: impl Into<String>) -> Token {
        Token {
            form: form.into(),
            lemma: None,
            cpos: None,
            pos: None,
            features: None,
        }
    }

    /// Get the word form or punctuation symbol.
    pub fn form(&self) -> &str {
        self.form.as_ref()
    }

    /// Get the lemma or stem of the word form.
    pub fn lemma(&self) -> Option<&str> {
        self.lemma.as_ref().map(String::as_ref)
    }

    /// Get the coarse-grained part-of-speech tag.
    pub fn cpos(&self) -> Option<&str> {
        self.cpos.as_ref().map(String::as_ref)
    }

    /// Get the fine-grained part-of-speech tag.
    pub fn pos(&self) -> Option<&str> {
        self.pos.as_ref().map(String::as_ref)
    }

    /// Get the syntactic and/or morphological features of the token.
    pub fn features(&self) -> Option<&Features> {
        self.features.as_ref()
    }

    /// Set the word form or punctuation symbol.
    ///
    /// Returns the form that is replaced.
    pub fn set_form(&mut self, form: impl Into<String>) -> String {
        mem::replace(&mut self.form, form.into())
    }

    /// Set the lemma or stem of the word form.
    ///
    /// Returns the lemma that is replaced.
    pub fn set_lemma<S>(&mut self, lemma: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.lemma, lemma.map(|i| i.into()))
    }

    /// Set the coarse-grained part-of-speech tag.
    ///
    /// Returns the coarse-grained part-of-speech tag that is replaced.
    pub fn set_cpos<S>(&mut self, cpos: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.cpos, cpos.map(|i| i.into()))
    }

    /// Set the fine-grained part-of-speech tag.
    ///
    /// Returns the fine-grained part-of-speech tag that is replaced.
    pub fn set_pos<S>(&mut self, pos: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.pos, pos.map(|i| i.into()))
    }

    /// Set the syntactic and/or morphological features of the token.
    ///
    /// Returns the features that are replaced.
    pub fn set_features(&mut self, features: Option<Features>) -> Option<Features> {
        mem::replace(&mut self.features, features)
    }
}

/// Token features.
///
/// In the CoNLL-X specification, these are morphological features of the
/// token. Typically, the features are a list or a key-value mapping.
pub struct Features {
    features: String,
    feature_map: Lazy<BTreeMap<String, Option<String>>>,
}

impl Features {
    /// Create features from a string. The casual format uses key-value
    /// pairs that are separated by a vertical bar (`|`) and keys and
    /// values using a colon (`:`). Arbitrary strings will also be accepted,
    /// however they will not give a nice feature-value mapping when using
    /// `as_map`.
    pub fn from_string(s: impl Into<String>) -> Self {
        Features {
            features: s.into(),
            feature_map: Lazy::new(),
        }
    }

    pub fn from_iter<I, S, T>(iter: I) -> Self
    where
        I: IntoIterator<Item = (S, Option<T>)>,
        S: Into<String>,
        T: Into<String>,
    {
        let feature_map =
            BTreeMap::from_iter(iter.into_iter().map(|(k, v)| (k.into(), v.map(Into::into))));
        let features = map_to_string(&feature_map);

        let lazy_feature_map = Lazy::new();
        lazy_feature_map.get_or_create(|| feature_map);

        Features {
            features,
            feature_map: lazy_feature_map,
        }
    }

    /// Get the features field as a key-value mapping. This assumes that
    /// the key-value pairs are separed using a vertical bar (`|`) and keys
    /// and values using a colon (`:`). If the value is absent, corresponding
    /// value in the mapping is `None`.
    ///
    /// The feature map is constructed lazily:
    ///
    /// * If `as_map` is never called, the feature map is never created.
    /// * If `as_map` is called once or more, the feature map is only created
    ///   once.
    pub fn as_map(&self) -> &BTreeMap<String, Option<String>> {
        self.feature_map.get_or_create(|| self.as_map_eager())
    }

    /// Get the features field.
    pub fn as_str(&self) -> &str {
        self.features.as_ref()
    }

    /// Unwrap the contained feature string and map. Since the feature map is
    /// initialized lazily, this will force initialization of the feature map
    /// when necessary.
    pub fn into_inner(self) -> (String, BTreeMap<String, Option<String>>) {
        let _ = self.feature_map.get_or_create(|| self.as_map_eager());
        (
            self.features,
            self.feature_map
                .into_inner()
                .expect("feature map should have been initialized"),
        )
    }

    /// Unwrap the contained feature map. Since the feature map is initialized
    /// lazily, this will force initialization of the feature map when necessary.
    pub fn into_inner_map(self) -> BTreeMap<String, Option<String>> {
        let _ = self.feature_map.get_or_create(|| self.as_map_eager());
        self.feature_map
            .into_inner()
            .expect("feature map should have been initialized")
    }

    /// Unwrap the contained feature string.
    pub fn into_inner_string(self) -> String {
        self.features
    }

    fn as_map_eager(&self) -> BTreeMap<String, Option<String>> {
        let mut features = BTreeMap::new();

        for fv in self.features.split('|') {
            let mut iter = fv.split(':');
            if let Some(k) = iter.next() {
                let v = iter.next().map(|s| s.to_owned());
                features.insert(k.to_owned(), v.to_owned());
            }
        }

        features
    }
}

impl Clone for Features {
    fn clone(&self) -> Self {
        Features {
            features: self.features.clone(),
            feature_map: Lazy::new(),
        }
    }
}

impl Debug for Features {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Features {{ features: {} }}", self.features)
    }
}

impl Display for Features {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.write_str(self.features.as_ref())
    }
}

impl Eq for Features {}

impl PartialEq for Features {
    fn eq(&self, other: &Features) -> bool {
        self.features.eq(&other.features)
    }
}

fn map_to_string(feature_map: &BTreeMap<String, Option<String>>) -> String {
    feature_map
        .iter()
        .map(|(k, v)| match *v {
            Some(ref v) => format!("{}:{}", k, v),
            None => k.to_owned(),
        })
        .join("|")
}

#[cfg(test)]
mod tests {
    use std::collections::BTreeMap;

    use maplit::btreemap;
    use quickcheck::quickcheck;

    use super::{Features, Token, TokenBuilder};

    quickcheck! {
        fn features_from_iter(feature_map: BTreeMap<String, Option<String>>) -> bool{
            &feature_map == Features::from_iter(feature_map.clone()).as_map()
        }
    }

    #[test]
    fn features_from_iter_as_string() {
        let feature_map = btreemap! {
            "feature2" => Some("y"),
            "feature3" => None,
            "feature1" => Some("x")
        };

        let features = Features::from_iter(feature_map);

        assert_eq!(features.as_str(), "feature1:x|feature2:y|feature3");
    }

    #[test]
    fn features() {
        let tokens = token_with_features();
        let features = features_correct();

        for (token, correct) in tokens.iter().zip(features) {
            let kv = token.features().as_ref().unwrap().as_map();
            assert_eq!(&correct, kv);
        }
    }

    fn token_with_features() -> Vec<Token> {
        vec![
            TokenBuilder::new("Gilles")
                .lemma("Gilles")
                .cpos("N")
                .pos("NE")
                .features(Features::from_string(
                    "case:nominative|number:singular|gender:masculine",
                ))
                .into(),
            TokenBuilder::new("Deleuze")
                .lemma("Deleuze")
                .cpos("N")
                .pos("NE")
                .features(Features::from_string("nominative|singular|masculine"))
                .into(),
        ]
    }

    fn features_correct() -> Vec<BTreeMap<String, Option<String>>> {
        let mut correct1 = BTreeMap::new();
        correct1.insert("case".to_owned(), Some("nominative".to_owned()));
        correct1.insert("number".to_owned(), Some("singular".to_owned()));
        correct1.insert("gender".to_owned(), Some("masculine".to_owned()));

        let mut correct2 = BTreeMap::new();
        correct2.insert("nominative".to_owned(), None);
        correct2.insert("singular".to_owned(), None);
        correct2.insert("masculine".to_owned(), None);

        vec![correct1, correct2]
    }
}
<commit_msg>Feature PartialEq: use map rather than string<commit_after>//! Tokens in the dependency graph.

use std::collections::BTreeMap;
use std::fmt;
use std::fmt::{Debug, Display};
use std::iter::FromIterator;
use std::mem;

use itertools::Itertools;
use lazy_init::Lazy;

pub const EMPTY_TOKEN: &'static str = "_";

/// A builder for `Token`s.
///
/// The `Token` type stores a CoNLL-X token. However, since this format
/// permits a large number of fields, construction of a token can get
/// tedious. This builder provides a fluent interface for creating `Token`s.
pub struct TokenBuilder {
    token: Token,
}

impl TokenBuilder {
    /// Create a `Token` builder with all non-form fields set to absent.
    pub fn new(form: impl Into<String>) -> TokenBuilder {
        TokenBuilder {
            token: Token::new(form),
        }
    }

    /// Set the word form or punctuation symbol.
    pub fn form(mut self, form: impl Into<String>) -> TokenBuilder {
        self.token.set_form(form);
        self
    }

    /// Set the lemma or stem of the word form.
    pub fn lemma(mut self, lemma: impl Into<String>) -> TokenBuilder {
        self.token.set_lemma(Some(lemma));
        self
    }

    /// Set the coarse-grained part-of-speech tag.
    pub fn cpos(mut self, cpos: impl Into<String>) -> TokenBuilder {
        self.token.set_cpos(Some(cpos));
        self
    }

    /// Set the fine-grained part-of-speech tag.
    pub fn pos(mut self, pos: impl Into<String>) -> TokenBuilder {
        self.token.set_pos(Some(pos));
        self
    }

    /// Set the syntactic and/or morphological features of the token.
    pub fn features(mut self, features: Features) -> TokenBuilder {
        self.token.set_features(Some(features));
        self
    }
}

impl From<Token> for TokenBuilder {
    fn from(token: Token) -> Self {
        TokenBuilder { token }
    }
}

impl From<TokenBuilder> for Token {
    fn from(builder: TokenBuilder) -> Self {
        builder.token
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Token {
    form: String,
    lemma: Option<String>,
    cpos: Option<String>,
    pos: Option<String>,
    features: Option<Features>,
}

impl Token {
    /// Create a new token where all the non-form fields are absent.
    pub fn new(form: impl Into<String>) -> Token {
        Token {
            form: form.into(),
            lemma: None,
            cpos: None,
            pos: None,
            features: None,
        }
    }

    /// Get the word form or punctuation symbol.
    pub fn form(&self) -> &str {
        self.form.as_ref()
    }

    /// Get the lemma or stem of the word form.
    pub fn lemma(&self) -> Option<&str> {
        self.lemma.as_ref().map(String::as_ref)
    }

    /// Get the coarse-grained part-of-speech tag.
    pub fn cpos(&self) -> Option<&str> {
        self.cpos.as_ref().map(String::as_ref)
    }

    /// Get the fine-grained part-of-speech tag.
    pub fn pos(&self) -> Option<&str> {
        self.pos.as_ref().map(String::as_ref)
    }

    /// Get the syntactic and/or morphological features of the token.
    pub fn features(&self) -> Option<&Features> {
        self.features.as_ref()
    }

    /// Set the word form or punctuation symbol.
    ///
    /// Returns the form that is replaced.
    pub fn set_form(&mut self, form: impl Into<String>) -> String {
        mem::replace(&mut self.form, form.into())
    }

    /// Set the lemma or stem of the word form.
    ///
    /// Returns the lemma that is replaced.
    pub fn set_lemma<S>(&mut self, lemma: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.lemma, lemma.map(|i| i.into()))
    }

    /// Set the coarse-grained part-of-speech tag.
    ///
    /// Returns the coarse-grained part-of-speech tag that is replaced.
    pub fn set_cpos<S>(&mut self, cpos: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.cpos, cpos.map(|i| i.into()))
    }

    /// Set the fine-grained part-of-speech tag.
    ///
    /// Returns the fine-grained part-of-speech tag that is replaced.
    pub fn set_pos<S>(&mut self, pos: Option<S>) -> Option<String>
    where
        S: Into<String>,
    {
        mem::replace(&mut self.pos, pos.map(|i| i.into()))
    }

    /// Set the syntactic and/or morphological features of the token.
    ///
    /// Returns the features that are replaced.
    pub fn set_features(&mut self, features: Option<Features>) -> Option<Features> {
        mem::replace(&mut self.features, features)
    }
}

/// Token features.
///
/// In the CoNLL-X specification, these are morphological features of the
/// token. Typically, the features are a list or a key-value mapping.
pub struct Features {
    features: String,
    feature_map: Lazy<BTreeMap<String, Option<String>>>,
}

impl Features {
    /// Create features from a string. The casual format uses key-value
    /// pairs that are separated by a vertical bar (`|`) and keys and
    /// values using a colon (`:`). Arbitrary strings will also be accepted,
    /// however they will not give a nice feature-value mapping when using
    /// `as_map`.
    pub fn from_string(s: impl Into<String>) -> Self {
        Features {
            features: s.into(),
            feature_map: Lazy::new(),
        }
    }

    pub fn from_iter<I, S, T>(iter: I) -> Self
    where
        I: IntoIterator<Item = (S, Option<T>)>,
        S: Into<String>,
        T: Into<String>,
    {
        let feature_map =
            BTreeMap::from_iter(iter.into_iter().map(|(k, v)| (k.into(), v.map(Into::into))));
        let features = map_to_string(&feature_map);

        let lazy_feature_map = Lazy::new();
        lazy_feature_map.get_or_create(|| feature_map);

        Features {
            features,
            feature_map: lazy_feature_map,
        }
    }

    /// Get the features field as a key-value mapping. This assumes that
    /// the key-value pairs are separed using a vertical bar (`|`) and keys
    /// and values using a colon (`:`). If the value is absent, corresponding
    /// value in the mapping is `None`.
    ///
    /// The feature map is constructed lazily:
    ///
    /// * If `as_map` is never called, the feature map is never created.
    /// * If `as_map` is called once or more, the feature map is only created
    ///   once.
    pub fn as_map(&self) -> &BTreeMap<String, Option<String>> {
        self.feature_map.get_or_create(|| self.as_map_eager())
    }

    /// Get the features field.
    pub fn as_str(&self) -> &str {
        self.features.as_ref()
    }

    /// Unwrap the contained feature string and map. Since the feature map is
    /// initialized lazily, this will force initialization of the feature map
    /// when necessary.
    pub fn into_inner(self) -> (String, BTreeMap<String, Option<String>>) {
        let _ = self.feature_map.get_or_create(|| self.as_map_eager());
        (
            self.features,
            self.feature_map
                .into_inner()
                .expect("feature map should have been initialized"),
        )
    }

    /// Unwrap the contained feature map. Since the feature map is initialized
    /// lazily, this will force initialization of the feature map when necessary.
    pub fn into_inner_map(self) -> BTreeMap<String, Option<String>> {
        let _ = self.feature_map.get_or_create(|| self.as_map_eager());
        self.feature_map
            .into_inner()
            .expect("feature map should have been initialized")
    }

    /// Unwrap the contained feature string.
    pub fn into_inner_string(self) -> String {
        self.features
    }

    fn as_map_eager(&self) -> BTreeMap<String, Option<String>> {
        let mut features = BTreeMap::new();

        for fv in self.features.split('|') {
            let mut iter = fv.split(':');
            if let Some(k) = iter.next() {
                let v = iter.next().map(|s| s.to_owned());
                features.insert(k.to_owned(), v.to_owned());
            }
        }

        features
    }
}

impl Clone for Features {
    fn clone(&self) -> Self {
        Features {
            features: self.features.clone(),
            feature_map: Lazy::new(),
        }
    }
}

impl Debug for Features {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Features {{ features: {} }}", self.features)
    }
}

impl Display for Features {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        f.write_str(self.features.as_ref())
    }
}

impl Eq for Features {}

impl PartialEq for Features {
    fn eq(&self, other: &Features) -> bool {
        self.feature_map.get_or_create(|| self.as_map_eager())
            == other.feature_map.get_or_create(|| other.as_map_eager())
    }
}

fn map_to_string(feature_map: &BTreeMap<String, Option<String>>) -> String {
    feature_map
        .iter()
        .map(|(k, v)| match *v {
            Some(ref v) => format!("{}:{}", k, v),
            None => k.to_owned(),
        })
        .join("|")
}

#[cfg(test)]
mod tests {
    use std::collections::BTreeMap;

    use maplit::btreemap;
    use quickcheck::quickcheck;

    use super::{Features, Token, TokenBuilder};

    quickcheck! {
        fn features_from_iter(feature_map: BTreeMap<String, Option<String>>) -> bool{
            &feature_map == Features::from_iter(feature_map.clone()).as_map()
        }
    }

    #[test]
    fn features_from_iter_as_string() {
        let feature_map = btreemap! {
            "feature2" => Some("y"),
            "feature3" => None,
            "feature1" => Some("x")
        };

        let features = Features::from_iter(feature_map);

        assert_eq!(features.as_str(), "feature1:x|feature2:y|feature3");
    }

    #[test]
    fn features() {
        let tokens = token_with_features();
        let features = features_correct();

        for (token, correct) in tokens.iter().zip(features) {
            let kv = token.features().as_ref().unwrap().as_map();
            assert_eq!(&correct, kv);
        }
    }

    fn token_with_features() -> Vec<Token> {
        vec![
            TokenBuilder::new("Gilles")
                .lemma("Gilles")
                .cpos("N")
                .pos("NE")
                .features(Features::from_string(
                    "case:nominative|number:singular|gender:masculine",
                ))
                .into(),
            TokenBuilder::new("Deleuze")
                .lemma("Deleuze")
                .cpos("N")
                .pos("NE")
                .features(Features::from_string("nominative|singular|masculine"))
                .into(),
        ]
    }

    fn features_correct() -> Vec<BTreeMap<String, Option<String>>> {
        let mut correct1 = BTreeMap::new();
        correct1.insert("case".to_owned(), Some("nominative".to_owned()));
        correct1.insert("number".to_owned(), Some("singular".to_owned()));
        correct1.insert("gender".to_owned(), Some("masculine".to_owned()));

        let mut correct2 = BTreeMap::new();
        correct2.insert("nominative".to_owned(), None);
        correct2.insert("singular".to_owned(), None);
        correct2.insert("masculine".to_owned(), None);

        vec![correct1, correct2]
    }

    #[test]
    fn eq_features_is_order_insensitive() {
        let token1: Token = TokenBuilder::new("a")
            .features(Features::from_string("a|b:c"))
            .into();
        let token2 = TokenBuilder::new("a")
            .features(Features::from_string("b:c|a"))
            .into();
        let token3: Token = TokenBuilder::new("a")
            .features(Features::from_string("b|a:c"))
            .into();

        assert_eq!(token1, token2);
        assert_ne!(token1, token3);
        assert_ne!(token2, token3);
    }
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2014 Jeremy Letang
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

pub struct NSRect {
    origin: NSPoint,
    size: NSSize
}

pub struct NSPoint {
    x: f64,
    y: f64
}

pub struct NSSize {
    width: f64,
    height: f64
}<commit_msg>make files public<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2014 Jeremy Letang
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

pub struct NSRect {
    pub origin: NSPoint,
    pub size: NSSize
}

pub struct NSPoint {
    pub x: f64,
    pub y: f64
}

pub struct NSSize {
    pub width: f64,
    pub height: f64
}<|endoftext|>


<commit_before>/*!
The types module provides a way of associating glob patterns on file names to
file types.
*/

use std::collections::HashMap;
use std::error::Error as StdError;
use std::fmt;
use std::path::Path;

use regex;

use gitignore::{Match, Pattern};
use globset::{self, GlobBuilder, GlobSet, GlobSetBuilder};

const TYPE_EXTENSIONS: &'static [(&'static str, &'static [&'static str])] = &[
    ("asm", &["*.asm", "*.s", "*.S"]),
    ("awk", &["*.awk"]),
    ("c", &["*.c", "*.h", "*.H"]),
    ("cbor", &["*.cbor"]),
    ("clojure", &["*.clj", "*.cljc", "*.cljs", "*.cljx"]),
    ("cmake", &["*.cmake", "CMakeLists.txt"]),
    ("coffeescript", &["*.coffee"]),
    ("config", &["*.config"]),
    ("cpp", &[
        "*.C", "*.cc", "*.cpp", "*.cxx",
        "*.h", "*.H", "*.hh", "*.hpp",
    ]),
    ("csharp", &["*.cs"]),
    ("css", &["*.css"]),
    ("cython", &["*.pyx"]),
    ("dart", &["*.dart"]),
    ("d", &["*.d"]),
    ("elisp", &["*.el"]),
    ("erlang", &["*.erl", "*.hrl"]),
    ("fortran", &[
        "*.f", "*.F", "*.f77", "*.F77", "*.pfo",
        "*.f90", "*.F90", "*.f95", "*.F95",
    ]),
    ("fsharp", &["*.fs", "*.fsx", "*.fsi"]),
    ("go", &["*.go"]),
    ("groovy", &["*.groovy", "*.gradle"]),
    ("hbs", &["*.hbs"]),
    ("haskell", &["*.hs", "*.lhs"]),
    ("html", &["*.htm", "*.html"]),
    ("java", &["*.java"]),
    ("jinja", &["*.jinja", "*.jinja2"]),
    ("js", &[
        "*.js", "*.jsx", "*.vue",
    ]),
    ("json", &["*.json"]),
    ("jsonl", &["*.jsonl"]),
    ("lisp", &["*.el", "*.jl", "*.lisp", "*.lsp", "*.sc", "*.scm"]),
    ("lua", &["*.lua"]),
    ("m4", &["*.ac", "*.m4"]),
    ("make", &["gnumakefile", "Gnumakefile", "makefile", "Makefile", "*.mk"]),
    ("markdown", &["*.md"]),
    ("md", &["*.md"]),
    ("matlab", &["*.m"]),
    ("mk", &["mkfile"]),
    ("ml", &["*.ml"]),
    ("nim", &["*.nim"]),
    ("objc", &["*.h", "*.m"]),
    ("objcpp", &["*.h", "*.mm"]),
    ("ocaml", &["*.ml", "*.mli", "*.mll", "*.mly"]),
    ("perl", &["*.perl", "*.pl", "*.PL", "*.plh", "*.plx", "*.pm"]),
    ("php", &["*.php", "*.php3", "*.php4", "*.php5", "*.phtml"]),
    ("py", &["*.py"]),
    ("readme", &["README*", "*README"]),
    ("r", &["*.R", "*.r", "*.Rmd", "*.Rnw"]),
    ("rst", &["*.rst"]),
    ("ruby", &["*.rb"]),
    ("rust", &["*.rs"]),
    ("scala", &["*.scala"]),
    ("sh", &["*.bash", "*.csh", "*.ksh", "*.sh", "*.tcsh"]),
    ("spark", &["*.spark"]),
    ("sql", &["*.sql"]),
    ("sv", &["*.v", "*.vg", "*.sv", "*.svh", "*.h"]),
    ("swift", &["*.swift"]),
    ("tcl", &["*.tcl"]),
    ("tex", &["*.tex", "*.cls", "*.sty"]),
    ("ts", &["*.ts", "*.tsx"]),
    ("txt", &["*.txt"]),
    ("toml", &["*.toml", "Cargo.lock"]),
    ("vala", &["*.vala"]),
    ("vb", &["*.vb"]),
    ("vimscript", &["*.vim"]),
    ("xml", &["*.xml"]),
    ("yacc", &["*.y"]),
    ("yaml", &["*.yaml", "*.yml"]),
    ("zsh", &["*.zsh", ".zshenv", ".zlogin", ".zprofile", ".zshrc"]),
];

/// Describes all the possible failure conditions for building a file type
/// matcher.
#[derive(Debug)]
pub enum Error {
    /// We tried to select (or negate) a file type that is not defined.
    UnrecognizedFileType(String),
    /// A user specified file type definition could not be parsed.
    InvalidDefinition,
    /// There was an error building the matcher (probably a bad glob).
    Glob(globset::Error),
    /// There was an error compiling a glob as a regex.
    Regex(regex::Error),
}

impl StdError for Error {
    fn description(&self) -> &str {
        match *self {
            Error::UnrecognizedFileType(_) => "unrecognized file type",
            Error::InvalidDefinition => "invalid definition",
            Error::Glob(ref err) => err.description(),
            Error::Regex(ref err) => err.description(),
        }
    }
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Error::UnrecognizedFileType(ref ty) => {
                write!(f, "unrecognized file type: {}", ty)
            }
            Error::InvalidDefinition => {
                write!(f, "invalid definition (format is type:glob, e.g., \
                           html:*.html)")
            }
            Error::Glob(ref err) => err.fmt(f),
            Error::Regex(ref err) => err.fmt(f),
        }
    }
}

impl From<globset::Error> for Error {
    fn from(err: globset::Error) -> Error {
        Error::Glob(err)
    }
}

impl From<regex::Error> for Error {
    fn from(err: regex::Error) -> Error {
        Error::Regex(err)
    }
}

/// A single file type definition.
#[derive(Clone, Debug)]
pub struct FileTypeDef {
    name: String,
    pats: Vec<String>,
}

impl FileTypeDef {
    /// Return the name of this file type.
    pub fn name(&self) -> &str {
        &self.name
    }

    /// Return the glob patterns used to recognize this file type.
    pub fn patterns(&self) -> &[String] {
        &self.pats
    }
}

/// Types is a file type matcher.
#[derive(Clone, Debug)]
pub struct Types {
    defs: Vec<FileTypeDef>,
    selected: Option<GlobSet>,
    negated: Option<GlobSet>,
    has_selected: bool,
    unmatched_pat: Pattern,
}

impl Types {
    /// Creates a new file type matcher from the given Gitignore matcher. If
    /// not Gitignore matcher is provided, then the file type matcher has no
    /// effect.
    ///
    /// If has_selected is true, then at least one file type was selected.
    /// Therefore, any non-matches should be ignored.
    fn new(
        selected: Option<GlobSet>,
        negated: Option<GlobSet>,
        has_selected: bool,
        defs: Vec<FileTypeDef>,
    ) -> Types {
        Types {
            defs: defs,
            selected: selected,
            negated: negated,
            has_selected: has_selected,
            unmatched_pat: Pattern {
                from: Path::new("<filetype>").to_path_buf(),
                original: "<N/A>".to_string(),
                pat: "<N/A>".to_string(),
                whitelist: false,
                only_dir: false,
            },
        }
    }

    /// Creates a new file type matcher that never matches.
    pub fn empty() -> Types {
        Types::new(None, None, false, vec![])
    }

    /// Returns a match for the given path against this file type matcher.
    ///
    /// The path is considered whitelisted if it matches a selected file type.
    /// The path is considered ignored if it matched a negated file type.
    /// If at least one file type is selected and path doesn't match, then
    /// the path is also considered ignored.
    pub fn matched<P: AsRef<Path>>(&self, path: P, is_dir: bool) -> Match {
        // If we don't have any matcher, then we can't do anything.
        if self.negated.is_none() && self.selected.is_none() {
            return Match::None;
        }
        // File types don't apply to directories.
        if is_dir {
            return Match::None;
        }
        let path = path.as_ref();
        let name = match path.file_name() {
            Some(name) => name.to_string_lossy(),
            None if self.has_selected => {
                return Match::Ignored(&self.unmatched_pat);
            }
            None => {
                return Match::None;
            }
        };
        if self.negated.as_ref().map(|s| s.is_match(&*name)).unwrap_or(false) {
            return Match::Ignored(&self.unmatched_pat);
        }
        if self.selected.as_ref().map(|s|s.is_match(&*name)).unwrap_or(false) {
            return Match::Whitelist(&self.unmatched_pat);
        }
        if self.has_selected {
            Match::Ignored(&self.unmatched_pat)
        } else {
            Match::None
        }
    }

    /// Return the set of current file type definitions.
    pub fn definitions(&self) -> &[FileTypeDef] {
        &self.defs
    }
}

/// TypesBuilder builds a type matcher from a set of file type definitions and
/// a set of file type selections.
pub struct TypesBuilder {
    types: HashMap<String, Vec<String>>,
    selected: Vec<String>,
    negated: Vec<String>,
}

impl TypesBuilder {
    /// Create a new builder for a file type matcher.
    pub fn new() -> TypesBuilder {
        TypesBuilder {
            types: HashMap::new(),
            selected: vec![],
            negated: vec![],
        }
    }

    /// Build the current set of file type definitions *and* selections into
    /// a file type matcher.
    pub fn build(&self) -> Result<Types, Error> {
        let selected_globs =
            if self.selected.is_empty() {
                None
            } else {
                let mut bset = GlobSetBuilder::new();
                for name in &self.selected {
                    let globs = match self.types.get(name) {
                        Some(globs) => globs,
                        None => {
                            let msg = name.to_string();
                            return Err(Error::UnrecognizedFileType(msg));
                        }
                    };
                    for glob in globs {
                        let pat = try!(
                            GlobBuilder::new(glob)
                                .literal_separator(true).build());
                        bset.add(pat);
                    }
                }
                Some(try!(bset.build()))
            };
        let negated_globs =
            if self.negated.is_empty() {
                None
            } else {
                let mut bset = GlobSetBuilder::new();
                for name in &self.negated {
                    let globs = match self.types.get(name) {
                        Some(globs) => globs,
                        None => {
                            let msg = name.to_string();
                            return Err(Error::UnrecognizedFileType(msg));
                        }
                    };
                    for glob in globs {
                        let pat = try!(
                            GlobBuilder::new(glob)
                                .literal_separator(true).build());
                        bset.add(pat);
                    }
                }
                Some(try!(bset.build()))
            };
        Ok(Types::new(
            selected_globs,
            negated_globs,
            !self.selected.is_empty(),
            self.definitions(),
        ))
    }

    /// Return the set of current file type definitions.
    pub fn definitions(&self) -> Vec<FileTypeDef> {
        let mut defs = vec![];
        for (ref name, ref pats) in &self.types {
            let mut pats = pats.to_vec();
            pats.sort();
            defs.push(FileTypeDef {
                name: name.to_string(),
                pats: pats,
            });
        }
        defs.sort_by(|def1, def2| def1.name().cmp(def2.name()));
        defs
    }

    /// Select the file type given by `name`.
    ///
    /// If `name` is `all`, then all file types are selected.
    pub fn select(&mut self, name: &str) -> &mut TypesBuilder {
        if name == "all" {
            for name in self.types.keys() {
                self.selected.push(name.to_string());
            }
        } else {
            self.selected.push(name.to_string());
        }
        self
    }

    /// Ignore the file type given by `name`.
    ///
    /// If `name` is `all`, then all file types are negated.
    pub fn negate(&mut self, name: &str) -> &mut TypesBuilder {
        if name == "all" {
            for name in self.types.keys() {
                self.negated.push(name.to_string());
            }
        } else {
            self.negated.push(name.to_string());
        }
        self
    }

    /// Clear any file type definitions for the type given.
    pub fn clear(&mut self, name: &str) -> &mut TypesBuilder {
        self.types.remove(name);
        self
    }

    /// Add a new file type definition. `name` can be arbitrary and `pat`
    /// should be a glob recognizing file paths belonging to the `name` type.
    pub fn add(&mut self, name: &str, pat: &str) -> &mut TypesBuilder {
        self.types.entry(name.to_string())
            .or_insert(vec![]).push(pat.to_string());
        self
    }

    /// Add a new file type definition specified in string form. The format
    /// is `name:glob`. Names may not include a colon.
    pub fn add_def(&mut self, def: &str) -> Result<(), Error> {
        let name: String = def.chars().take_while(|&c| c != ':').collect();
        let pat: String = def.chars().skip(name.chars().count() + 1).collect();
        if name.is_empty() || pat.is_empty() {
            return Err(Error::InvalidDefinition);
        }
        self.add(&name, &pat);
        Ok(())
    }

    /// Add a set of default file type definitions.
    pub fn add_defaults(&mut self) -> &mut TypesBuilder {
        for &(name, exts) in TYPE_EXTENSIONS {
            for ext in exts {
                self.add(name, ext);
            }
        }
        self
    }
}

#[cfg(test)]
mod tests {
    use super::TypesBuilder;

    macro_rules! matched {
        ($name:ident, $types:expr, $sel:expr, $selnot:expr,
         $path:expr) => {
            matched!($name, $types, $sel, $selnot, $path, true);
        };
        (not, $name:ident, $types:expr, $sel:expr, $selnot:expr,
         $path:expr) => {
            matched!($name, $types, $sel, $selnot, $path, false);
        };
        ($name:ident, $types:expr, $sel:expr, $selnot:expr,
         $path:expr, $matched:expr) => {
            #[test]
            fn $name() {
                let mut btypes = TypesBuilder::new();
                for tydef in $types {
                    btypes.add_def(tydef).unwrap();
                }
                for sel in $sel {
                    btypes.select(sel);
                }
                for selnot in $selnot {
                    btypes.negate(selnot);
                }
                let types = btypes.build().unwrap();
                let mat = types.matched($path, false);
                assert_eq!($matched, !mat.is_ignored());
            }
        };
    }

    fn types() -> Vec<&'static str> {
        vec![
            "html:*.html",
            "html:*.htm",
            "rust:*.rs",
            "js:*.js",
        ]
    }

    matched!(match1, types(), vec!["rust"], vec![], "lib.rs");
    matched!(match2, types(), vec!["html"], vec![], "index.html");
    matched!(match3, types(), vec!["html"], vec![], "index.htm");
    matched!(match4, types(), vec!["html", "rust"], vec![], "main.rs");
    matched!(match5, types(), vec![], vec![], "index.html");
    matched!(match6, types(), vec![], vec!["rust"], "index.html");

    matched!(not, matchnot1, types(), vec!["rust"], vec![], "index.html");
    matched!(not, matchnot2, types(), vec![], vec!["rust"], "main.rs");
}
<commit_msg>update python types to include Cython files<commit_after>/*!
The types module provides a way of associating glob patterns on file names to
file types.
*/

use std::collections::HashMap;
use std::error::Error as StdError;
use std::fmt;
use std::path::Path;

use regex;

use gitignore::{Match, Pattern};
use globset::{self, GlobBuilder, GlobSet, GlobSetBuilder};

const TYPE_EXTENSIONS: &'static [(&'static str, &'static [&'static str])] = &[
    ("asm", &["*.asm", "*.s", "*.S"]),
    ("awk", &["*.awk"]),
    ("c", &["*.c", "*.h", "*.H"]),
    ("cbor", &["*.cbor"]),
    ("clojure", &["*.clj", "*.cljc", "*.cljs", "*.cljx"]),
    ("cmake", &["*.cmake", "CMakeLists.txt"]),
    ("coffeescript", &["*.coffee"]),
    ("config", &["*.config"]),
    ("cpp", &[
        "*.C", "*.cc", "*.cpp", "*.cxx",
        "*.h", "*.H", "*.hh", "*.hpp",
    ]),
    ("csharp", &["*.cs"]),
    ("css", &["*.css"]),
    ("cython", &["*.pyx"]),
    ("dart", &["*.dart"]),
    ("d", &["*.d"]),
    ("elisp", &["*.el"]),
    ("erlang", &["*.erl", "*.hrl"]),
    ("fortran", &[
        "*.f", "*.F", "*.f77", "*.F77", "*.pfo",
        "*.f90", "*.F90", "*.f95", "*.F95",
    ]),
    ("fsharp", &["*.fs", "*.fsx", "*.fsi"]),
    ("go", &["*.go"]),
    ("groovy", &["*.groovy", "*.gradle"]),
    ("hbs", &["*.hbs"]),
    ("haskell", &["*.hs", "*.lhs"]),
    ("html", &["*.htm", "*.html"]),
    ("java", &["*.java"]),
    ("jinja", &["*.jinja", "*.jinja2"]),
    ("js", &[
        "*.js", "*.jsx", "*.vue",
    ]),
    ("json", &["*.json"]),
    ("jsonl", &["*.jsonl"]),
    ("lisp", &["*.el", "*.jl", "*.lisp", "*.lsp", "*.sc", "*.scm"]),
    ("lua", &["*.lua"]),
    ("m4", &["*.ac", "*.m4"]),
    ("make", &["gnumakefile", "Gnumakefile", "makefile", "Makefile", "*.mk"]),
    ("markdown", &["*.md"]),
    ("md", &["*.md"]),
    ("matlab", &["*.m"]),
    ("mk", &["mkfile"]),
    ("ml", &["*.ml"]),
    ("nim", &["*.nim"]),
    ("objc", &["*.h", "*.m"]),
    ("objcpp", &["*.h", "*.mm"]),
    ("ocaml", &["*.ml", "*.mli", "*.mll", "*.mly"]),
    ("perl", &["*.perl", "*.pl", "*.PL", "*.plh", "*.plx", "*.pm"]),
    ("php", &["*.php", "*.php3", "*.php4", "*.php5", "*.phtml"]),
    ("py", &["*.py", "*.pyx"]),
    ("readme", &["README*", "*README"]),
    ("r", &["*.R", "*.r", "*.Rmd", "*.Rnw"]),
    ("rst", &["*.rst"]),
    ("ruby", &["*.rb"]),
    ("rust", &["*.rs"]),
    ("scala", &["*.scala"]),
    ("sh", &["*.bash", "*.csh", "*.ksh", "*.sh", "*.tcsh"]),
    ("spark", &["*.spark"]),
    ("sql", &["*.sql"]),
    ("sv", &["*.v", "*.vg", "*.sv", "*.svh", "*.h"]),
    ("swift", &["*.swift"]),
    ("tcl", &["*.tcl"]),
    ("tex", &["*.tex", "*.cls", "*.sty"]),
    ("ts", &["*.ts", "*.tsx"]),
    ("txt", &["*.txt"]),
    ("toml", &["*.toml", "Cargo.lock"]),
    ("vala", &["*.vala"]),
    ("vb", &["*.vb"]),
    ("vimscript", &["*.vim"]),
    ("xml", &["*.xml"]),
    ("yacc", &["*.y"]),
    ("yaml", &["*.yaml", "*.yml"]),
    ("zsh", &["*.zsh", ".zshenv", ".zlogin", ".zprofile", ".zshrc"]),
];

/// Describes all the possible failure conditions for building a file type
/// matcher.
#[derive(Debug)]
pub enum Error {
    /// We tried to select (or negate) a file type that is not defined.
    UnrecognizedFileType(String),
    /// A user specified file type definition could not be parsed.
    InvalidDefinition,
    /// There was an error building the matcher (probably a bad glob).
    Glob(globset::Error),
    /// There was an error compiling a glob as a regex.
    Regex(regex::Error),
}

impl StdError for Error {
    fn description(&self) -> &str {
        match *self {
            Error::UnrecognizedFileType(_) => "unrecognized file type",
            Error::InvalidDefinition => "invalid definition",
            Error::Glob(ref err) => err.description(),
            Error::Regex(ref err) => err.description(),
        }
    }
}

impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            Error::UnrecognizedFileType(ref ty) => {
                write!(f, "unrecognized file type: {}", ty)
            }
            Error::InvalidDefinition => {
                write!(f, "invalid definition (format is type:glob, e.g., \
                           html:*.html)")
            }
            Error::Glob(ref err) => err.fmt(f),
            Error::Regex(ref err) => err.fmt(f),
        }
    }
}

impl From<globset::Error> for Error {
    fn from(err: globset::Error) -> Error {
        Error::Glob(err)
    }
}

impl From<regex::Error> for Error {
    fn from(err: regex::Error) -> Error {
        Error::Regex(err)
    }
}

/// A single file type definition.
#[derive(Clone, Debug)]
pub struct FileTypeDef {
    name: String,
    pats: Vec<String>,
}

impl FileTypeDef {
    /// Return the name of this file type.
    pub fn name(&self) -> &str {
        &self.name
    }

    /// Return the glob patterns used to recognize this file type.
    pub fn patterns(&self) -> &[String] {
        &self.pats
    }
}

/// Types is a file type matcher.
#[derive(Clone, Debug)]
pub struct Types {
    defs: Vec<FileTypeDef>,
    selected: Option<GlobSet>,
    negated: Option<GlobSet>,
    has_selected: bool,
    unmatched_pat: Pattern,
}

impl Types {
    /// Creates a new file type matcher from the given Gitignore matcher. If
    /// not Gitignore matcher is provided, then the file type matcher has no
    /// effect.
    ///
    /// If has_selected is true, then at least one file type was selected.
    /// Therefore, any non-matches should be ignored.
    fn new(
        selected: Option<GlobSet>,
        negated: Option<GlobSet>,
        has_selected: bool,
        defs: Vec<FileTypeDef>,
    ) -> Types {
        Types {
            defs: defs,
            selected: selected,
            negated: negated,
            has_selected: has_selected,
            unmatched_pat: Pattern {
                from: Path::new("<filetype>").to_path_buf(),
                original: "<N/A>".to_string(),
                pat: "<N/A>".to_string(),
                whitelist: false,
                only_dir: false,
            },
        }
    }

    /// Creates a new file type matcher that never matches.
    pub fn empty() -> Types {
        Types::new(None, None, false, vec![])
    }

    /// Returns a match for the given path against this file type matcher.
    ///
    /// The path is considered whitelisted if it matches a selected file type.
    /// The path is considered ignored if it matched a negated file type.
    /// If at least one file type is selected and path doesn't match, then
    /// the path is also considered ignored.
    pub fn matched<P: AsRef<Path>>(&self, path: P, is_dir: bool) -> Match {
        // If we don't have any matcher, then we can't do anything.
        if self.negated.is_none() && self.selected.is_none() {
            return Match::None;
        }
        // File types don't apply to directories.
        if is_dir {
            return Match::None;
        }
        let path = path.as_ref();
        let name = match path.file_name() {
            Some(name) => name.to_string_lossy(),
            None if self.has_selected => {
                return Match::Ignored(&self.unmatched_pat);
            }
            None => {
                return Match::None;
            }
        };
        if self.negated.as_ref().map(|s| s.is_match(&*name)).unwrap_or(false) {
            return Match::Ignored(&self.unmatched_pat);
        }
        if self.selected.as_ref().map(|s|s.is_match(&*name)).unwrap_or(false) {
            return Match::Whitelist(&self.unmatched_pat);
        }
        if self.has_selected {
            Match::Ignored(&self.unmatched_pat)
        } else {
            Match::None
        }
    }

    /// Return the set of current file type definitions.
    pub fn definitions(&self) -> &[FileTypeDef] {
        &self.defs
    }
}

/// TypesBuilder builds a type matcher from a set of file type definitions and
/// a set of file type selections.
pub struct TypesBuilder {
    types: HashMap<String, Vec<String>>,
    selected: Vec<String>,
    negated: Vec<String>,
}

impl TypesBuilder {
    /// Create a new builder for a file type matcher.
    pub fn new() -> TypesBuilder {
        TypesBuilder {
            types: HashMap::new(),
            selected: vec![],
            negated: vec![],
        }
    }

    /// Build the current set of file type definitions *and* selections into
    /// a file type matcher.
    pub fn build(&self) -> Result<Types, Error> {
        let selected_globs =
            if self.selected.is_empty() {
                None
            } else {
                let mut bset = GlobSetBuilder::new();
                for name in &self.selected {
                    let globs = match self.types.get(name) {
                        Some(globs) => globs,
                        None => {
                            let msg = name.to_string();
                            return Err(Error::UnrecognizedFileType(msg));
                        }
                    };
                    for glob in globs {
                        let pat = try!(
                            GlobBuilder::new(glob)
                                .literal_separator(true).build());
                        bset.add(pat);
                    }
                }
                Some(try!(bset.build()))
            };
        let negated_globs =
            if self.negated.is_empty() {
                None
            } else {
                let mut bset = GlobSetBuilder::new();
                for name in &self.negated {
                    let globs = match self.types.get(name) {
                        Some(globs) => globs,
                        None => {
                            let msg = name.to_string();
                            return Err(Error::UnrecognizedFileType(msg));
                        }
                    };
                    for glob in globs {
                        let pat = try!(
                            GlobBuilder::new(glob)
                                .literal_separator(true).build());
                        bset.add(pat);
                    }
                }
                Some(try!(bset.build()))
            };
        Ok(Types::new(
            selected_globs,
            negated_globs,
            !self.selected.is_empty(),
            self.definitions(),
        ))
    }

    /// Return the set of current file type definitions.
    pub fn definitions(&self) -> Vec<FileTypeDef> {
        let mut defs = vec![];
        for (ref name, ref pats) in &self.types {
            let mut pats = pats.to_vec();
            pats.sort();
            defs.push(FileTypeDef {
                name: name.to_string(),
                pats: pats,
            });
        }
        defs.sort_by(|def1, def2| def1.name().cmp(def2.name()));
        defs
    }

    /// Select the file type given by `name`.
    ///
    /// If `name` is `all`, then all file types are selected.
    pub fn select(&mut self, name: &str) -> &mut TypesBuilder {
        if name == "all" {
            for name in self.types.keys() {
                self.selected.push(name.to_string());
            }
        } else {
            self.selected.push(name.to_string());
        }
        self
    }

    /// Ignore the file type given by `name`.
    ///
    /// If `name` is `all`, then all file types are negated.
    pub fn negate(&mut self, name: &str) -> &mut TypesBuilder {
        if name == "all" {
            for name in self.types.keys() {
                self.negated.push(name.to_string());
            }
        } else {
            self.negated.push(name.to_string());
        }
        self
    }

    /// Clear any file type definitions for the type given.
    pub fn clear(&mut self, name: &str) -> &mut TypesBuilder {
        self.types.remove(name);
        self
    }

    /// Add a new file type definition. `name` can be arbitrary and `pat`
    /// should be a glob recognizing file paths belonging to the `name` type.
    pub fn add(&mut self, name: &str, pat: &str) -> &mut TypesBuilder {
        self.types.entry(name.to_string())
            .or_insert(vec![]).push(pat.to_string());
        self
    }

    /// Add a new file type definition specified in string form. The format
    /// is `name:glob`. Names may not include a colon.
    pub fn add_def(&mut self, def: &str) -> Result<(), Error> {
        let name: String = def.chars().take_while(|&c| c != ':').collect();
        let pat: String = def.chars().skip(name.chars().count() + 1).collect();
        if name.is_empty() || pat.is_empty() {
            return Err(Error::InvalidDefinition);
        }
        self.add(&name, &pat);
        Ok(())
    }

    /// Add a set of default file type definitions.
    pub fn add_defaults(&mut self) -> &mut TypesBuilder {
        for &(name, exts) in TYPE_EXTENSIONS {
            for ext in exts {
                self.add(name, ext);
            }
        }
        self
    }
}

#[cfg(test)]
mod tests {
    use super::TypesBuilder;

    macro_rules! matched {
        ($name:ident, $types:expr, $sel:expr, $selnot:expr,
         $path:expr) => {
            matched!($name, $types, $sel, $selnot, $path, true);
        };
        (not, $name:ident, $types:expr, $sel:expr, $selnot:expr,
         $path:expr) => {
            matched!($name, $types, $sel, $selnot, $path, false);
        };
        ($name:ident, $types:expr, $sel:expr, $selnot:expr,
         $path:expr, $matched:expr) => {
            #[test]
            fn $name() {
                let mut btypes = TypesBuilder::new();
                for tydef in $types {
                    btypes.add_def(tydef).unwrap();
                }
                for sel in $sel {
                    btypes.select(sel);
                }
                for selnot in $selnot {
                    btypes.negate(selnot);
                }
                let types = btypes.build().unwrap();
                let mat = types.matched($path, false);
                assert_eq!($matched, !mat.is_ignored());
            }
        };
    }

    fn types() -> Vec<&'static str> {
        vec![
            "html:*.html",
            "html:*.htm",
            "rust:*.rs",
            "js:*.js",
        ]
    }

    matched!(match1, types(), vec!["rust"], vec![], "lib.rs");
    matched!(match2, types(), vec!["html"], vec![], "index.html");
    matched!(match3, types(), vec!["html"], vec![], "index.htm");
    matched!(match4, types(), vec!["html", "rust"], vec![], "main.rs");
    matched!(match5, types(), vec![], vec![], "index.html");
    matched!(match6, types(), vec![], vec!["rust"], "index.html");

    matched!(not, matchnot1, types(), vec!["rust"], vec![], "index.html");
    matched!(not, matchnot2, types(), vec![], vec!["rust"], "main.rs");
}
<|endoftext|>