<commit_before>use crate::utils::{is_allowed, snippet, span_help_and_lint};
use rustc::hir::*;
use rustc::lint::{LateContext, LateLintPass, LintArray, LintPass};
use rustc::{declare_lint_pass, declare_tool_lint};
use syntax::ast::LitKind;
use syntax::source_map::Span;
use unicode_normalization::UnicodeNormalization;

declare_clippy_lint! {
    /// **What it does:** Checks for the Unicode zero-width space in the code.
    ///
    /// **Why is this bad?** Having an invisible character in the code makes for all
    /// sorts of April fools, but otherwise is very much frowned upon.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:** You don't see it, but there may be a zero-width space
    /// somewhere in this text.
    pub ZERO_WIDTH_SPACE,
    correctness,
    "using a zero-width space in a string literal, which is confusing"
}

declare_clippy_lint! {
    /// **What it does:** Checks for non-ASCII characters in string literals.
    ///
    /// **Why is this bad?** Yeah, we know, the 90's called and wanted their charset
    /// back. Even so, there still are editors and other programs out there that
    /// don't work well with Unicode. So if the code is meant to be used
    /// internationally, on multiple operating systems, or has other portability
    /// requirements, activating this lint could be useful.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:**
    /// ```rust
    /// let x = String::from("€");
    /// ```
    /// Could be written as:
    /// ```rust
    /// let x = String::from("\u{20ac}");
    /// ```
    pub NON_ASCII_LITERAL,
    pedantic,
    "using any literal non-ASCII chars in a string literal instead of using the `\\u` escape"
}

declare_clippy_lint! {
    /// **What it does:** Checks for string literals that contain Unicode in a form
    /// that is not equal to its
    /// [NFC-recomposition](http://www.unicode.org/reports/tr15/#Norm_Forms).
    ///
    /// **Why is this bad?** If such a string is compared to another, the results
    /// may be surprising.
    ///
    /// **Known problems** None.
    ///
    /// **Example:** You may not see it, but “à” and “à” aren't the same string. The
    /// former when escaped is actually `"a\u{300}"` while the latter is `"\u{e0}"`.
    pub UNICODE_NOT_NFC,
    pedantic,
    "using a unicode literal not in NFC normal form (see [unicode tr15](http://www.unicode.org/reports/tr15/) for further information)"
}

declare_lint_pass!(Unicode => [ZERO_WIDTH_SPACE, NON_ASCII_LITERAL, UNICODE_NOT_NFC]);

impl<'a, 'tcx> LateLintPass<'a, 'tcx> for Unicode {
    fn check_expr(&mut self, cx: &LateContext<'a, 'tcx>, expr: &'tcx Expr) {
        if let ExprKind::Lit(ref lit) = expr.node {
            if let LitKind::Str(_, _) = lit.node {
                check_str(cx, lit.span, expr.hir_id)
            }
        }
    }
}

fn escape<T: Iterator<Item = char>>(s: T) -> String {
    let mut result = String::new();
    for c in s {
        if c as u32 > 0x7F {
            for d in c.escape_unicode() {
                result.push(d)
            }
        } else {
            result.push(c);
        }
    }
    result
}

fn check_str(cx: &LateContext<'_, '_>, span: Span, id: HirId) {
    let string = snippet(cx, span, "");
    if string.contains('\u{200B}') {
        span_help_and_lint(
            cx,
            ZERO_WIDTH_SPACE,
            span,
            "zero-width space detected",
            &format!(
                "Consider replacing the string with:\n\"{}\"",
                string.replace("\u{200B}", "\\u{200B}")
            ),
        );
    }
    if string.chars().any(|c| c as u32 > 0x7F) {
        span_help_and_lint(
            cx,
            NON_ASCII_LITERAL,
            span,
            "literal non-ASCII character detected",
            &format!(
                "Consider replacing the string with:\n\"{}\"",
                if is_allowed(cx, UNICODE_NOT_NFC, id) {
                    escape(string.chars())
                } else {
                    escape(string.nfc())
                }
            ),
        );
    }
    if is_allowed(cx, NON_ASCII_LITERAL, id) && string.chars().zip(string.nfc()).any(|(a, b)| a != b) {
        span_help_and_lint(
            cx,
            UNICODE_NOT_NFC,
            span,
            "non-nfc unicode sequence detected",
            &format!(
                "Consider replacing the string with:\n\"{}\"",
                string.nfc().collect::<String>()
            ),
        );
    }
}
<commit_msg>Make non_ascii_literal auto-fixable<commit_after>use crate::utils::{is_allowed, snippet, span_help_and_lint, span_lint_and_sugg};
use rustc::hir::*;
use rustc::lint::{LateContext, LateLintPass, LintArray, LintPass};
use rustc::{declare_lint_pass, declare_tool_lint};
use rustc_errors::Applicability;
use syntax::ast::LitKind;
use syntax::source_map::Span;
use unicode_normalization::UnicodeNormalization;

declare_clippy_lint! {
    /// **What it does:** Checks for the Unicode zero-width space in the code.
    ///
    /// **Why is this bad?** Having an invisible character in the code makes for all
    /// sorts of April fools, but otherwise is very much frowned upon.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:** You don't see it, but there may be a zero-width space
    /// somewhere in this text.
    pub ZERO_WIDTH_SPACE,
    correctness,
    "using a zero-width space in a string literal, which is confusing"
}

declare_clippy_lint! {
    /// **What it does:** Checks for non-ASCII characters in string literals.
    ///
    /// **Why is this bad?** Yeah, we know, the 90's called and wanted their charset
    /// back. Even so, there still are editors and other programs out there that
    /// don't work well with Unicode. So if the code is meant to be used
    /// internationally, on multiple operating systems, or has other portability
    /// requirements, activating this lint could be useful.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:**
    /// ```rust
    /// let x = String::from("€");
    /// ```
    /// Could be written as:
    /// ```rust
    /// let x = String::from("\u{20ac}");
    /// ```
    pub NON_ASCII_LITERAL,
    pedantic,
    "using any literal non-ASCII chars in a string literal instead of using the `\\u` escape"
}

declare_clippy_lint! {
    /// **What it does:** Checks for string literals that contain Unicode in a form
    /// that is not equal to its
    /// [NFC-recomposition](http://www.unicode.org/reports/tr15/#Norm_Forms).
    ///
    /// **Why is this bad?** If such a string is compared to another, the results
    /// may be surprising.
    ///
    /// **Known problems** None.
    ///
    /// **Example:** You may not see it, but “à” and “à” aren't the same string. The
    /// former when escaped is actually `"a\u{300}"` while the latter is `"\u{e0}"`.
    pub UNICODE_NOT_NFC,
    pedantic,
    "using a unicode literal not in NFC normal form (see [unicode tr15](http://www.unicode.org/reports/tr15/) for further information)"
}

declare_lint_pass!(Unicode => [ZERO_WIDTH_SPACE, NON_ASCII_LITERAL, UNICODE_NOT_NFC]);

impl<'a, 'tcx> LateLintPass<'a, 'tcx> for Unicode {
    fn check_expr(&mut self, cx: &LateContext<'a, 'tcx>, expr: &'tcx Expr) {
        if let ExprKind::Lit(ref lit) = expr.node {
            if let LitKind::Str(_, _) = lit.node {
                check_str(cx, lit.span, expr.hir_id)
            }
        }
    }
}

fn escape<T: Iterator<Item = char>>(s: T) -> String {
    let mut result = String::new();
    for c in s {
        if c as u32 > 0x7F {
            for d in c.escape_unicode() {
                result.push(d)
            }
        } else {
            result.push(c);
        }
    }
    result
}

fn check_str(cx: &LateContext<'_, '_>, span: Span, id: HirId) {
    let string = snippet(cx, span, "");
    if string.contains('\u{200B}') {
        span_help_and_lint(
            cx,
            ZERO_WIDTH_SPACE,
            span,
            "zero-width space detected",
            &format!(
                "Consider replacing the string with:\n\"{}\"",
                string.replace("\u{200B}", "\\u{200B}")
            ),
        );
    }
    if string.chars().any(|c| c as u32 > 0x7F) {
        span_lint_and_sugg(
            cx,
            NON_ASCII_LITERAL,
            span,
            "literal non-ASCII character detected",
            &format!(
                "Consider replacing the string with:\n\"{}\"",
                if is_allowed(cx, UNICODE_NOT_NFC, id) {
                    escape(string.chars())
                } else {
                    escape(string.nfc())
                }
            ),
            format!(
                "{}",
                if is_allowed(cx, UNICODE_NOT_NFC, id) {
                    escape(string.chars())
                } else {
                    escape(string.nfc())
                }
            ),
            Applicability::MachineApplicable,
        );
    }
    if is_allowed(cx, NON_ASCII_LITERAL, id) && string.chars().zip(string.nfc()).any(|(a, b)| a != b) {
        span_help_and_lint(
            cx,
            UNICODE_NOT_NFC,
            span,
            "non-nfc unicode sequence detected",
            &format!(
                "Consider replacing the string with:\n\"{}\"",
                string.nfc().collect::<String>()
            ),
        );
    }
}
<|endoftext|>


<commit_before>///! File system utilities, particularly functionality that requires
///! alternative implementations for alternative operating systems.
use eyre::Result;
use std::{os, path::Path};

/// Set permissions on a file
#[allow(unused_variables)]
pub fn set_perms<File: AsRef<Path>>(path: File, mode: u32) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    {
        use os::unix::fs::PermissionsExt;
        std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode))?;
    }

    Ok(())
}

/// Create a symbolic (soft) link to a file
pub fn symlink_file<Original: AsRef<Path>, Link: AsRef<Path>>(
    original: Original,
    link: Link,
) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    os::unix::fs::symlink(original, link)?;

    #[cfg(target_os = "windows")]
    os::windows::fs::symlink_file(original, link)?;

    Ok(())
}

/// Create a symbolic (soft) link to a directory
pub fn symlink_dir<Original: AsRef<Path>, Link: AsRef<Path>>(
    original: Original,
    link: Link,
) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    os::unix::fs::symlink(original, link)?;

    #[cfg(target_os = "windows")]
    os::windows::fs::symlink_dir(original, link)?;

    Ok(())
}
<commit_msg>feat(Filesystem utilities): Add `copy_dir_all` function<commit_after>///! File system utilities, particularly functionality that requires
///! alternative implementations for alternative operating systems.
use eyre::Result;
use std::{fs, os, path::Path};

/// Set permissions on a file
#[allow(unused_variables)]
pub fn set_perms<File: AsRef<Path>>(path: File, mode: u32) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    {
        use os::unix::fs::PermissionsExt;
        fs::set_permissions(path, fs::Permissions::from_mode(mode))?;
    }

    Ok(())
}

/// Create a symbolic (soft) link to a file
pub fn symlink_file<Original: AsRef<Path>, Link: AsRef<Path>>(
    original: Original,
    link: Link,
) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    os::unix::fs::symlink(original, link)?;

    #[cfg(target_os = "windows")]
    os::windows::fs::symlink_file(original, link)?;

    Ok(())
}

/// Create a symbolic (soft) link to a directory
pub fn symlink_dir<Original: AsRef<Path>, Link: AsRef<Path>>(
    original: Original,
    link: Link,
) -> Result<()> {
    #[cfg(any(target_os = "linux", target_os = "macos"))]
    os::unix::fs::symlink(original, link)?;

    #[cfg(target_os = "windows")]
    os::windows::fs::symlink_dir(original, link)?;

    Ok(())
}

/// Recursively copy a directory to another
pub fn copy_dir_all(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> Result<()> {
    fs::create_dir_all(&dest)?;
    for entry in fs::read_dir(src)? {
        let entry = entry?;
        if entry.file_type()?.is_dir() {
            copy_dir_all(entry.path(), dest.as_ref().join(entry.file_name()))?;
        } else {
            fs::copy(entry.path(), dest.as_ref().join(entry.file_name()))?;
        }
    }
    Ok(())
}
<|endoftext|>


<commit_before>use libc;

use glib_sys;
use glib::translate::*;

use std::collections::HashMap;
use std::collections::hash_map;
use std::ffi::{CStr, CString};
use std::ops::Deref;
use std::ptr;
use std::str::{self, FromStr};

use attributes::Attribute;

pub struct PropertyBag<'a>(HashMap<&'a CStr, (Attribute, &'a CStr)>);

pub struct OwnedPropertyBag(HashMap<CString, (Attribute, CString)>);

pub struct PropertyBagIter<'a>(PropertyBagCStrIter<'a>);

pub struct PropertyBagCStrIter<'a>(hash_map::Iter<'a, &'a CStr, (Attribute, &'a CStr)>);

trait Utf8CStrToStr {
    fn to_str_utf8(&self) -> &str;
}

impl Utf8CStrToStr for CStr {
    fn to_str_utf8(&self) -> &str {
        // We can *only* do this when the CStr comes from a C string that was validated
        // as UTF-8 on the C side of things.  In our case, the C strings from libxml2 and
        // are valid UTF-8.
        unsafe { str::from_utf8_unchecked(self.to_bytes()) }
    }
}

impl<'a> PropertyBag<'a> {
    pub unsafe fn new_from_key_value_pairs(pairs: *const *const libc::c_char) -> PropertyBag<'a> {
        let mut map = HashMap::new();

        if !pairs.is_null() {
            let mut i = 0;
            loop {
                let key = *pairs.offset(i);
                if !key.is_null() {
                    let val = *pairs.offset(i + 1);
                    assert!(!val.is_null());

                    let key_str = CStr::from_ptr(key);
                    let val_str = CStr::from_ptr(val);

                    // We silently drop unknown attributes.  New attributes should be added in
                    // build.rs.
                    if let Ok(attr) = Attribute::from_str(key_str.to_str_utf8()) {
                        map.insert(key_str, (attr, val_str));
                    }
                } else {
                    break;
                }

                i += 2;
            }
        }

        PropertyBag(map)
    }

    pub fn from_owned(owned: &OwnedPropertyBag) -> PropertyBag {
        let mut map = HashMap::new();

        for (k, &(a, ref v)) in &owned.0 {
            map.insert(k.deref(), (a, v.deref()));
        }

        PropertyBag(map)
    }

    pub fn to_owned(&self) -> OwnedPropertyBag {
        let mut map = HashMap::<CString, (Attribute, CString)>::new();

        for (k, &(a, v)) in &self.0 {
            map.insert((*k).to_owned(), (a, (*v).to_owned()));
        }

        OwnedPropertyBag(map)
    }

    pub fn ffi(&self) -> *const PropertyBag {
        self as *const PropertyBag
    }

    pub fn len(&self) -> usize {
        self.0.len()
    }

    pub fn iter(&self) -> PropertyBagIter {
        PropertyBagIter(self.cstr_iter())
    }

    pub fn cstr_iter(&self) -> PropertyBagCStrIter {
        PropertyBagCStrIter(self.0.iter())
    }
}

impl<'a> Iterator for PropertyBagIter<'a> {
    type Item = (&'a str, Attribute, &'a str);

    fn next(&mut self) -> Option<Self::Item> {
        self.0.next().map(|(k, a, v)| (k.to_str_utf8(), a, v.to_str_utf8()))
    }
}

impl<'a> Iterator for PropertyBagCStrIter<'a> {
    type Item = (&'a CStr, Attribute, &'a CStr);

    fn next(&mut self) -> Option<Self::Item> {
        self.0.next().map(|(k, &(a, v))| (*k, a, v))
    }
}

#[no_mangle]
pub extern fn rsvg_property_bag_new<'a>(atts: *const *const libc::c_char) -> *const PropertyBag<'a> {
    let pbag = unsafe { PropertyBag::new_from_key_value_pairs(atts) };
    Box::into_raw(Box::new(pbag))
}

#[no_mangle]
pub extern fn rsvg_property_bag_free(pbag: *mut PropertyBag) {
    unsafe {
        let _ = Box::from_raw(pbag);
    }
}

#[no_mangle]
pub extern fn rsvg_property_bag_iter_begin(pbag: *const PropertyBag) -> *mut PropertyBagCStrIter {
    assert!(!pbag.is_null());
    let pbag = unsafe { &*pbag };

    Box::into_raw(Box::new(pbag.cstr_iter()))
}

#[no_mangle]
pub extern fn rsvg_property_bag_iter_next(iter: *mut PropertyBagCStrIter,
                                          out_key: *mut *const libc::c_char,
                                          out_attr: *mut Attribute,
                                          out_value: *mut *const libc::c_char)
                                          -> glib_sys::gboolean
{
    assert!(!iter.is_null());
    let iter = unsafe { &mut *iter };

    if let Some((key, attr, val)) = iter.next() {
        unsafe {
            *out_key = key.as_ptr();
            *out_attr = attr;
            *out_value = val.as_ptr();
        }
        true.to_glib()
    } else {
        unsafe {
            *out_key = ptr::null();
            *out_value = ptr::null();
        }
        false.to_glib()
    }
}

#[no_mangle]
pub extern fn rsvg_property_bag_iter_end(iter: *mut PropertyBagCStrIter) {
    assert!(!iter.is_null());

    unsafe { Box::from_raw(iter) };
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::ffi::CString;
    use std::mem;

    #[test]
    fn empty_property_bag() {
        let map = unsafe { PropertyBag::new_from_key_value_pairs(ptr::null()) };
        assert_eq!(map.len(), 0);
    }

    #[test]
    fn property_bag_iters() {
        let pairs = [
            CString::new("rx").unwrap(),
            CString::new("1").unwrap(),
            CString::new("ry").unwrap(),
            CString::new("2").unwrap(),
        ];

        let mut v = Vec::new();

        for x in &pairs {
            v.push(x.as_ptr() as *const libc::c_char);
        }

        v.push(ptr::null());

        let pbag = unsafe { PropertyBag::new_from_key_value_pairs(v.as_ptr()) };

        let mut had_rx: bool = false;
        let mut had_ry: bool = false;

        for (k, a, v) in pbag.iter() {
            if k == "rx" {
                assert!(a == Attribute::Rx);
                assert!(v == "1");
                had_rx = true;
            } else if k == "ry" {
                assert!(a == Attribute::Ry);
                assert!(v == "2");
                had_ry = true;
            } else {
                unreachable!();
            }
        }

        assert!(had_rx);
        assert!(had_ry);
    }

    #[test]
    fn property_bag_can_iterate_from_c() {
        let pairs = [
            CString::new("rx").unwrap(),
            CString::new("1").unwrap(),
            CString::new("ry").unwrap(),
            CString::new("2").unwrap(),
        ];

        let mut v = Vec::new();

        for x in &pairs {
            v.push(x.as_ptr() as *const libc::c_char);
        }

        v.push(ptr::null());

        let pbag = unsafe { PropertyBag::new_from_key_value_pairs(v.as_ptr()) };

        let mut had_rx: bool = false;
        let mut had_ry: bool = false;

        let iter = rsvg_property_bag_iter_begin(&pbag as *const PropertyBag);

        let mut key = unsafe { mem::uninitialized() };
        let mut att = unsafe { mem::uninitialized() };
        let mut val = unsafe { mem::uninitialized() };

        while from_glib(rsvg_property_bag_iter_next(iter,
                                                    &mut key as *mut _,
                                                    &mut att as *mut _,
                                                    &mut val as *mut _)) {
            let k = unsafe { CStr::from_ptr(key).to_str_utf8() };
            let v = unsafe { CStr::from_ptr(val).to_str_utf8() };

            if k == "rx" {
                assert!(att == Attribute::Rx);
                assert!(v == "1");
                had_rx = true;
            } else if k == "ry" {
                assert!(att == Attribute::Ry);
                assert!(v == "2");
                had_ry = true;
            }
        }

        rsvg_property_bag_iter_end(iter);

        assert!(had_rx);
        assert!(had_ry);
    }
}
<commit_msg>PropertyBag: Use a Vec, not a HashMap<commit_after>use libc;

use glib_sys;
use glib::translate::*;

use std::ffi::{CStr, CString};
use std::ops::Deref;
use std::ptr;
use std::slice;
use std::str::{self, FromStr};

use attributes::Attribute;

// We store (key, attribute, value)
pub struct PropertyBag<'a>(Vec<(&'a CStr, Attribute, &'a CStr)>);

pub struct OwnedPropertyBag(Vec<(CString, Attribute, CString)>);

pub struct PropertyBagIter<'a>(PropertyBagCStrIter<'a>);

pub struct PropertyBagCStrIter<'a>(slice::Iter<'a, (&'a CStr, Attribute, &'a CStr)>);

trait Utf8CStrToStr {
    fn to_str_utf8(&self) -> &str;
}

impl Utf8CStrToStr for CStr {
    fn to_str_utf8(&self) -> &str {
        // We can *only* do this when the CStr comes from a C string that was validated
        // as UTF-8 on the C side of things.  In our case, the C strings from libxml2 and
        // are valid UTF-8.
        unsafe { str::from_utf8_unchecked(self.to_bytes()) }
    }
}

impl<'a> PropertyBag<'a> {
    /// Creates an iterable `PropertyBag` from a C array of borrowed C strings.
    ///
    /// With libxml2's SAX parser, the caller's callback for "element start"
    /// gets passed a `xmlChar **` of attribute/value pairs.  Even indices
    /// in the array are pointers to attribute names; odd indices are
    /// pointers to attribute values.  The array terminates with a NULL
    /// element in an even index.
    ///
    /// This function is unsafe because the caller must guarantee the following:
    ///
    /// * `pairs` is a valid pointer, or NULL for an empty array
    ///
    /// * `pairs` has key/value pairs and is NULL terminated
    ///
    /// * Both keys and values are valid UTF-8, nul-terminated C strings
    ///
    /// The lifetime of the `PropertyBag` should be considered the same as the lifetime of the
    /// `pairs` array, as the property bag does not copy the strings - it directly stores pointers
    /// into that array's strings.
    pub unsafe fn new_from_key_value_pairs(pairs: *const *const libc::c_char) -> PropertyBag<'a> {
        let mut array = Vec::new();

        if !pairs.is_null() {
            let mut i = 0;
            loop {
                let key = *pairs.offset(i);
                if !key.is_null() {
                    let val = *pairs.offset(i + 1);
                    assert!(!val.is_null());

                    let key_str = CStr::from_ptr(key);
                    let val_str = CStr::from_ptr(val);

                    // We silently drop unknown attributes.  New attributes should be added in
                    // build.rs.
                    if let Ok(attr) = Attribute::from_str(key_str.to_str_utf8()) {
                        array.push((key_str, attr, val_str));
                    }
                } else {
                    break;
                }

                i += 2;
            }
        }

        PropertyBag(array)
    }

    pub fn from_owned(owned: &OwnedPropertyBag) -> PropertyBag {
        let mut array = Vec::new();

        for &(ref k, a, ref v) in &owned.0 {
            array.push((k.deref(), a, v.deref()));
        }

        PropertyBag(array)
    }

    pub fn to_owned(&self) -> OwnedPropertyBag {
        let mut array = Vec::<(CString, Attribute, CString)>::new();

        for &(ref k, a, ref v) in &self.0 {
            array.push(((*k).to_owned(), a, (*v).to_owned()));
        }

        OwnedPropertyBag(array)
    }

    pub fn ffi(&self) -> *const PropertyBag {
        self as *const PropertyBag
    }

    pub fn len(&self) -> usize {
        self.0.len()
    }

    pub fn iter(&self) -> PropertyBagIter {
        PropertyBagIter(self.cstr_iter())
    }

    pub fn cstr_iter(&self) -> PropertyBagCStrIter {
        PropertyBagCStrIter(self.0.iter())
    }
}

impl<'a> Iterator for PropertyBagIter<'a> {
    type Item = (&'a str, Attribute, &'a str);

    fn next(&mut self) -> Option<Self::Item> {
        self.0.next().map(|(k, a, v)| (k.to_str_utf8(), a, v.to_str_utf8()))
    }
}

impl<'a> Iterator for PropertyBagCStrIter<'a> {
    type Item = (&'a CStr, Attribute, &'a CStr);

    fn next(&mut self) -> Option<Self::Item> {
        self.0.next().map(|&(k, a, v)| (k, a, v))
    }
}

#[no_mangle]
pub extern fn rsvg_property_bag_new<'a>(atts: *const *const libc::c_char) -> *const PropertyBag<'a> {
    let pbag = unsafe { PropertyBag::new_from_key_value_pairs(atts) };
    Box::into_raw(Box::new(pbag))
}

#[no_mangle]
pub extern fn rsvg_property_bag_free(pbag: *mut PropertyBag) {
    unsafe {
        let _ = Box::from_raw(pbag);
    }
}

#[no_mangle]
pub extern fn rsvg_property_bag_iter_begin(pbag: *const PropertyBag) -> *mut PropertyBagCStrIter {
    assert!(!pbag.is_null());
    let pbag = unsafe { &*pbag };

    Box::into_raw(Box::new(pbag.cstr_iter()))
}

#[no_mangle]
pub extern fn rsvg_property_bag_iter_next(iter: *mut PropertyBagCStrIter,
                                          out_key: *mut *const libc::c_char,
                                          out_attr: *mut Attribute,
                                          out_value: *mut *const libc::c_char)
                                          -> glib_sys::gboolean
{
    assert!(!iter.is_null());
    let iter = unsafe { &mut *iter };

    if let Some((key, attr, val)) = iter.next() {
        unsafe {
            *out_key = key.as_ptr();
            *out_attr = attr;
            *out_value = val.as_ptr();
        }
        true.to_glib()
    } else {
        unsafe {
            *out_key = ptr::null();
            *out_value = ptr::null();
        }
        false.to_glib()
    }
}

#[no_mangle]
pub extern fn rsvg_property_bag_iter_end(iter: *mut PropertyBagCStrIter) {
    assert!(!iter.is_null());

    unsafe { Box::from_raw(iter) };
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::ffi::CString;
    use std::mem;

    #[test]
    fn empty_property_bag() {
        let map = unsafe { PropertyBag::new_from_key_value_pairs(ptr::null()) };
        assert_eq!(map.len(), 0);
    }

    #[test]
    fn property_bag_iters() {
        let pairs = [
            CString::new("rx").unwrap(),
            CString::new("1").unwrap(),
            CString::new("ry").unwrap(),
            CString::new("2").unwrap(),
        ];

        let mut v = Vec::new();

        for x in &pairs {
            v.push(x.as_ptr() as *const libc::c_char);
        }

        v.push(ptr::null());

        let pbag = unsafe { PropertyBag::new_from_key_value_pairs(v.as_ptr()) };

        let mut had_rx: bool = false;
        let mut had_ry: bool = false;

        for (k, a, v) in pbag.iter() {
            if k == "rx" {
                assert!(a == Attribute::Rx);
                assert!(v == "1");
                had_rx = true;
            } else if k == "ry" {
                assert!(a == Attribute::Ry);
                assert!(v == "2");
                had_ry = true;
            } else {
                unreachable!();
            }
        }

        assert!(had_rx);
        assert!(had_ry);
    }

    #[test]
    fn property_bag_can_iterate_from_c() {
        let pairs = [
            CString::new("rx").unwrap(),
            CString::new("1").unwrap(),
            CString::new("ry").unwrap(),
            CString::new("2").unwrap(),
        ];

        let mut v = Vec::new();

        for x in &pairs {
            v.push(x.as_ptr() as *const libc::c_char);
        }

        v.push(ptr::null());

        let pbag = unsafe { PropertyBag::new_from_key_value_pairs(v.as_ptr()) };

        let mut had_rx: bool = false;
        let mut had_ry: bool = false;

        let iter = rsvg_property_bag_iter_begin(&pbag as *const PropertyBag);

        let mut key = unsafe { mem::uninitialized() };
        let mut att = unsafe { mem::uninitialized() };
        let mut val = unsafe { mem::uninitialized() };

        while from_glib(rsvg_property_bag_iter_next(iter,
                                                    &mut key as *mut _,
                                                    &mut att as *mut _,
                                                    &mut val as *mut _)) {
            let k = unsafe { CStr::from_ptr(key).to_str_utf8() };
            let v = unsafe { CStr::from_ptr(val).to_str_utf8() };

            if k == "rx" {
                assert!(att == Attribute::Rx);
                assert!(v == "1");
                had_rx = true;
            } else if k == "ry" {
                assert!(att == Attribute::Ry);
                assert!(v == "2");
                had_ry = true;
            }
        }

        rsvg_property_bag_iter_end(iter);

        assert!(had_rx);
        assert!(had_ry);
    }
}
<|endoftext|>


<commit_before>use std::thread;

struct Philosopher {
    name: String,
}

impl Philosopher {
    fn new(name: &str) -> Philosopher {
        Philosopher {
            name: name.to_string(),
        }
    }

    fn eat(&self) {
        println!("{} is eating.", self.name);

        thread::sleep_ms(1000);

        println!("{} is done eating.", self.name);
    }
}

fn main() {
    let philosophers = vec![
        Philosopher::new("Judith Butler"),
        Philosopher::new("Gilles Deleuze"),
        Philosopher::new("Karl Marx"),
        Philosopher::new("Emma Goldman"),
        Philosopher::new("Michel Foucault"),
    ];

    for p in &philosophers {
        p.eat();
    }
}
<commit_msg>Add concurrency<commit_after>use std::thread;
use std::sync::{Mutex, Arc};

struct Table {
    forks: Vec<Mutex<()>>,
}

struct Philosopher {
    name: String,
    left: usize,
    right: usize,
}

impl Philosopher {
    fn new(name: &str, left: usize, right: usize) -> Philosopher {
        Philosopher {
            name: name.to_string(),
            left: left,
            right: right,
        }
    }

    fn eat(&self, table: &Table) {
        let _left = table.forks[self.left].lock().unwrap();
        let _right = table.forks[self.right].lock().unwrap();

        println!("{} is eating.", self.name);

        thread::sleep_ms(1000);

        println!("{} is done eating.", self.name);
    }
}

fn main() {
    let table = Arc::new(Table { forks: vec![
        Mutex::new(()),
        Mutex::new(()),
        Mutex::new(()),
        Mutex::new(()),
        Mutex::new(()),
    ]});

    let philosophers = vec![
        Philosopher::new("Judith Butler", 0, 1),
        Philosopher::new("Gilles Deleuze", 1, 2),
        Philosopher::new("Karl Marx", 2, 3),
        Philosopher::new("Emma Goldman", 3, 4),
        Philosopher::new("Michel Foucault", 0, 4),
    ];

    let handles: Vec<_> = philosophers.into_iter().map(|p| {
        let table = table.clone();

        thread::spawn(move || {
            p.eat(&table);
        })
    }).collect();

    for h in handles {
        h.join().unwrap();
    }
}
<|endoftext|>


<commit_before><commit_msg>write helper function tests<commit_after>use ::{is_year_leap, number_of_month_days, time_to_string};

#[test]
fn it_has_a_valid_leap_year_indicator() {
  assert!(is_year_leap(1370));
  assert!(! is_year_leap(1371));
}

#[test]
fn it_has_a_valid_number_of_month_days_indicator() {
  assert_eq!(number_of_month_days(1370, 11), 30);
  assert_eq!(number_of_month_days(1371, 11), 29);
}

#[test]
fn it_has_a_valid_time_to_string_converter() {
  assert_eq!(time_to_string(719425800), "Yek Meh 26 20:00:00 1371\n".to_string());
}
<|endoftext|>


<commit_before>use super::stacker::{Addr, MemoryArena, TermHashMap};

use postings::recorder::{
    BufferLender, NothingRecorder, Recorder, TFAndPositionRecorder, TermFrequencyRecorder,
};
use postings::UnorderedTermId;
use postings::{FieldSerializer, InvertedIndexSerializer};
use schema::IndexRecordOption;
use schema::{Field, FieldEntry, FieldType, Schema, Term};
use std::collections::HashMap;
use std::io;
use std::marker::PhantomData;
use std::ops::DerefMut;
use termdict::TermOrdinal;
use tokenizer::TokenStream;
use tokenizer::{Token, MAX_TOKEN_LEN};
use DocId;
use Result;

fn posting_from_field_entry(field_entry: &FieldEntry) -> Box<PostingsWriter> {
    match *field_entry.field_type() {
        FieldType::Str(ref text_options) => text_options
            .get_indexing_options()
            .map(|indexing_options| match indexing_options.index_option() {
                IndexRecordOption::Basic => {
                    SpecializedPostingsWriter::<NothingRecorder>::new_boxed()
                }
                IndexRecordOption::WithFreqs => {
                    SpecializedPostingsWriter::<TermFrequencyRecorder>::new_boxed()
                }
                IndexRecordOption::WithFreqsAndPositions => {
                    SpecializedPostingsWriter::<TFAndPositionRecorder>::new_boxed()
                }
            })
            .unwrap_or_else(|| SpecializedPostingsWriter::<NothingRecorder>::new_boxed()),
        FieldType::U64(_)
        | FieldType::I64(_)
        | FieldType::Date(_)
        | FieldType::HierarchicalFacet => SpecializedPostingsWriter::<NothingRecorder>::new_boxed(),
        FieldType::Bytes => {
            // FieldType::Bytes cannot actually be indexed.
            // TODO fix during the indexer refactoring described in #276
            SpecializedPostingsWriter::<NothingRecorder>::new_boxed()
        }
    }
}

pub struct MultiFieldPostingsWriter {
    heap: MemoryArena,
    schema: Schema,
    term_index: TermHashMap,
    per_field_postings_writers: Vec<Box<PostingsWriter>>,
}

fn make_field_partition(
    term_offsets: &[(&[u8], Addr, UnorderedTermId)],
) -> Vec<(Field, usize, usize)> {
    let term_offsets_it = term_offsets
        .iter()
        .map(|(key, _, _)| Term::wrap(key).field())
        .enumerate();
    let mut prev_field = Field(u32::max_value());
    let mut fields = vec![];
    let mut offsets = vec![];
    for (offset, field) in term_offsets_it {
        if field != prev_field {
            prev_field = field;
            fields.push(field);
            offsets.push(offset);
        }
    }
    offsets.push(term_offsets.len());
    let mut field_offsets = vec![];
    for i in 0..fields.len() {
        field_offsets.push((fields[i], offsets[i], offsets[i + 1]));
    }
    field_offsets
}

impl MultiFieldPostingsWriter {
    /// Create a new `MultiFieldPostingsWriter` given
    /// a schema and a heap.
    pub fn new(schema: &Schema, table_bits: usize) -> MultiFieldPostingsWriter {
        let term_index = TermHashMap::new(table_bits);
        let per_field_postings_writers: Vec<_> = schema
            .fields()
            .iter()
            .map(|field_entry| posting_from_field_entry(field_entry))
            .collect();
        MultiFieldPostingsWriter {
            heap: MemoryArena::new(),
            schema: schema.clone(),
            term_index,
            per_field_postings_writers,
        }
    }

    pub fn mem_usage(&self) -> usize {
        self.term_index.mem_usage() + self.heap.mem_usage()
    }

    pub fn index_text(&mut self, doc: DocId, field: Field, token_stream: &mut TokenStream) -> u32 {
        let postings_writer = self.per_field_postings_writers[field.0 as usize].deref_mut();
        postings_writer.index_text(
            &mut self.term_index,
            doc,
            field,
            token_stream,
            &mut self.heap,
        )
    }

    pub fn subscribe(&mut self, doc: DocId, term: &Term) -> UnorderedTermId {
        let postings_writer = self.per_field_postings_writers[term.field().0 as usize].deref_mut();
        postings_writer.subscribe(&mut self.term_index, doc, 0u32, term, &mut self.heap)
    }

    /// Serialize the inverted index.
    /// It pushes all term, one field at a time, towards the
    /// postings serializer.
    pub fn serialize(
        &self,
        serializer: &mut InvertedIndexSerializer,
    ) -> Result<HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>> {
        let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> =
            self.term_index.iter().collect();
        term_offsets.sort_unstable_by_key(|&(k, _, _)| k);

        let mut unordered_term_mappings: HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>> =
            HashMap::new();

        let field_offsets = make_field_partition(&term_offsets);

        for (field, start, stop) in field_offsets {
            let field_entry = self.schema.get_field_entry(field);

            match *field_entry.field_type() {
                FieldType::Str(_) | FieldType::HierarchicalFacet => {
                    // populating the (unordered term ord) -> (ordered term ord) mapping
                    // for the field.
                    let mut unordered_term_ids = term_offsets[start..stop]
                        .iter()
                        .map(|&(_, _, bucket)| bucket);
                    let mut mapping: HashMap<UnorderedTermId, TermOrdinal> = unordered_term_ids
                        .enumerate()
                        .map(|(term_ord, unord_term_id)| {
                            (unord_term_id as UnorderedTermId, term_ord as TermOrdinal)
                        })
                        .collect();
                    unordered_term_mappings.insert(field, mapping);
                }
                FieldType::U64(_) | FieldType::I64(_) | FieldType::Date(_) => {}
                FieldType::Bytes => {}
            }

            let postings_writer = &self.per_field_postings_writers[field.0 as usize];
            let mut field_serializer =
                serializer.new_field(field, postings_writer.total_num_tokens())?;
            postings_writer.serialize(
                &term_offsets[start..stop],
                &mut field_serializer,
                &self.term_index.heap,
                &self.heap,
            )?;
            field_serializer.close()?;
        }
        Ok(unordered_term_mappings)
    }
}

/// The `PostingsWriter` is in charge of receiving documenting
/// and building a `Segment` in anonymous memory.
///
/// `PostingsWriter` writes in a `MemoryArena`.
pub trait PostingsWriter {
    /// Record that a document contains a term at a given position.
    ///
    /// * doc  - the document id
    /// * pos  - the term position (expressed in tokens)
    /// * term - the term
    /// * heap - heap used to store the postings informations as well as the terms
    /// in the hashmap.
    fn subscribe(
        &mut self,
        term_index: &mut TermHashMap,
        doc: DocId,
        pos: u32,
        term: &Term,
        heap: &mut MemoryArena,
    ) -> UnorderedTermId;

    /// Serializes the postings on disk.
    /// The actual serialization format is handled by the `PostingsSerializer`.
    fn serialize(
        &self,
        term_addrs: &[(&[u8], Addr, UnorderedTermId)],
        serializer: &mut FieldSerializer,
        term_heap: &MemoryArena,
        heap: &MemoryArena,
    ) -> io::Result<()>;

    /// Tokenize a text and subscribe all of its token.
    fn index_text(
        &mut self,
        term_index: &mut TermHashMap,
        doc_id: DocId,
        field: Field,
        token_stream: &mut TokenStream,
        heap: &mut MemoryArena,
    ) -> u32 {
        let mut term = Term::for_field(field);
        let mut sink = |token: &Token| {
            // We skip all tokens with a len greater than u16.
            if token.text.len() <= MAX_TOKEN_LEN {
                term.set_text(token.text.as_str());
                self.subscribe(term_index, doc_id, token.position as u32, &term, heap);
            }
        };
        token_stream.process(&mut sink)
    }

    fn total_num_tokens(&self) -> u64;
}

/// The `SpecializedPostingsWriter` is just here to remove dynamic
/// dispatch to the recorder information.
pub(crate) struct SpecializedPostingsWriter<Rec: Recorder + 'static> {
    total_num_tokens: u64,
    _recorder_type: PhantomData<Rec>,
}

impl<Rec: Recorder + 'static> SpecializedPostingsWriter<Rec> {
    /// constructor
    pub fn new() -> SpecializedPostingsWriter<Rec> {
        SpecializedPostingsWriter {
            total_num_tokens: 0u64,
            _recorder_type: PhantomData,
        }
    }

    /// Builds a `SpecializedPostingsWriter` storing its data in a heap.
    pub fn new_boxed() -> Box<PostingsWriter> {
        Box::new(SpecializedPostingsWriter::<Rec>::new())
    }
}

impl<Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<Rec> {
    fn subscribe(
        &mut self,
        term_index: &mut TermHashMap,
        doc: DocId,
        position: u32,
        term: &Term,
        heap: &mut MemoryArena,
    ) -> UnorderedTermId {
        debug_assert!(term.as_slice().len() >= 4);
        self.total_num_tokens += 1;
        term_index.mutate_or_create(term, |opt_recorder: Option<Rec>| {
            if let Some(mut recorder) = opt_recorder {
                let current_doc = recorder.current_doc();
                if current_doc != doc {
                    recorder.close_doc(heap);
                    recorder.new_doc(doc, heap);
                }
                recorder.record_position(position, heap);
                recorder
            } else {
                let mut recorder = Rec::new();
                recorder.new_doc(doc, heap);
                recorder.record_position(position, heap);
                recorder
            }
        }) as UnorderedTermId
    }

    fn serialize(
        &self,
        term_addrs: &[(&[u8], Addr, UnorderedTermId)],
        serializer: &mut FieldSerializer,
        termdict_heap: &MemoryArena,
        heap: &MemoryArena,
    ) -> io::Result<()> {
        let mut buffer_lender = BufferLender::default();
        for &(term_bytes, addr, _) in term_addrs {
            let recorder: Rec = termdict_heap.read(addr);
            serializer.new_term(&term_bytes[4..])?;
            recorder.serialize(&mut buffer_lender, serializer, heap)?;
            serializer.close_term()?;
        }
        Ok(())
    }

    fn total_num_tokens(&self) -> u64 {
        self.total_num_tokens
    }
}
<commit_msg>Added logging when token is dropped. (#538)<commit_after>use super::stacker::{Addr, MemoryArena, TermHashMap};

use postings::recorder::{
    BufferLender, NothingRecorder, Recorder, TFAndPositionRecorder, TermFrequencyRecorder,
};
use postings::UnorderedTermId;
use postings::{FieldSerializer, InvertedIndexSerializer};
use schema::IndexRecordOption;
use schema::{Field, FieldEntry, FieldType, Schema, Term};
use std::collections::HashMap;
use std::io;
use std::marker::PhantomData;
use std::ops::DerefMut;
use termdict::TermOrdinal;
use tokenizer::TokenStream;
use tokenizer::{Token, MAX_TOKEN_LEN};
use DocId;
use Result;

fn posting_from_field_entry(field_entry: &FieldEntry) -> Box<PostingsWriter> {
    match *field_entry.field_type() {
        FieldType::Str(ref text_options) => text_options
            .get_indexing_options()
            .map(|indexing_options| match indexing_options.index_option() {
                IndexRecordOption::Basic => {
                    SpecializedPostingsWriter::<NothingRecorder>::new_boxed()
                }
                IndexRecordOption::WithFreqs => {
                    SpecializedPostingsWriter::<TermFrequencyRecorder>::new_boxed()
                }
                IndexRecordOption::WithFreqsAndPositions => {
                    SpecializedPostingsWriter::<TFAndPositionRecorder>::new_boxed()
                }
            })
            .unwrap_or_else(|| SpecializedPostingsWriter::<NothingRecorder>::new_boxed()),
        FieldType::U64(_)
        | FieldType::I64(_)
        | FieldType::Date(_)
        | FieldType::HierarchicalFacet => SpecializedPostingsWriter::<NothingRecorder>::new_boxed(),
        FieldType::Bytes => {
            // FieldType::Bytes cannot actually be indexed.
            // TODO fix during the indexer refactoring described in #276
            SpecializedPostingsWriter::<NothingRecorder>::new_boxed()
        }
    }
}

pub struct MultiFieldPostingsWriter {
    heap: MemoryArena,
    schema: Schema,
    term_index: TermHashMap,
    per_field_postings_writers: Vec<Box<PostingsWriter>>,
}

fn make_field_partition(
    term_offsets: &[(&[u8], Addr, UnorderedTermId)],
) -> Vec<(Field, usize, usize)> {
    let term_offsets_it = term_offsets
        .iter()
        .map(|(key, _, _)| Term::wrap(key).field())
        .enumerate();
    let mut prev_field = Field(u32::max_value());
    let mut fields = vec![];
    let mut offsets = vec![];
    for (offset, field) in term_offsets_it {
        if field != prev_field {
            prev_field = field;
            fields.push(field);
            offsets.push(offset);
        }
    }
    offsets.push(term_offsets.len());
    let mut field_offsets = vec![];
    for i in 0..fields.len() {
        field_offsets.push((fields[i], offsets[i], offsets[i + 1]));
    }
    field_offsets
}

impl MultiFieldPostingsWriter {
    /// Create a new `MultiFieldPostingsWriter` given
    /// a schema and a heap.
    pub fn new(schema: &Schema, table_bits: usize) -> MultiFieldPostingsWriter {
        let term_index = TermHashMap::new(table_bits);
        let per_field_postings_writers: Vec<_> = schema
            .fields()
            .iter()
            .map(|field_entry| posting_from_field_entry(field_entry))
            .collect();
        MultiFieldPostingsWriter {
            heap: MemoryArena::new(),
            schema: schema.clone(),
            term_index,
            per_field_postings_writers,
        }
    }

    pub fn mem_usage(&self) -> usize {
        self.term_index.mem_usage() + self.heap.mem_usage()
    }

    pub fn index_text(&mut self, doc: DocId, field: Field, token_stream: &mut TokenStream) -> u32 {
        let postings_writer = self.per_field_postings_writers[field.0 as usize].deref_mut();
        postings_writer.index_text(
            &mut self.term_index,
            doc,
            field,
            token_stream,
            &mut self.heap,
        )
    }

    pub fn subscribe(&mut self, doc: DocId, term: &Term) -> UnorderedTermId {
        let postings_writer = self.per_field_postings_writers[term.field().0 as usize].deref_mut();
        postings_writer.subscribe(&mut self.term_index, doc, 0u32, term, &mut self.heap)
    }

    /// Serialize the inverted index.
    /// It pushes all term, one field at a time, towards the
    /// postings serializer.
    pub fn serialize(
        &self,
        serializer: &mut InvertedIndexSerializer,
    ) -> Result<HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>>> {
        let mut term_offsets: Vec<(&[u8], Addr, UnorderedTermId)> =
            self.term_index.iter().collect();
        term_offsets.sort_unstable_by_key(|&(k, _, _)| k);

        let mut unordered_term_mappings: HashMap<Field, HashMap<UnorderedTermId, TermOrdinal>> =
            HashMap::new();

        let field_offsets = make_field_partition(&term_offsets);

        for (field, start, stop) in field_offsets {
            let field_entry = self.schema.get_field_entry(field);

            match *field_entry.field_type() {
                FieldType::Str(_) | FieldType::HierarchicalFacet => {
                    // populating the (unordered term ord) -> (ordered term ord) mapping
                    // for the field.
                    let mut unordered_term_ids = term_offsets[start..stop]
                        .iter()
                        .map(|&(_, _, bucket)| bucket);
                    let mut mapping: HashMap<UnorderedTermId, TermOrdinal> = unordered_term_ids
                        .enumerate()
                        .map(|(term_ord, unord_term_id)| {
                            (unord_term_id as UnorderedTermId, term_ord as TermOrdinal)
                        })
                        .collect();
                    unordered_term_mappings.insert(field, mapping);
                }
                FieldType::U64(_) | FieldType::I64(_) | FieldType::Date(_) => {}
                FieldType::Bytes => {}
            }

            let postings_writer = &self.per_field_postings_writers[field.0 as usize];
            let mut field_serializer =
                serializer.new_field(field, postings_writer.total_num_tokens())?;
            postings_writer.serialize(
                &term_offsets[start..stop],
                &mut field_serializer,
                &self.term_index.heap,
                &self.heap,
            )?;
            field_serializer.close()?;
        }
        Ok(unordered_term_mappings)
    }
}

/// The `PostingsWriter` is in charge of receiving documenting
/// and building a `Segment` in anonymous memory.
///
/// `PostingsWriter` writes in a `MemoryArena`.
pub trait PostingsWriter {
    /// Record that a document contains a term at a given position.
    ///
    /// * doc  - the document id
    /// * pos  - the term position (expressed in tokens)
    /// * term - the term
    /// * heap - heap used to store the postings informations as well as the terms
    /// in the hashmap.
    fn subscribe(
        &mut self,
        term_index: &mut TermHashMap,
        doc: DocId,
        pos: u32,
        term: &Term,
        heap: &mut MemoryArena,
    ) -> UnorderedTermId;

    /// Serializes the postings on disk.
    /// The actual serialization format is handled by the `PostingsSerializer`.
    fn serialize(
        &self,
        term_addrs: &[(&[u8], Addr, UnorderedTermId)],
        serializer: &mut FieldSerializer,
        term_heap: &MemoryArena,
        heap: &MemoryArena,
    ) -> io::Result<()>;

    /// Tokenize a text and subscribe all of its token.
    fn index_text(
        &mut self,
        term_index: &mut TermHashMap,
        doc_id: DocId,
        field: Field,
        token_stream: &mut TokenStream,
        heap: &mut MemoryArena,
    ) -> u32 {
        let mut term = Term::for_field(field);
        let mut sink = |token: &Token| {
            // We skip all tokens with a len greater than u16.
            if token.text.len() <= MAX_TOKEN_LEN {
                term.set_text(token.text.as_str());
                self.subscribe(term_index, doc_id, token.position as u32, &term, heap);
            } else {
                info!(
                    "A token exceeding MAX_TOKEN_LEN ({}>{}) was dropped. Search for \
                     MAX_TOKEN_LEN in the documentation for more information.",
                    token.text.len(),
                    MAX_TOKEN_LEN
                );
            }
        };
        token_stream.process(&mut sink)
    }

    fn total_num_tokens(&self) -> u64;
}

/// The `SpecializedPostingsWriter` is just here to remove dynamic
/// dispatch to the recorder information.
pub(crate) struct SpecializedPostingsWriter<Rec: Recorder + 'static> {
    total_num_tokens: u64,
    _recorder_type: PhantomData<Rec>,
}

impl<Rec: Recorder + 'static> SpecializedPostingsWriter<Rec> {
    /// constructor
    pub fn new() -> SpecializedPostingsWriter<Rec> {
        SpecializedPostingsWriter {
            total_num_tokens: 0u64,
            _recorder_type: PhantomData,
        }
    }

    /// Builds a `SpecializedPostingsWriter` storing its data in a heap.
    pub fn new_boxed() -> Box<PostingsWriter> {
        Box::new(SpecializedPostingsWriter::<Rec>::new())
    }
}

impl<Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<Rec> {
    fn subscribe(
        &mut self,
        term_index: &mut TermHashMap,
        doc: DocId,
        position: u32,
        term: &Term,
        heap: &mut MemoryArena,
    ) -> UnorderedTermId {
        debug_assert!(term.as_slice().len() >= 4);
        self.total_num_tokens += 1;
        term_index.mutate_or_create(term, |opt_recorder: Option<Rec>| {
            if let Some(mut recorder) = opt_recorder {
                let current_doc = recorder.current_doc();
                if current_doc != doc {
                    recorder.close_doc(heap);
                    recorder.new_doc(doc, heap);
                }
                recorder.record_position(position, heap);
                recorder
            } else {
                let mut recorder = Rec::new();
                recorder.new_doc(doc, heap);
                recorder.record_position(position, heap);
                recorder
            }
        }) as UnorderedTermId
    }

    fn serialize(
        &self,
        term_addrs: &[(&[u8], Addr, UnorderedTermId)],
        serializer: &mut FieldSerializer,
        termdict_heap: &MemoryArena,
        heap: &MemoryArena,
    ) -> io::Result<()> {
        let mut buffer_lender = BufferLender::default();
        for &(term_bytes, addr, _) in term_addrs {
            let recorder: Rec = termdict_heap.read(addr);
            serializer.new_term(&term_bytes[4..])?;
            recorder.serialize(&mut buffer_lender, serializer, heap)?;
            serializer.close_term()?;
        }
        Ok(())
    }

    fn total_num_tokens(&self) -> u64 {
        self.total_num_tokens
    }
}
<|endoftext|>


<commit_before>use il;

pub fn gen(globals: Vec<il::Global>) {
    for g in globals.iter() {
        match *g {
            il::TypedefVar(ref v) => {
                println!("pub type {} = {};", v.name, v.dst);
            },
            il::StructVar(ref v) => {
                println!("#[repr(C)]");
                println!("pub struct {} {{", v.name);
                if v.fields.len() == 0 {
                    println!("    _dummy: (),");
                }
                for f in v.fields.iter() {
                    print!("    ");
                    if f.public {
                        print!("pub ");
                    }
                    println!("{}: {},", f.name, f.ty);
                }
                println!("}}");
                println!("new!({})", v.name);
            },
            il::ConstantVar(ref v) => {
                println!("pub static {}: {} = {};", v.name, v.ty, v.val);
            },
        }
    }
}
<commit_msg>update rust<commit_after>use il;

pub fn gen(globals: Vec<il::Global>) {
    for g in globals.iter() {
        match *g {
            il::TypedefVar(ref v) => {
                println!("pub type {} = {};", v.name, v.dst);
            },
            il::StructVar(ref v) => {
                println!("#[repr(C)]");
                println!("pub struct {} {{", v.name);
                if v.fields.len() == 0 {
                    println!("    _dummy: (),");
                }
                for f in v.fields.iter() {
                    print!("    ");
                    if f.public {
                        print!("pub ");
                    }
                    println!("{}: {},", f.name, f.ty);
                }
                println!("}}");
                println!("new!({})", v.name);
            },
            il::ConstantVar(ref v) => {
                println!("pub const {}: {} = {};", v.name, v.ty, v.val);
            },
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use driver::session::Session;

use middle::save::generated_code;

use std::cell::Cell;

use syntax::ast;
use syntax::codemap::*;
use syntax::parse::lexer;
use syntax::parse::lexer::{Reader,StringReader};
use syntax::parse::token;
use syntax::parse::token::{keywords, Token};

pub struct SpanUtils<'a> {
    pub sess: &'a Session,
    pub err_count: Cell<int>,
}

impl<'a> SpanUtils<'a> {
    // Standard string for extents/location.
    pub fn extent_str(&self, span: Span) -> String {
        let lo_loc = self.sess.codemap().lookup_char_pos(span.lo);
        let hi_loc = self.sess.codemap().lookup_char_pos(span.hi);
        let lo_pos = self.sess.codemap().lookup_byte_offset(span.lo).pos;
        let hi_pos = self.sess.codemap().lookup_byte_offset(span.hi).pos;

        format!("file_name,{},file_line,{},file_col,{},extent_start,{},\
                 file_line_end,{},file_col_end,{},extent_end,{}",
                lo_loc.file.name, lo_loc.line, lo_loc.col.to_uint(), lo_pos.to_uint(),
                hi_loc.line, hi_loc.col.to_uint(), hi_pos.to_uint())
    }

    // sub_span starts at span.lo, so we need to adjust the positions etc.
    // If sub_span is None, we don't need to adjust.
    pub fn make_sub_span(&self, span: Span, sub_span: Option<Span>) -> Option<Span> {
        let loc = self.sess.codemap().lookup_char_pos(span.lo);
        assert!(!generated_code(span),
                "generated code; we should not be processing this `{}` in {}, line {}",
                 self.snippet(span), loc.file.name, loc.line);

        match sub_span {
            None => None,
            Some(sub) => {
                let FileMapAndBytePos {fm, pos} =
                    self.sess.codemap().lookup_byte_offset(span.lo);
                let base = pos + fm.start_pos;
                Some(Span {
                    lo: base + self.sess.codemap().lookup_byte_offset(sub.lo).pos,
                    hi: base + self.sess.codemap().lookup_byte_offset(sub.hi).pos,
                    expn_id: NO_EXPANSION,
                })
            }
        }
    }

    pub fn snippet(&self, span: Span) -> String {
        match self.sess.codemap().span_to_snippet(span) {
            Some(s) => s,
            None => String::new(),
        }
    }

    pub fn retokenise_span(&self, span: Span) -> StringReader<'a> {
        // sadness - we don't have spans for sub-expressions nor access to the tokens
        // so in order to get extents for the function name itself (which dxr expects)
        // we need to re-tokenise the fn definition

        // Note: this is a bit awful - it adds the contents of span to the end of
        // the codemap as a new filemap. This is mostly OK, but means we should
        // not iterate over the codemap. Also, any spans over the new filemap
        // are incompatible with spans over other filemaps.
        let filemap = self.sess.codemap().new_filemap(String::from_str("<anon-dxr>"),
                                                      self.snippet(span));
        let s = self.sess;
        lexer::StringReader::new(s.diagnostic(), filemap)
    }

    // Re-parses a path and returns the span for the last identifier in the path
    pub fn span_for_last_ident(&self, span: Span) -> Option<Span> {
        let mut result = None;

        let mut toks = self.retokenise_span(span);
        let mut bracket_count = 0u;
        loop {
            let ts = toks.real_token();
            if ts.tok == token::Eof {
                return self.make_sub_span(span, result)
            }
            if bracket_count == 0 &&
               (ts.tok.is_ident() || ts.tok.is_keyword(keywords::Self)) {
                result = Some(ts.sp);
            }

            bracket_count += match ts.tok {
                token::Lt => 1,
                token::Gt => -1,
                token::BinOp(token::Shr) => -2,
                _ => 0
            }
        }
    }

    // Return the span for the first identifier in the path.
    pub fn span_for_first_ident(&self, span: Span) -> Option<Span> {
        let mut toks = self.retokenise_span(span);
        let mut bracket_count = 0u;
        loop {
            let ts = toks.real_token();
            if ts.tok == token::Eof {
                return None;
            }
            if bracket_count == 0 &&
               (ts.tok.is_ident() || ts.tok.is_keyword(keywords::Self)) {
                return self.make_sub_span(span, Some(ts.sp));
            }

            bracket_count += match ts.tok {
                token::Lt => 1,
                token::Gt => -1,
                token::BinOp(token::Shr) => -2,
                _ => 0
            }
        }
    }

    // Return the span for the last ident before a `(` or `<` or '::<' and outside any
    // any brackets, or the last span.
    pub fn sub_span_for_meth_name(&self, span: Span) -> Option<Span> {
        let mut toks = self.retokenise_span(span);
        let mut prev = toks.real_token();
        let mut result = None;
        let mut bracket_count = 0u;
        let mut last_span = None;
        while prev.tok != token::Eof {
            last_span = None;
            let mut next = toks.real_token();

            if (next.tok == token::OpenDelim(token::Paren) ||
                next.tok == token::Lt) &&
               bracket_count == 0 &&
               prev.tok.is_ident() {
                result = Some(prev.sp);
            }

            if bracket_count == 0 &&
                next.tok == token::ModSep {
                let old = prev;
                prev = next;
                next = toks.real_token();
                if next.tok == token::Lt &&
                   old.tok.is_ident() {
                    result = Some(old.sp);
                }
            }

            bracket_count += match prev.tok {
                token::OpenDelim(token::Paren) | token::Lt => 1,
                token::CloseDelim(token::Paren) | token::Gt => -1,
                token::BinOp(token::Shr) => -2,
                _ => 0
            };

            if prev.tok.is_ident() && bracket_count == 0 {
                last_span = Some(prev.sp);
            }
            prev = next;
        }
        if result.is_none() && last_span.is_some() {
            return self.make_sub_span(span, last_span);
        }
        return self.make_sub_span(span, result);
    }

    // Return the span for the last ident before a `<` and outside any
    // brackets, or the last span.
    pub fn sub_span_for_type_name(&self, span: Span) -> Option<Span> {
        let mut toks = self.retokenise_span(span);
        let mut prev = toks.real_token();
        let mut result = None;
        let mut bracket_count = 0u;
        loop {
            let next = toks.real_token();

            if (next.tok == token::Lt ||
                next.tok == token::Colon) &&
               bracket_count == 0 &&
               prev.tok.is_ident() {
                result = Some(prev.sp);
            }

            bracket_count += match prev.tok {
                token::Lt => 1,
                token::Gt => -1,
                token::BinOp(token::Shr) => -2,
                _ => 0
            };

            if next.tok == token::Eof {
                break;
            }
            prev = next;
        }
        if bracket_count != 0 {
            let loc = self.sess.codemap().lookup_char_pos(span.lo);
            self.sess.span_bug(span,
                format!("Mis-counted brackets when breaking path? Parsing '{}' in {}, line {}",
                        self.snippet(span), loc.file.name, loc.line).as_slice());
        }
        if result.is_none() && prev.tok.is_ident() && bracket_count == 0 {
            return self.make_sub_span(span, Some(prev.sp));
        }
        self.make_sub_span(span, result)
    }

    // Reparse span and return an owned vector of sub spans of the first limit
    // identifier tokens in the given nesting level.
    // example with Foo<Bar<T,V>, Bar<T,V>>
    // Nesting = 0: all idents outside of brackets: ~[Foo]
    // Nesting = 1: idents within one level of brackets: ~[Bar, Bar]
    pub fn spans_with_brackets(&self, span: Span, nesting: int, limit: int) -> Vec<Span> {
        let mut result: Vec<Span> = vec!();

        let mut toks = self.retokenise_span(span);
        // We keep track of how many brackets we're nested in
        let mut bracket_count = 0i;
        loop {
            let ts = toks.real_token();
            if ts.tok == token::Eof {
                if bracket_count != 0 {
                    let loc = self.sess.codemap().lookup_char_pos(span.lo);
                    self.sess.span_bug(span, format!(
                        "Mis-counted brackets when breaking path? Parsing '{}' in {}, line {}",
                         self.snippet(span), loc.file.name, loc.line).as_slice());
                }
                return result
            }
            if (result.len() as int) == limit {
                return result;
            }
            bracket_count += match ts.tok {
                token::Lt => 1,
                token::Gt => -1,
                token::BinOp(token::Shl) => 2,
                token::BinOp(token::Shr) => -2,
                _ => 0
            };
            if ts.tok.is_ident() &&
               bracket_count == nesting {
                result.push(self.make_sub_span(span, Some(ts.sp)).unwrap());
            }
        }
    }

    pub fn sub_span_before_token(&self, span: Span, tok: Token) -> Option<Span> {
        let mut toks = self.retokenise_span(span);
        let mut prev = toks.real_token();
        loop {
            if prev.tok == token::Eof {
                return None;
            }
            let next = toks.real_token();
            if next.tok == tok {
                return self.make_sub_span(span, Some(prev.sp));
            }
            prev = next;
        }
    }

    pub fn sub_span_after_keyword(&self,
                              span: Span,
                              keyword: keywords::Keyword) -> Option<Span> {
        let mut toks = self.retokenise_span(span);
        loop {
            let ts = toks.real_token();
            if ts.tok == token::Eof {
                return None;
            }
            if ts.tok.is_keyword(keyword) {
                let ts = toks.real_token();
                if ts.tok == token::Eof {
                    return None
                } else {
                    println!("found keyword: {} at {}", ts, ts.sp);
                    return self.make_sub_span(span, Some(ts.sp));
                }
            }
        }
    }

    // Returns a list of the spans of idents in a patch.
    // E.g., For foo::bar<x,t>::baz, we return [foo, bar, baz] (well, their spans)
    pub fn spans_for_path_segments(&self, path: &ast::Path) -> Vec<Span> {
        if generated_code(path.span) {
            return vec!();
        }

        self.spans_with_brackets(path.span, 0, -1)
    }

    // Return an owned vector of the subspans of the param identifier
    // tokens found in span.
    pub fn spans_for_ty_params(&self, span: Span, number: int) -> Vec<Span> {
        if generated_code(span) {
            return vec!();
        }
        // Type params are nested within one level of brackets:
        // i.e. we want ~[A, B] from Foo<A, B<T,U>>
        self.spans_with_brackets(span, 1, number)
    }

    pub fn report_span_err(&self, kind: &str, span: Span) {
        let loc = self.sess.codemap().lookup_char_pos(span.lo);
        info!("({}) Could not find sub_span in `{}` in {}, line {}",
              kind, self.snippet(span), loc.file.name, loc.line);
        self.err_count.set(self.err_count.get()+1);
        if self.err_count.get() > 1000 {
            self.sess.bug("span errors reached 1000, giving up");
        }
    }
}
<commit_msg>Dump char and byte positions in save_analysis<commit_after>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use driver::session::Session;

use middle::save::generated_code;

use std::cell::Cell;

use syntax::ast;
use syntax::codemap::*;
use syntax::parse::lexer;
use syntax::parse::lexer::{Reader,StringReader};
use syntax::parse::token;
use syntax::parse::token::{keywords, Token};

pub struct SpanUtils<'a> {
    pub sess: &'a Session,
    pub err_count: Cell<int>,
}

impl<'a> SpanUtils<'a> {
    // Standard string for extents/location.
    pub fn extent_str(&self, span: Span) -> String {
        let lo_loc = self.sess.codemap().lookup_char_pos(span.lo);
        let hi_loc = self.sess.codemap().lookup_char_pos(span.hi);
        let lo_pos = self.sess.codemap().bytepos_to_file_charpos(span.lo);
        let hi_pos = self.sess.codemap().bytepos_to_file_charpos(span.hi);
        let lo_pos_byte = self.sess.codemap().lookup_byte_offset(span.lo).pos;
        let hi_pos_byte = self.sess.codemap().lookup_byte_offset(span.hi).pos;

        format!("file_name,{},file_line,{},file_col,{},extent_start,{},extent_start_bytes,{},\
                 file_line_end,{},file_col_end,{},extent_end,{},extent_end_bytes,{}",
                lo_loc.file.name,
                lo_loc.line, lo_loc.col.to_uint(), lo_pos.to_uint(), lo_pos_byte.to_uint(),
                hi_loc.line, hi_loc.col.to_uint(), hi_pos.to_uint(), hi_pos_byte.to_uint())
    }

    // sub_span starts at span.lo, so we need to adjust the positions etc.
    // If sub_span is None, we don't need to adjust.
    pub fn make_sub_span(&self, span: Span, sub_span: Option<Span>) -> Option<Span> {
        let loc = self.sess.codemap().lookup_char_pos(span.lo);
        assert!(!generated_code(span),
                "generated code; we should not be processing this `{}` in {}, line {}",
                 self.snippet(span), loc.file.name, loc.line);

        match sub_span {
            None => None,
            Some(sub) => {
                let FileMapAndBytePos {fm, pos} =
                    self.sess.codemap().lookup_byte_offset(span.lo);
                let base = pos + fm.start_pos;
                Some(Span {
                    lo: base + self.sess.codemap().lookup_byte_offset(sub.lo).pos,
                    hi: base + self.sess.codemap().lookup_byte_offset(sub.hi).pos,
                    expn_id: NO_EXPANSION,
                })
            }
        }
    }

    pub fn snippet(&self, span: Span) -> String {
        match self.sess.codemap().span_to_snippet(span) {
            Some(s) => s,
            None => String::new(),
        }
    }

    pub fn retokenise_span(&self, span: Span) -> StringReader<'a> {
        // sadness - we don't have spans for sub-expressions nor access to the tokens
        // so in order to get extents for the function name itself (which dxr expects)
        // we need to re-tokenise the fn definition

        // Note: this is a bit awful - it adds the contents of span to the end of
        // the codemap as a new filemap. This is mostly OK, but means we should
        // not iterate over the codemap. Also, any spans over the new filemap
        // are incompatible with spans over other filemaps.
        let filemap = self.sess.codemap().new_filemap(String::from_str("<anon-dxr>"),
                                                      self.snippet(span));
        let s = self.sess;
        lexer::StringReader::new(s.diagnostic(), filemap)
    }

    // Re-parses a path and returns the span for the last identifier in the path
    pub fn span_for_last_ident(&self, span: Span) -> Option<Span> {
        let mut result = None;

        let mut toks = self.retokenise_span(span);
        let mut bracket_count = 0u;
        loop {
            let ts = toks.real_token();
            if ts.tok == token::Eof {
                return self.make_sub_span(span, result)
            }
            if bracket_count == 0 &&
               (ts.tok.is_ident() || ts.tok.is_keyword(keywords::Self)) {
                result = Some(ts.sp);
            }

            bracket_count += match ts.tok {
                token::Lt => 1,
                token::Gt => -1,
                token::BinOp(token::Shr) => -2,
                _ => 0
            }
        }
    }

    // Return the span for the first identifier in the path.
    pub fn span_for_first_ident(&self, span: Span) -> Option<Span> {
        let mut toks = self.retokenise_span(span);
        let mut bracket_count = 0u;
        loop {
            let ts = toks.real_token();
            if ts.tok == token::Eof {
                return None;
            }
            if bracket_count == 0 &&
               (ts.tok.is_ident() || ts.tok.is_keyword(keywords::Self)) {
                return self.make_sub_span(span, Some(ts.sp));
            }

            bracket_count += match ts.tok {
                token::Lt => 1,
                token::Gt => -1,
                token::BinOp(token::Shr) => -2,
                _ => 0
            }
        }
    }

    // Return the span for the last ident before a `(` or `<` or '::<' and outside any
    // any brackets, or the last span.
    pub fn sub_span_for_meth_name(&self, span: Span) -> Option<Span> {
        let mut toks = self.retokenise_span(span);
        let mut prev = toks.real_token();
        let mut result = None;
        let mut bracket_count = 0u;
        let mut last_span = None;
        while prev.tok != token::Eof {
            last_span = None;
            let mut next = toks.real_token();

            if (next.tok == token::OpenDelim(token::Paren) ||
                next.tok == token::Lt) &&
               bracket_count == 0 &&
               prev.tok.is_ident() {
                result = Some(prev.sp);
            }

            if bracket_count == 0 &&
                next.tok == token::ModSep {
                let old = prev;
                prev = next;
                next = toks.real_token();
                if next.tok == token::Lt &&
                   old.tok.is_ident() {
                    result = Some(old.sp);
                }
            }

            bracket_count += match prev.tok {
                token::OpenDelim(token::Paren) | token::Lt => 1,
                token::CloseDelim(token::Paren) | token::Gt => -1,
                token::BinOp(token::Shr) => -2,
                _ => 0
            };

            if prev.tok.is_ident() && bracket_count == 0 {
                last_span = Some(prev.sp);
            }
            prev = next;
        }
        if result.is_none() && last_span.is_some() {
            return self.make_sub_span(span, last_span);
        }
        return self.make_sub_span(span, result);
    }

    // Return the span for the last ident before a `<` and outside any
    // brackets, or the last span.
    pub fn sub_span_for_type_name(&self, span: Span) -> Option<Span> {
        let mut toks = self.retokenise_span(span);
        let mut prev = toks.real_token();
        let mut result = None;
        let mut bracket_count = 0u;
        loop {
            let next = toks.real_token();

            if (next.tok == token::Lt ||
                next.tok == token::Colon) &&
               bracket_count == 0 &&
               prev.tok.is_ident() {
                result = Some(prev.sp);
            }

            bracket_count += match prev.tok {
                token::Lt => 1,
                token::Gt => -1,
                token::BinOp(token::Shr) => -2,
                _ => 0
            };

            if next.tok == token::Eof {
                break;
            }
            prev = next;
        }
        if bracket_count != 0 {
            let loc = self.sess.codemap().lookup_char_pos(span.lo);
            self.sess.span_bug(span,
                format!("Mis-counted brackets when breaking path? Parsing '{}' in {}, line {}",
                        self.snippet(span), loc.file.name, loc.line).as_slice());
        }
        if result.is_none() && prev.tok.is_ident() && bracket_count == 0 {
            return self.make_sub_span(span, Some(prev.sp));
        }
        self.make_sub_span(span, result)
    }

    // Reparse span and return an owned vector of sub spans of the first limit
    // identifier tokens in the given nesting level.
    // example with Foo<Bar<T,V>, Bar<T,V>>
    // Nesting = 0: all idents outside of brackets: ~[Foo]
    // Nesting = 1: idents within one level of brackets: ~[Bar, Bar]
    pub fn spans_with_brackets(&self, span: Span, nesting: int, limit: int) -> Vec<Span> {
        let mut result: Vec<Span> = vec!();

        let mut toks = self.retokenise_span(span);
        // We keep track of how many brackets we're nested in
        let mut bracket_count = 0i;
        loop {
            let ts = toks.real_token();
            if ts.tok == token::Eof {
                if bracket_count != 0 {
                    let loc = self.sess.codemap().lookup_char_pos(span.lo);
                    self.sess.span_bug(span, format!(
                        "Mis-counted brackets when breaking path? Parsing '{}' in {}, line {}",
                         self.snippet(span), loc.file.name, loc.line).as_slice());
                }
                return result
            }
            if (result.len() as int) == limit {
                return result;
            }
            bracket_count += match ts.tok {
                token::Lt => 1,
                token::Gt => -1,
                token::BinOp(token::Shl) => 2,
                token::BinOp(token::Shr) => -2,
                _ => 0
            };
            if ts.tok.is_ident() &&
               bracket_count == nesting {
                result.push(self.make_sub_span(span, Some(ts.sp)).unwrap());
            }
        }
    }

    pub fn sub_span_before_token(&self, span: Span, tok: Token) -> Option<Span> {
        let mut toks = self.retokenise_span(span);
        let mut prev = toks.real_token();
        loop {
            if prev.tok == token::Eof {
                return None;
            }
            let next = toks.real_token();
            if next.tok == tok {
                return self.make_sub_span(span, Some(prev.sp));
            }
            prev = next;
        }
    }

    pub fn sub_span_after_keyword(&self,
                              span: Span,
                              keyword: keywords::Keyword) -> Option<Span> {
        let mut toks = self.retokenise_span(span);
        loop {
            let ts = toks.real_token();
            if ts.tok == token::Eof {
                return None;
            }
            if ts.tok.is_keyword(keyword) {
                let ts = toks.real_token();
                if ts.tok == token::Eof {
                    return None
                } else {
                    println!("found keyword: {} at {}", ts, ts.sp);
                    return self.make_sub_span(span, Some(ts.sp));
                }
            }
        }
    }

    // Returns a list of the spans of idents in a patch.
    // E.g., For foo::bar<x,t>::baz, we return [foo, bar, baz] (well, their spans)
    pub fn spans_for_path_segments(&self, path: &ast::Path) -> Vec<Span> {
        if generated_code(path.span) {
            return vec!();
        }

        self.spans_with_brackets(path.span, 0, -1)
    }

    // Return an owned vector of the subspans of the param identifier
    // tokens found in span.
    pub fn spans_for_ty_params(&self, span: Span, number: int) -> Vec<Span> {
        if generated_code(span) {
            return vec!();
        }
        // Type params are nested within one level of brackets:
        // i.e. we want ~[A, B] from Foo<A, B<T,U>>
        self.spans_with_brackets(span, 1, number)
    }

    pub fn report_span_err(&self, kind: &str, span: Span) {
        let loc = self.sess.codemap().lookup_char_pos(span.lo);
        info!("({}) Could not find sub_span in `{}` in {}, line {}",
              kind, self.snippet(span), loc.file.name, loc.line);
        self.err_count.set(self.err_count.get()+1);
        if self.err_count.get() > 1000 {
            self.sess.bug("span errors reached 1000, giving up");
        }
    }
}
<|endoftext|>


<commit_before>use view::{View,SizeRequest,DimensionRequest};
use vec::Vec2;
use printer::Printer;
use orientation::Orientation;
use event::{Event,EventResult,Key};

/// Arranges its children linearly according to its orientation.
pub struct LinearLayout {
    children: Vec<Child>,
    orientation: Orientation,
    focus: usize,
}

struct Child {
    view: Box<View>,
    size: Vec2,
    weight: usize,
}

impl LinearLayout {
    /// Creates a new layout with the given orientation.
    pub fn new(orientation: Orientation) -> Self {
        LinearLayout {
            children: Vec::new(),
            orientation: orientation,
            focus: 0,
        }
    }

    /// Modifies the weight of the last child added.
    ///
    /// It is an error to call this before adding a child (and it will panic).
    pub fn weight(mut self, weight: usize) -> Self {
        self.children.last_mut().unwrap().weight = weight;

        self
    }

    /// Adds a child to the layout.
    pub fn child<V: View + 'static>(mut self, view: V) -> Self {
        self.children.push(Child {
            view: Box::new(view),
            size: Vec2::zero(),
            weight: 0,
        });

        self
    }

    /// Creates a new vertical layout.
    pub fn vertical() -> Self {
        LinearLayout::new(Orientation::Vertical)
    }

    /// Creates a new horizontal layout.
    pub fn horizontal() -> Self {
        LinearLayout::new(Orientation::Horizontal)
    }
}

/// Returns the index of the maximum element.
/// WTF isn't it part of standard library??
fn find_max(list: &Vec<usize>) -> usize {
    let mut max_value = 0;
    let mut max = 0;
    for (i,&x) in list.iter().enumerate() {
        if x > max_value {
            max_value = x;
            max = i;
        }
    }
    max
}

/// Given a total number of points and a list of weights,
/// try to share the points according to the weight,
/// rounding properly and conserving the sum of points.
fn share(total: usize, weights: Vec<usize>) -> Vec<usize> {
    // It first give a base value to everyone, which is their truncated share.
    // Then, it gives the rest to the most deserving.
    if weights.len() == 0 { return Vec::new(); }

    let sum_weight = weights.iter().fold(0,|a,b| a+b);
    if sum_weight == 0 {
        return (0..weights.len()).map(|_| 0).collect();
    }

    let mut base = Vec::with_capacity(weights.len());
    let mut rest = Vec::with_capacity(weights.len());
    let mut extra = total;

    for weight in weights.iter() {
        let b = total * weight / sum_weight;
        extra -= b;
        base.push(b);
        rest.push(total * weight - b*sum_weight);
    }

    // TODO: better to sort (base,rest) as one array and pick the extra first.
    for _ in 0..extra {
        let i = find_max(&rest);
        rest[i] = 0;
        base[i] += 1;
    }

    base
}

impl View for LinearLayout {
    fn draw(&mut self, printer: &Printer) {
        // Use pre-computed sizes
        let mut offset = Vec2::zero();
        for child in self.children.iter_mut() {
            child.view.draw(&printer.sub_printer(offset, child.size, true));

            *self.orientation.get_ref(&mut offset) += self.orientation.get(&child.size);
        }
    }

    fn layout(&mut self, size: Vec2) {
        // Compute the very minimal required size
        let req = SizeRequest{
            w: DimensionRequest::AtMost(size.x),
            h: DimensionRequest::AtMost(size.y),
        };
        let min_sizes: Vec<Vec2> = self.children.iter().map(|child| child.view.get_min_size(req)).collect();
        let min_size = self.orientation.stack(min_sizes.iter());

        // Emulate 'non-strict inequality' on integers
        // (default comparison on Vec2 is strict)
        if !(min_size < size+(1,1)) {
            // Error! Not enough space! Emergency procedures!
            return
        }

        // Now share this extra space among everyone

        let extras = {
            let extra = size - min_size;
            let space = self.orientation.get(&extra);
            share(space, self.children.iter().map(|child| child.weight).collect())
        };

        for (child,(child_size,extra)) in self.children.iter_mut().zip(min_sizes.iter().zip(extras.iter())) {
            let mut child_size = *child_size;
            *self.orientation.get_ref(&mut child_size) += *extra;
            *self.orientation.swap().get_ref(&mut child_size) = self.orientation.swap().get(&size);
            child.size = child_size;
            child.view.layout(child_size);
        }
    }

    fn get_min_size(&self, req: SizeRequest) -> Vec2 {
        // First, make a naive scenario: everything will work fine.
        let sizes: Vec<Vec2> = self.children.iter().map(|view| view.view.get_min_size(req)).collect();
        self.orientation.stack(sizes.iter())

        // Did it work? Champagne!

        // Ok, so maybe it didn't.
        // Last chance: did someone lie about his needs?
        // Could we squash him a little?

        // Find out who's fluid, if any.
    }

    fn on_event(&mut self, event: Event) -> EventResult {
        match self.children[self.focus].view.on_event(event) {
            EventResult::Ignored => match event {
                Event::KeyEvent(Key::Tab) if self.focus > 0 => {
                    self.focus -= 1;
                    EventResult::Consumed(None)
                },
                Event::KeyEvent(Key::ShiftTab) if self.focus+1 < self.children.len() => {
                    self.focus += 1;
                    EventResult::Consumed(None)
                }
                Event::KeyEvent(Key::Left) if self.orientation == Orientation::Horizontal && self.focus > 0 => {
                    self.focus -= 1;
                    EventResult::Consumed(None)
                },
                Event::KeyEvent(Key::Up) if self.orientation == Orientation::Vertical && self.focus > 0 => {
                    self.focus -= 1;
                    EventResult::Consumed(None)
                },
                Event::KeyEvent(Key::Right) if self.orientation == Orientation::Horizontal && self.focus+1 < self.children.len() => {
                    self.focus += 1;
                    EventResult::Consumed(None)
                },
                Event::KeyEvent(Key::Down) if self.orientation == Orientation::Vertical && self.focus+1 < self.children.len() => {
                    self.focus += 1;
                    EventResult::Consumed(None)
                },
                _ => EventResult::Ignored,
            },
            res => res,
        }
    }
}
<commit_msg>Fix focus state in linear layout<commit_after>use view::{View,SizeRequest,DimensionRequest};
use vec::Vec2;
use printer::Printer;
use orientation::Orientation;
use event::{Event,EventResult,Key};

/// Arranges its children linearly according to its orientation.
pub struct LinearLayout {
    children: Vec<Child>,
    orientation: Orientation,
    focus: usize,
}

struct Child {
    view: Box<View>,
    size: Vec2,
    weight: usize,
}

impl LinearLayout {
    /// Creates a new layout with the given orientation.
    pub fn new(orientation: Orientation) -> Self {
        LinearLayout {
            children: Vec::new(),
            orientation: orientation,
            focus: 0,
        }
    }

    /// Modifies the weight of the last child added.
    ///
    /// It is an error to call this before adding a child (and it will panic).
    pub fn weight(mut self, weight: usize) -> Self {
        self.children.last_mut().unwrap().weight = weight;

        self
    }

    /// Adds a child to the layout.
    pub fn child<V: View + 'static>(mut self, view: V) -> Self {
        self.children.push(Child {
            view: Box::new(view),
            size: Vec2::zero(),
            weight: 0,
        });

        self
    }

    /// Creates a new vertical layout.
    pub fn vertical() -> Self {
        LinearLayout::new(Orientation::Vertical)
    }

    /// Creates a new horizontal layout.
    pub fn horizontal() -> Self {
        LinearLayout::new(Orientation::Horizontal)
    }
}

/// Returns the index of the maximum element.
/// WTF isn't it part of standard library??
fn find_max(list: &Vec<usize>) -> usize {
    let mut max_value = 0;
    let mut max = 0;
    for (i,&x) in list.iter().enumerate() {
        if x > max_value {
            max_value = x;
            max = i;
        }
    }
    max
}

/// Given a total number of points and a list of weights,
/// try to share the points according to the weight,
/// rounding properly and conserving the sum of points.
fn share(total: usize, weights: Vec<usize>) -> Vec<usize> {
    // It first give a base value to everyone, which is their truncated share.
    // Then, it gives the rest to the most deserving.
    if weights.len() == 0 { return Vec::new(); }

    let sum_weight = weights.iter().fold(0,|a,b| a+b);
    if sum_weight == 0 {
        return (0..weights.len()).map(|_| 0).collect();
    }

    let mut base = Vec::with_capacity(weights.len());
    let mut rest = Vec::with_capacity(weights.len());
    let mut extra = total;

    for weight in weights.iter() {
        let b = total * weight / sum_weight;
        extra -= b;
        base.push(b);
        rest.push(total * weight - b*sum_weight);
    }

    // TODO: better to sort (base,rest) as one array and pick the extra first.
    for _ in 0..extra {
        let i = find_max(&rest);
        rest[i] = 0;
        base[i] += 1;
    }

    base
}

impl View for LinearLayout {
    fn draw(&mut self, printer: &Printer) {
        // Use pre-computed sizes
        let mut offset = Vec2::zero();
        for (i,child) in self.children.iter_mut().enumerate() {
            child.view.draw(&printer.sub_printer(offset, child.size, i == self.focus));

            *self.orientation.get_ref(&mut offset) += self.orientation.get(&child.size);
        }
    }

    fn layout(&mut self, size: Vec2) {
        // Compute the very minimal required size
        let req = SizeRequest{
            w: DimensionRequest::AtMost(size.x),
            h: DimensionRequest::AtMost(size.y),
        };
        let min_sizes: Vec<Vec2> = self.children.iter().map(|child| child.view.get_min_size(req)).collect();
        let min_size = self.orientation.stack(min_sizes.iter());

        // Emulate 'non-strict inequality' on integers
        // (default comparison on Vec2 is strict)
        if !(min_size < size+(1,1)) {
            // Error! Not enough space! Emergency procedures!
            return
        }

        // Now share this extra space among everyone

        let extras = {
            let extra = size - min_size;
            let space = self.orientation.get(&extra);
            share(space, self.children.iter().map(|child| child.weight).collect())
        };

        for (child,(child_size,extra)) in self.children.iter_mut().zip(min_sizes.iter().zip(extras.iter())) {
            let mut child_size = *child_size;
            *self.orientation.get_ref(&mut child_size) += *extra;
            *self.orientation.swap().get_ref(&mut child_size) = self.orientation.swap().get(&size);
            child.size = child_size;
            child.view.layout(child_size);
        }
    }

    fn get_min_size(&self, req: SizeRequest) -> Vec2 {
        // First, make a naive scenario: everything will work fine.
        let sizes: Vec<Vec2> = self.children.iter().map(|view| view.view.get_min_size(req)).collect();
        self.orientation.stack(sizes.iter())

        // Did it work? Champagne!

        // Ok, so maybe it didn't.
        // Last chance: did someone lie about his needs?
        // Could we squash him a little?

        // Find out who's fluid, if any.
    }

    fn on_event(&mut self, event: Event) -> EventResult {
        match self.children[self.focus].view.on_event(event) {
            EventResult::Ignored => match event {
                Event::KeyEvent(Key::Tab) if self.focus > 0 => {
                    self.focus -= 1;
                    EventResult::Consumed(None)
                },
                Event::KeyEvent(Key::ShiftTab) if self.focus+1 < self.children.len() => {
                    self.focus += 1;
                    EventResult::Consumed(None)
                }
                Event::KeyEvent(Key::Left) if self.orientation == Orientation::Horizontal && self.focus > 0 => {
                    self.focus -= 1;
                    EventResult::Consumed(None)
                },
                Event::KeyEvent(Key::Up) if self.orientation == Orientation::Vertical && self.focus > 0 => {
                    self.focus -= 1;
                    EventResult::Consumed(None)
                },
                Event::KeyEvent(Key::Right) if self.orientation == Orientation::Horizontal && self.focus+1 < self.children.len() => {
                    self.focus += 1;
                    EventResult::Consumed(None)
                },
                Event::KeyEvent(Key::Down) if self.orientation == Orientation::Vertical && self.focus+1 < self.children.len() => {
                    self.focus += 1;
                    EventResult::Consumed(None)
                },
                _ => EventResult::Ignored,
            },
            res => res,
        }
    }
}
<|endoftext|>


<commit_before><commit_msg>Kata05: Make bloom do something<commit_after><|endoftext|>


<commit_before>#![crate_name = "xpath"]

extern crate document;

use document::{Any,ToAny};
use document::{Nodeset};
use std::collections::HashMap;

#[deriving(PartialEq,Show,Clone)]
pub enum XPathValue {
    Boolean(bool),
    Number(f64),
    String(String),
    Nodes(Nodeset), // rename as Nodeset
}

impl XPathValue {
    fn boolean(&self) -> bool {
        match *self {
            Boolean(val) => val,
            Number(n) => n != 0.0 && ! n.is_nan(),
            String(ref s) => ! s.is_empty(),
            Nodes(ref nodeset) => nodeset.size() > 0,
        }
    }

    fn number(&self) -> f64 {
        match *self {
            Number(val) => val,
            _ => -42.0
        }
    }

    fn string(&self) -> String {
        match *self {
            String(ref val) => val.clone(),
            _ => "Unimplemented".to_string(),
        }
    }

    fn nodeset(&self) -> Nodeset {
        match *self {
            Nodes(ref ns) => ns.clone(),
            _ => fail!("Did not evaluate to a nodeset!"),
        }
    }
}

pub trait XPathFunction {
    fn evaluate(&self,
                context: &XPathEvaluationContext,
                args: Vec<XPathValue>) -> XPathValue;
}

type BoxFunc = Box<XPathFunction + 'static>;
pub type Functions = HashMap<String, BoxFunc>;
pub type Variables = HashMap<String, XPathValue>;

pub struct XPathEvaluationContext<'a> {
    node: Any,
    functions: & 'a Functions,
    variables: & 'a Variables,
    position: uint,
}

impl<'a> XPathEvaluationContext<'a> {
    pub fn new<A: ToAny>(node: A,
                         functions: &'a Functions,
                         variables: &'a Variables) -> XPathEvaluationContext<'a>
    {
        XPathEvaluationContext {
            node: node.to_any(),
            functions: functions,
            variables: variables,
            position: 0,
        }
    }

    fn node(&self) -> &Any {
        &self.node
    }

    fn new_context_for(& self, _size: uint) -> XPathEvaluationContext<'a> {
        XPathEvaluationContext {
            node: self.node.clone(),
            functions: self.functions,
            variables: self.variables,
            position: 0,
        }
    }

    fn next<A: ToAny>(& mut self, _node: A) {
        self.position += 1;
    }

    fn position(&self) -> uint {
        self.position
    }

    fn function_for_name(&self, name: &str) -> Option<& 'a BoxFunc> {
        self.functions.find(&name.to_string())
    }

    fn value_of(&self, name: &str) -> Option<&XPathValue> {
        self.variables.find(&name.to_string())
    }
}

pub trait XPathNodeTest {
    fn test(&self, context: &XPathEvaluationContext, result: &mut Nodeset);
}

pub mod token;
pub mod tokenizer;
pub mod deabbreviator;
pub mod disambiguator;
pub mod axis;
pub mod expression;
<commit_msg>Mark XPathValue methods public<commit_after>#![crate_name = "xpath"]

extern crate document;

use document::{Any,ToAny};
use document::{Nodeset};
use std::collections::HashMap;

#[deriving(PartialEq,Show,Clone)]
pub enum XPathValue {
    Boolean(bool),
    Number(f64),
    String(String),
    Nodes(Nodeset), // rename as Nodeset
}

impl XPathValue {
    pub fn boolean(&self) -> bool {
        match *self {
            Boolean(val) => val,
            Number(n) => n != 0.0 && ! n.is_nan(),
            String(ref s) => ! s.is_empty(),
            Nodes(ref nodeset) => nodeset.size() > 0,
        }
    }

    pub fn number(&self) -> f64 {
        match *self {
            Number(val) => val,
            _ => -42.0
        }
    }

    pub fn string(&self) -> String {
        match *self {
            String(ref val) => val.clone(),
            _ => "Unimplemented".to_string(),
        }
    }

    pub fn nodeset(&self) -> Nodeset {
        match *self {
            Nodes(ref ns) => ns.clone(),
            _ => fail!("Did not evaluate to a nodeset!"),
        }
    }
}

pub trait XPathFunction {
    fn evaluate(&self,
                context: &XPathEvaluationContext,
                args: Vec<XPathValue>) -> XPathValue;
}

type BoxFunc = Box<XPathFunction + 'static>;
pub type Functions = HashMap<String, BoxFunc>;
pub type Variables = HashMap<String, XPathValue>;

pub struct XPathEvaluationContext<'a> {
    node: Any,
    functions: & 'a Functions,
    variables: & 'a Variables,
    position: uint,
}

impl<'a> XPathEvaluationContext<'a> {
    pub fn new<A: ToAny>(node: A,
                         functions: &'a Functions,
                         variables: &'a Variables) -> XPathEvaluationContext<'a>
    {
        XPathEvaluationContext {
            node: node.to_any(),
            functions: functions,
            variables: variables,
            position: 0,
        }
    }

    fn node(&self) -> &Any {
        &self.node
    }

    fn new_context_for(& self, _size: uint) -> XPathEvaluationContext<'a> {
        XPathEvaluationContext {
            node: self.node.clone(),
            functions: self.functions,
            variables: self.variables,
            position: 0,
        }
    }

    pub fn next<A: ToAny>(& mut self, _node: A) {
        self.position += 1;
    }

    fn position(&self) -> uint {
        self.position
    }

    fn function_for_name(&self, name: &str) -> Option<& 'a BoxFunc> {
        self.functions.find(&name.to_string())
    }

    fn value_of(&self, name: &str) -> Option<&XPathValue> {
        self.variables.find(&name.to_string())
    }
}

pub trait XPathNodeTest {
    fn test(&self, context: &XPathEvaluationContext, result: &mut Nodeset);
}

pub mod token;
pub mod tokenizer;
pub mod deabbreviator;
pub mod disambiguator;
pub mod axis;
pub mod expression;
<|endoftext|>