<commit_before>use analysis::symbols;
use regex::{Captures, Regex};

const LANGUAGE_SEP_BEGIN : &'static str = "<!-- language=\"";
const LANGUAGE_SEP_END : &'static str = "\" -->";
const LANGUAGE_BLOCK_BEGIN : &'static str = "|[";
const LANGUAGE_BLOCK_END : &'static str = "\n]|";

pub fn reformat_doc(input: &str, symbols: &symbols::Info) -> String {
    code_blocks_transformation(input, symbols)
}

fn try_split<'a>(src: &'a str, needle: &str) -> (&'a str, Option<&'a str>) {
    match src.find(needle) {
        Some(pos) => (&src[..pos], Some(&src[pos + needle.len()..])),
        None => (src, None),
    }
}

fn code_blocks_transformation(mut input: &str, symbols: &symbols::Info) -> String {
    let mut out = String::new();

    loop {
        input = match try_split(input, LANGUAGE_BLOCK_BEGIN) {
            (before, Some(after)) => {
                out.push_str(&replace_c_types(before, symbols));
                if let (before, Some(after)) = try_split(get_language(after, &mut out),
                                                         LANGUAGE_BLOCK_END) {
                    out.push_str(before);
                    out.push_str("\n```");
                    after
                } else {
                    after
                }
            }
            (before, None) => {
                out.push_str(&replace_c_types(before, symbols));
                return out
            }
        };
    }
}

fn get_language<'a>(entry: &'a str, out: &mut String) -> &'a str {
    if let (_, Some(after)) = try_split(entry, LANGUAGE_SEP_BEGIN) {
        if let (before, Some(after)) = try_split(after, LANGUAGE_SEP_END) {
            out.push_str(&format!("\n```{}", before));
            return after
        }
    }
    out.push_str("\n```text");
    entry
}

lazy_static! {
    static ref SYMBOL: Regex = Regex::new(r"(^|[^\\])[@#%]([\w]+\b)([:.]+[\w_-]+\b)?") .unwrap();
    static ref FUNCTION: Regex = Regex::new(r"(\b[a-z0-9_]+)\(\)") .unwrap();
    static ref GDK_GTK: Regex = Regex::new(r"(G[dt]k[A-Z][\w]+\b)").unwrap();
    static ref SPACES: Regex = Regex::new(r"[ ][ ]+").unwrap();
}

fn replace_c_types(entry: &str, symbols: &symbols::Info) -> String {
    let lookup = |s: &str| -> String {
        symbols.by_c_name(s)
            .map(|s| s.full_rust_name())
            .unwrap_or(s.into())
    };
    let out = SYMBOL.replace_all(entry, |caps: &Captures| {
        format!("{}`{}{}`", &caps[1], lookup(&caps[2]), caps.at(3).unwrap_or(""))
    });
    let out = FUNCTION.replace_all(&out, |caps: &Captures| {
        format!("`{}`", lookup(&caps[1]))
    });
    let out = GDK_GTK.replace_all(&out, |caps: &Captures| {
        format!("`{}`", lookup(&caps[1]))
    });
    SPACES.replace_all(&out, " ")
}
<commit_msg>Avoid mangling inline code in backquotes<commit_after>use analysis::symbols;
use regex::{Captures, Regex};

const LANGUAGE_SEP_BEGIN : &'static str = "<!-- language=\"";
const LANGUAGE_SEP_END : &'static str = "\" -->";
const LANGUAGE_BLOCK_BEGIN : &'static str = "|[";
const LANGUAGE_BLOCK_END : &'static str = "\n]|";

pub fn reformat_doc(input: &str, symbols: &symbols::Info) -> String {
    code_blocks_transformation(input, symbols)
}

fn try_split<'a>(src: &'a str, needle: &str) -> (&'a str, Option<&'a str>) {
    match src.find(needle) {
        Some(pos) => (&src[..pos], Some(&src[pos + needle.len()..])),
        None => (src, None),
    }
}

fn code_blocks_transformation(mut input: &str, symbols: &symbols::Info) -> String {
    let mut out = String::with_capacity(input.len());

    loop {
        input = match try_split(input, LANGUAGE_BLOCK_BEGIN) {
            (before, Some(after)) => {
                out.push_str(&format(before, symbols));
                if let (before, Some(after)) = try_split(get_language(after, &mut out),
                                                         LANGUAGE_BLOCK_END) {
                    out.push_str(before);
                    out.push_str("\n```");
                    after
                } else {
                    after
                }
            }
            (before, None) => {
                out.push_str(&format(before, symbols));
                return out
            }
        };
    }
}

fn get_language<'a>(entry: &'a str, out: &mut String) -> &'a str {
    if let (_, Some(after)) = try_split(entry, LANGUAGE_SEP_BEGIN) {
        if let (before, Some(after)) = try_split(after, LANGUAGE_SEP_END) {
            out.push_str(&format!("\n```{}", before));
            return after
        }
    }
    out.push_str("\n```text");
    entry
}

lazy_static! {
    static ref SYMBOL: Regex = Regex::new(r"(^|[^\\])[@#%]([\w]+\b)([:.]+[\w_-]+\b)?") .unwrap();
    static ref FUNCTION: Regex = Regex::new(r"(\b[a-z0-9_]+)\(\)") .unwrap();
    static ref GDK_GTK: Regex = Regex::new(r"(G[dt]k[A-Z][\w]+\b)").unwrap();
    static ref SPACES: Regex = Regex::new(r"[ ][ ]+").unwrap();
}

fn format(mut input: &str, symbols: &symbols::Info) -> String {
    let mut ret = String::with_capacity(input.len());
    loop {
        let (before, after) = try_split(input, "`");
        ret.push_str(&replace_c_types(before, symbols));
        if let Some(after) = after {
            ret.push_str("`");
            let (before, after) = try_split(after, "`");
            // don't touch anything enclosed in backticks
            ret.push_str(before);
            if let Some(after) = after {
                ret.push_str("`");
                input = after;
            } else {
                return ret;
            }
        } else {
            return ret;
        }
    }
}

fn replace_c_types(entry: &str, symbols: &symbols::Info) -> String {
    let lookup = |s: &str| -> String {
        symbols.by_c_name(s)
            .map(|s| s.full_rust_name())
            .unwrap_or(s.into())
    };
    let out = SYMBOL.replace_all(entry, |caps: &Captures| {
        format!("{}`{}{}`", &caps[1], lookup(&caps[2]), caps.at(3).unwrap_or(""))
    });
    let out = FUNCTION.replace_all(&out, |caps: &Captures| {
        format!("`{}`", lookup(&caps[1]))
    });
    let out = GDK_GTK.replace_all(&out, |caps: &Captures| {
        format!("`{}`", lookup(&caps[1]))
    });
    SPACES.replace_all(&out, " ")
}
<|endoftext|>


<commit_before>//! # nalgebra-lapack
//!
//! Rust library for linear algebra using nalgebra and LAPACK.
//!
//! ## Documentation
//!
//! Documentation is available [here](https://docs.rs/nalgebra-lapack/).
//!
//! ## License
//!
//! MIT
//!
//! ## Cargo features to select lapack provider
//!
//! Like the [lapack crate](https://crates.io/crates/lapack) from which this
//! behavior is inherited, nalgebra-lapack uses [cargo
//! features](https://doc.crates.io/manifest.html#the-[features]-section) to select
//! which lapack provider (or implementation) is used. Command line arguments to
//! cargo are the easiest way to do this, and the best provider depends on your
//! particular system. In some cases, the providers can be further tuned with
//! environment variables.
//!
//! Below are given examples of how to invoke `cargo build` on two different systems
//! using two different providers. The `--no-default-features --features "provider"`
//! arguments will be consistent for other `cargo` commands.
//!
//! ### Ubuntu
//!
//! As tested on Ubuntu 12.04, do this to build the lapack package against
//! the system installation of netlib without LAPACKE (note the E) or
//! CBLAS:
//!
//! ```.ignore
//! sudo apt-get install gfortran libblas3gf liblapack3gf
//! export CARGO_FEATURE_SYSTEM_NETLIB=1
//! export CARGO_FEATURE_EXCLUDE_LAPACKE=1
//! export CARGO_FEATURE_EXCLUDE_CBLAS=1
//!
//! export CARGO_FEATURES="--no-default-features --features netlib"
//! cargo build ${CARGO_FEATURES}
//! ```
//!
//! ### macOS
//!
//! On macOS, do this to use Apple's Accelerate framework:
//!
//! ```.ignore
//! export CARGO_FEATURES="--no-default-features --features accelerate"
//! cargo build ${CARGO_FEATURES}
//! ```
//!
//! [version-img]: https://img.shields.io/crates/v/nalgebra-lapack.svg
//! [version-url]: https://crates.io/crates/nalgebra-lapack
//! [status-img]: https://travis-ci.org/strawlab/nalgebra-lapack.svg?branch=master
//! [status-url]: https://travis-ci.org/strawlab/nalgebra-lapack
//! [doc-img]: https://docs.rs/nalgebra-lapack/badge.svg
//! [doc-url]: https://docs.rs/nalgebra-lapack/
//!
//! ## Contributors
//! This integration of LAPACK on nalgebra was
//! [initiated](https://github.com/strawlab/nalgebra-lapack) by Andrew Straw. It
//! then became officially supported and integrated to the main nalgebra
//! repository.

#![deny(non_camel_case_types)]
#![deny(unused_parens)]
#![deny(non_upper_case_globals)]
#![deny(unused_qualifications)]
#![deny(unused_results)]
#![deny(missing_docs)]
#![doc(
    html_favicon_url = "https://nalgebra.org/img/favicon.ico",
    html_root_url = "https://nalgebra.org/rustdoc"
)]

extern crate alga;
extern crate lapack;
extern crate lapack_src;
extern crate nalgebra as na;
extern crate num_complex;
extern crate num_traits as num;

mod lapack_check;

mod cholesky;
mod eigen;
mod hessenberg;
mod lu;
mod qr;
mod schur;
mod svd;
mod symmetric_eigen;

use num_complex::Complex;

pub use self::cholesky::{Cholesky, CholeskyScalar};
pub use self::eigen::Eigen;
pub use self::hessenberg::Hessenberg;
pub use self::lu::{LUScalar, LU};
pub use self::qr::QR;
pub use self::schur::Schur;
pub use self::svd::SVD;
pub use self::symmetric_eigen::SymmetricEigen;

trait ComplexHelper {
    type RealPart;

    fn real_part(self) -> Self::RealPart;
}

impl ComplexHelper for f32 {
    type RealPart = f32;

    #[inline]
    fn real_part(self) -> Self::RealPart {
        self
    }
}

impl ComplexHelper for f64 {
    type RealPart = f64;

    #[inline]
    fn real_part(self) -> Self::RealPart {
        self
    }
}

impl ComplexHelper for Complex<f32> {
    type RealPart = f32;

    #[inline]
    fn real_part(self) -> Self::RealPart {
        self.re
    }
}

impl ComplexHelper for Complex<f64> {
    type RealPart = f64;

    #[inline]
    fn real_part(self) -> Self::RealPart {
        self.re
    }
}

unsafe fn uninitialized_vec<T: Copy>(n: usize) -> Vec<T> {
    let mut res = Vec::new();
    res.reserve_exact(n);
    res.set_len(n);
    res
}
<commit_msg>nalgebra-lapack: remove useless `extern crate`.<commit_after>//! # nalgebra-lapack
//!
//! Rust library for linear algebra using nalgebra and LAPACK.
//!
//! ## Documentation
//!
//! Documentation is available [here](https://docs.rs/nalgebra-lapack/).
//!
//! ## License
//!
//! MIT
//!
//! ## Cargo features to select lapack provider
//!
//! Like the [lapack crate](https://crates.io/crates/lapack) from which this
//! behavior is inherited, nalgebra-lapack uses [cargo
//! features](https://doc.crates.io/manifest.html#the-[features]-section) to select
//! which lapack provider (or implementation) is used. Command line arguments to
//! cargo are the easiest way to do this, and the best provider depends on your
//! particular system. In some cases, the providers can be further tuned with
//! environment variables.
//!
//! Below are given examples of how to invoke `cargo build` on two different systems
//! using two different providers. The `--no-default-features --features "provider"`
//! arguments will be consistent for other `cargo` commands.
//!
//! ### Ubuntu
//!
//! As tested on Ubuntu 12.04, do this to build the lapack package against
//! the system installation of netlib without LAPACKE (note the E) or
//! CBLAS:
//!
//! ```.ignore
//! sudo apt-get install gfortran libblas3gf liblapack3gf
//! export CARGO_FEATURE_SYSTEM_NETLIB=1
//! export CARGO_FEATURE_EXCLUDE_LAPACKE=1
//! export CARGO_FEATURE_EXCLUDE_CBLAS=1
//!
//! export CARGO_FEATURES="--no-default-features --features netlib"
//! cargo build ${CARGO_FEATURES}
//! ```
//!
//! ### macOS
//!
//! On macOS, do this to use Apple's Accelerate framework:
//!
//! ```.ignore
//! export CARGO_FEATURES="--no-default-features --features accelerate"
//! cargo build ${CARGO_FEATURES}
//! ```
//!
//! [version-img]: https://img.shields.io/crates/v/nalgebra-lapack.svg
//! [version-url]: https://crates.io/crates/nalgebra-lapack
//! [status-img]: https://travis-ci.org/strawlab/nalgebra-lapack.svg?branch=master
//! [status-url]: https://travis-ci.org/strawlab/nalgebra-lapack
//! [doc-img]: https://docs.rs/nalgebra-lapack/badge.svg
//! [doc-url]: https://docs.rs/nalgebra-lapack/
//!
//! ## Contributors
//! This integration of LAPACK on nalgebra was
//! [initiated](https://github.com/strawlab/nalgebra-lapack) by Andrew Straw. It
//! then became officially supported and integrated to the main nalgebra
//! repository.

#![deny(non_camel_case_types)]
#![deny(unused_parens)]
#![deny(non_upper_case_globals)]
#![deny(unused_qualifications)]
#![deny(unused_results)]
#![deny(missing_docs)]
#![doc(
    html_favicon_url = "https://nalgebra.org/img/favicon.ico",
    html_root_url = "https://nalgebra.org/rustdoc"
)]

extern crate nalgebra as na;
extern crate num_traits as num;

mod lapack_check;

mod cholesky;
mod eigen;
mod hessenberg;
mod lu;
mod qr;
mod schur;
mod svd;
mod symmetric_eigen;

use num_complex::Complex;

pub use self::cholesky::{Cholesky, CholeskyScalar};
pub use self::eigen::Eigen;
pub use self::hessenberg::Hessenberg;
pub use self::lu::{LUScalar, LU};
pub use self::qr::QR;
pub use self::schur::Schur;
pub use self::svd::SVD;
pub use self::symmetric_eigen::SymmetricEigen;

trait ComplexHelper {
    type RealPart;

    fn real_part(self) -> Self::RealPart;
}

impl ComplexHelper for f32 {
    type RealPart = f32;

    #[inline]
    fn real_part(self) -> Self::RealPart {
        self
    }
}

impl ComplexHelper for f64 {
    type RealPart = f64;

    #[inline]
    fn real_part(self) -> Self::RealPart {
        self
    }
}

impl ComplexHelper for Complex<f32> {
    type RealPart = f32;

    #[inline]
    fn real_part(self) -> Self::RealPart {
        self.re
    }
}

impl ComplexHelper for Complex<f64> {
    type RealPart = f64;

    #[inline]
    fn real_part(self) -> Self::RealPart {
        self.re
    }
}

unsafe fn uninitialized_vec<T: Copy>(n: usize) -> Vec<T> {
    let mut res = Vec::new();
    res.reserve_exact(n);
    res.set_len(n);
    res
}
<|endoftext|>


<commit_before>// Issue #945
// error-pattern:non-exhaustive match failure
fn test_box() {
    @0;
}
fn test_str() {
    let res = alt false { true { "happy" } };
}
fn main() {
    test_box();
    test_str();
}<commit_msg>Add an assert to run-fail/unwind-alt<commit_after>// Issue #945
// error-pattern:non-exhaustive match failure
fn test_box() {
    @0;
}
fn test_str() {
    let res = alt false { true { "happy" } };
    assert res == "happy";
}
fn main() {
    test_box();
    test_str();
}<|endoftext|>


<commit_before>

// -*- rust -*-
use std;
import std::str;
import std::vec;

// FIXME: import std::dbg::const_refcount. Currently
// cross-crate const references don't work.
const const_refcount: uint = 0x7bad_face_u;

fn fast_growth() {
    let v: vec[int] = [1, 2, 3, 4, 5];
    v += [6, 7, 8, 9, 0];
    log v.(9);
    assert (v.(0) == 1);
    assert (v.(7) == 8);
    assert (v.(9) == 0);
}

fn slow_growth() {
    let v: vec[int] = [];
    let u: vec[int] = v;
    v += [17];
    log v.(0);
    assert (v.(0) == 17);
}

fn slow_growth2_helper(s: str) { // ref up: s

    obj acc(mutable v: vec[str]) {
        fn add(s: &str) { v += [s]; }
    }
    let ss: str = s; // ref up: s

    let mumble: str = "mrghrm"; // ref up: mumble

    {
        /**
         * Within this block, mumble goes into a vec that is referenced
         * both by the local slot v and the acc's v data field.  When we
         * add(s) on the acc, its v undergoes a slow append (allocate a
         * new vec, copy over existing elements).  Here we're testing to
         * see that this slow path goes over well.  In particular, the
         * copy of existing elements should increment the ref count of
         * mumble, the existing str in the originally- shared vec.
         */

        let v: vec[str] = [mumble]; // ref up: v, mumble

        log vec::refcount[str](v);
        let a: acc = acc(v); // ref up: a, v

        log vec::refcount[str](v);
        assert (vec::refcount[str](v) == 2u);
        a.add(s); // ref up: mumble, s.  ref down: v

        log vec::refcount[str](v);
        log str::refcount(s);
        log str::refcount(mumble);
        assert (vec::refcount[str](v) == 1u);
        assert (str::refcount(s) == const_refcount);
        assert (str::refcount(mumble) == const_refcount);
        log v.(0);
        log vec::len[str](v);
        assert (str::eq(v.(0), mumble));
        assert (vec::len[str](v) == 1u);
    } // ref down: a, mumble, s, v

    log str::refcount(s);
    log str::refcount(mumble);
    assert (str::refcount(s) == const_refcount);
    assert (str::refcount(mumble) == const_refcount);
    log mumble;
    log ss;
}

// ref down
fn slow_growth2() {
    let s: str = "hi"; // ref up: s

    slow_growth2_helper(s);
    log str::refcount(s);
    assert (str::refcount(s) == const_refcount);
}

fn main() { fast_growth(); slow_growth(); slow_growth2(); }<commit_msg>Convert run-pass/vec-append to ivecs<commit_after>

// -*- rust -*-
use std;
import std::str;
import std::ivec;

// FIXME: import std::dbg::const_refcount. Currently
// cross-crate const references don't work.
const const_refcount: uint = 0x7bad_face_u;

fn fast_growth() {
    let v: [int] = ~[1, 2, 3, 4, 5];
    v += ~[6, 7, 8, 9, 0];
    log v.(9);
    assert (v.(0) == 1);
    assert (v.(7) == 8);
    assert (v.(9) == 0);
}

fn slow_growth() {
    let v: [int] = ~[];
    let u: [int] = v;
    v += ~[17];
    log v.(0);
    assert (v.(0) == 17);
}

fn slow_growth2_helper(s: str) { // ref up: s

    obj acc(mutable v: [str]) {
        fn add(s: &str) { v += ~[s]; }
    }
    let ss: str = s; // ref up: s

    let mumble: str = "mrghrm"; // ref up: mumble

    {
        /**
         * Within this block, mumble goes into a vec that is referenced
         * both by the local slot v and the acc's v data field.  When we
         * add(s) on the acc, its v undergoes a slow append (allocate a
         * new vec, copy over existing elements).  Here we're testing to
         * see that this slow path goes over well.  In particular, the
         * copy of existing elements should increment the ref count of
         * mumble, the existing str in the originally- shared vec.
         */

        let v: [str] = ~[mumble]; // ref up: mumble

        let a: acc = acc(v);

        a.add(s); // ref up: mumble, s

        log str::refcount(s);
        log str::refcount(mumble);
        assert (str::refcount(s) == const_refcount);
        assert (str::refcount(mumble) == const_refcount);
        log v.(0);
        log ivec::len[str](v);
        assert (str::eq(v.(0), mumble));
        assert (ivec::len[str](v) == 1u);
    } // ref down: mumble, s,

    log str::refcount(s);
    log str::refcount(mumble);
    assert (str::refcount(s) == const_refcount);
    assert (str::refcount(mumble) == const_refcount);
    log mumble;
    log ss;
}

// ref down
fn slow_growth2() {
    let s: str = "hi"; // ref up: s

    slow_growth2_helper(s);
    log str::refcount(s);
    assert (str::refcount(s) == const_refcount);
}

fn main() { fast_growth(); slow_growth(); slow_growth2(); }<|endoftext|>


<commit_before>use models::*;
use std::iter::repeat;

pub fn print_headline_with_author(item: &HnItem, index: &i32) {
    let s = create_headline_with_author(item, index).unwrap(); // Not handling errs
    println!("{}", s);
}

fn create_headline_with_author(item: &HnItem, index: &i32) -> Result<String, String> {
    match item.title {
        Some(_) => {
            let s = format!("[{:3}] {:80} ~ {}",
                            index,
                            item.title.as_ref().unwrap(),
                            item.by);
            Ok(s)

        }
        None => Err(String::from("Not headline")),
    }
}

fn create_comment_string(item:&HnItem, depth:&usize) -> Result<String, String> {
    let depthStr = repeat("-").take(*depth).collect::<String>();
    if(item.text.is_some()) {
        return Ok(format!("{} {} ~ {} on {}", depthStr, item.text.as_ref().unwrap(), &item.by, &item.time))
    }
    else {
        return Err(String::from("Could not get text for comment"))
    }
}

pub fn no_comments_for(numb:&usize) {
    println!("No comments for [{}]", numb);
}

pub fn could_not_get_any_commments_for_item(item: &HnItem) {
    println!("Could not get comments for item with id {}", item.id)
}

pub fn print_filename_of_loaded_page(filen: &str, title: &str) {
    println!("{} {} {} {}", "Downloaded page", title, "into file", filen);
}
pub fn could_not_load_page(title: &str) {
    println!("Could not download to file with title {}", title);
}

pub fn print_comments(item: &HnItem, vec: &Vec<HnItem>) {
    // FIXME No error handling
    println!("Comments for item id {} with title {}",
             &item.id,
             item.title.as_ref().unwrap());
    let mut comment_index = 0;
    for comment in vec {
        comment_index += 1; // Should be from main?
        let res = create_comment_row(&comment_index, &comment);
        if (res.is_some()) {
            println!("{}", res.unwrap());
        }
    }
}
fn create_comment_row(index: &i32, item: &HnItem) -> Option<String> {
    match item.text {
        Some(ref text) => {
            let s = format!("[{:3}] {:80} ~{}", index, text, &item.by); // TODO This needs to handle unicode characters to utf8 or something similar (snap's -> snap's)
            Some(s)
        }
        None => None,
    }

}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn create_headline_with_author_test() {
        use std::fs::File;
        use std::io::prelude::*;
        use serde_json;
        let mut contents = String::new();
        File::open("res/test/item.json")
            .and_then(|mut file| file.read_to_string(&mut contents))
            .unwrap();
        let deserialized: HnItem = serde_json::from_str(&contents).unwrap();
        let index = 1;
        let s: String = create_headline_with_author(&deserialized, &index).unwrap();
        assert!(s.len() != 0);
        assert!(s.contains("1"));
        assert!(s.contains("dhouston"));
        assert!(s.contains("My YC app: Dropbox - Throw away your USB drive"));
        assert!(deserialized.by.len() != 0);
        assert!(deserialized.title.unwrap().len() != 0);
    }

    #[test]
    fn create_comment_string_test() {
        use std::fs::File;
        use std::io::prelude::*;
        use serde_json;
        let mut contents = String::new();
        File::open("res/test/children-item.json")
            .and_then(|mut file| file.read_to_string(&mut contents))
            .unwrap();
        let deserialized: HnItem = serde_json::from_str(&contents).unwrap();
        let commentStr = create_comment_string(&deserialized, &2).unwrap();
        assert!(commentStr.contains("is not a valid concern. Unless you are planning"));
        assert!(commentStr.contains("cholantesh"));
        assert!(commentStr.contains("-- "));

    }
}<commit_msg>Fixes the panicking when title is None and prints if no comments available<commit_after>use models::*;
use std::iter::repeat;

pub fn print_headline_with_author(item: &HnItem, index: &i32) {
    let s = create_headline_with_author(item, index).unwrap(); // Not handling errs
    println!("{}", s);
}

fn create_headline_with_author(item: &HnItem, index: &i32) -> Result<String, String> {
    match item.title {
        Some(_) => {
            let s = format!("[{:3}] {:80} ~ {}",
                            index,
                            item.title.as_ref().unwrap(),
                            item.by);
            Ok(s)

        }
        None => Err(String::from("Not headline")),
    }
}

fn create_comment_string(item: &HnItem, depth: &usize) -> Result<String, String> {
    let depthStr = repeat("-").take(*depth).collect::<String>();
    if (item.text.is_some()) {
        return Ok(format!("{} {} ~ {} on {}",
                          depthStr,
                          item.text.as_ref().unwrap(),
                          &item.by,
                          &item.time));
    } else {
        return Err(String::from("Could not get text for comment"));
    }
}

pub fn no_comments_for(numb: &usize) {
    println!("No comments for [{}]", numb);
}

pub fn could_not_get_any_commments_for_item(item: &HnItem) {
    println!("Could not get comments for item with id {}", item.id)
}

pub fn print_filename_of_loaded_page(filen: &str, title: &str) {
    println!("{} {} {} {}", "Downloaded page", title, "into file", filen);
}
pub fn could_not_load_page(title: &str) {
    println!("Could not download to file with title {}", title);
}

pub fn print_comments(item: &HnItem, comments: &Vec<HnItem>) {
    // FIXME No error handling
    if (comments.len() > 0) {
        match item.title {
            Some(ref title) => {
                println!("Comments for item id {} with title {}",
                         &item.id,
                         item.title.as_ref().unwrap())
            }
            None => println!("Comments for item id {}", &item.id),
        }
        let mut comment_index = 0;
        for comment in comments {
            comment_index += 1; // Should be from main?
            let res = create_comment_row(&comment_index, &comment);
            if (res.is_some()) {
                println!("{}", res.unwrap());
            }
        }
    }
    else {
        println!("No comments for {}", item.id);
    }
}

fn create_comment_row(index: &i32, item: &HnItem) -> Option<String> {
    match item.text {
        Some(ref text) => {
            let s = format!("[{:3}] {:80} ~{}", index, text, &item.by); // TODO This needs to handle unicode characters to utf8 or something similar (snap's -> snap's)
            Some(s)
        }
        None => None,
    }

}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn create_headline_with_author_test() {
        use std::fs::File;
        use std::io::prelude::*;
        use serde_json;
        let mut contents = String::new();
        File::open("res/test/item.json")
            .and_then(|mut file| file.read_to_string(&mut contents))
            .unwrap();
        let deserialized: HnItem = serde_json::from_str(&contents).unwrap();
        let index = 1;
        let s: String = create_headline_with_author(&deserialized, &index).unwrap();
        assert!(s.len() != 0);
        assert!(s.contains("1"));
        assert!(s.contains("dhouston"));
        assert!(s.contains("My YC app: Dropbox - Throw away your USB drive"));
        assert!(deserialized.by.len() != 0);
        assert!(deserialized.title.unwrap().len() != 0);
    }

    #[test]
    fn create_comment_string_test() {
        use std::fs::File;
        use std::io::prelude::*;
        use serde_json;
        let mut contents = String::new();
        File::open("res/test/children-item.json")
            .and_then(|mut file| file.read_to_string(&mut contents))
            .unwrap();
        let deserialized: HnItem = serde_json::from_str(&contents).unwrap();
        let commentStr = create_comment_string(&deserialized, &2).unwrap();
        assert!(commentStr.contains("is not a valid concern. Unless you are planning"));
        assert!(commentStr.contains("cholantesh"));
        assert!(commentStr.contains("-- "));

    }
}<|endoftext|>


<commit_before>use flowgger::config::Config;
use flowgger::record::{Record, StructuredData, SDValue, SDValueType};
use chrono::DateTime;
use std::collections::HashMap;
use super::Decoder;

#[derive(Clone)]
struct Suffixes {
    s_bool: Option<String>,
    s_f64: Option<String>,
    s_i64: Option<String>,
    s_u64: Option<String>,
}

#[derive(Clone)]
pub struct LTSVDecoder {
    schema: Option<HashMap<String, SDValueType>>,
    suffixes: Suffixes,
}

impl LTSVDecoder {
    pub fn new(config: &Config) -> LTSVDecoder {
        let schema = match config.lookup("input.ltsv_schema") {
            None => None,
            Some(pairs) => {
                let mut schema = HashMap::new();
                for (name, sdtype) in pairs.as_table()
                    .expect("input.ltsv_schema must be a list of key/type pairs") {
                    let sdtype = match sdtype.as_str()
                        .expect("input.ltsv_schema types must be strings")
                        .to_lowercase()
                        .as_ref() {
                        "string" => SDValueType::String,
                        "bool" => SDValueType::Bool,
                        "f64" => SDValueType::F64,
                        "i64" => SDValueType::I64,
                        "u64" => SDValueType::U64,
                        _ => {
                            panic!(format!("Unsupported type in input.ltsv_schema for name [{}]",
                                           name))
                        }
                    };
                    schema.insert(name.to_owned(), sdtype);
                }
                Some(schema)
            }
        };
        let mut suffixes = Suffixes {
            s_bool: None,
            s_f64: None,
            s_i64: None,
            s_u64: None,
        };
        match config.lookup("input.ltsv_suffixes") {
            None => {}
            Some(pairs) => {
                for (sdtype, suffix) in pairs.as_table()
                    .expect("input.ltsv_suffixes must be a list of type/suffixes pairs") {
                    let suffix = suffix.as_str()
                        .expect("input.ltsv_suffixes suffixes must be strings")
                        .to_owned();
                    match sdtype.to_lowercase().as_ref() {
                        "string" => panic!("Strings cannot be suffixed"),
                        "bool" => suffixes.s_bool = Some(suffix),
                        "f64" => suffixes.s_f64 = Some(suffix),
                        "i64" => suffixes.s_i64 = Some(suffix),
                        "u64" => suffixes.s_u64 = Some(suffix),
                        _ => {
                            panic!(format!("Unsupported type in input.ltsv_suffixes for type [{}]",
                                           sdtype))
                        }
                    }
                }
            }
        };
        LTSVDecoder {
            schema: schema,
            suffixes: suffixes,
        }
    }
}

impl Decoder for LTSVDecoder {
    fn decode(&self, line: &str) -> Result<Record, &'static str> {
        let mut sd = StructuredData::new(None);
        let mut ts = None;
        let mut hostname = None;
        let mut msg = None;
        let mut severity = None;

        for part in line.split('\t') {
            let mut pair = part.splitn(2, ':');
            let name = try!(pair.next().ok_or("Missing name in an LTSV record"));
            let value = try!(pair.next().ok_or("Missing value in an LTSV record"));
            match name {
                "time" => {
                    let ts_s = if value.starts_with('[') && value.ends_with(']') {
                        &value[1..(value.len() - 1)]
                    } else {
                        value
                    };
                    ts = Some(try!(parse_ts(ts_s)));
                }
                "host" => hostname = Some(value.to_owned()),
                "message" => msg = Some(value.to_owned()),
                "level" => {
                    let severity_given: u8 = try!(value.parse().or(Err("Invalid severity level")));
                    if severity_given > 7 {
                        return Err("Severity level should be <= 7");
                    }
                    severity = Some(severity_given);
                }
                name => {
                    let (final_name, value): (String, SDValue) = if let Some(ref schema) =
                                                                        self.schema {
                        match schema.get(name) {
                            None |
                            Some(&SDValueType::String) => {
                                (format!("_{}", name), SDValue::String(value.to_owned()))
                            }
                            Some(&SDValueType::Bool) => {
                                let final_name = match self.suffixes.s_bool {
                                    Some(ref suffix) if !name.ends_with(suffix) => {
                                        format!("_{}{}", name, suffix)
                                    }
                                    _ => format!("_{}", name),
                                };
                                (final_name,
                                 SDValue::Bool(try!(value.parse::<bool>()
                                    .or(Err("Type error; boolean was expected")))))
                            }
                            Some(&SDValueType::F64) => {
                                let final_name = match self.suffixes.s_f64 {
                                    Some(ref suffix) if !name.ends_with(suffix) => {
                                        format!("_{}{}", name, suffix)
                                    }
                                    _ => format!("_{}", name),
                                };
                                (final_name,
                                 SDValue::F64(try!(value.parse::<f64>()
                                    .or(Err("Type error; f64 was expected")))))
                            }
                            Some(&SDValueType::I64) => {
                                let final_name = match self.suffixes.s_i64 {
                                    Some(ref suffix) if !name.ends_with(suffix) => {
                                        format!("_{}{}", name, suffix)
                                    }
                                    _ => format!("_{}", name),
                                };
                                (final_name,
                                 SDValue::I64(try!(value.parse::<i64>()
                                    .or(Err("Type error; i64 was expected")))))
                            }
                            Some(&SDValueType::U64) => {
                                let final_name = match self.suffixes.s_u64 {
                                    Some(ref suffix) if !name.ends_with(suffix) => {
                                        format!("_{}{}", name, suffix)
                                    }
                                    _ => format!("_{}", name),
                                };
                                (final_name,
                                 SDValue::U64(try!(value.parse::<u64>()
                                    .or(Err("Type error; u64 was expected")))))
                            }
                        }
                    } else {
                        (format!("_{}", name), SDValue::String(value.to_owned()))
                    };
                    sd.pairs.push((final_name, value));
                }
            };
        }
        let record = Record {
            ts: try!(ts.ok_or("Missing timestamp")),
            hostname: try!(hostname.ok_or("Missing hostname")),
            facility: None,
            severity: severity,
            appname: None,
            procid: None,
            msgid: None,
            sd: if sd.pairs.is_empty() { None } else { Some(sd) },
            msg: msg,
            full_msg: None,
        };
        Ok(record)
    }
}

fn rfc3339_to_unix(rfc3339: &str) -> Result<f64, &'static str> {
    match DateTime::parse_from_rfc3339(rfc3339) {
        Ok(date) => Ok(date.timestamp() as f64),
        Err(_) => Err("Unable to parse the date"),
    }
}

fn english_time_to_unix(et: &str) -> Result<f64, &'static str> {
    match DateTime::parse_from_str(et, "%e/%b/%Y:%H:%M:%S %z") {
        Ok(date) => Ok(date.timestamp() as f64),
        Err(_) => Err("Unable to parse the date"),
    }
}

fn unix_strtime_to_unix(et: &str) -> Result<f64, &'static str> {
    match et.parse::<f64>() {
        Ok(ts) => Ok(ts),
        Err(_) => Err("Unable to parse the date"),
    }
}

fn parse_ts(line: &str) -> Result<f64, &'static str> {
    unix_strtime_to_unix(line).or(rfc3339_to_unix(line)).or(english_time_to_unix(line))
}

#[test]
fn test_ltsv_suffixes() {
    let config = Config::from_string("[input]\n[input.ltsv_schema]\ncounter = \"U64\"\nscore = \
                                      \"I64\"\nmean = \"f64\"\ndone = \
                                      \"bool\"\n[input.ltsv_suffixes]\nu64 = \"_u64\"\ni64 = \
                                      \"_i64\"\nF64 = \"_f64\"\nBool = \"_bool\"\n");
    let ltsv_decoder = LTSVDecoder::new(&config.unwrap());
    let msg = "time:[10/Oct/2000:13:55:36 \
               -0700]\tdone:true\tscore:-1\tmean:0.42\tcounter:42\tlevel:3\thost:\
               testhostname\tname1:value1\tname 2: value 2\tn3:v3\tmessage:this is a test";
    let res = ltsv_decoder.decode(msg).unwrap();
    let sd = res.sd.unwrap();
    let pairs = sd.pairs;
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::U64(v) = v {
        k == "_counter_u64" && v == 42
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::I64(v) = v {
        k == "_score_i64" && v == -1
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::F64(v) = v {
        k == "_mean_f64" && f64::abs(v - 0.42) < 1e-5
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::Bool(v) = v {
        k == "_done_bool" && v == true
    } else {
        false
    }));
}

#[test]
fn test_ltsv_suffixes_2() {
    let config = Config::from_string("[input]\n[input.ltsv_schema]\ncounter_u64 = \
                                      \"U64\"\nscore_i64 = \"I64\"\nmean_f64 = \
                                      \"f64\"\ndone_bool = \"bool\"\n[input.ltsv_suffixes]\nu64 \
                                      = \"_u64\"\ni64 = \"_i64\"\nf64 = \"_f64\"\nbool = \
                                      \"_bool\"\n");
    let ltsv_decoder = LTSVDecoder::new(&config.unwrap());
    let msg = "time:[10/Oct/2000:13:55:36 \
               -0700]\tdone_bool:true\tscore_i64:-1\tmean_f64:0.42\tcounter_u64:42\tlevel:3\thost:\
               testhostname\tname1:value1\tname 2: value 2\tn3:v3\tmessage:this is a test";
    let res = ltsv_decoder.decode(msg).unwrap();
    let sd = res.sd.unwrap();
    let pairs = sd.pairs;
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::U64(v) = v {
        k == "_counter_u64" && v == 42
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::I64(v) = v {
        k == "_score_i64" && v == -1
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::F64(v) = v {
        k == "_mean_f64" && f64::abs(v - 0.42) < 1e-5
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::Bool(v) = v {
        k == "_done_bool" && v == true
    } else {
        false
    }));
}

#[test]
fn test_ltsv() {
    let config = Config::from_string("[input]\n[input.ltsv_schema]\ncounter = \"u64\"\nscore = \
                                      \"i64\"\nmean = \"f64\"\ndone = \"bool\"\n");
    let ltsv_decoder = LTSVDecoder::new(&config.unwrap());
    let msg = "time:1438790025\thost:testhostname\tname1:value1\tname 2: value \
               2\tn3:v3";
    let res = ltsv_decoder.decode(msg).unwrap();
    assert!(res.ts as i64 == 1438790025);
}

#[test]
fn test_ltsv2() {
    let config = Config::from_string("[input]\n[input.ltsv_schema]\ncounter = \"u64\"\nscore = \
                                      \"i64\"\nmean = \"f64\"\ndone = \"bool\"\n");
    let ltsv_decoder = LTSVDecoder::new(&config.unwrap());
    let msg = "time:[2015-08-05T15:53:45.637824Z]\thost:testhostname\tname1:value1\tname 2: value \
               2\tn3:v3";
    let res = ltsv_decoder.decode(msg).unwrap();
    assert!(res.ts as i64 == 1438790025);
}

#[test]
fn test_ltsv_3() {
    let config = Config::from_string("[input]\n[input.ltsv_schema]\ncounter = \"u64\"\nscore = \
                                      \"i64\"\nmean = \"f64\"\ndone = \"bool\"\n");
    let ltsv_decoder = LTSVDecoder::new(&config.unwrap());
    let msg = "time:[10/Oct/2000:13:55:36 \
               -0700]\tdone:true\tscore:-1\tmean:0.42\tcounter:42\tlevel:3\thost:\
               testhostname\tname1:value1\tname 2: value 2\tn3:v3\tmessage:this is a test";
    let res = ltsv_decoder.decode(msg).unwrap();
    assert!(res.ts as i64 == 971211336);
    assert!(res.severity.unwrap() == 3);

    assert!(res.hostname == "testhostname");
    assert!(res.msg.unwrap() == "this is a test");
    let sd = res.sd.unwrap();
    let pairs = sd.pairs;
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::String(v) = v {
        k == "_name1" && v == "value1"
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::String(v) = v {
        k == "_name 2" && v == " value 2"
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::String(v) = v {
        k == "_n3" && v == "v3"
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::U64(v) = v {
        k == "_counter" && v == 42
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::I64(v) = v {
        k == "_score" && v == -1
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::F64(v) = v {
        k == "_mean" && f64::abs(v - 0.42) < 1e-5
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::Bool(v) = v {
        k == "_done" && v == true
    } else {
        false
    }));
}
<commit_msg>The "English" time format can now accept real numbers as a timestamp<commit_after>use flowgger::config::Config;
use flowgger::record::{Record, StructuredData, SDValue, SDValueType};
use chrono::DateTime;
use std::collections::HashMap;
use super::Decoder;

#[derive(Clone)]
struct Suffixes {
    s_bool: Option<String>,
    s_f64: Option<String>,
    s_i64: Option<String>,
    s_u64: Option<String>,
}

#[derive(Clone)]
pub struct LTSVDecoder {
    schema: Option<HashMap<String, SDValueType>>,
    suffixes: Suffixes,
}

impl LTSVDecoder {
    pub fn new(config: &Config) -> LTSVDecoder {
        let schema = match config.lookup("input.ltsv_schema") {
            None => None,
            Some(pairs) => {
                let mut schema = HashMap::new();
                for (name, sdtype) in pairs.as_table()
                    .expect("input.ltsv_schema must be a list of key/type pairs") {
                    let sdtype = match sdtype.as_str()
                        .expect("input.ltsv_schema types must be strings")
                        .to_lowercase()
                        .as_ref() {
                        "string" => SDValueType::String,
                        "bool" => SDValueType::Bool,
                        "f64" => SDValueType::F64,
                        "i64" => SDValueType::I64,
                        "u64" => SDValueType::U64,
                        _ => {
                            panic!(format!("Unsupported type in input.ltsv_schema for name [{}]",
                                           name))
                        }
                    };
                    schema.insert(name.to_owned(), sdtype);
                }
                Some(schema)
            }
        };
        let mut suffixes = Suffixes {
            s_bool: None,
            s_f64: None,
            s_i64: None,
            s_u64: None,
        };
        match config.lookup("input.ltsv_suffixes") {
            None => {}
            Some(pairs) => {
                for (sdtype, suffix) in pairs.as_table()
                    .expect("input.ltsv_suffixes must be a list of type/suffixes pairs") {
                    let suffix = suffix.as_str()
                        .expect("input.ltsv_suffixes suffixes must be strings")
                        .to_owned();
                    match sdtype.to_lowercase().as_ref() {
                        "string" => panic!("Strings cannot be suffixed"),
                        "bool" => suffixes.s_bool = Some(suffix),
                        "f64" => suffixes.s_f64 = Some(suffix),
                        "i64" => suffixes.s_i64 = Some(suffix),
                        "u64" => suffixes.s_u64 = Some(suffix),
                        _ => {
                            panic!(format!("Unsupported type in input.ltsv_suffixes for type [{}]",
                                           sdtype))
                        }
                    }
                }
            }
        };
        LTSVDecoder {
            schema: schema,
            suffixes: suffixes,
        }
    }
}

impl Decoder for LTSVDecoder {
    fn decode(&self, line: &str) -> Result<Record, &'static str> {
        let mut sd = StructuredData::new(None);
        let mut ts = None;
        let mut hostname = None;
        let mut msg = None;
        let mut severity = None;

        for part in line.split('\t') {
            let mut pair = part.splitn(2, ':');
            let name = try!(pair.next().ok_or("Missing name in an LTSV record"));
            let value = try!(pair.next().ok_or("Missing value in an LTSV record"));
            match name {
                "time" => {
                    let ts_s = if value.starts_with('[') && value.ends_with(']') {
                        &value[1..(value.len() - 1)]
                    } else {
                        value
                    };
                    ts = Some(try!(parse_ts(ts_s)));
                }
                "host" => hostname = Some(value.to_owned()),
                "message" => msg = Some(value.to_owned()),
                "level" => {
                    let severity_given: u8 = try!(value.parse().or(Err("Invalid severity level")));
                    if severity_given > 7 {
                        return Err("Severity level should be <= 7");
                    }
                    severity = Some(severity_given);
                }
                name => {
                    let (final_name, value): (String, SDValue) = if let Some(ref schema) =
                                                                        self.schema {
                        match schema.get(name) {
                            None |
                            Some(&SDValueType::String) => {
                                (format!("_{}", name), SDValue::String(value.to_owned()))
                            }
                            Some(&SDValueType::Bool) => {
                                let final_name = match self.suffixes.s_bool {
                                    Some(ref suffix) if !name.ends_with(suffix) => {
                                        format!("_{}{}", name, suffix)
                                    }
                                    _ => format!("_{}", name),
                                };
                                (final_name,
                                 SDValue::Bool(try!(value.parse::<bool>()
                                    .or(Err("Type error; boolean was expected")))))
                            }
                            Some(&SDValueType::F64) => {
                                let final_name = match self.suffixes.s_f64 {
                                    Some(ref suffix) if !name.ends_with(suffix) => {
                                        format!("_{}{}", name, suffix)
                                    }
                                    _ => format!("_{}", name),
                                };
                                (final_name,
                                 SDValue::F64(try!(value.parse::<f64>()
                                    .or(Err("Type error; f64 was expected")))))
                            }
                            Some(&SDValueType::I64) => {
                                let final_name = match self.suffixes.s_i64 {
                                    Some(ref suffix) if !name.ends_with(suffix) => {
                                        format!("_{}{}", name, suffix)
                                    }
                                    _ => format!("_{}", name),
                                };
                                (final_name,
                                 SDValue::I64(try!(value.parse::<i64>()
                                    .or(Err("Type error; i64 was expected")))))
                            }
                            Some(&SDValueType::U64) => {
                                let final_name = match self.suffixes.s_u64 {
                                    Some(ref suffix) if !name.ends_with(suffix) => {
                                        format!("_{}{}", name, suffix)
                                    }
                                    _ => format!("_{}", name),
                                };
                                (final_name,
                                 SDValue::U64(try!(value.parse::<u64>()
                                    .or(Err("Type error; u64 was expected")))))
                            }
                        }
                    } else {
                        (format!("_{}", name), SDValue::String(value.to_owned()))
                    };
                    sd.pairs.push((final_name, value));
                }
            };
        }
        let record = Record {
            ts: try!(ts.ok_or("Missing timestamp")),
            hostname: try!(hostname.ok_or("Missing hostname")),
            facility: None,
            severity: severity,
            appname: None,
            procid: None,
            msgid: None,
            sd: if sd.pairs.is_empty() { None } else { Some(sd) },
            msg: msg,
            full_msg: None,
        };
        Ok(record)
    }
}

fn rfc3339_to_unix(rfc3339: &str) -> Result<f64, &'static str> {
    match DateTime::parse_from_rfc3339(rfc3339) {
        Ok(date) => Ok(date.timestamp() as f64),
        Err(_) => Err("Unable to parse the date"),
    }
}

fn english_time_to_unix(et: &str) -> Result<f64, &'static str> {
    match DateTime::parse_from_str(et, "%e/%b/%Y:%H:%M:%S%.f %z") {
        Ok(date) => Ok(date.timestamp() as f64),
        Err(_) => Err("Unable to parse the date"),
    }
}

fn unix_strtime_to_unix(et: &str) -> Result<f64, &'static str> {
    match et.parse::<f64>() {
        Ok(ts) => Ok(ts),
        Err(_) => Err("Unable to parse the date"),
    }
}

fn parse_ts(line: &str) -> Result<f64, &'static str> {
    unix_strtime_to_unix(line).or(rfc3339_to_unix(line)).or(english_time_to_unix(line))
}

#[test]
fn test_ltsv_suffixes() {
    let config = Config::from_string("[input]\n[input.ltsv_schema]\ncounter = \"U64\"\nscore = \
                                      \"I64\"\nmean = \"f64\"\ndone = \
                                      \"bool\"\n[input.ltsv_suffixes]\nu64 = \"_u64\"\ni64 = \
                                      \"_i64\"\nF64 = \"_f64\"\nBool = \"_bool\"\n");
    let ltsv_decoder = LTSVDecoder::new(&config.unwrap());
    let msg = "time:[10/Oct/2000:13:55:36 \
               -0700]\tdone:true\tscore:-1\tmean:0.42\tcounter:42\tlevel:3\thost:\
               testhostname\tname1:value1\tname 2: value 2\tn3:v3\tmessage:this is a test";
    let res = ltsv_decoder.decode(msg).unwrap();
    let sd = res.sd.unwrap();
    let pairs = sd.pairs;
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::U64(v) = v {
        k == "_counter_u64" && v == 42
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::I64(v) = v {
        k == "_score_i64" && v == -1
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::F64(v) = v {
        k == "_mean_f64" && f64::abs(v - 0.42) < 1e-5
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::Bool(v) = v {
        k == "_done_bool" && v == true
    } else {
        false
    }));
}

#[test]
fn test_ltsv_suffixes_2() {
    let config = Config::from_string("[input]\n[input.ltsv_schema]\ncounter_u64 = \
                                      \"U64\"\nscore_i64 = \"I64\"\nmean_f64 = \
                                      \"f64\"\ndone_bool = \"bool\"\n[input.ltsv_suffixes]\nu64 \
                                      = \"_u64\"\ni64 = \"_i64\"\nf64 = \"_f64\"\nbool = \
                                      \"_bool\"\n");
    let ltsv_decoder = LTSVDecoder::new(&config.unwrap());
    let msg = "time:[10/Oct/2000:13:55:36 \
               -0700]\tdone_bool:true\tscore_i64:-1\tmean_f64:0.42\tcounter_u64:42\tlevel:3\thost:\
               testhostname\tname1:value1\tname 2: value 2\tn3:v3\tmessage:this is a test";
    let res = ltsv_decoder.decode(msg).unwrap();
    let sd = res.sd.unwrap();
    let pairs = sd.pairs;
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::U64(v) = v {
        k == "_counter_u64" && v == 42
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::I64(v) = v {
        k == "_score_i64" && v == -1
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::F64(v) = v {
        k == "_mean_f64" && f64::abs(v - 0.42) < 1e-5
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::Bool(v) = v {
        k == "_done_bool" && v == true
    } else {
        false
    }));
}

#[test]
fn test_ltsv() {
    let config = Config::from_string("[input]\n[input.ltsv_schema]\ncounter = \"u64\"\nscore = \
                                      \"i64\"\nmean = \"f64\"\ndone = \"bool\"\n");
    let ltsv_decoder = LTSVDecoder::new(&config.unwrap());
    let msg = "time:1438790025\thost:testhostname\tname1:value1\tname 2: value \
               2\tn3:v3";
    let res = ltsv_decoder.decode(msg).unwrap();
    assert!(res.ts as i64 == 1438790025);
}

#[test]
fn test_ltsv2() {
    let config = Config::from_string("[input]\n[input.ltsv_schema]\ncounter = \"u64\"\nscore = \
                                      \"i64\"\nmean = \"f64\"\ndone = \"bool\"\n");
    let ltsv_decoder = LTSVDecoder::new(&config.unwrap());
    let msg = "time:[2015-08-05T15:53:45.637824Z]\thost:testhostname\tname1:value1\tname 2: value \
               2\tn3:v3";
    let res = ltsv_decoder.decode(msg).unwrap();
    assert!(res.ts as i64 == 1438790025);
}

#[test]
fn test_ltsv_3() {
    let config = Config::from_string("[input]\n[input.ltsv_schema]\ncounter = \"u64\"\nscore = \
                                      \"i64\"\nmean = \"f64\"\ndone = \"bool\"\n");
    let ltsv_decoder = LTSVDecoder::new(&config.unwrap());
    let msg = "time:[10/Oct/2000:13:55:36 \
               -0700]\tdone:true\tscore:-1\tmean:0.42\tcounter:42\tlevel:3\thost:\
               testhostname\tname1:value1\tname 2: value 2\tn3:v3\tmessage:this is a test";
    let res = ltsv_decoder.decode(msg).unwrap();
    assert!(res.ts as i64 == 971211336);
    assert!(res.severity.unwrap() == 3);

    assert!(res.hostname == "testhostname");
    assert!(res.msg.unwrap() == "this is a test");
    let sd = res.sd.unwrap();
    let pairs = sd.pairs;
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::String(v) = v {
        k == "_name1" && v == "value1"
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::String(v) = v {
        k == "_name 2" && v == " value 2"
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::String(v) = v {
        k == "_n3" && v == "v3"
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::U64(v) = v {
        k == "_counter" && v == 42
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::I64(v) = v {
        k == "_score" && v == -1
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::F64(v) = v {
        k == "_mean" && f64::abs(v - 0.42) < 1e-5
    } else {
        false
    }));
    assert!(pairs.iter().cloned().any(|(k, v)| if let SDValue::Bool(v) = v {
        k == "_done" && v == true
    } else {
        false
    }));
}
<|endoftext|>


<commit_before>use crate::core::{
    db::{EntryGateway, EntryIndex, EntryIndexQuery, EntryIndexer},
    entities::Entry,
    util::geo::{LatCoord, LngCoord},
};

use failure::Fallible;
use std::ops::Bound;
use std::path::Path;
use tantivy::{
    collector::{Count, TopDocs},
    query::{BooleanQuery, Occur, Query, QueryParser, RangeQuery, TermQuery},
    schema::*,
    tokenizer::{LowerCaser, RawTokenizer, Tokenizer},
    DocAddress, Document, Index, IndexWriter, Score,
};

const OVERALL_INDEX_HEAP_SIZE_IN_BYTES: usize = 50_000_000;

struct TantivyEntryFields {
    id: Field,
    title: Field,
    description: Field,
    category: Field,
    lat: Field,
    lng: Field,
    tag: Field,
}

pub(crate) struct TantivyEntryIndex {
    fields: TantivyEntryFields,
    index: Index,
    writer: IndexWriter,
    text_query_parser: QueryParser,
}

const ID_TOKENIZER: &str = "raw";
const TAG_TOKENIZER: &str = "tag";
const TEXT_TOKENIZER: &str = "default";

fn build_schema() -> (Schema, TantivyEntryFields) {
    let id_options = TextOptions::default()
        .set_indexing_options(
            TextFieldIndexing::default()
                .set_tokenizer(ID_TOKENIZER)
                .set_index_option(IndexRecordOption::Basic),
        )
        .set_stored();
    let category_options = TextOptions::default().set_indexing_options(
        TextFieldIndexing::default()
            .set_tokenizer(ID_TOKENIZER)
            .set_index_option(IndexRecordOption::WithFreqs),
    );
    let tag_options = TextOptions::default().set_indexing_options(
        TextFieldIndexing::default()
            .set_tokenizer(TAG_TOKENIZER)
            .set_index_option(IndexRecordOption::WithFreqs),
    );
    let text_options = TextOptions::default().set_indexing_options(
        TextFieldIndexing::default()
            .set_tokenizer(TEXT_TOKENIZER)
            .set_index_option(IndexRecordOption::WithFreqsAndPositions),
    );
    let mut schema_builder = SchemaBuilder::default();
    let id = schema_builder.add_text_field("id", id_options);
    let lat = schema_builder.add_i64_field("lat", INT_INDEXED);
    let lng = schema_builder.add_i64_field("lng", INT_INDEXED);
    let title = schema_builder.add_text_field("title", text_options.clone());
    let description = schema_builder.add_text_field("desc", text_options);
    let category = schema_builder.add_text_field("cat", category_options.clone());
    let tag = schema_builder.add_text_field("tag", tag_options);
    let schema = schema_builder.build();
    let fields = TantivyEntryFields {
        id,
        lat,
        lng,
        title,
        description,
        category,
        tag,
    };
    (schema, fields)
}

fn register_tokenizers(index: &Index) {
    // Predefined tokenizers
    debug_assert!(index.tokenizers().get(ID_TOKENIZER).is_some());
    debug_assert!(index.tokenizers().get(TEXT_TOKENIZER).is_some());
    // Custom tokenizer(s)
    debug_assert!(index.tokenizers().get(TAG_TOKENIZER).is_none());
    index
        .tokenizers()
        .register(TAG_TOKENIZER, RawTokenizer.filter(LowerCaser));
}

impl TantivyEntryIndex {
    pub fn create_in_ram() -> Fallible<Self> {
        let no_path: Option<&Path> = None;
        Self::create(no_path)
    }

    pub fn create<P: AsRef<Path>>(path: Option<P>) -> Fallible<Self> {
        let (schema, fields) = build_schema();

        // TODO: Open index from existing directory
        let index = if let Some(path) = path {
            info!(
                "Creating full-text search index in directory: {}",
                path.as_ref().to_string_lossy()
            );
            Index::create_in_dir(path, schema)?
        } else {
            warn!("Creating full-text search index in RAM");
            Index::create_in_ram(schema)
        };

        register_tokenizers(&index);

        let writer = index.writer(OVERALL_INDEX_HEAP_SIZE_IN_BYTES)?;
        let text_query_parser =
            QueryParser::for_index(&index, vec![fields.title, fields.description]);
        Ok(Self {
            fields,
            index,
            writer,
            text_query_parser,
        })
    }
}

impl EntryIndexer for TantivyEntryIndex {
    fn add_or_update_entry(&mut self, entry: &Entry) -> Fallible<()> {
        let id_term = Term::from_field_text(self.fields.id, &entry.id);
        self.writer.delete_term(id_term);
        let mut doc = Document::default();
        doc.add_text(self.fields.id, &entry.id);
        doc.add_i64(
            self.fields.lat,
            i64::from(LatCoord::from_deg(entry.location.lat).to_raw()),
        );
        doc.add_i64(
            self.fields.lng,
            i64::from(LngCoord::from_deg(entry.location.lng).to_raw()),
        );
        doc.add_text(self.fields.title, &entry.title);
        doc.add_text(self.fields.description, &entry.description);
        for category in &entry.categories {
            doc.add_text(self.fields.category, category);
        }
        for tag in &entry.tags {
            doc.add_text(self.fields.tag, tag);
        }
        self.writer.add_document(doc);
        Ok(())
    }

    fn remove_entry_by_id(&mut self, id: &str) -> Fallible<()> {
        let id_term = Term::from_field_text(self.fields.id, id);
        self.writer.delete_term(id_term);
        Ok(())
    }

    fn flush(&mut self) -> Fallible<()> {
        self.writer.commit()?;
        self.index.load_searchers()?;
        Ok(())
    }
}

// Upper bound for the expected number of results for which
// memory is pre-allocated.
const MAX_LIMIT: usize = 100;

impl EntryIndex for TantivyEntryIndex {
    fn query_entries(
        &self,
        entries: &EntryGateway,
        query: &EntryIndexQuery,
        limit: usize,
    ) -> Fallible<Vec<Entry>> {
        let mut sub_queries: Vec<(Occur, Box<Query>)> =
            Vec::with_capacity(2 + 2 + query.categories.len() + query.tags.len());
        if let Some(ref bbox) = query.bbox {
            debug_assert!(bbox.is_valid());
            debug_assert!(!bbox.is_empty());
            let lat_query = RangeQuery::new_i64_bounds(
                self.fields.lat,
                Bound::Included(i64::from(bbox.south_west().lat().to_raw())),
                Bound::Included(i64::from(bbox.north_east().lat().to_raw())),
            );
            sub_queries.push((Occur::Must, Box::new(lat_query)));
            if bbox.south_west().lng() <= bbox.north_east().lng() {
                // regular (inclusive)
                let lng_query = RangeQuery::new_i64_bounds(
                    self.fields.lng,
                    Bound::Included(i64::from(bbox.south_west().lng().to_raw())),
                    Bound::Included(i64::from(bbox.north_east().lng().to_raw())),
                );
                sub_queries.push((Occur::Must, Box::new(lng_query)));
            } else {
                // inverse (exclusive)
                let lng_query = RangeQuery::new_i64_bounds(
                    self.fields.lng,
                    Bound::Excluded(i64::from(bbox.north_east().lng().to_raw())),
                    Bound::Excluded(i64::from(bbox.south_west().lng().to_raw())),
                );
                sub_queries.push((Occur::MustNot, Box::new(lng_query)));
            }
        }
        if let Some(ref text) = query.text {
            debug_assert!(!text.trim().is_empty());
            match self.text_query_parser.parse_query(&text.to_lowercase()) {
                Ok(query) => {
                    sub_queries.push((Occur::Should, Box::new(query)));
                }
                Err(err) => {
                    warn!("Failed to parse query text '{}': {:?}", text, err);
                }
            }
        }
        for category in &query.categories {
            debug_assert!(!category.trim().is_empty());
            let category_term =
                Term::from_field_text(self.fields.category, &category.to_lowercase());
            let category_query = TermQuery::new(category_term, IndexRecordOption::Basic);
            sub_queries.push((Occur::Must, Box::new(category_query)));
        }
        for tag in &query.tags {
            debug_assert!(!tag.trim().is_empty());
            let tag_term = Term::from_field_text(self.fields.tag, &tag.to_lowercase());
            let tag_query = TermQuery::new(tag_term, IndexRecordOption::Basic);
            sub_queries.push((Occur::Should, Box::new(tag_query)));
        }
        let query = BooleanQuery::from(sub_queries);
        let searcher = self.index.searcher();
        let (_doc_count, top_docs): (usize, Vec<(Score, DocAddress)>) =
            searcher.search(&query, &(Count, TopDocs::with_limit(limit.min(MAX_LIMIT))))?;
        let mut top_entries = Vec::with_capacity(top_docs.len());
        for (_score, doc_addr) in top_docs {
            match searcher.doc(doc_addr) {
                Ok(doc) => {
                    if let Some(id) = doc.get_first(self.fields.id).and_then(Value::text) {
                        match entries.get_entry(id) {
                            Ok(entry) => {
                                top_entries.push(entry);
                            }
                            Err(err) => {
                                warn!("Entry {} not found: {}", id, err);
                            }
                        }
                    } else {
                        warn!("Mising entry id in document {:?}", doc_addr);
                    }
                }
                Err(err) => {
                    warn!("Failed to load document {:?}: {}", doc_addr, err);
                }
            }
        }
        Ok(top_entries)
    }
}
<commit_msg>Use strict filtering when searching for text<commit_after>use crate::core::{
    db::{EntryGateway, EntryIndex, EntryIndexQuery, EntryIndexer},
    entities::Entry,
    util::geo::{LatCoord, LngCoord},
};

use failure::Fallible;
use std::ops::Bound;
use std::path::Path;
use tantivy::{
    collector::{Count, TopDocs},
    query::{BooleanQuery, Occur, Query, QueryParser, RangeQuery, TermQuery},
    schema::*,
    tokenizer::{LowerCaser, RawTokenizer, Tokenizer},
    DocAddress, Document, Index, IndexWriter, Score,
};

const OVERALL_INDEX_HEAP_SIZE_IN_BYTES: usize = 50_000_000;

struct TantivyEntryFields {
    id: Field,
    title: Field,
    description: Field,
    category: Field,
    lat: Field,
    lng: Field,
    tag: Field,
}

pub(crate) struct TantivyEntryIndex {
    fields: TantivyEntryFields,
    index: Index,
    writer: IndexWriter,
    text_query_parser: QueryParser,
}

const ID_TOKENIZER: &str = "raw";
const TAG_TOKENIZER: &str = "tag";
const TEXT_TOKENIZER: &str = "default";

fn build_schema() -> (Schema, TantivyEntryFields) {
    let id_options = TextOptions::default()
        .set_indexing_options(
            TextFieldIndexing::default()
                .set_tokenizer(ID_TOKENIZER)
                .set_index_option(IndexRecordOption::Basic),
        )
        .set_stored();
    let category_options = TextOptions::default().set_indexing_options(
        TextFieldIndexing::default()
            .set_tokenizer(ID_TOKENIZER)
            .set_index_option(IndexRecordOption::WithFreqs),
    );
    let tag_options = TextOptions::default().set_indexing_options(
        TextFieldIndexing::default()
            .set_tokenizer(TAG_TOKENIZER)
            .set_index_option(IndexRecordOption::WithFreqs),
    );
    let text_options = TextOptions::default().set_indexing_options(
        TextFieldIndexing::default()
            .set_tokenizer(TEXT_TOKENIZER)
            .set_index_option(IndexRecordOption::WithFreqsAndPositions),
    );
    let mut schema_builder = SchemaBuilder::default();
    let id = schema_builder.add_text_field("id", id_options);
    let lat = schema_builder.add_i64_field("lat", INT_INDEXED);
    let lng = schema_builder.add_i64_field("lng", INT_INDEXED);
    let title = schema_builder.add_text_field("title", text_options.clone());
    let description = schema_builder.add_text_field("desc", text_options);
    let category = schema_builder.add_text_field("cat", category_options.clone());
    let tag = schema_builder.add_text_field("tag", tag_options);
    let schema = schema_builder.build();
    let fields = TantivyEntryFields {
        id,
        lat,
        lng,
        title,
        description,
        category,
        tag,
    };
    (schema, fields)
}

fn register_tokenizers(index: &Index) {
    // Predefined tokenizers
    debug_assert!(index.tokenizers().get(ID_TOKENIZER).is_some());
    debug_assert!(index.tokenizers().get(TEXT_TOKENIZER).is_some());
    // Custom tokenizer(s)
    debug_assert!(index.tokenizers().get(TAG_TOKENIZER).is_none());
    index
        .tokenizers()
        .register(TAG_TOKENIZER, RawTokenizer.filter(LowerCaser));
}

impl TantivyEntryIndex {
    pub fn create_in_ram() -> Fallible<Self> {
        let no_path: Option<&Path> = None;
        Self::create(no_path)
    }

    pub fn create<P: AsRef<Path>>(path: Option<P>) -> Fallible<Self> {
        let (schema, fields) = build_schema();

        // TODO: Open index from existing directory
        let index = if let Some(path) = path {
            info!(
                "Creating full-text search index in directory: {}",
                path.as_ref().to_string_lossy()
            );
            Index::create_in_dir(path, schema)?
        } else {
            warn!("Creating full-text search index in RAM");
            Index::create_in_ram(schema)
        };

        register_tokenizers(&index);

        let writer = index.writer(OVERALL_INDEX_HEAP_SIZE_IN_BYTES)?;
        let text_query_parser =
            QueryParser::for_index(&index, vec![fields.title, fields.description]);
        Ok(Self {
            fields,
            index,
            writer,
            text_query_parser,
        })
    }
}

impl EntryIndexer for TantivyEntryIndex {
    fn add_or_update_entry(&mut self, entry: &Entry) -> Fallible<()> {
        let id_term = Term::from_field_text(self.fields.id, &entry.id);
        self.writer.delete_term(id_term);
        let mut doc = Document::default();
        doc.add_text(self.fields.id, &entry.id);
        doc.add_i64(
            self.fields.lat,
            i64::from(LatCoord::from_deg(entry.location.lat).to_raw()),
        );
        doc.add_i64(
            self.fields.lng,
            i64::from(LngCoord::from_deg(entry.location.lng).to_raw()),
        );
        doc.add_text(self.fields.title, &entry.title);
        doc.add_text(self.fields.description, &entry.description);
        for category in &entry.categories {
            doc.add_text(self.fields.category, category);
        }
        for tag in &entry.tags {
            doc.add_text(self.fields.tag, tag);
        }
        self.writer.add_document(doc);
        Ok(())
    }

    fn remove_entry_by_id(&mut self, id: &str) -> Fallible<()> {
        let id_term = Term::from_field_text(self.fields.id, id);
        self.writer.delete_term(id_term);
        Ok(())
    }

    fn flush(&mut self) -> Fallible<()> {
        self.writer.commit()?;
        self.index.load_searchers()?;
        Ok(())
    }
}

// Upper bound for the expected number of results for which
// memory is pre-allocated.
const MAX_LIMIT: usize = 100;

impl EntryIndex for TantivyEntryIndex {
    fn query_entries(
        &self,
        entries: &EntryGateway,
        query: &EntryIndexQuery,
        limit: usize,
    ) -> Fallible<Vec<Entry>> {
        let mut sub_queries: Vec<(Occur, Box<Query>)> =
            Vec::with_capacity(2 + 2 + query.categories.len() + query.tags.len());
        if let Some(ref bbox) = query.bbox {
            debug_assert!(bbox.is_valid());
            debug_assert!(!bbox.is_empty());
            let lat_query = RangeQuery::new_i64_bounds(
                self.fields.lat,
                Bound::Included(i64::from(bbox.south_west().lat().to_raw())),
                Bound::Included(i64::from(bbox.north_east().lat().to_raw())),
            );
            sub_queries.push((Occur::Must, Box::new(lat_query)));
            if bbox.south_west().lng() <= bbox.north_east().lng() {
                // regular (inclusive)
                let lng_query = RangeQuery::new_i64_bounds(
                    self.fields.lng,
                    Bound::Included(i64::from(bbox.south_west().lng().to_raw())),
                    Bound::Included(i64::from(bbox.north_east().lng().to_raw())),
                );
                sub_queries.push((Occur::Must, Box::new(lng_query)));
            } else {
                // inverse (exclusive)
                let lng_query = RangeQuery::new_i64_bounds(
                    self.fields.lng,
                    Bound::Excluded(i64::from(bbox.north_east().lng().to_raw())),
                    Bound::Excluded(i64::from(bbox.south_west().lng().to_raw())),
                );
                sub_queries.push((Occur::MustNot, Box::new(lng_query)));
            }
        }
        if let Some(ref text) = query.text {
            debug_assert!(!text.trim().is_empty());
            match self.text_query_parser.parse_query(&text.to_lowercase()) {
                Ok(query) => {
                    sub_queries.push((Occur::Must, Box::new(query)));
                }
                Err(err) => {
                    warn!("Failed to parse query text '{}': {:?}", text, err);
                }
            }
        }
        for category in &query.categories {
            debug_assert!(!category.trim().is_empty());
            let category_term =
                Term::from_field_text(self.fields.category, &category.to_lowercase());
            let category_query = TermQuery::new(category_term, IndexRecordOption::Basic);
            sub_queries.push((Occur::Must, Box::new(category_query)));
        }
        for tag in &query.tags {
            debug_assert!(!tag.trim().is_empty());
            let tag_term = Term::from_field_text(self.fields.tag, &tag.to_lowercase());
            let tag_query = TermQuery::new(tag_term, IndexRecordOption::Basic);
            sub_queries.push((Occur::Should, Box::new(tag_query)));
        }
        let query = BooleanQuery::from(sub_queries);
        let searcher = self.index.searcher();
        let (_doc_count, top_docs): (usize, Vec<(Score, DocAddress)>) =
            searcher.search(&query, &(Count, TopDocs::with_limit(limit.min(MAX_LIMIT))))?;
        let mut top_entries = Vec::with_capacity(top_docs.len());
        for (_score, doc_addr) in top_docs {
            match searcher.doc(doc_addr) {
                Ok(doc) => {
                    if let Some(id) = doc.get_first(self.fields.id).and_then(Value::text) {
                        match entries.get_entry(id) {
                            Ok(entry) => {
                                top_entries.push(entry);
                            }
                            Err(err) => {
                                warn!("Entry {} not found: {}", id, err);
                            }
                        }
                    } else {
                        warn!("Mising entry id in document {:?}", doc_addr);
                    }
                }
                Err(err) => {
                    warn!("Failed to load document {:?}: {}", doc_addr, err);
                }
            }
        }
        Ok(top_entries)
    }
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::fmt::Error as FmtError;
use std::fmt::{Display, Formatter};

generate_error_types!(ListError, ListErrorKind,
    FormatError    => "FormatError",
    EntryError     => "EntryError",
    IterationError => "IterationError",
    CLIError       => "No CLI subcommand for listing entries"
);

<commit_msg>libimagentrylist: Replace error module imports with macro helper<commit_after>generate_error_imports!();

generate_error_types!(ListError, ListErrorKind,
    FormatError    => "FormatError",
    EntryError     => "EntryError",
    IterationError => "IterationError",
    CLIError       => "No CLI subcommand for listing entries"
);

<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use ast;
use parse::{ParseSess,PResult,filemap_to_tts};
use parse::new_parser_from_source_str;
use parse::parser::Parser;
use parse::token;
use ptr::P;
use str::char_at;

/// Map a string to tts, using a made-up filename:
pub fn string_to_tts(source_str: String) -> Vec<ast::TokenTree> {
    let ps = ParseSess::new();
    filemap_to_tts(&ps, ps.codemap().new_filemap("bogofile".to_string(), source_str))
}

/// Map string to parser (via tts)
pub fn string_to_parser<'a>(ps: &'a ParseSess, source_str: String) -> Parser<'a> {
    new_parser_from_source_str(ps,
                               Vec::new(),
                               "bogofile".to_string(),
                               source_str)
}

fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where
    F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>,
{
    let mut p = string_to_parser(&ps, s);
    let x = panictry!(f(&mut p));
    p.abort_if_errors();
    x
}

/// Parse a string, return a crate.
pub fn string_to_crate (source_str : String) -> ast::Crate {
    let ps = ParseSess::new();
    with_error_checking_parse(source_str, &ps, |p| {
        p.parse_crate_mod()
    })
}

/// Parse a string, return an expr
pub fn string_to_expr (source_str : String) -> P<ast::Expr> {
    let ps = ParseSess::new();
    with_error_checking_parse(source_str, &ps, |p| {
        p.parse_expr()
    })
}

/// Parse a string, return an item
pub fn string_to_item (source_str : String) -> Option<P<ast::Item>> {
    let ps = ParseSess::new();
    with_error_checking_parse(source_str, &ps, |p| {
        p.parse_item()
    })
}

/// Parse a string, return a stmt
pub fn string_to_stmt(source_str : String) -> Option<P<ast::Stmt>> {
    let ps = ParseSess::new();
    with_error_checking_parse(source_str, &ps, |p| {
        p.parse_stmt()
    })
}

/// Parse a string, return a pat. Uses "irrefutable"... which doesn't
/// (currently) affect parsing.
pub fn string_to_pat(source_str: String) -> P<ast::Pat> {
    let ps = ParseSess::new();
    with_error_checking_parse(source_str, &ps, |p| {
        p.parse_pat()
    })
}

/// Convert a vector of strings to a vector of ast::Ident's
pub fn strs_to_idents(ids: Vec<&str> ) -> Vec<ast::Ident> {
    ids.iter().map(|u| token::str_to_ident(*u)).collect()
}

/// Does the given string match the pattern? whitespace in the first string
/// may be deleted or replaced with other whitespace to match the pattern.
/// this function is Unicode-ignorant; fortunately, the careful design of
/// UTF-8 mitigates this ignorance.  In particular, this function only collapses
/// sequences of \n, \r, ' ', and \t, but it should otherwise tolerate Unicode
/// chars. Unsurprisingly, it doesn't do NKF-normalization(?).
pub fn matches_codepattern(a : &str, b : &str) -> bool {
    let mut idx_a = 0;
    let mut idx_b = 0;
    loop {
        if idx_a == a.len() && idx_b == b.len() {
            return true;
        }
        else if idx_a == a.len() {return false;}
        else if idx_b == b.len() {
            // maybe the stuff left in a is all ws?
            if is_whitespace(char_at(a, idx_a)) {
                return scan_for_non_ws_or_end(a,idx_a) == a.len();
            } else {
                return false;
            }
        }
        // ws in both given and pattern:
        else if is_whitespace(char_at(a, idx_a))
           && is_whitespace(char_at(b, idx_b)) {
            idx_a = scan_for_non_ws_or_end(a,idx_a);
            idx_b = scan_for_non_ws_or_end(b,idx_b);
        }
        // ws in given only:
        else if is_whitespace(char_at(a, idx_a)) {
            idx_a = scan_for_non_ws_or_end(a,idx_a);
        }
        // *don't* silently eat ws in expected only.
        else if char_at(a, idx_a) == char_at(b, idx_b) {
            idx_a += 1;
            idx_b += 1;
        }
        else {
            return false;
        }
    }
}

/// Given a string and an index, return the first usize >= idx
/// that is a non-ws-char or is outside of the legal range of
/// the string.
fn scan_for_non_ws_or_end(a : &str, idx: usize) -> usize {
    let mut i = idx;
    let len = a.len();
    while (i < len) && (is_whitespace(char_at(a, i))) {
        i += 1;
    }
    i
}

pub fn is_whitespace(c: char) -> bool {
    c.is_whitespace()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test] fn eqmodws() {
        assert_eq!(matches_codepattern("",""),true);
        assert_eq!(matches_codepattern("","a"),false);
        assert_eq!(matches_codepattern("a",""),false);
        assert_eq!(matches_codepattern("a","a"),true);
        assert_eq!(matches_codepattern("a b","a   \n\t\r  b"),true);
        assert_eq!(matches_codepattern("a b ","a   \n\t\r  b"),true);
        assert_eq!(matches_codepattern("a b","a   \n\t\r  b "),false);
        assert_eq!(matches_codepattern("a   b","a b"),true);
        assert_eq!(matches_codepattern("ab","a b"),false);
        assert_eq!(matches_codepattern("a   b","ab"),true);
    }
}
<commit_msg>libsyntax: make matches_codepattern unicode aware<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use ast;
use parse::{ParseSess,PResult,filemap_to_tts};
use parse::new_parser_from_source_str;
use parse::parser::Parser;
use parse::token;
use ptr::P;
use std::iter::Peekable;

/// Map a string to tts, using a made-up filename:
pub fn string_to_tts(source_str: String) -> Vec<ast::TokenTree> {
    let ps = ParseSess::new();
    filemap_to_tts(&ps, ps.codemap().new_filemap("bogofile".to_string(), source_str))
}

/// Map string to parser (via tts)
pub fn string_to_parser<'a>(ps: &'a ParseSess, source_str: String) -> Parser<'a> {
    new_parser_from_source_str(ps,
                               Vec::new(),
                               "bogofile".to_string(),
                               source_str)
}

fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where
    F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>,
{
    let mut p = string_to_parser(&ps, s);
    let x = panictry!(f(&mut p));
    p.abort_if_errors();
    x
}

/// Parse a string, return a crate.
pub fn string_to_crate (source_str : String) -> ast::Crate {
    let ps = ParseSess::new();
    with_error_checking_parse(source_str, &ps, |p| {
        p.parse_crate_mod()
    })
}

/// Parse a string, return an expr
pub fn string_to_expr (source_str : String) -> P<ast::Expr> {
    let ps = ParseSess::new();
    with_error_checking_parse(source_str, &ps, |p| {
        p.parse_expr()
    })
}

/// Parse a string, return an item
pub fn string_to_item (source_str : String) -> Option<P<ast::Item>> {
    let ps = ParseSess::new();
    with_error_checking_parse(source_str, &ps, |p| {
        p.parse_item()
    })
}

/// Parse a string, return a stmt
pub fn string_to_stmt(source_str : String) -> Option<P<ast::Stmt>> {
    let ps = ParseSess::new();
    with_error_checking_parse(source_str, &ps, |p| {
        p.parse_stmt()
    })
}

/// Parse a string, return a pat. Uses "irrefutable"... which doesn't
/// (currently) affect parsing.
pub fn string_to_pat(source_str: String) -> P<ast::Pat> {
    let ps = ParseSess::new();
    with_error_checking_parse(source_str, &ps, |p| {
        p.parse_pat()
    })
}

/// Convert a vector of strings to a vector of ast::Ident's
pub fn strs_to_idents(ids: Vec<&str> ) -> Vec<ast::Ident> {
    ids.iter().map(|u| token::str_to_ident(*u)).collect()
}

/// Does the given string match the pattern? whitespace in the first string
/// may be deleted or replaced with other whitespace to match the pattern.
/// This function is relatively Unicode-ignorant; fortunately, the careful design
/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?).
pub fn matches_codepattern(a : &str, b : &str) -> bool {
    let mut a_iter = a.chars().peekable();
    let mut b_iter = b.chars().peekable();

    loop {
        let (a, b) = match (a_iter.peek(), b_iter.peek()) {
            (None, None) => return true,
            (None, _) => return false,
            (Some(a), None) => {
                if a.is_whitespace() {
                    break // trailing whitespace check is out of loop for borrowck
                } else {
                    return false
                }
            }
            (Some(&a), Some(&b)) => (a, b)
        };

        if a.is_whitespace() && b.is_whitespace() {
            // skip whitespace for a and b
            scan_for_non_ws_or_end(&mut a_iter);
            scan_for_non_ws_or_end(&mut b_iter);
        } else if a.is_whitespace() {
            // skip whitespace for a
            scan_for_non_ws_or_end(&mut a_iter);
        } else if a == b {
            a_iter.next();
            b_iter.next();
        } else {
            return false
        }
    }

    // check if a has *only* trailing whitespace
    a_iter.all(|c| c.is_whitespace())
}

/// Advances the given peekable `Iterator` until it reaches a non-whitespace character
fn scan_for_non_ws_or_end<I: Iterator<Item= char>>(iter: &mut Peekable<I>) {
    loop {
        match iter.peek() {
            Some(c) if c.is_whitespace() => {} // fall through; borrowck
            _ => return
        }

        iter.next();
    }
}

pub fn is_whitespace(c: char) -> bool {
    c.is_whitespace()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn eqmodws() {
        assert_eq!(matches_codepattern("",""),true);
        assert_eq!(matches_codepattern("","a"),false);
        assert_eq!(matches_codepattern("a",""),false);
        assert_eq!(matches_codepattern("a","a"),true);
        assert_eq!(matches_codepattern("a b","a   \n\t\r  b"),true);
        assert_eq!(matches_codepattern("a b ","a   \n\t\r  b"),true);
        assert_eq!(matches_codepattern("a b","a   \n\t\r  b "),false);
        assert_eq!(matches_codepattern("a   b","a b"),true);
        assert_eq!(matches_codepattern("ab","a b"),false);
        assert_eq!(matches_codepattern("a   b","ab"),true);
        assert_eq!(matches_codepattern(" a   b","ab"),true);
    }

    #[test]
    fn more_whitespace() {
        assert_eq!(matches_codepattern("","\x0C"), false);
        assert_eq!(matches_codepattern("a b","a\u{2002}b"),true);
        assert_eq!(matches_codepattern("a b ","a   \u{0085}\n\t\r  b"),true);
        assert_eq!(matches_codepattern("a b","a   \u{0085}\n\t\r  b "),false);
        assert_eq!(matches_codepattern("a   b","a\u{2002}b"),true);
        assert_eq!(matches_codepattern("ab","a\u{2003}b"),false);
        assert_eq!(matches_codepattern("a  \u{3000}b","ab"),true);
        assert_eq!(matches_codepattern("\u{205F}a   b","ab"),true);
    }
}
<|endoftext|>


<commit_before>//! Basic 2D object and shader.

use std::rc::Rc;
use glium::{ Display, Frame };
use texture::{ Texture, TextureRef };
use mesh::Mesh;
use shader::ShaderType;
use transform::Transform;
use renderer::{ Renderer, Renderable };
use rect::Rect;
use cgmath::Matrix4;
use id::Id;

/// 2D Sprite vertex type
#[derive(Copy, Clone)]
pub struct Vertex
{
    pub position: [f32; 2],
    pub texture_position: [f32; 2],
}

implement_vertex!{ Vertex, position, texture_position }

/// Simple sprite shader
pub struct Shader;

impl ShaderType for Shader
{
    type Vertex = Vertex;
    type Uniforms = Uniforms;

    fn vertex() -> &'static str
    {
        include_str!("sprite.vert")
    }

    fn fragment() -> &'static str
    {
        include_str!("sprite.frag")
    }
}

/// Sprite rendering uniforms.
pub struct Uniforms
{
    pub matrix: [[f32; 4]; 4],
    pub opacity: f32,
    pub image: TextureRef,
}

implement_uniforms! { Uniforms, matrix, opacity, image }

/// Basic 2D object.
pub struct Sprite<'display>
{
    pub id: Id,
    display: &'display Display,
    texture: TextureRef,
    width: f32,
    height: f32,
    opacity: f32,
    rect: Rect,
    mesh: Mesh<Vertex>,
    pub transform: Transform,
}

impl<'display> Sprite<'display>
{
    pub fn new(display: &'display Display, tex: Rc<Texture>, width: u32, height: u32)
        -> Sprite<'display>
    {
        let rect = Rect::new(0, 0, tex.width as i32, tex.height as i32);
        Sprite::with_rect(display, tex, rect, width, height)
    }

    /// Create a sprite with size and
    /// [texture atlas](https://en.wikipedia.org/wiki/Texture_atlas).
    pub fn with_rect(display: &'display Display, tex: Rc<Texture>,
                     rect: Rect, width: u32, height: u32)
        -> Sprite<'display>
    {
        let (w, h) = (width as f32, height as f32);
        Sprite
        {
            id: Id::new(),
            texture: TextureRef(tex),
            width: w,
            height: h,
            transform: Transform::new(),
            opacity: 1.0,
            mesh: Sprite::build_mesh(display, w, h, &rect),
            rect: rect,
            display: display,
        }
    }

    fn build_mesh(display: &Display, width: f32, height: f32, rect: &Rect)
        -> Mesh<Vertex>
    {
        let x = rect.x as f32;
        let y = rect.y as f32;
        let w = rect.width as f32;
        let h = rect.height as f32;
        let verties = [
            Vertex { position: [  0.0, height], texture_position: [x, y+h] },
            Vertex { position: [  0.0,    0.0], texture_position: [x, y] },
            Vertex { position: [width,    0.0], texture_position: [x+w, y] },
            Vertex { position: [width, height], texture_position: [x+w,y+h] },
        ];

        Mesh::with_indices(display, &verties, &[0, 1, 2, 2, 3, 0])
    }

    pub fn resize(&mut self, width: f32, height: f32)
    {
        self.mesh = Sprite::build_mesh(self.display, width, height, &self.rect);
    }

    /// Change the texture atlas rectangle.
    pub fn rect(&mut self, rect: Rect)
    {
        self.mesh = Sprite::build_mesh(self.display, self.width, self.height, &rect);
        self.rect = rect;
    }
}

impl<'a> Renderable<Shader> for Sprite<'a>
{
    fn draw(&self, renderer: &Renderer<Shader>, target: &mut Frame, parent: &Matrix4<f32>)
    {
        let uniforms = Uniforms
        {
            image: self.texture.clone(),
            opacity: self.opacity,
            matrix: (parent * self.transform.matrix()).into(),
        };
        renderer.draw(target, &self.mesh, &uniforms);
    }
}

<commit_msg>Remove `Display` point in `Sprite`<commit_after>//! Basic 2D object and shader.

use std::rc::Rc;
use glium::{ Display, Frame };
use texture::{ Texture, TextureRef };
use mesh::Mesh;
use shader::ShaderType;
use transform::Transform;
use renderer::{ Renderer, Renderable };
use rect::Rect;
use cgmath::Matrix4;
use id::Id;

/// 2D Sprite vertex type
#[derive(Copy, Clone)]
pub struct Vertex
{
    pub position: [f32; 2],
    pub texture_position: [f32; 2],
}

implement_vertex!{ Vertex, position, texture_position }

/// Simple sprite shader
pub struct Shader;

impl ShaderType for Shader
{
    type Vertex = Vertex;
    type Uniforms = Uniforms;

    fn vertex() -> &'static str
    {
        include_str!("sprite.vert")
    }

    fn fragment() -> &'static str
    {
        include_str!("sprite.frag")
    }
}

/// Sprite rendering uniforms.
pub struct Uniforms
{
    pub matrix: [[f32; 4]; 4],
    pub opacity: f32,
    pub image: TextureRef,
}

implement_uniforms! { Uniforms, matrix, opacity, image }

/// Basic 2D object.
pub struct Sprite
{
    pub id: Id,
    texture: TextureRef,
    width: f32,
    height: f32,
    opacity: f32,
    rect: Rect,
    mesh: Mesh<Vertex>,
    pub transform: Transform,
}

impl Sprite
{
    pub fn new(display: &Display, tex: Rc<Texture>, width: u32, height: u32)
        -> Sprite
    {
        let rect = Rect::new(0, 0, tex.width as i32, tex.height as i32);
        Sprite::with_rect(display, tex, rect, width, height)
    }

    /// Create a sprite with size and
    /// [texture atlas](https://en.wikipedia.org/wiki/Texture_atlas).
    pub fn with_rect(display: &Display, tex: Rc<Texture>,
                     rect: Rect, width: u32, height: u32)
        -> Sprite
    {
        let (w, h) = (width as f32, height as f32);
        Sprite
        {
            id: Id::new(),
            texture: TextureRef(tex),
            width: w,
            height: h,
            transform: Transform::new(),
            opacity: 1.0,
            mesh: Sprite::build_mesh(display, w, h, &rect),
            rect: rect,
        }
    }

    fn build_mesh(display: &Display, width: f32, height: f32, rect: &Rect)
        -> Mesh<Vertex>
    {
        let x = rect.x as f32;
        let y = rect.y as f32;
        let w = rect.width as f32;
        let h = rect.height as f32;
        let verties = [
            Vertex { position: [  0.0, height], texture_position: [  x, y+h] },
            Vertex { position: [  0.0,    0.0], texture_position: [  x,   y] },
            Vertex { position: [width,    0.0], texture_position: [x+w,   y] },
            Vertex { position: [width, height], texture_position: [x+w, y+h] },
        ];

        Mesh::with_indices(display, &verties, &[0, 1, 2, 2, 3, 0])
    }

    pub fn resize(&mut self, display: &Display, width: f32, height: f32)
    {
        self.mesh = Sprite::build_mesh(display, width, height, &self.rect);
    }

    /// Change the texture atlas rectangle.
    pub fn rect(&mut self, display: &Display, rect: Rect)
    {
        self.mesh = Sprite::build_mesh(display, self.width, self.height, &rect);
        self.rect = rect;
    }
}

impl Renderable<Shader> for Sprite
{
    fn draw(&self, renderer: &Renderer<Shader>, target: &mut Frame, parent: &Matrix4<f32>)
    {
        let uniforms = Uniforms
        {
            image: self.texture.clone(),
            opacity: self.opacity,
            matrix: (parent * self.transform.matrix()).into(),
        };
        renderer.draw(target, &self.mesh, &uniforms);
    }
}

<|endoftext|>