<commit_before>use std::borrow::Cow;
use std::char;
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::convert::TryFrom;
use std::env;
use std::fs::File;
use std::io::{self, BufWriter, Write};
use std::path::Path;

const RAW_DATA: &str = include_str!(concat!(
    env!("CARGO_MANIFEST_DIR"),
    "/pinyin-data/pinyin.txt"
));

const PHONETIC_SYMBOL_MAP: &[(char, char, u8)] = &[
    ('ā', 'a', 1),
    ('á', 'a', 2),
    ('ǎ', 'a', 3),
    ('à', 'a', 4),
    ('ē', 'e', 1),
    ('é', 'e', 2),
    ('ě', 'e', 3),
    ('è', 'e', 4),
    ('ế', 'ê', 2),
    ('ề', 'ê', 4),
    ('ō', 'o', 1),
    ('ó', 'o', 2),
    ('ǒ', 'o', 3),
    ('ò', 'o', 4),
    ('ī', 'i', 1),
    ('í', 'i', 2),
    ('ǐ', 'i', 3),
    ('ì', 'i', 4),
    ('ū', 'u', 1),
    ('ú', 'u', 2),
    ('ǔ', 'u', 3),
    ('ù', 'u', 4),
    ('ü', 'v', 0),
    ('ǘ', 'v', 2),
    ('ǚ', 'v', 3),
    ('ǜ', 'v', 4),
    ('ń', 'n', 2),
    ('ň', 'n', 3),
    ('ǹ', 'n', 4),
    ('ḿ', 'm', 2),
];

#[cfg(any(feature = "with_tone_num", feature = "with_tone_num_end"))]
const TONE_NUMS: &[char] = &['0', '1', '2', '3', '4'];
const TONE_MAP: &[(char, u8)] = &[('\u{304}', 1), ('\u{30c}', 3), ('\u{300}', 4)];
const NON_PINYIN_TONE: &[char] = &['ê'];

type Style = (&'static str, fn(&str) -> Cow<'_, str>);
type InputData = Vec<(u32, Vec<&'static str>)>;
type PinyinDataIndex = HashMap<&'static str, usize>;
type HeteronymDataIndex = HashMap<u32, usize>;

fn main() -> io::Result<()> {
    let data = build_data();
    let pinyin_index = generate_pinyin_data(&data)?;
    let heteronym_index = generate_heteronym_table(&data, &pinyin_index)?;
    generate_char_table(&data, &pinyin_index, &heteronym_index)?;
    // 输出这行以保证改动项目的其他文件不会触发编译脚本重新执行
    println!("cargo:rerun-if-changed=build.rs");
    Ok(())
}

fn build_data() -> InputData {
    let mut input_data = RAW_DATA
        .lines()
        .enumerate()
        // 移除注释和空格
        .map(|(i, mut line)| {
            if let Some(hash_pos) = line.find('#') {
                line = &line[..hash_pos];
            }
            (i, line.trim())
        })
        // 移除空行
        .filter(|(_, line)| !line.is_empty())
        .map(|(i, line)| {
            // Split the line by colon
            let colon_pos = match line.find(':') {
                Some(pos) => pos,
                None => unreachable!("no colon found in line {}", i),
            };
            let code_point = line[..colon_pos].trim();
            let pinyin = line[colon_pos + 1..].trim();
            // 检查我们理解拼音数据中的每个字符
            for ch in pinyin.chars() {
                let is_known = ch.is_ascii()
                    || PHONETIC_SYMBOL_MAP.iter().any(|(c, _, _)| *c == ch)
                    || TONE_MAP.iter().any(|(c, _)| *c == ch)
                    || NON_PINYIN_TONE.iter().any(|c| *c == ch);
                assert!(
                    is_known,
                    "unknown character {:?} at line {}: {}",
                    ch, i, line,
                );
            }
            // 解析码位
            const CODE_POINT_PREFIX: &str = "U+";
            assert!(code_point.starts_with(CODE_POINT_PREFIX));
            let code = &code_point[CODE_POINT_PREFIX.len()..];
            let code = match u32::from_str_radix(code, 16) {
                Ok(code) => code,
                Err(_) => unreachable!("invalid code point {} at line {}", code, i),
            };
            (code, pinyin.split(',').collect())
        })
        .collect::<Vec<_>>();
    input_data.sort_by_key(|(code, _)| *code);
    input_data
}

const STYLES: &[Style] = &[
    #[cfg(feature = "plain")]
    ("plain", |input| {
        input
            .chars()
            .map(|c| match get_phonetic_info(c) {
                Some((base, _)) => base,
                None => c,
            })
            .collect()
    }),
    #[cfg(feature = "with_tone")]
    ("with_tone", |input| Cow::from(input)),
    #[cfg(feature = "with_tone_num")]
    ("with_tone_num", |input| {
        let mut result = String::new();
        for ch in input.chars() {
            match get_phonetic_info(ch) {
                Some((base, tone)) => {
                    result.push(base);
                    if tone > 0 {
                        result.push(TONE_NUMS[usize::try_from(tone).unwrap()]);
                    }
                }
                None => result.push(ch),
            }
        }
        result.into()
    }),
    #[cfg(feature = "with_tone_num_end")]
    ("with_tone_num_end", |input| {
        let mut result = String::new();
        let mut output_tone = None;
        for ch in input.chars() {
            match get_phonetic_info(ch) {
                Some((base, tone)) => {
                    result.push(base);
                    if tone > 0 {
                        assert!(output_tone.is_none());
                        output_tone = Some(TONE_NUMS[usize::try_from(tone).unwrap()]);
                    }
                }
                None => result.push(ch),
            }
        }
        if let Some(tone) = output_tone {
            result.push(tone);
        }
        result.into()
    }),
];

fn generate_pinyin_data(data: &InputData) -> io::Result<PinyinDataIndex> {
    let mut output = create_out_file("pinyin_data.rs")?;
    let mut pinyin_data = HashMap::new();
    writeln!(output, "&[")?;
    let mut process_pinyin = |pinyin| {
        let index = pinyin_data.len();
        match pinyin_data.entry(pinyin) {
            Entry::Occupied(_) => return Ok(()),
            Entry::Vacant(entry) => {
                entry.insert(index);
            }
        }
        write!(output, "    PinyinData {{ ")?;
        for (field, converter) in STYLES.iter() {
            write!(output, r#"{}: "{}", "#, field, converter(pinyin))?;
        }
        #[cfg(feature = "compat")]
        {
            // 计算切分声母和韵母的位置
            const INITIALS: &[&str] = &[
                "b", "p", "m", "f", "d", "t", "n", "l", "g", "k", "h", "j", "q", "x", "r", "zh",
                "ch", "sh", "z", "c", "s",
            ];
            let split = INITIALS
                .iter()
                .find(|initial| pinyin.starts_with(*initial))
                .map_or(0, |initial| initial.len());
            write!(output, "split: {}, ", split)?;
        }
        writeln!(output, "}},")?;
        Ok(())
    };
    // 插入一个空的拼音数据作为零位
    process_pinyin("")?;
    data.iter()
        .flat_map(|(_, list)| list.iter().map(|s| *s))
        .map(process_pinyin)
        .collect::<io::Result<()>>()?;
    writeln!(output, "]")?;
    Ok(pinyin_data)
}

#[cfg(feature = "heteronym")]
fn generate_heteronym_table(
    data: &InputData,
    index: &PinyinDataIndex,
) -> io::Result<HeteronymDataIndex> {
    let mut heteronym_list_index = HashMap::new();
    let mut heteronym_index = HashMap::new();
    let mut output = create_out_file("heteronym_table.rs")?;
    writeln!(output, "&[")?;
    writeln!(output, "    &[],")?;
    heteronym_list_index.insert(vec![].into_boxed_slice(), 0);
    data.iter()
        .map(|(code, list)| {
            let list = list[1..]
                .iter()
                .map(|pinyin| *index.get(pinyin).unwrap())
                .collect::<Box<[_]>>();
            let new_idx = heteronym_list_index.len();
            let idx = match heteronym_list_index.entry(list) {
                Entry::Occupied(entry) => *entry.get(),
                Entry::Vacant(entry) => {
                    write!(output, "    &[")?;
                    for i in entry.key().iter() {
                        write!(output, "{}, ", i)?;
                    }
                    writeln!(output, "],")?;
                    entry.insert(new_idx);
                    new_idx
                }
            };
            heteronym_index.insert(*code, idx);
            Ok(())
        })
        .collect::<io::Result<()>>()?;
    writeln!(output, "]")?;
    Ok(heteronym_index)
}

#[cfg(not(feature = "heteronym"))]
fn generate_heteronym_table(
    _data: &InputData,
    _index: &PinyinDataIndex,
) -> io::Result<HeteronymDataIndex> {
    Ok(HashMap::new())
}

fn generate_char_table(
    data: &InputData,
    pinyin_index: &PinyinDataIndex,
    heteronym_index: &HeteronymDataIndex,
) -> io::Result<()> {
    // 将码位分入数据块
    const GAP_THRESHOLD: u32 = 2048;
    let mut block_ranges = vec![];
    data.iter()
        .for_each(|(code, _)| match block_ranges.last_mut() {
            Some((_, end)) if *end + GAP_THRESHOLD > *code => *end = *code + 1,
            _ => block_ranges.push((*code, *code + 1)),
        });
    // 当我们允许最大2048个空位时，我们目前会切出3个块。如果这个数字在未来增加了，我们也许会希望调整策略。
    assert_eq!(block_ranges.len(), 3);

    // 输出字符表
    let mut output = create_out_file("char_blocks.rs")?;
    writeln!(output, "&[")?;
    let mut data_iter = data.iter().peekable();
    for (start, end) in block_ranges {
        let len = usize::try_from(end - start).unwrap();
        let mut block = vec![0; len];
        let mut heteronym = vec![0; len];
        while let Some((code, list)) = data_iter.peek() {
            let idx = usize::try_from(*code - start).unwrap();
            if idx >= len {
                break;
            }
            block[idx] = *pinyin_index.get(list[0]).unwrap();
            if cfg!(feature = "heteronym") {
                heteronym[idx] = *heteronym_index.get(code).unwrap();
            }
            data_iter.next();
        }
        write!(output, "    CharBlock {{ start_code: {}, data: &[", start)?;
        for idx in block {
            write!(output, "{}, ", idx)?;
        }
        write!(output, "], ")?;
        if cfg!(feature = "heteronym") {
            write!(output, "heteronym: &[")?;
            for idx in heteronym {
                write!(output, "{}, ", idx)?;
            }
            write!(output, "], ")?;
        }
        writeln!(output, "}},")?;
    }
    writeln!(output, "]")?;
    Ok(())
}

fn create_out_file(name: &str) -> io::Result<impl Write> {
    let path = Path::new(&env::var("OUT_DIR").unwrap()).join(name);
    Ok(BufWriter::new(File::create(&path)?))
}

#[cfg(any(
    feature = "plain",
    feature = "with_tone_num",
    feature = "with_tone_num_end"
))]
fn get_phonetic_info(ch: char) -> Option<(char, u8)> {
    PHONETIC_SYMBOL_MAP
        .iter()
        .find(|(c, _, _)| *c == ch)
        .map(|(_, base, tone)| (*base, *tone))
}
<commit_msg>确保音节数据的字符全部在我们预料之中<commit_after>use std::borrow::Cow;
use std::char;
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::convert::TryFrom;
use std::env;
use std::fs::File;
use std::io::{self, BufWriter, Write};
use std::path::Path;

const RAW_DATA: &str = include_str!(concat!(
    env!("CARGO_MANIFEST_DIR"),
    "/pinyin-data/pinyin.txt"
));

const PHONETIC_SYMBOL_MAP: &[(char, char, u8)] = &[
    ('ā', 'a', 1),
    ('á', 'a', 2),
    ('ǎ', 'a', 3),
    ('à', 'a', 4),
    ('ē', 'e', 1),
    ('é', 'e', 2),
    ('ě', 'e', 3),
    ('è', 'e', 4),
    ('ế', 'ê', 2),
    ('ề', 'ê', 4),
    ('ō', 'o', 1),
    ('ó', 'o', 2),
    ('ǒ', 'o', 3),
    ('ò', 'o', 4),
    ('ī', 'i', 1),
    ('í', 'i', 2),
    ('ǐ', 'i', 3),
    ('ì', 'i', 4),
    ('ū', 'u', 1),
    ('ú', 'u', 2),
    ('ǔ', 'u', 3),
    ('ù', 'u', 4),
    ('ü', 'v', 0),
    ('ǘ', 'v', 2),
    ('ǚ', 'v', 3),
    ('ǜ', 'v', 4),
    ('ń', 'n', 2),
    ('ň', 'n', 3),
    ('ǹ', 'n', 4),
    ('ḿ', 'm', 2),
];

#[rustfmt::skip]
const LETTER_TABLE: &[char] = &[
    'b', 'p', 'm', 'f', 'd',
    't', 'n', 'l', 'g', 'k',
    'h', 'j', 'q', 'x', 'r',
    'z', 'c', 's',
                   'w', 'y',
    // 因为数据源里面不会使用 `v` 以及其它的 简写字母，所以这里注释掉
    // 'v', 'ẑ', 'ĉ', 'ŝ', 'ŋ',

    '\u{0304}', '\u{030C}', '\u{0300}', // Unicode 声调连字符
    'a', 'ā', 'á', 'ǎ', 'à',
    'e', 'ē', 'é', 'ě', 'è',
    'i', 'ī', 'í', 'ǐ', 'ì',
    //   "m̄"       "m̌"  "m̀"
    'm',      'ḿ',
    //   "n̄"
    'n',      'ń', 'ň', 'ǹ',
    'o', 'ō', 'ó', 'ǒ', 'ò',
    'u', 'ū', 'ú', 'ǔ', 'ù',
    //   "ê̄"       "ê̌"
    'ê',      'ế',      'ề',
    //   'ǖ'
    'ü',      'ǘ', 'ǚ', 'ǜ',
];

#[cfg(any(feature = "with_tone_num", feature = "with_tone_num_end"))]
const TONE_NUMS: &[char] = &['0', '1', '2', '3', '4'];

type Style = (&'static str, fn(&str) -> Cow<'_, str>);
type InputData = Vec<(u32, Vec<&'static str>)>;
type PinyinDataIndex = HashMap<&'static str, usize>;
type HeteronymDataIndex = HashMap<u32, usize>;

fn main() -> io::Result<()> {
    let data = build_data();
    let pinyin_index = generate_pinyin_data(&data)?;
    let heteronym_index = generate_heteronym_table(&data, &pinyin_index)?;
    generate_char_table(&data, &pinyin_index, &heteronym_index)?;
    // 输出这行以保证改动项目的其他文件不会触发编译脚本重新执行
    println!("cargo:rerun-if-changed=build.rs");
    Ok(())
}

fn build_data() -> InputData {
    let mut input_data = RAW_DATA
        .lines()
        .enumerate()
        // 移除注释和空格
        .map(|(i, mut line)| {
            if let Some(hash_pos) = line.find('#') {
                line = &line[..hash_pos];
            }
            (i, line.trim())
        })
        // 移除空行
        .filter(|(_, line)| !line.is_empty())
        .map(|(i, line)| {
            // Split the line by colon
            let colon_pos = match line.find(':') {
                Some(pos) => pos,
                None => unreachable!("no colon found in line {}", i),
            };
            let code_point = line[..colon_pos].trim();
            let pinyin_list: Vec<_> = line[colon_pos + 1..].trim().split(',').collect();

            // 确保输入数据的字符全部在我们预料之中。
            // 同时也可以提前知道一些被遗弃的码位，如: U+E7C8 和 U+E7C7
            for pinyin in pinyin_list.iter() {
                for ch in pinyin.chars() {
                    let is_known = LETTER_TABLE.contains(&ch);
                    assert!(
                        is_known,
                        "unknown character {:?} at line {}: {}",
                        ch, i, line,
                    );
                }
            }

            // 解析码位
            const CODE_POINT_PREFIX: &str = "U+";
            assert!(code_point.starts_with(CODE_POINT_PREFIX));
            let code = &code_point[CODE_POINT_PREFIX.len()..];
            let code = match u32::from_str_radix(code, 16) {
                Ok(code) => code,
                Err(_) => unreachable!("invalid code point {} at line {}", code, i),
            };
            (code, pinyin_list)
        })
        .collect::<Vec<_>>();
    input_data.sort_by_key(|(code, _)| *code);
    input_data
}

const STYLES: &[Style] = &[
    #[cfg(feature = "plain")]
    ("plain", |input| {
        input
            .chars()
            .map(|c| match get_phonetic_info(c) {
                Some((base, _)) => base,
                None => c,
            })
            .collect()
    }),
    #[cfg(feature = "with_tone")]
    ("with_tone", |input| Cow::from(input)),
    #[cfg(feature = "with_tone_num")]
    ("with_tone_num", |input| {
        let mut result = String::new();
        for ch in input.chars() {
            match get_phonetic_info(ch) {
                Some((base, tone)) => {
                    result.push(base);
                    if tone > 0 {
                        result.push(TONE_NUMS[usize::try_from(tone).unwrap()]);
                    }
                }
                None => result.push(ch),
            }
        }
        result.into()
    }),
    #[cfg(feature = "with_tone_num_end")]
    ("with_tone_num_end", |input| {
        let mut result = String::new();
        let mut output_tone = None;
        for ch in input.chars() {
            match get_phonetic_info(ch) {
                Some((base, tone)) => {
                    result.push(base);
                    if tone > 0 {
                        assert!(output_tone.is_none());
                        output_tone = Some(TONE_NUMS[usize::try_from(tone).unwrap()]);
                    }
                }
                None => result.push(ch),
            }
        }
        if let Some(tone) = output_tone {
            result.push(tone);
        }
        result.into()
    }),
];

fn generate_pinyin_data(data: &InputData) -> io::Result<PinyinDataIndex> {
    let mut output = create_out_file("pinyin_data.rs")?;
    let mut pinyin_data = HashMap::new();
    writeln!(output, "&[")?;
    let mut process_pinyin = |pinyin| {
        let index = pinyin_data.len();
        match pinyin_data.entry(pinyin) {
            Entry::Occupied(_) => return Ok(()),
            Entry::Vacant(entry) => {
                entry.insert(index);
            }
        }
        write!(output, "    PinyinData {{ ")?;
        for (field, converter) in STYLES.iter() {
            write!(output, r#"{}: "{}", "#, field, converter(pinyin))?;
        }
        #[cfg(feature = "compat")]
        {
            // 计算切分声母和韵母的位置
            const INITIALS: &[&str] = &[
                "b", "p", "m", "f", "d", "t", "n", "l", "g", "k", "h", "j", "q", "x", "r", "zh",
                "ch", "sh", "z", "c", "s",
            ];
            let split = INITIALS
                .iter()
                .find(|initial| pinyin.starts_with(*initial))
                .map_or(0, |initial| initial.len());
            write!(output, "split: {}, ", split)?;
        }
        writeln!(output, "}},")?;
        Ok(())
    };
    // 插入一个空的拼音数据作为零位
    process_pinyin("")?;
    data.iter()
        .flat_map(|(_, list)| list.iter().map(|s| *s))
        .map(process_pinyin)
        .collect::<io::Result<()>>()?;
    writeln!(output, "]")?;
    Ok(pinyin_data)
}

#[cfg(feature = "heteronym")]
fn generate_heteronym_table(
    data: &InputData,
    index: &PinyinDataIndex,
) -> io::Result<HeteronymDataIndex> {
    let mut heteronym_list_index = HashMap::new();
    let mut heteronym_index = HashMap::new();
    let mut output = create_out_file("heteronym_table.rs")?;
    writeln!(output, "&[")?;
    writeln!(output, "    &[],")?;
    heteronym_list_index.insert(vec![].into_boxed_slice(), 0);
    data.iter()
        .map(|(code, list)| {
            let list = list[1..]
                .iter()
                .map(|pinyin| *index.get(pinyin).unwrap())
                .collect::<Box<[_]>>();
            let new_idx = heteronym_list_index.len();
            let idx = match heteronym_list_index.entry(list) {
                Entry::Occupied(entry) => *entry.get(),
                Entry::Vacant(entry) => {
                    write!(output, "    &[")?;
                    for i in entry.key().iter() {
                        write!(output, "{}, ", i)?;
                    }
                    writeln!(output, "],")?;
                    entry.insert(new_idx);
                    new_idx
                }
            };
            heteronym_index.insert(*code, idx);
            Ok(())
        })
        .collect::<io::Result<()>>()?;
    writeln!(output, "]")?;
    Ok(heteronym_index)
}

#[cfg(not(feature = "heteronym"))]
fn generate_heteronym_table(
    _data: &InputData,
    _index: &PinyinDataIndex,
) -> io::Result<HeteronymDataIndex> {
    Ok(HashMap::new())
}

fn generate_char_table(
    data: &InputData,
    pinyin_index: &PinyinDataIndex,
    heteronym_index: &HeteronymDataIndex,
) -> io::Result<()> {
    // 将码位分入数据块
    const GAP_THRESHOLD: u32 = 2048;
    let mut block_ranges = vec![];
    data.iter()
        .for_each(|(code, _)| match block_ranges.last_mut() {
            Some((_, end)) if *end + GAP_THRESHOLD > *code => *end = *code + 1,
            _ => block_ranges.push((*code, *code + 1)),
        });
    // 当我们允许最大2048个空位时，我们目前会切出3个块。如果这个数字在未来增加了，我们也许会希望调整策略。
    assert_eq!(block_ranges.len(), 3);

    // 输出字符表
    let mut output = create_out_file("char_blocks.rs")?;
    writeln!(output, "&[")?;
    let mut data_iter = data.iter().peekable();
    for (start, end) in block_ranges {
        let len = usize::try_from(end - start).unwrap();
        let mut block = vec![0; len];
        let mut heteronym = vec![0; len];
        while let Some((code, list)) = data_iter.peek() {
            let idx = usize::try_from(*code - start).unwrap();
            if idx >= len {
                break;
            }
            block[idx] = *pinyin_index.get(list[0]).unwrap();
            if cfg!(feature = "heteronym") {
                heteronym[idx] = *heteronym_index.get(code).unwrap();
            }
            data_iter.next();
        }
        write!(output, "    CharBlock {{ start_code: {}, data: &[", start)?;
        for idx in block {
            write!(output, "{}, ", idx)?;
        }
        write!(output, "], ")?;
        if cfg!(feature = "heteronym") {
            write!(output, "heteronym: &[")?;
            for idx in heteronym {
                write!(output, "{}, ", idx)?;
            }
            write!(output, "], ")?;
        }
        writeln!(output, "}},")?;
    }
    writeln!(output, "]")?;
    Ok(())
}

fn create_out_file(name: &str) -> io::Result<impl Write> {
    let path = Path::new(&env::var("OUT_DIR").unwrap()).join(name);
    Ok(BufWriter::new(File::create(&path)?))
}

#[cfg(any(
    feature = "plain",
    feature = "with_tone_num",
    feature = "with_tone_num_end"
))]
fn get_phonetic_info(ch: char) -> Option<(char, u8)> {
    PHONETIC_SYMBOL_MAP
        .iter()
        .find(|(c, _, _)| *c == ch)
        .map(|(_, base, tone)| (*base, *tone))
}
<|endoftext|>


<commit_before>use value::{Value};
use value;
use self::EveFn::*;
use value::Value::Float;

// Enums...
// Expression Enum ------------------------------------------------------------
#[derive(Clone, Debug)]
pub enum Expression {
	Constant(Constant),
	Variable(Variable),
	Call(Call),
	Match(Box<Match>),
	Value(Value),
}

// End Expression Enum --------------------------------------------------------

#[derive(Clone,Debug)]
pub enum EveFn {
	// Basic ops
	Add,Subtract,Multiply,Divide,Exponentiate,

	// General math
	Sqrt,Log,Log10,Log2,Ln,Abs,Sign,Exp,

	//Trig
	Sin,Cos,Tan,ASin,ACos,ATan,ATan2,

	// Aggregates
	Sum,Prod,

	// Strings
	StrConcat,StrUpper,StrLower,StrLength,StrReplace,StrSplit,
}

#[derive(Clone,Debug)]
pub struct Match {
	pub input: Expression,
	pub patterns: PatternVec,
	pub handlers: ExpressionVec,
}

// Begin Pattern Enum ---------------------------------------------------------
#[derive(Clone,Debug)]
pub enum Pattern {
	Constant(Constant),
	Variable(Variable),
	Tuple(PatternVec),
}

impl<'a> PartialEq<Value> for &'a Pattern {
	fn eq(&self, other: &Value) -> bool {
		match self {
			&&Pattern::Constant(ref x) => x == other,
			&&Pattern::Variable(_) => panic!("Cannot match Variable against Value"),
			&&Pattern::Tuple(_) => panic!("Cannot match Tuple against Value"),
		}
	}
}
// End Pattern Enum -----------------------------------------------------------

#[derive(Clone,Debug)]
pub struct Call {
	pub fun: EveFn,
	pub args: ExpressionVec,
}

// Some type aliases
pub type Constant = Value;
pub type Variable = String;
pub type PatternVec = Vec<Pattern>;
pub type ExpressionVec = Vec<Expression>;

// This is the main interface to the interpreter. Pass in an expression, get
// back a value
pub fn evaluate(e: & Expression) -> Value {

	eval_expression(e)

}

fn eval_expression(e: &Expression) -> Value {

	match *e {
		Expression::Call(ref x) => eval_call(x),
		Expression::Constant(ref x) => x.clone(),
		Expression::Value(ref x) => x.clone(),
		Expression::Match(ref x) => eval_match(x),
		_ => unimplemented!(),
	}
}

fn eval_match(m: &Match) -> Value {

	// Before we do anything, make sure we have the same number of patterns and
	// handlers
	assert_eq!(m.patterns.len(),m.handlers.len());

	let input = eval_expression(&m.input);
	let tests: Vec<Value> = m.patterns.iter()
						  			  .zip(m.handlers.iter())
						  			  .filter_map(|(pattern,handler)| -> Option<Value> {
													if pattern == input { Some(eval_expression(&handler)) }
													else { None }
												})
						  			  .take(1)
						  			  .collect();

	Value::Tuple(tests)
}

fn eval_call(c: &Call) -> Value {

	let args: Vec<Value> = c.args.iter().map(eval_expression).collect();

	match(&c.fun, &args[..]) {

		// Basic Math
		(&Add,[Float(x),Float(y)]) => Float(x+y),
		(&Subtract,[Float(x),Float(y)]) => Float(x-y),
		(&Multiply,[Float(x),Float(y)]) => Float(x*y),
		(&Divide,[Float(x),Float(y)]) => {
			match (x,y) {
				(_,0f64) => panic!("Error: Division by 0"),
				(x,y) => Float(x/y),
			}
		},
		(&Exponentiate,[Float(x),Float(y)]) => Float(x.powf(y)),

		// Some general math functions
		(&Abs,[Float(x)]) => Float(x.abs()),
		(&Sqrt,[Float(x)]) => Float(x.sqrt()),
		(&Sign,[Float(x)]) => Float(x.signum()),
		(&Exp,[Float(x)]) => Float(x.exp()),
		(&Ln,[Float(x)]) => {
			if x <= 0f64 {panic!("Error: ln(x<=0) is undefined")}
			else {Float(x.ln())}
		},
		(&Log10,[Float(x)]) => {
			if x <= 0f64 {panic!("Error: log10(x<=0) is undefined")}
			else {Float(x.log10())}
		},
		(&Log2,[Float(x)]) => {
			if x <= 0f64 {panic!("Error: log2(x<=0) is undefined")}
			else {Float(x.log2())}
		},

		// Trig functions
		(&Sin,[Float(x)]) => Float(x.sin()),
		(&Cos,[Float(x)]) => Float(x.cos()),
		(&Tan,[Float(x)]) => Float(x.tan()),
		(&ASin,[Float(x)]) => Float(x.asin()),
		(&ACos,[Float(x)]) => Float(x.acos()),
		(&ATan,[Float(x)]) => Float(x.atan()),
		(&ATan2,[Float(x),Float(y)]) => Float(x.atan2(y)),

		// String functions
		(&StrConcat,[Value::String(ref s1),Value::String(ref s2)]) => Value::String(s1.to_string()+&s2[..]),
		(&StrUpper,[Value::String(ref s)]) => Value::String(s.to_uppercase()),
		(&StrLower,[Value::String(ref s)]) => Value::String(s.to_lowercase()),
		(&StrLength,[Value::String(ref s)]) => Float(s.len() as f64),
		(&StrReplace,[Value::String(ref s),Value::String(ref q),Value::String(ref r)]) => Value::String(s.replace(&q[..],&r[..])),
		(&StrSplit,[Value::String(ref s)]) => {
			let w: Vec<Value> = s.words().map(|x| Value::String(x.to_string())).collect();
			Value::Tuple(w)
		},

		// Aggregate functions
		(&Sum,[Value::Tuple(ref x)]) => general_agg(x),
		//&Prod => general_agg(|x,y|{x*y},1f64,&c.args),

		// Returns an empty string for the purpose of handling incomplete function
		(_, _) => Value::String("No Result".to_string()),
	}
}

// Aggregate Functions --------------------------------------------------------

//fn general_agg<F: Fn(f64,f64) -> f64>(f: F, base: f64, args: &ExpressionVec) -> Value {
fn general_agg(x: &value::Tuple) -> Value {

	// Some fold magic!
	println!("{:?}",x);

	Value::Float(10f64)

}
// End Aggregate Functions ----------------------------------------------------
<commit_msg>Remove to_str from interpreter<commit_after>use value::*;
use value::Value::Float;
use self::EveFn::*;

// Enums...
// Expression Enum ------------------------------------------------------------
#[derive(Clone, Debug)]
pub enum Expression {
	Constant(Constant),
	Variable(Variable),
	Call(Call),
	Match(Box<Match>),
	Value(Value),
}

// End Expression Enum --------------------------------------------------------

#[derive(Clone,Debug)]
pub enum EveFn {
	// Basic ops
	Add,Subtract,Multiply,Divide,Exponentiate,

	// General math
	Sqrt,Log,Log10,Log2,Ln,Abs,Sign,Exp,

	//Trig
	Sin,Cos,Tan,ASin,ACos,ATan,ATan2,

	// Aggregates
	Sum,Prod,

	// Strings
	StrConcat,StrUpper,StrLower,StrLength,StrReplace,StrSplit,
}

#[derive(Clone,Debug)]
pub struct Match {
	pub input: Expression,
	pub patterns: PatternVec,
	pub handlers: ExpressionVec,
}

// Begin Pattern Enum ---------------------------------------------------------
#[derive(Clone,Debug)]
pub enum Pattern {
	Constant(Constant),
	Variable(Variable),
	Tuple(PatternVec),
}

impl<'a> PartialEq<Value> for &'a Pattern {
	fn eq(&self, other: &Value) -> bool {
		match self {
			&&Pattern::Constant(ref x) => x == other,
			&&Pattern::Variable(_) => panic!("Cannot match Variable against Value"),
			&&Pattern::Tuple(_) => panic!("Cannot match Tuple against Value"),
		}
	}
}
// End Pattern Enum -----------------------------------------------------------

#[derive(Clone,Debug)]
pub struct Call {
	pub fun: EveFn,
	pub args: ExpressionVec,
}

// Some type aliases
pub type Constant = Value;
pub type Variable = String;
pub type PatternVec = Vec<Pattern>;
pub type ExpressionVec = Vec<Expression>;

// This is the main interface to the interpreter. Pass in an expression, get
// back a value
pub fn evaluate(e: & Expression) -> Value {

	eval_expression(e)

}

fn eval_expression(e: &Expression) -> Value {

	match *e {
		Expression::Call(ref x) => eval_call(x),
		Expression::Constant(ref x) => x.clone(),
		Expression::Value(ref x) => x.clone(),
		Expression::Match(ref x) => eval_match(x),
		_ => unimplemented!(),
	}
}

fn eval_match(m: &Match) -> Value {

	// Before we do anything, make sure we have the same number of patterns and
	// handlers
	assert_eq!(m.patterns.len(),m.handlers.len());

	let input = eval_expression(&m.input);
	let tests: Vec<Value> = m.patterns.iter()
						  			  .zip(m.handlers.iter())
						  			  .filter_map(|(pattern,handler)| -> Option<Value> {
													if pattern == input { Some(eval_expression(&handler)) }
													else { None }
												})
						  			  .take(1)
						  			  .collect();

	Value::Tuple(tests)
}

fn eval_call(c: &Call) -> Value {

	let args: Vec<Value> = c.args.iter().map(eval_expression).collect();

	match(&c.fun, &args[..]) {

		// Basic Math
		(&Add,[Float(x),Float(y)]) => Float(x+y),
		(&Subtract,[Float(x),Float(y)]) => Float(x-y),
		(&Multiply,[Float(x),Float(y)]) => Float(x*y),
		(&Divide,[Float(x),Float(y)]) => {
			match (x,y) {
				(_,0f64) => panic!("Error: Division by 0"),
				(x,y) => Float(x/y),
			}
		},
		(&Exponentiate,[Float(x),Float(y)]) => Float(x.powf(y)),

		// Some general math functions
		(&Abs,[Float(x)]) => Float(x.abs()),
		(&Sqrt,[Float(x)]) => Float(x.sqrt()),
		(&Sign,[Float(x)]) => Float(x.signum()),
		(&Exp,[Float(x)]) => Float(x.exp()),
		(&Ln,[Float(x)]) => {
			if x <= 0f64 {panic!("Error: ln(x<=0) is undefined")}
			else {Float(x.ln())}
		},
		(&Log10,[Float(x)]) => {
			if x <= 0f64 {panic!("Error: log10(x<=0) is undefined")}
			else {Float(x.log10())}
		},
		(&Log2,[Float(x)]) => {
			if x <= 0f64 {panic!("Error: log2(x<=0) is undefined")}
			else {Float(x.log2())}
		},

		// Trig functions
		(&Sin,[Float(x)]) => Float(x.sin()),
		(&Cos,[Float(x)]) => Float(x.cos()),
		(&Tan,[Float(x)]) => Float(x.tan()),
		(&ASin,[Float(x)]) => Float(x.asin()),
		(&ACos,[Float(x)]) => Float(x.acos()),
		(&ATan,[Float(x)]) => Float(x.atan()),
		(&ATan2,[Float(x),Float(y)]) => Float(x.atan2(y)),

		// String functions
		(&StrConcat,[Value::String(ref s1),Value::String(ref s2)]) => Value::String(s1.to_string()+&s2[..]),
		(&StrUpper,[Value::String(ref s)]) => Value::String(s.to_uppercase()),
		(&StrLower,[Value::String(ref s)]) => Value::String(s.to_lowercase()),
		(&StrLength,[Value::String(ref s)]) => Float(s.len() as f64),
		(&StrReplace,[Value::String(ref s),Value::String(ref q),Value::String(ref r)]) => Value::String(s.replace(&q[..],&r[..])),
		(&StrSplit,[Value::String(ref s)]) => {
			let w: Vec<Value> = s.words().map(|x| Value::String(String::from_str(x))).collect();
			Value::Tuple(w)
		},

		/*
		// Aggregate functions
		(&Sum,[Value::Tuple(ref x)]) => {
			Value::Float(x.iter().fold(0f64, |acc: f64, ref item| {
				match item {
					&&Value::Float(ref y) => acc+y,
					x => panic!("Cannot aggregate {:?}",x),
				}
			}))
		},
		(&Prod,[Value::Tuple(ref x)]) => {
			Value::Float(x.iter().fold(1f64, |acc: f64, ref item| {
				match item {
					&&Value::Float(ref y) => acc*y,
					x => panic!("Cannot aggregate {:?}",x),
				}
			}))
		},
		*/

		// Returns an empty string for the purpose of handling incomplete function
		(_, _) => Value::String(String::from_str("Could not match with any function")),
	}
}<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013-2014, David Renshaw (dwrenshaw@gmail.com)
 *
 * See the LICENSE file in the capnproto-rust root directory.
 */

use std;
use common::*;
use arena::*;
use layout;

pub struct ReaderOptions {
    traversalLimitInWords : u64,
    nestingLimit : uint
}

pub static DEFAULT_READER_OPTIONS : ReaderOptions =
    ReaderOptions { traversalLimitInWords : 8 * 1024 * 1024, nestingLimit : 64 };

pub struct MessageReader<'a> {
    segments : &'a [ &'a [Word]],
    options : ReaderOptions,
    arena : ReaderArena
}

type SegmentId = u32;

impl <'a> MessageReader<'a> {

    #[inline]
    pub fn get_options<'b>(&'b self) -> &'b ReaderOptions {
        return &self.options;
    }
}

impl <'a> MessageReader<'a> {

    pub fn new<'b>(segments : &'b [&'b [Word]], options : ReaderOptions) -> ~MessageReader<'b> {

        assert!(segments.len() > 0);
        let mut result = ~MessageReader {
            segments : segments,
            arena : ReaderArena {
                segment0 : SegmentReader {
                    arena : Null,
                    ptr : unsafe { segments[0].unsafe_ref(0) },
                    size : segments[0].len()
                },
                more_segments : None
            },
            options : options
        };

        let arena_ptr = ReaderArenaPtr (std::ptr::to_unsafe_ptr(&result.arena));

        result.arena.segment0.arena = arena_ptr;

        if segments.len() > 1 {
            let mut moreSegmentReaders = ~[];
            for segment in segments.slice_from(1).iter() {
                let segmentReader = SegmentReader {
                    arena : arena_ptr,
                    ptr : unsafe { segment.unsafe_ref(0) },
                    size : segment.len()
                };
                moreSegmentReaders.push(segmentReader);
            }
            result.arena.more_segments = Some(moreSegmentReaders);
        }

        result

    }

    pub fn get_root<T : layout::FromStructReader<'a>>(&self) -> T {
        unsafe {
            let segment : *SegmentReader = std::ptr::to_unsafe_ptr(&self.arena.segment0);

            let pointer_reader = layout::PointerReader::get_root::<'a>(
                segment, (*segment).get_start_ptr(), self.options.nestingLimit as int);

            let result : T = layout::FromStructReader::from_struct_reader(
                pointer_reader.get_struct::<'a>(std::ptr::null()));

            result
        }
    }
}

pub enum AllocationStrategy {
    FIXED_SIZE,
    GROW_HEURISTICALLY
}

pub static SUGGESTED_FIRST_SEGMENT_WORDS : uint = 1024;
pub static SUGGESTED_ALLOCATION_STRATEGY : AllocationStrategy = GROW_HEURISTICALLY;

pub struct MessageBuilder {
    nextSize : uint,
    allocation_strategy : AllocationStrategy,
    arena : ~BuilderArena,
    own_first_segment : bool,
    first_segment : *mut Word,
    more_segments : Option<~[*mut Word]>
}

impl Drop for MessageBuilder {
    fn drop(&mut self) {
        if self.own_first_segment {
            unsafe { std::libc::free(std::cast::transmute(self.first_segment)) }
        } else {
            self.get_segments_for_output(|segments| {
                    unsafe {
                        std::ptr::zero_memory(self.first_segment, segments[0].len());
                    }
                });
        }

        match self.more_segments {
            None => {},
            Some(ref mut segs) => {
                for &segment_ptr in segs.iter() {
                    unsafe { std::libc::free(std::cast::transmute(segment_ptr)); }
                }
            }
        }
    }
}

pub enum FirstSegment<'a> {
    NumWords(uint),
    ZeroedWords(&'a mut [Word])
}

impl MessageBuilder {

    // TODO: maybe when Rust issue #5121 is fixed we can safely get away with not passing
    //  a closure here.
    pub fn new<'a, T>(first_segment_arg : FirstSegment<'a>,
                  allocationStrategy : AllocationStrategy,
                  cont : |&mut MessageBuilder| -> T) -> T {

        let (first_segment, num_words, own_first_segment) : (*mut Word, uint, bool) = unsafe {
            match first_segment_arg {
                NumWords(n) =>
                    (std::cast::transmute(
                        std::libc::calloc(n as std::libc::size_t,
                                          BYTES_PER_WORD as std::libc::size_t)),
                     n, true),
                ZeroedWords(w) => (w.as_mut_ptr(), w.len(), false)
            }};

        let mut arena = ~BuilderArena::<'a> {
            message : std::ptr::mut_null(),
            segment0 : SegmentBuilder {
                reader : SegmentReader {
                    ptr : first_segment as * Word,
                    size : num_words,
                    arena : Null },
                id : 0,
                pos : first_segment
            },
            more_segments : None };

        let arena_ptr = std::ptr::to_mut_unsafe_ptr(arena);
        arena.segment0.reader.arena = BuilderArenaPtr(arena_ptr);

        let mut result = ~MessageBuilder {
            nextSize : num_words,
            allocation_strategy : allocationStrategy,
            arena : arena,
            own_first_segment: own_first_segment,
            first_segment : first_segment,
            more_segments : None
        };

        (*result.arena).message = std::ptr::to_mut_unsafe_ptr(result);

        cont(result)
    }

    pub fn new_default<T>(cont : |&mut MessageBuilder| -> T) -> T {
        MessageBuilder::new(NumWords(SUGGESTED_FIRST_SEGMENT_WORDS), SUGGESTED_ALLOCATION_STRATEGY, cont)
    }

    pub fn allocate_segment(&mut self, minimumSize : WordCount) -> (*mut Word, WordCount) {
        let size = std::cmp::max(minimumSize, self.nextSize);
        let new_words : *mut Word = unsafe {
            std::cast::transmute(std::libc::calloc(size as std::libc::size_t,
                                                   BYTES_PER_WORD as std::libc::size_t)) };

        match self.more_segments {
            None => self.more_segments = Some(~[new_words]),
            Some(ref mut segs) => segs.push(new_words)
        }

        match self.allocation_strategy {
            GROW_HEURISTICALLY => { self.nextSize += size; }
            _ => { }
        }
        (new_words, size)
    }

    pub fn get_segments_for_output<T>(&self, cont : |&[&[Word]]| -> T) -> T {
        self.arena.get_segments_for_output(cont)
    }
}

impl MessageBuilder {
    // Note: This type signature ought to prevent a MessageBuilder
    // from being initted twice simultaneously. It currently does not
    // fulfill that goal, perhaps due to Rust issue #5121.
    pub fn init_root<'a, T : layout::HasStructSize + layout::FromStructBuilder<'a>>(&mut self) -> T {
        // Rolled in this stuff form getRootSegment.
        let rootSegment = std::ptr::to_mut_unsafe_ptr(&mut self.arena.segment0);

        match self.arena.segment0.allocate(WORDS_PER_POINTER) {
            None => {fail!("could not allocate root pointer") }
            Some(location) => {
                //assert!(location == 0,
                //        "First allocated word of new segment was not at offset 0");

                let pb = layout::PointerBuilder::get_root(rootSegment, location);

                return layout::FromStructBuilder::from_struct_builder(
                    pb.init_struct(layout::HasStructSize::struct_size(None::<T>)));
            }
        }
    }

}
<commit_msg>starting on some big changes<commit_after>/*
 * Copyright (c) 2013-2014, David Renshaw (dwrenshaw@gmail.com)
 *
 * See the LICENSE file in the capnproto-rust root directory.
 */

use std;
use common::*;
use arena::*;
use layout;

pub struct ReaderOptions {
    traversalLimitInWords : u64,
    nestingLimit : uint
}

pub static DEFAULT_READER_OPTIONS : ReaderOptions =
    ReaderOptions { traversalLimitInWords : 8 * 1024 * 1024, nestingLimit : 64 };

pub struct MessageReader<'a> {
    segments : &'a [ &'a [Word]],
    options : ReaderOptions,
    arena : ReaderArena
}

type SegmentId = u32;

pub trait MessageReader {
    fn get_segment<'a>(&'a self, id : uint) -> &'a [Word];
    fn arena(&self) -> ReaderArena
    fn get_root<T : layout::FromStructReader<'a>>(&self) -> T {
        unsafe {
            let segment : *SegmentReader<'a> = std::ptr::to_unsafe_ptr(&self.arena.segment0);

            let pointer_reader = layout::PointerReader::get_root::<'a>(
                segment, (*segment).get_start_ptr(), self.options.nestingLimit as int);

            let result : T = layout::FromStructReader::from_struct_reader(
                pointer_reader.get_struct::<'a>(std::ptr::null()));

            result
        }

    }
}

impl <'a> MessageReader<'a> {

    #[inline]
    pub fn get_options<'b>(&'b self) -> &'b ReaderOptions {
        return &self.options;
    }
}

impl <'a> MessageReader<'a> {

    pub fn new<'b>(segments : &'b [&'b [Word]], options : ReaderOptions) -> ~MessageReader<'b> {

        assert!(segments.len() > 0);
        let mut result = ~MessageReader {
            segments : segments,
            arena : ReaderArena {
                segment0 : SegmentReader {
                    arena : Null,
                    ptr : unsafe { segments[0].unsafe_ref(0) },
                    size : segments[0].len()
                },
                more_segments : None
            },
            options : options
        };

        let arena_ptr = ReaderArenaPtr (std::ptr::to_unsafe_ptr(&result.arena));

        result.arena.segment0.arena = arena_ptr;

        if segments.len() > 1 {
            let mut moreSegmentReaders = ~[];
            for segment in segments.slice_from(1).iter() {
                let segmentReader = SegmentReader {
                    arena : arena_ptr,
                    ptr : unsafe { segment.unsafe_ref(0) },
                    size : segment.len()
                };
                moreSegmentReaders.push(segmentReader);
            }
            result.arena.more_segments = Some(moreSegmentReaders);
        }

        result

    }

    pub fn get_root<T : layout::FromStructReader<'a>>(&self) -> T {
        unsafe {
            let segment : *SegmentReader = std::ptr::to_unsafe_ptr(&self.arena.segment0);

            let pointer_reader = layout::PointerReader::get_root::<'a>(
                segment, (*segment).get_start_ptr(), self.options.nestingLimit as int);

            let result : T = layout::FromStructReader::from_struct_reader(
                pointer_reader.get_struct::<'a>(std::ptr::null()));

            result
        }
    }
}

pub enum AllocationStrategy {
    FIXED_SIZE,
    GROW_HEURISTICALLY
}

pub static SUGGESTED_FIRST_SEGMENT_WORDS : uint = 1024;
pub static SUGGESTED_ALLOCATION_STRATEGY : AllocationStrategy = GROW_HEURISTICALLY;

pub struct MessageBuilder {
    nextSize : uint,
    allocation_strategy : AllocationStrategy,
    arena : ~BuilderArena,
    own_first_segment : bool,
    first_segment : *mut Word,
    more_segments : Option<~[*mut Word]>
}

impl Drop for MessageBuilder {
    fn drop(&mut self) {
        if self.own_first_segment {
            unsafe { std::libc::free(std::cast::transmute(self.first_segment)) }
        } else {
            self.get_segments_for_output(|segments| {
                    unsafe {
                        std::ptr::zero_memory(self.first_segment, segments[0].len());
                    }
                });
        }

        match self.more_segments {
            None => {},
            Some(ref mut segs) => {
                for &segment_ptr in segs.iter() {
                    unsafe { std::libc::free(std::cast::transmute(segment_ptr)); }
                }
            }
        }
    }
}

pub enum FirstSegment<'a> {
    NumWords(uint),
    ZeroedWords(&'a mut [Word])
}

impl MessageBuilder {

    // TODO: maybe when Rust issue #5121 is fixed we can safely get away with not passing
    //  a closure here.
    pub fn new<'a, T>(first_segment_arg : FirstSegment<'a>,
                  allocationStrategy : AllocationStrategy,
                  cont : |&mut MessageBuilder| -> T) -> T {

        let (first_segment, num_words, own_first_segment) : (*mut Word, uint, bool) = unsafe {
            match first_segment_arg {
                NumWords(n) =>
                    (std::cast::transmute(
                        std::libc::calloc(n as std::libc::size_t,
                                          BYTES_PER_WORD as std::libc::size_t)),
                     n, true),
                ZeroedWords(w) => (w.as_mut_ptr(), w.len(), false)
            }};

        let mut arena = ~BuilderArena::<'a> {
            message : std::ptr::mut_null(),
            segment0 : SegmentBuilder {
                reader : SegmentReader {
                    ptr : first_segment as * Word,
                    size : num_words,
                    arena : Null },
                id : 0,
                pos : first_segment
            },
            more_segments : None };

        let arena_ptr = std::ptr::to_mut_unsafe_ptr(arena);
        arena.segment0.reader.arena = BuilderArenaPtr(arena_ptr);

        let mut result = ~MessageBuilder {
            nextSize : num_words,
            allocation_strategy : allocationStrategy,
            arena : arena,
            own_first_segment: own_first_segment,
            first_segment : first_segment,
            more_segments : None
        };

        (*result.arena).message = std::ptr::to_mut_unsafe_ptr(result);

        cont(result)
    }

    pub fn new_default<T>(cont : |&mut MessageBuilder| -> T) -> T {
        MessageBuilder::new(NumWords(SUGGESTED_FIRST_SEGMENT_WORDS), SUGGESTED_ALLOCATION_STRATEGY, cont)
    }

    pub fn allocate_segment(&mut self, minimumSize : WordCount) -> (*mut Word, WordCount) {
        let size = std::cmp::max(minimumSize, self.nextSize);
        let new_words : *mut Word = unsafe {
            std::cast::transmute(std::libc::calloc(size as std::libc::size_t,
                                                   BYTES_PER_WORD as std::libc::size_t)) };

        match self.more_segments {
            None => self.more_segments = Some(~[new_words]),
            Some(ref mut segs) => segs.push(new_words)
        }

        match self.allocation_strategy {
            GROW_HEURISTICALLY => { self.nextSize += size; }
            _ => { }
        }
        (new_words, size)
    }

    pub fn get_segments_for_output<T>(&self, cont : |&[&[Word]]| -> T) -> T {
        self.arena.get_segments_for_output(cont)
    }
}

impl MessageBuilder {
    // Note: This type signature ought to prevent a MessageBuilder
    // from being initted twice simultaneously. It currently does not
    // fulfill that goal, perhaps due to Rust issue #5121.
    pub fn init_root<'a, T : layout::HasStructSize + layout::FromStructBuilder<'a>>(&mut self) -> T {
        // Rolled in this stuff form getRootSegment.
        let rootSegment = std::ptr::to_mut_unsafe_ptr(&mut self.arena.segment0);

        match self.arena.segment0.allocate(WORDS_PER_POINTER) {
            None => {fail!("could not allocate root pointer") }
            Some(location) => {
                //assert!(location == 0,
                //        "First allocated word of new segment was not at offset 0");

                let pb = layout::PointerBuilder::get_root(rootSegment, location);

                return layout::FromStructBuilder::from_struct_builder(
                    pb.init_struct(layout::HasStructSize::struct_size(None::<T>)));
            }
        }
    }

}
<|endoftext|>


<commit_before>// Implements the `zakopane checksum` subcommand.

use std::convert::TryInto;
use std::io::Read;
use std::io::Write;

use crate::structs::ChecksumCliOptions;
use crate::structs::ZakopaneError;

const READ_SIZE: usize = 2 << 20;
const BIG_FILE_BYTES: usize = 2 << 27;

struct ChecksumWithPath {
    checksum: String,
    path: std::path::PathBuf,
}

impl ChecksumWithPath {
    pub fn new(checksum: String, path: std::path::PathBuf) -> Self {
        Self { checksum, path }
    }
}

type ChecksumResult = Result<ChecksumWithPath, ZakopaneError>;
type ChecksumTaskJoinHandle = tokio::task::JoinHandle<Vec<ChecksumWithPath>>;

struct ChecksumTaskDispatcherData {
    // Rate-limiter for spawned checksum tasks.
    semaphore: std::sync::Arc<tokio::sync::Semaphore>,

    cli_options: ChecksumCliOptions,

    // Counts total number of spawned checksum tasks.
    spawn_counter: std::sync::Arc<std::sync::atomic::AtomicUsize>,

    sender: tokio::sync::mpsc::Sender<ChecksumResult>,
}

impl ChecksumTaskDispatcherData {
    pub fn new(
        cli_options: ChecksumCliOptions,
        spawn_counter: std::sync::Arc<std::sync::atomic::AtomicUsize>,
        sender: tokio::sync::mpsc::Sender<ChecksumResult>,
    ) -> Self {
        Self {
            semaphore: std::sync::Arc::new(tokio::sync::Semaphore::new(cli_options.max_tasks)),
            cli_options,
            spawn_counter,
            sender,
        }
    }
}

struct FileDetails {
    is_file: bool,
    is_big: bool,
}

fn get_file_details(path: &std::path::PathBuf) -> FileDetails {
    let metadata = match std::fs::symlink_metadata(path) {
        Ok(m) => m,
        Err(_) => {
            return FileDetails {
                is_file: false,
                is_big: false,
            }
        }
    };

    if !metadata.file_type().is_file() {
        return FileDetails {
            is_file: false,
            is_big: false,
        };
    }
    if metadata.len() > BIG_FILE_BYTES.try_into().unwrap() {
        return FileDetails {
            is_file: true,
            is_big: true,
        };
    }
    FileDetails {
        is_file: true,
        is_big: false,
    }
}

// Sends a checksum task result downstream to the collector task.
fn checksum_task_send_result(
    result: ChecksumResult,
    sender: tokio::sync::mpsc::Sender<ChecksumResult>,
) {
    if let Err(e) = sender.blocking_send(result) {
        eprintln!("BUG: Sender::send() failed: {}", e);
    }
}

fn checksum_task_impl(path: std::path::PathBuf) -> ChecksumResult {
    let mut hasher = crypto_hash::Hasher::new(crypto_hash::Algorithm::SHA256);
    let mut buffer: Vec<u8> = vec![0; READ_SIZE];
    let mut file = std::fs::File::open(&path).map_err(ZakopaneError::Io)?;
    loop {
        let read_bytes = file.read(&mut buffer).map_err(ZakopaneError::Io)?;
        if read_bytes == 0 {
            let checksum = hasher
                .finish()
                .into_iter()
                .map(|byte| format!("{:02x}", byte))
                .collect::<Vec<String>>()
                .join("");
            return Ok(ChecksumWithPath::new(checksum, path));
        }
        hasher
            .write_all(&buffer[..read_bytes])
            .map_err(ZakopaneError::Io)?;
    }
}

// Represents a spawned checksum task.
//
// `_permit` is moved into this checksum task simply to hold onto the
// semaphore-dispensed resource for the duration of this task.
fn checksum_task(
    path: std::path::PathBuf,
    sender: tokio::sync::mpsc::Sender<ChecksumResult>,
    _permit: tokio::sync::OwnedSemaphorePermit,
) {
    let result = checksum_task_impl(path);
    checksum_task_send_result(result, sender);
}

// Represents the spawned collection task.
async fn collector_task(
    mut receiver: tokio::sync::mpsc::Receiver<ChecksumResult>,
    spawn_counter: std::sync::Arc<std::sync::atomic::AtomicUsize>,
) -> Vec<ChecksumWithPath> {
    let mut sums: Vec<ChecksumWithPath> = Vec::new();
    let mut errors: usize = 0;
    loop {
        while let Some(result) = receiver.recv().await {
            match result {
                Ok(digest_with_path) => sums.push(digest_with_path),
                Err(_) => errors += 1,
            }
        }
        if sums.len() + errors == spawn_counter.load(std::sync::atomic::Ordering::SeqCst) {
            break;
        }
    }
    if errors != 0 {
        eprintln!("WARNING: {} errors collected", errors);
    }
    sums
}

// Spawns the collector task that listens for checksum results
// provided by the checksum tasks.
async fn spawn_collector(
    options: ChecksumCliOptions,
) -> (ChecksumTaskDispatcherData, ChecksumTaskJoinHandle) {
    let (sender, receiver) = tokio::sync::mpsc::channel(options.max_tasks);
    let spawn_counter = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
    let join_handle = tokio::task::spawn(collector_task(receiver, spawn_counter.clone()));
    (
        ChecksumTaskDispatcherData::new(options, spawn_counter, sender),
        join_handle,
    )
}

async fn spawn_checksum_tasks(context: ChecksumTaskDispatcherData) {
    let walk_iter = walkdir::WalkDir::new(&context.cli_options.path).into_iter();
    // The `filter_entry()` call is crafted s.t.
    // *    we skip and don't descend into hidden directories
    // *    unless the hidden directory is the target directory, because
    //      the target directory is always the first yielded value from
    //      the `WalkDir`.
    for entry in walk_iter.filter_entry(|e| {
        e.path() == &context.cli_options.path
            || !e
                .file_name()
                .to_str()
                .map(|path| path.starts_with("."))
                .unwrap_or(false)
    }) {
        if let Ok(direntry) = entry {
            let path = direntry.into_path();
            let file_details = get_file_details(&path);
            if !file_details.is_file {
                continue;
            }

            // If the file is "big" (arbitrarily defined), attempt to
            // seize all permits to force us momentarily to work with a
            // bit less I/O contention.
            let permit = if file_details.is_big {
                context.semaphore.clone().acquire_owned().await.unwrap()
            } else {
                context
                    .semaphore
                    .clone()
                    .acquire_many_owned(context.cli_options.max_tasks.try_into().unwrap())
                    .await
                    .unwrap()
            };

            context
                .spawn_counter
                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
            let sender = context.sender.clone();
            tokio::task::spawn_blocking(move || checksum_task(path, sender, permit));
        }
    }
}

// Pretty-prints the sorted `checksums` in a format much like what the
// `sha256sum` binary outputs.
//
// Note that the standard zakopane snapshot header is not added here.
fn pretty_format_checksums(path: std::path::PathBuf, checksums: Vec<ChecksumWithPath>) -> String {
    let mut buffer: Vec<String> = checksums
        .into_iter()
        .map(|e| {
            format!(
                "{}  ./{}",
                e.checksum,
                e.path.strip_prefix(&path).unwrap().to_str().unwrap()
            )
        })
        .collect();
    buffer.push(String::new());
    buffer.join("\n")
}

async fn checksum_impl(options: ChecksumCliOptions) -> String {
    let path = options.path.clone();
    let (dispatcher_data, join_handle) = spawn_collector(options).await;
    spawn_checksum_tasks(dispatcher_data).await;
    let mut checksums: Vec<ChecksumWithPath> = join_handle.await.unwrap();
    checksums.sort_by(|a, b| a.path.cmp(&b.path));
    pretty_format_checksums(path, checksums)
}

pub fn checksum(options: ChecksumCliOptions) -> String {
    let runtime = tokio::runtime::Runtime::new().unwrap();
    runtime.block_on(checksum_impl(options))
}
<commit_msg>checksum: Constant bugfixes<commit_after>// Implements the `zakopane checksum` subcommand.

use std::convert::TryInto;
use std::io::Read;
use std::io::Write;

use crate::structs::ChecksumCliOptions;
use crate::structs::ZakopaneError;

const READ_SIZE: usize = 1 << 20;
const BIG_FILE_BYTES: usize = 1 << 27;

struct ChecksumWithPath {
    checksum: String,
    path: std::path::PathBuf,
}

impl ChecksumWithPath {
    pub fn new(checksum: String, path: std::path::PathBuf) -> Self {
        Self { checksum, path }
    }
}

type ChecksumResult = Result<ChecksumWithPath, ZakopaneError>;
type ChecksumTaskJoinHandle = tokio::task::JoinHandle<Vec<ChecksumWithPath>>;

struct ChecksumTaskDispatcherData {
    // Rate-limiter for spawned checksum tasks.
    semaphore: std::sync::Arc<tokio::sync::Semaphore>,

    cli_options: ChecksumCliOptions,

    // Counts total number of spawned checksum tasks.
    spawn_counter: std::sync::Arc<std::sync::atomic::AtomicUsize>,

    sender: tokio::sync::mpsc::Sender<ChecksumResult>,
}

impl ChecksumTaskDispatcherData {
    pub fn new(
        cli_options: ChecksumCliOptions,
        spawn_counter: std::sync::Arc<std::sync::atomic::AtomicUsize>,
        sender: tokio::sync::mpsc::Sender<ChecksumResult>,
    ) -> Self {
        Self {
            semaphore: std::sync::Arc::new(tokio::sync::Semaphore::new(cli_options.max_tasks)),
            cli_options,
            spawn_counter,
            sender,
        }
    }
}

struct FileDetails {
    is_file: bool,
    is_big: bool,
}

fn get_file_details(path: &std::path::PathBuf) -> FileDetails {
    let metadata = match std::fs::symlink_metadata(path) {
        Ok(m) => m,
        Err(_) => {
            return FileDetails {
                is_file: false,
                is_big: false,
            }
        }
    };

    if !metadata.file_type().is_file() {
        return FileDetails {
            is_file: false,
            is_big: false,
        };
    }
    if metadata.len() > BIG_FILE_BYTES.try_into().unwrap() {
        return FileDetails {
            is_file: true,
            is_big: true,
        };
    }
    FileDetails {
        is_file: true,
        is_big: false,
    }
}

// Sends a checksum task result downstream to the collector task.
fn checksum_task_send_result(
    result: ChecksumResult,
    sender: tokio::sync::mpsc::Sender<ChecksumResult>,
) {
    if let Err(e) = sender.blocking_send(result) {
        eprintln!("BUG: Sender::send() failed: {}", e);
    }
}

fn checksum_task_impl(path: std::path::PathBuf) -> ChecksumResult {
    let mut hasher = crypto_hash::Hasher::new(crypto_hash::Algorithm::SHA256);
    let mut buffer: Vec<u8> = vec![0; READ_SIZE];
    let mut file = std::fs::File::open(&path).map_err(ZakopaneError::Io)?;
    loop {
        let read_bytes = file.read(&mut buffer).map_err(ZakopaneError::Io)?;
        if read_bytes == 0 {
            let checksum = hasher
                .finish()
                .into_iter()
                .map(|byte| format!("{:02x}", byte))
                .collect::<Vec<String>>()
                .join("");
            return Ok(ChecksumWithPath::new(checksum, path));
        }
        hasher
            .write_all(&buffer[..read_bytes])
            .map_err(ZakopaneError::Io)?;
    }
}

// Represents a spawned checksum task.
//
// `_permit` is moved into this checksum task simply to hold onto the
// semaphore-dispensed resource for the duration of this task.
fn checksum_task(
    path: std::path::PathBuf,
    sender: tokio::sync::mpsc::Sender<ChecksumResult>,
    _permit: tokio::sync::OwnedSemaphorePermit,
) {
    let result = checksum_task_impl(path);
    checksum_task_send_result(result, sender);
}

// Represents the spawned collection task.
async fn collector_task(
    mut receiver: tokio::sync::mpsc::Receiver<ChecksumResult>,
    spawn_counter: std::sync::Arc<std::sync::atomic::AtomicUsize>,
) -> Vec<ChecksumWithPath> {
    let mut sums: Vec<ChecksumWithPath> = Vec::new();
    let mut errors: usize = 0;
    loop {
        while let Some(result) = receiver.recv().await {
            match result {
                Ok(digest_with_path) => sums.push(digest_with_path),
                Err(_) => errors += 1,
            }
        }
        if sums.len() + errors == spawn_counter.load(std::sync::atomic::Ordering::SeqCst) {
            break;
        }
    }
    if errors != 0 {
        eprintln!("WARNING: {} errors collected", errors);
    }
    sums
}

// Spawns the collector task that listens for checksum results
// provided by the checksum tasks.
async fn spawn_collector(
    options: ChecksumCliOptions,
) -> (ChecksumTaskDispatcherData, ChecksumTaskJoinHandle) {
    let (sender, receiver) = tokio::sync::mpsc::channel(options.max_tasks);
    let spawn_counter = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
    let join_handle = tokio::task::spawn(collector_task(receiver, spawn_counter.clone()));
    (
        ChecksumTaskDispatcherData::new(options, spawn_counter, sender),
        join_handle,
    )
}

async fn spawn_checksum_tasks(context: ChecksumTaskDispatcherData) {
    let walk_iter = walkdir::WalkDir::new(&context.cli_options.path).into_iter();
    // The `filter_entry()` call is crafted s.t.
    // *    we skip and don't descend into hidden directories
    // *    unless the hidden directory is the target directory, because
    //      the target directory is always the first yielded value from
    //      the `WalkDir`.
    for entry in walk_iter.filter_entry(|e| {
        e.path() == &context.cli_options.path
            || !e
                .file_name()
                .to_str()
                .map(|path| path.starts_with("."))
                .unwrap_or(false)
    }) {
        if let Ok(direntry) = entry {
            let path = direntry.into_path();
            let file_details = get_file_details(&path);
            if !file_details.is_file {
                continue;
            }

            // If the file is "big" (arbitrarily defined), attempt to
            // seize all permits to force us momentarily to work with a
            // bit less I/O contention.
            let permit = if file_details.is_big {
                context
                    .semaphore
                    .clone()
                    .acquire_many_owned(context.cli_options.max_tasks.try_into().unwrap())
                    .await
                    .unwrap()
            } else {
                context.semaphore.clone().acquire_owned().await.unwrap()
            };

            context
                .spawn_counter
                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
            let sender = context.sender.clone();
            tokio::task::spawn_blocking(move || checksum_task(path, sender, permit));
        }
    }
}

// Pretty-prints the sorted `checksums` in a format much like what the
// `sha256sum` binary outputs.
//
// Note that the standard zakopane snapshot header is not added here.
fn pretty_format_checksums(path: std::path::PathBuf, checksums: Vec<ChecksumWithPath>) -> String {
    let mut buffer: Vec<String> = checksums
        .into_iter()
        .map(|e| {
            format!(
                "{}  ./{}",
                e.checksum,
                e.path.strip_prefix(&path).unwrap().to_str().unwrap()
            )
        })
        .collect();
    buffer.push(String::new());
    buffer.join("\n")
}

async fn checksum_impl(options: ChecksumCliOptions) -> String {
    let path = options.path.clone();
    let (dispatcher_data, join_handle) = spawn_collector(options).await;
    spawn_checksum_tasks(dispatcher_data).await;
    let mut checksums: Vec<ChecksumWithPath> = join_handle.await.unwrap();
    checksums.sort_by(|a, b| a.path.cmp(&b.path));
    pretty_format_checksums(path, checksums)
}

pub fn checksum(options: ChecksumCliOptions) -> String {
    let runtime = tokio::runtime::Runtime::new().unwrap();
    runtime.block_on(checksum_impl(options))
}
<|endoftext|>


<commit_before>#![allow(deprecated)]

#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate criterion;

use std::mem;
use std::slice;

use criterion::{black_box, Criterion, ParameterizedBenchmark, Throughput};

use fasthash::*;

const KB: usize = 1024;
const SEED: u64 = 0x0123456789ABCDEF;
const PARAMS: [usize; 7] = [7, 8, 32, 256, KB, 4 * KB, 16 * KB];

lazy_static! {
    static ref DATA: Vec<u8> = (0..16 * KB).map(|b| b as u8).collect::<Vec<_>>();
}

fn bench_memory(c: &mut Criterion) {
    c.bench(
        "memory",
        ParameterizedBenchmark::new(
            "sum",
            move |b, &&size| {
                let s = unsafe {
                    slice::from_raw_parts(DATA.as_ptr() as *mut u32, size / mem::size_of::<u32>())
                };

                b.iter(|| {
                    black_box(s.iter().fold(0u64, |acc, &x| acc + x as u64));
                })
            },
            &PARAMS,
        )
        .throughput(|&&size| Throughput::Bytes(size as u64)),
    );
}

fn bench_hash32(c: &mut Criterion) {
    c.bench(
        "hash32",
        ParameterizedBenchmark::new(
            "city::hash32",
            move |b, &&size| {
                b.iter(|| city::hash32_with_seed(&DATA.as_slice()[..size], SEED as u32));
            },
            &PARAMS,
        )
        .with_function("farm::hash32", move |b, &&size| {
            b.iter(|| farm::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("farm::finterprint32", move |b, &&size| {
            b.iter(|| farm::fingerprint32(&DATA[..size]));
        })
        .with_function("lookup3::hash32", move |b, &&size| {
            b.iter(|| lookup3::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur::hash32", move |b, &&size| {
            b.iter(|| murmur::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur::hash32_aligned", move |b, &&size| {
            b.iter(|| murmur::hash32_aligned_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur2::hash32", move |b, &&size| {
            b.iter(|| murmur2::Hash32::hash_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur2::hash32_a", move |b, &&size| {
            b.iter(|| murmur2::Hash32A::hash_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur2::hash32_neutral", move |b, &&size| {
            b.iter(|| murmur2::Hash32Neutral::hash_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur2::hash32_aligned", move |b, &&size| {
            b.iter(|| murmur2::Hash32Aligned::hash_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur3::hash32", move |b, &&size| {
            b.iter(|| murmur3::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("spooky::hash32", move |b, &&size| {
            b.iter(|| spooky::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("xx::hash32", move |b, &&size| {
            b.iter(|| xx::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .throughput(|&&size| Throughput::Bytes(size as u64)),
    );
}

fn bench_hash64(c: &mut Criterion) {
    let mut bench = ParameterizedBenchmark::new(
        "city::hash64",
        move |b, &&size| {
            b.iter(|| city::hash64_with_seed(&DATA.as_slice()[..size], SEED));
        },
        &PARAMS,
    )
    .with_function("farm::hash64", move |b, &&size| {
        b.iter(|| farm::hash64_with_seed(&DATA[..size], SEED));
    })
    .with_function("farm::finterprint64", move |b, &&size| {
        b.iter(|| farm::fingerprint32(&DATA[..size]));
    })
    .with_function("metro::hash64_1", move |b, &&size| {
        b.iter(|| metro::Hash64_1::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("metro::hash64_2", move |b, &&size| {
        b.iter(|| metro::Hash64_2::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("mum::hash64", move |b, &&size| {
        b.iter(|| mum::hash64_with_seed(&DATA[..size], SEED));
    })
    .with_function("murmur2::hash64_x64", move |b, &&size| {
        b.iter(|| murmur2::Hash64_x64::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("murmur2::hash64_x86", move |b, &&size| {
        b.iter(|| murmur2::Hash64_x86::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("sea::hash64", move |b, &&size| {
        b.iter(|| sea::hash64_with_seeds(&DATA[..size], SEED, SEED, SEED, SEED));
    })
    .with_function("spooky::hash64", move |b, &&size| {
        b.iter(|| spooky::hash64_with_seed(&DATA[..size], SEED));
    })
    .with_function("t1ha0::hash64", move |b, &&size| {
        b.iter(|| t1ha0::Hash64::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("t1ha1::hash64", move |b, &&size| {
        b.iter(|| t1ha1::Hash64::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("t1ha2::hash64_atonce", move |b, &&size| {
        b.iter(|| t1ha2::Hash64AtOnce::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("xx::hash64", move |b, &&size| {
        b.iter(|| xx::hash64_with_seed(&DATA[..size], SEED));
    })
    .with_function("xxh3::hash64", move |b, &&size| {
        b.iter(|| xxh3::hash64_with_seed(&DATA[..size], SEED));
    });

    if cfg!(any(feature = "sse4.2", target_feature = "sse4.2")) {
        bench = bench
            .with_function("metro::crc::hash64_1", move |b, &&size| {
                b.iter(|| metro::crc::Hash64_1::hash_with_seed(&DATA[..size], SEED as u32));
            })
            .with_function("metro::crc::hash64_2", move |b, &&size| {
                b.iter(|| metro::crc::Hash64_2::hash_with_seed(&DATA[..size], SEED as u32));
            });
    }

    c.bench(
        "hash64",
        bench.throughput(|&&size| Throughput::Bytes(size as u64)),
    );
}

fn bench_hash128(c: &mut Criterion) {
    let mut bench = ParameterizedBenchmark::new(
        "city::hash128",
        move |b, &&size| {
            b.iter(|| city::Hash128::hash_with_seed(&DATA[..size], SEED as u128));
        },
        &PARAMS,
    )
    .with_function("farm::hash128", move |b, &&size| {
        b.iter(|| farm::hash128_with_seed(&DATA[..size], SEED as u128));
    })
    .with_function("farm::fingerprint128", move |b, &&size| {
        b.iter(|| farm::fingerprint128(&DATA[..size]));
    })
    .with_function("metro::Hash128_1", move |b, &&size| {
        b.iter(|| metro::Hash128_1::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("metro::Hash128_2", move |b, &&size| {
        b.iter(|| metro::Hash128_2::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("murmur3::hash128_x64", move |b, &&size| {
        b.iter(|| murmur3::Hash128_x64::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("murmur3::hash128_x86", move |b, &&size| {
        b.iter(|| murmur3::Hash128_x86::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("spooky::hash128", move |b, &&size| {
        b.iter(|| spooky::hash128_with_seed(&DATA[..size], SEED as u128));
    })
    .with_function("t1ha2::hash128_atonce", move |b, &&size| {
        b.iter(|| t1ha2::Hash128AtOnce::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("xxh3::hash128", move |b, &&size| {
        b.iter(|| t1ha2::Hash128AtOnce::hash_with_seed(&DATA[..size], SEED));
    });

    if cfg!(any(feature = "sse4.2", target_feature = "sse4.2")) {
        bench = bench
            .with_function("city::crc::hash128", move |b, &&size| {
                b.iter(|| city::crc::Hash128::hash_with_seed(&DATA[..size], SEED as u128));
            })
            .with_function("metro::crc::hash128_1", move |b, &&size| {
                b.iter(|| metro::crc::Hash128_1::hash_with_seed(&DATA[..size], SEED as u32));
            })
            .with_function("metro::crc::hash128_2", move |b, &&size| {
                b.iter(|| metro::crc::Hash128_2::hash_with_seed(&DATA[..size], SEED as u32));
            });
    }

    c.bench(
        "hash128",
        bench.throughput(|&&size| Throughput::Bytes(size as u64)),
    );
}

criterion_group!(
    benches,
    bench_memory,
    bench_hash32,
    bench_hash64,
    bench_hash128,
);
criterion_main!(benches);
<commit_msg>add highway hash<commit_after>#![allow(deprecated)]

#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate criterion;

use std::mem;
use std::slice;

use criterion::{black_box, Criterion, ParameterizedBenchmark, Throughput};

use fasthash::*;

const KB: usize = 1024;
const SEED: u64 = 0x0123456789ABCDEF;
const PARAMS: [usize; 7] = [7, 8, 32, 256, KB, 4 * KB, 16 * KB];

lazy_static! {
    static ref DATA: Vec<u8> = (0..16 * KB).map(|b| b as u8).collect::<Vec<_>>();
}

fn bench_memory(c: &mut Criterion) {
    c.bench(
        "memory",
        ParameterizedBenchmark::new(
            "sum",
            move |b, &&size| {
                let s = unsafe {
                    slice::from_raw_parts(DATA.as_ptr() as *mut u32, size / mem::size_of::<u32>())
                };

                b.iter(|| {
                    black_box(s.iter().fold(0u64, |acc, &x| acc + x as u64));
                })
            },
            &PARAMS,
        )
        .throughput(|&&size| Throughput::Bytes(size as u64)),
    );
}

fn bench_hash32(c: &mut Criterion) {
    c.bench(
        "hash32",
        ParameterizedBenchmark::new(
            "city::hash32",
            move |b, &&size| {
                b.iter(|| city::hash32_with_seed(&DATA.as_slice()[..size], SEED as u32));
            },
            &PARAMS,
        )
        .with_function("farm::hash32", move |b, &&size| {
            b.iter(|| farm::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("farm::finterprint32", move |b, &&size| {
            b.iter(|| farm::fingerprint32(&DATA[..size]));
        })
        .with_function("lookup3::hash32", move |b, &&size| {
            b.iter(|| lookup3::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur::hash32", move |b, &&size| {
            b.iter(|| murmur::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur::hash32_aligned", move |b, &&size| {
            b.iter(|| murmur::hash32_aligned_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur2::hash32", move |b, &&size| {
            b.iter(|| murmur2::Hash32::hash_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur2::hash32_a", move |b, &&size| {
            b.iter(|| murmur2::Hash32A::hash_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur2::hash32_neutral", move |b, &&size| {
            b.iter(|| murmur2::Hash32Neutral::hash_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur2::hash32_aligned", move |b, &&size| {
            b.iter(|| murmur2::Hash32Aligned::hash_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("murmur3::hash32", move |b, &&size| {
            b.iter(|| murmur3::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("spooky::hash32", move |b, &&size| {
            b.iter(|| spooky::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .with_function("xx::hash32", move |b, &&size| {
            b.iter(|| xx::hash32_with_seed(&DATA[..size], SEED as u32));
        })
        .throughput(|&&size| Throughput::Bytes(size as u64)),
    );
}

fn bench_hash64(c: &mut Criterion) {
    let mut bench = ParameterizedBenchmark::new(
        "city::hash64",
        move |b, &&size| {
            b.iter(|| city::hash64_with_seed(&DATA.as_slice()[..size], SEED));
        },
        &PARAMS,
    )
    .with_function("farm::hash64", move |b, &&size| {
        b.iter(|| farm::hash64_with_seed(&DATA[..size], SEED));
    })
    .with_function("farm::finterprint64", move |b, &&size| {
        b.iter(|| farm::fingerprint32(&DATA[..size]));
    })
    .with_function("metro::hash64_1", move |b, &&size| {
        b.iter(|| metro::Hash64_1::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("metro::hash64_2", move |b, &&size| {
        b.iter(|| metro::Hash64_2::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("mum::hash64", move |b, &&size| {
        b.iter(|| mum::hash64_with_seed(&DATA[..size], SEED));
    })
    .with_function("murmur2::hash64_x64", move |b, &&size| {
        b.iter(|| murmur2::Hash64_x64::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("murmur2::hash64_x86", move |b, &&size| {
        b.iter(|| murmur2::Hash64_x86::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("sea::hash64", move |b, &&size| {
        b.iter(|| sea::hash64_with_seeds(&DATA[..size], SEED, SEED, SEED, SEED));
    })
    .with_function("spooky::hash64", move |b, &&size| {
        b.iter(|| spooky::hash64_with_seed(&DATA[..size], SEED));
    })
    .with_function("t1ha0::hash64", move |b, &&size| {
        b.iter(|| t1ha0::Hash64::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("t1ha1::hash64", move |b, &&size| {
        b.iter(|| t1ha1::Hash64::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("t1ha2::hash64_atonce", move |b, &&size| {
        b.iter(|| t1ha2::Hash64AtOnce::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("xx::hash64", move |b, &&size| {
        b.iter(|| xx::hash64_with_seed(&DATA[..size], SEED));
    })
    .with_function("xxh3::hash64", move |b, &&size| {
        b.iter(|| xxh3::hash64_with_seed(&DATA[..size], SEED));
    })
    .with_function("highway::hash64", move |b, &&size| {
        b.iter(|| highway::hash64_with_seed(&DATA[..size], [SEED, SEED, SEED, SEED]));
    });

    if cfg!(any(feature = "sse4.2", target_feature = "sse4.2")) {
        bench = bench
            .with_function("metro::crc::hash64_1", move |b, &&size| {
                b.iter(|| metro::crc::Hash64_1::hash_with_seed(&DATA[..size], SEED as u32));
            })
            .with_function("metro::crc::hash64_2", move |b, &&size| {
                b.iter(|| metro::crc::Hash64_2::hash_with_seed(&DATA[..size], SEED as u32));
            });
    }

    c.bench(
        "hash64",
        bench.throughput(|&&size| Throughput::Bytes(size as u64)),
    );
}

fn bench_hash128(c: &mut Criterion) {
    let mut bench = ParameterizedBenchmark::new(
        "city::hash128",
        move |b, &&size| {
            b.iter(|| city::Hash128::hash_with_seed(&DATA[..size], SEED as u128));
        },
        &PARAMS,
    )
    .with_function("farm::hash128", move |b, &&size| {
        b.iter(|| farm::hash128_with_seed(&DATA[..size], SEED as u128));
    })
    .with_function("farm::fingerprint128", move |b, &&size| {
        b.iter(|| farm::fingerprint128(&DATA[..size]));
    })
    .with_function("metro::Hash128_1", move |b, &&size| {
        b.iter(|| metro::Hash128_1::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("metro::Hash128_2", move |b, &&size| {
        b.iter(|| metro::Hash128_2::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("murmur3::hash128_x64", move |b, &&size| {
        b.iter(|| murmur3::Hash128_x64::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("murmur3::hash128_x86", move |b, &&size| {
        b.iter(|| murmur3::Hash128_x86::hash_with_seed(&DATA[..size], SEED as u32));
    })
    .with_function("spooky::hash128", move |b, &&size| {
        b.iter(|| spooky::hash128_with_seed(&DATA[..size], SEED as u128));
    })
    .with_function("t1ha2::hash128_atonce", move |b, &&size| {
        b.iter(|| t1ha2::Hash128AtOnce::hash_with_seed(&DATA[..size], SEED));
    })
    .with_function("xxh3::hash128", move |b, &&size| {
        b.iter(|| xxh3::hash128_with_seed(&DATA[..size], SEED));
    })
    .with_function("highway::hash128", move |b, &&size| {
        b.iter(|| highway::hash128_with_seed(&DATA[..size], [SEED, SEED, SEED, SEED]));
    });

    if cfg!(any(feature = "sse4.2", target_feature = "sse4.2")) {
        bench = bench
            .with_function("city::crc::hash128", move |b, &&size| {
                b.iter(|| city::crc::Hash128::hash_with_seed(&DATA[..size], SEED as u128));
            })
            .with_function("metro::crc::hash128_1", move |b, &&size| {
                b.iter(|| metro::crc::Hash128_1::hash_with_seed(&DATA[..size], SEED as u32));
            })
            .with_function("metro::crc::hash128_2", move |b, &&size| {
                b.iter(|| metro::crc::Hash128_2::hash_with_seed(&DATA[..size], SEED as u32));
            });
    }

    c.bench(
        "hash128",
        bench.throughput(|&&size| Throughput::Bytes(size as u64)),
    );
}

criterion_group!(
    benches,
    bench_memory,
    bench_hash32,
    bench_hash64,
    bench_hash128,
);
criterion_main!(benches);
<|endoftext|>


<commit_before>#![crate_name = "uutils"]

/*
 * This file is part of the uutils coreutils package.
 *
 * (c) Michael Gehring <mg@ebfe.org>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

include!(concat!(env!("OUT_DIR"), "/uutils_crates.rs"));

use std::collections::hash_map::HashMap;
use std::path::Path;
use std::env;

static NAME: &'static str = "uutils";
static VERSION: &'static str = env!("CARGO_PKG_VERSION");

include!(concat!(env!("OUT_DIR"), "/uutils_map.rs"));

fn usage(cmap: &UtilityMap) {
    println!("{} {}", NAME, VERSION);
    println!("");
    println!("Usage:");
    println!("  {} [util [arguments...]]\n", NAME);
    println!("Currently defined functions:");
    let mut utils: Vec<&str> = cmap.keys().map(|&s| s).collect();
    utils.sort();
    for util in utils {
        println!("\t{}", util);
    }
}

fn main() {
    let umap = util_map();
    let mut args : Vec<String> = env::args().collect();

    // try binary name as util name.
    let args0 = args[0].clone();
    let binary = Path::new(&args0[..]);
    let binary_as_util = binary.file_name().unwrap().to_str().unwrap();

    match umap.get(binary_as_util) {
        Some(&uumain) => {
            std::process::exit(uumain(args));
        }
        None => (),
    }

    if binary_as_util.ends_with("uutils") || binary_as_util.starts_with("uutils") ||
     binary_as_util.ends_with("busybox") || binary_as_util.starts_with("busybox") {
        args.remove(0);
    } else {
        let mut found = false;
        for util in umap.keys() {
            if binary_as_util.ends_with(util) {
                args[0] = util.clone().to_owned();
                found = true;
                break;
            }
        }
        if ! found {
            println!("{}: applet not found", binary_as_util);
            std::process::exit(1);
        }
    }

    // try first arg as util name.
    if args.len() >= 1 {

        let util = &args[0][..];

        match umap.get(util) {
            Some(&uumain) => {
                std::process::exit(uumain(args.clone()));
            }
            None => {
                if &args[0][..] == "--help" {
                    // see if they want help on a specific util
                    if args.len() >= 2 {
                        let util = &args[1][..];
                        match umap.get(util) {
                            Some(&uumain) => {
                                std::process::exit(uumain(vec![util.to_owned(), "--help".to_owned()]));
                            }
                            None => {
                                println!("{}: applet not found", util);
                                std::process::exit(1);
                            }
                        }
                    }
                    usage(&umap);
                    std::process::exit(0);
                } else {
                    println!("{}: applet not found", util);
                    std::process::exit(1);
                }
            }
        }
    } else {
        // no arguments provided
        usage(&umap);
        std::process::exit(0);
    }
}
<commit_msg>fix two clippy warnings<commit_after>#![crate_name = "uutils"]

/*
 * This file is part of the uutils coreutils package.
 *
 * (c) Michael Gehring <mg@ebfe.org>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

include!(concat!(env!("OUT_DIR"), "/uutils_crates.rs"));

use std::collections::hash_map::HashMap;
use std::path::Path;
use std::env;

static NAME: &'static str = "uutils";
static VERSION: &'static str = env!("CARGO_PKG_VERSION");

include!(concat!(env!("OUT_DIR"), "/uutils_map.rs"));

fn usage(cmap: &UtilityMap) {
    println!("{} {}", NAME, VERSION);
    println!("");
    println!("Usage:");
    println!("  {} [util [arguments...]]\n", NAME);
    println!("Currently defined functions:");
    let mut utils: Vec<&str> = cmap.keys().map(|&s| s).collect();
    utils.sort();
    for util in utils {
        println!("\t{}", util);
    }
}

fn main() {
    let umap = util_map();
    let mut args : Vec<String> = env::args().collect();

    // try binary name as util name.
    let args0 = args[0].clone();
    let binary = Path::new(&args0[..]);
    let binary_as_util = binary.file_name().unwrap().to_str().unwrap();

    if let Some(&uumain) = umap.get(binary_as_util) {
        std::process::exit(uumain(args));
    }

    if binary_as_util.ends_with("uutils") || binary_as_util.starts_with("uutils") ||
     binary_as_util.ends_with("busybox") || binary_as_util.starts_with("busybox") {
        args.remove(0);
    } else {
        let mut found = false;
        for util in umap.keys() {
            if binary_as_util.ends_with(util) {
                args[0] = (*util).to_owned();
                found = true;
                break;
            }
        }
        if ! found {
            println!("{}: applet not found", binary_as_util);
            std::process::exit(1);
        }
    }

    // try first arg as util name.
    if args.len() >= 1 {

        let util = &args[0][..];

        match umap.get(util) {
            Some(&uumain) => {
                std::process::exit(uumain(args.clone()));
            }
            None => {
                if &args[0][..] == "--help" {
                    // see if they want help on a specific util
                    if args.len() >= 2 {
                        let util = &args[1][..];
                        match umap.get(util) {
                            Some(&uumain) => {
                                std::process::exit(uumain(vec![util.to_owned(), "--help".to_owned()]));
                            }
                            None => {
                                println!("{}: applet not found", util);
                                std::process::exit(1);
                            }
                        }
                    }
                    usage(&umap);
                    std::process::exit(0);
                } else {
                    println!("{}: applet not found", util);
                    std::process::exit(1);
                }
            }
        }
    } else {
        // no arguments provided
        usage(&umap);
        std::process::exit(0);
    }
}
<|endoftext|>