<commit_before>use std;
use std::num::FromPrimitive;
use std::ffi::CStr;
use std::str;
use std::borrow::ToOwned;
use ffi;
use {
    FtResult,
    GlyphSlot,
    Matrix,
    Vector,
};

#[repr(u32)]
#[derive(Copy)]
pub enum KerningMode {
    KerningDefault = ffi::FT_KERNING_DEFAULT,
    KerningUnfitted = ffi::FT_KERNING_UNFITTED,
    KerningUnscaled = ffi::FT_KERNING_UNSCALED
}

bitflags!(
flags LoadFlag: i32 {
    const DEFAULT                    = ffi::FT_LOAD_DEFAULT,
    const NO_SCALE                   = ffi::FT_LOAD_NO_SCALE,
    const NO_HINTING                 = ffi::FT_LOAD_NO_HINTING,
    const RENDER                     = ffi::FT_LOAD_RENDER,
    const NO_BITMAP                  = ffi::FT_LOAD_NO_BITMAP,
    const VERTICAL_LAYOUT            = ffi::FT_LOAD_VERTICAL_LAYOUT,
    const FORCE_AUTOHINT             = ffi::FT_LOAD_FORCE_AUTOHINT,
    const CROP_BITMAP                = ffi::FT_LOAD_CROP_BITMAP,
    const PEDANTIC                   = ffi::FT_LOAD_PEDANTIC,
    const IGNORE_GLOBAL_ADVANCE_WITH = ffi::FT_LOAD_IGNORE_GLOBAL_ADVANCE_WIDTH,
    const NO_RECURSE                 = ffi::FT_LOAD_NO_RECURSE,
    const IGNORE_TRANSFORM           = ffi::FT_LOAD_IGNORE_TRANSFORM,
    const MONOCHROME                 = ffi::FT_LOAD_MONOCHROME,
    const LINEAR_DESIGN              = ffi::FT_LOAD_LINEAR_DESIGN,
    const NO_AUTOHINT                = ffi::FT_LOAD_NO_AUTOHINT,
    const TARGET_NORMAL              = ffi::FT_LOAD_TARGET_NORMAL,
    const TARGET_LIGHT               = ffi::FT_LOAD_TARGET_LIGHT,
    const TARGET_MONO                = ffi::FT_LOAD_TARGET_MONO,
    const TARGET_LCD                 = ffi::FT_LOAD_TARGET_LCD,
    const TARGET_LCD_V               = ffi::FT_LOAD_TARGET_LCD_V,
    const COLOR                      = ffi::FT_LOAD_COLOR
});

#[derive(Eq, PartialEq, Hash)]
pub struct Face {
    library_raw: ffi::FT_Library,
    raw: ffi::FT_Face,
    glyph: GlyphSlot,
}

impl Clone for Face {
    fn clone(&self) -> Face {
        unsafe {
            ffi::FT_Reference_Library(self.library_raw);
            ffi::FT_Reference_Face(self.raw);
        }
        Face { ..*self }
    }
}

impl Face {
    pub fn from_raw(library_raw: ffi::FT_Library, raw: ffi::FT_Face) -> Face {
        unsafe {
            ffi::FT_Reference_Library(library_raw);
        }

        Face {
            library_raw: library_raw,
            raw: raw,
            glyph: unsafe { GlyphSlot::from_raw(library_raw, (*raw).glyph) },
        }
    }

    pub fn attach_file(&self, filepathname: &str) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Attach_File(self.raw, filepathname.as_ptr() as *const i8);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn reference(&self) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Reference_Face(self.raw);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn set_char_size(&self, char_width: isize, char_height: isize, horz_resolution: u32, vert_resolution: u32) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Set_Char_Size(self.raw, char_width as ffi::FT_F26Dot6,
                                            char_height as ffi::FT_F26Dot6, horz_resolution,
                                            vert_resolution);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn set_pixel_sizes(&self, pixel_width: u32, pixel_height: u32) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Set_Pixel_Sizes(self.raw, pixel_width, pixel_height);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn load_glyph(&self, glyph_index: u32, load_flags: LoadFlag) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Load_Glyph(self.raw, glyph_index, load_flags.bits);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn load_char(&self, char_code: usize, load_flags: LoadFlag) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Load_Char(self.raw, char_code as ffi::FT_ULong, load_flags.bits);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn set_transform(&self, matrix: &Matrix, delta: &Vector) {
        unsafe {
            ffi::FT_Set_Transform(self.raw, matrix, delta);
        }
    }

    pub fn get_char_index(&self, charcode: usize) -> u32 {
        unsafe {
            ffi::FT_Get_Char_Index(self.raw, charcode as ffi::FT_ULong)
        }
    }

    pub fn get_kerning(&self, left_char_index: u32, right_char_index: u32, kern_mode: KerningMode)
        -> FtResult<Vector> {

        let vec = Vector { x: 0, y: 0 };

        let err_code = unsafe {
            ffi::FT_Get_Kerning(self.raw, left_char_index, right_char_index,
                                kern_mode as u32, std::mem::transmute(&vec))
        };

        if err_code == ffi::FT_Err_Ok {
            Ok(vec)
        } else {
            Err(FromPrimitive::from_i32(err_code).unwrap())
        }
    }

    // According to FreeType doc, each time you load a new glyph image,
    // the previous one is erased from the glyph slot.
    #[inline(always)]
    pub fn glyph(&self) -> &GlyphSlot {
        &self.glyph
    }

    #[inline(always)]
    pub fn has_horizontal(&self) -> bool {
        ffi::FT_HAS_HORIZONTAL(self.raw)
    }

    #[inline(always)]
    pub fn has_vertical(&self) -> bool {
        ffi::FT_HAS_VERTICAL(self.raw)
    }

    #[inline(always)]
    pub fn has_kerning(&self) -> bool {
        ffi::FT_HAS_KERNING(self.raw)
    }

    #[inline(always)]
    pub fn is_scalable(&self) -> bool {
        ffi::FT_IS_SCALABLE(self.raw)
    }

    #[inline(always)]
    pub fn is_sfnt(&self) -> bool {
        ffi::FT_IS_SFNT(self.raw)
    }

    #[inline(always)]
    pub fn is_fixed_width(&self) -> bool {
        ffi::FT_IS_FIXED_WIDTH(self.raw)
    }

    #[inline(always)]
    pub fn has_fixed_sizes(&self) -> bool {
        ffi::FT_HAS_FIXED_SIZES(self.raw)
    }

    #[inline(always)]
    pub fn has_glyph_names(&self) -> bool {
        ffi::FT_HAS_GLYPH_NAMES(self.raw)
    }

    #[inline(always)]
    pub fn is_cid_keyed(&self) -> bool {
        ffi::FT_IS_CID_KEYED(self.raw)
    }

    #[inline(always)]
    pub fn is_tricky(&self) -> bool {
        ffi::FT_IS_TRICKY(self.raw)
    }

    #[inline(always)]
    pub fn has_color(&self) -> bool {
        ffi::FT_HAS_COLOR(self.raw)
    }

    #[inline(always)]
    pub fn raw(&self) -> &ffi::FT_FaceRec {
        unsafe {
            &*self.raw
        }
    }

    #[inline(always)]
    pub fn ascender(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).ascender
        }
    }

    #[inline(always)]
    pub fn descender(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).descender
        }
    }

    #[inline(always)]
    pub fn height(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).height
        }
    }

    #[inline(always)]
    pub fn max_advance_width(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).max_advance_width
        }
    }

    #[inline(always)]
    pub fn max_advance_height(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).max_advance_height
        }
    }

    #[inline(always)]
    pub fn underline_position(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).underline_position
        }
    }

    #[inline(always)]
    pub fn underline_thickness(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).underline_thickness
        }
    }

    pub fn family_name(&self) -> Option<String> {

        let family_name_ptr = unsafe { (*self.raw).family_name };

        if family_name_ptr.is_null() {
            None
        } else {
            let family_name = unsafe { CStr::from_ptr(family_name_ptr).to_bytes() };
            match str::from_utf8(family_name) {
                Ok(string)  => Some(string.to_owned()),
                _           => None
            }
        }
    }

    pub fn style_name(&self) -> Option<String> {

        let style_name_ptr = unsafe { (*self.raw).style_name };

        if style_name_ptr.is_null() {
            None
        } else {
            let style_name = unsafe { CStr::from_ptr(style_name_ptr).to_bytes() };
            match str::from_utf8(style_name) {
                Ok(string)  => Some(string.to_owned()),
                _           => None
            }
        }
    }
}

impl Drop for Face {
    fn drop(&mut self) {
        unsafe {
            match ffi::FT_Done_Face(self.raw) {
                ffi::FT_Err_Ok => {
                    if ffi::FT_Done_Library(self.library_raw) != ffi::FT_Err_Ok {
                        panic!("Failed to deref library");
                    }
                },
                _ => panic!("Failed to drop face"),
            }
        }
    }
}
<commit_msg>implemented Debug for Face<commit_after>use std;
use std::num::FromPrimitive;
use std::ffi::CStr;
use std::str;
use std::borrow::ToOwned;
use ffi;
use {
    FtResult,
    GlyphSlot,
    Matrix,
    Vector,
};

#[repr(u32)]
#[derive(Copy)]
pub enum KerningMode {
    KerningDefault = ffi::FT_KERNING_DEFAULT,
    KerningUnfitted = ffi::FT_KERNING_UNFITTED,
    KerningUnscaled = ffi::FT_KERNING_UNSCALED
}

bitflags!(
flags LoadFlag: i32 {
    const DEFAULT                    = ffi::FT_LOAD_DEFAULT,
    const NO_SCALE                   = ffi::FT_LOAD_NO_SCALE,
    const NO_HINTING                 = ffi::FT_LOAD_NO_HINTING,
    const RENDER                     = ffi::FT_LOAD_RENDER,
    const NO_BITMAP                  = ffi::FT_LOAD_NO_BITMAP,
    const VERTICAL_LAYOUT            = ffi::FT_LOAD_VERTICAL_LAYOUT,
    const FORCE_AUTOHINT             = ffi::FT_LOAD_FORCE_AUTOHINT,
    const CROP_BITMAP                = ffi::FT_LOAD_CROP_BITMAP,
    const PEDANTIC                   = ffi::FT_LOAD_PEDANTIC,
    const IGNORE_GLOBAL_ADVANCE_WITH = ffi::FT_LOAD_IGNORE_GLOBAL_ADVANCE_WIDTH,
    const NO_RECURSE                 = ffi::FT_LOAD_NO_RECURSE,
    const IGNORE_TRANSFORM           = ffi::FT_LOAD_IGNORE_TRANSFORM,
    const MONOCHROME                 = ffi::FT_LOAD_MONOCHROME,
    const LINEAR_DESIGN              = ffi::FT_LOAD_LINEAR_DESIGN,
    const NO_AUTOHINT                = ffi::FT_LOAD_NO_AUTOHINT,
    const TARGET_NORMAL              = ffi::FT_LOAD_TARGET_NORMAL,
    const TARGET_LIGHT               = ffi::FT_LOAD_TARGET_LIGHT,
    const TARGET_MONO                = ffi::FT_LOAD_TARGET_MONO,
    const TARGET_LCD                 = ffi::FT_LOAD_TARGET_LCD,
    const TARGET_LCD_V               = ffi::FT_LOAD_TARGET_LCD_V,
    const COLOR                      = ffi::FT_LOAD_COLOR
});

#[derive(Eq, PartialEq, Hash)]
pub struct Face {
    library_raw: ffi::FT_Library,
    raw: ffi::FT_Face,
    glyph: GlyphSlot,
}

impl Clone for Face {
    fn clone(&self) -> Face {
        unsafe {
            ffi::FT_Reference_Library(self.library_raw);
            ffi::FT_Reference_Face(self.raw);
        }
        Face { ..*self }
    }
}

impl Face {
    pub fn from_raw(library_raw: ffi::FT_Library, raw: ffi::FT_Face) -> Face {
        unsafe {
            ffi::FT_Reference_Library(library_raw);
        }

        Face {
            library_raw: library_raw,
            raw: raw,
            glyph: unsafe { GlyphSlot::from_raw(library_raw, (*raw).glyph) },
        }
    }

    pub fn attach_file(&self, filepathname: &str) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Attach_File(self.raw, filepathname.as_ptr() as *const i8);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn reference(&self) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Reference_Face(self.raw);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn set_char_size(&self, char_width: isize, char_height: isize, horz_resolution: u32, vert_resolution: u32) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Set_Char_Size(self.raw, char_width as ffi::FT_F26Dot6,
                                            char_height as ffi::FT_F26Dot6, horz_resolution,
                                            vert_resolution);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn set_pixel_sizes(&self, pixel_width: u32, pixel_height: u32) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Set_Pixel_Sizes(self.raw, pixel_width, pixel_height);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn load_glyph(&self, glyph_index: u32, load_flags: LoadFlag) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Load_Glyph(self.raw, glyph_index, load_flags.bits);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn load_char(&self, char_code: usize, load_flags: LoadFlag) -> FtResult<()> {
        unsafe {
            let err = ffi::FT_Load_Char(self.raw, char_code as ffi::FT_ULong, load_flags.bits);
            if err == ffi::FT_Err_Ok {
                Ok(())
            } else {
                Err(FromPrimitive::from_i32(err).unwrap())
            }
        }
    }

    pub fn set_transform(&self, matrix: &Matrix, delta: &Vector) {
        unsafe {
            ffi::FT_Set_Transform(self.raw, matrix, delta);
        }
    }

    pub fn get_char_index(&self, charcode: usize) -> u32 {
        unsafe {
            ffi::FT_Get_Char_Index(self.raw, charcode as ffi::FT_ULong)
        }
    }

    pub fn get_kerning(&self, left_char_index: u32, right_char_index: u32, kern_mode: KerningMode)
        -> FtResult<Vector> {

        let vec = Vector { x: 0, y: 0 };

        let err_code = unsafe {
            ffi::FT_Get_Kerning(self.raw, left_char_index, right_char_index,
                                kern_mode as u32, std::mem::transmute(&vec))
        };

        if err_code == ffi::FT_Err_Ok {
            Ok(vec)
        } else {
            Err(FromPrimitive::from_i32(err_code).unwrap())
        }
    }

    // According to FreeType doc, each time you load a new glyph image,
    // the previous one is erased from the glyph slot.
    #[inline(always)]
    pub fn glyph(&self) -> &GlyphSlot {
        &self.glyph
    }

    #[inline(always)]
    pub fn has_horizontal(&self) -> bool {
        ffi::FT_HAS_HORIZONTAL(self.raw)
    }

    #[inline(always)]
    pub fn has_vertical(&self) -> bool {
        ffi::FT_HAS_VERTICAL(self.raw)
    }

    #[inline(always)]
    pub fn has_kerning(&self) -> bool {
        ffi::FT_HAS_KERNING(self.raw)
    }

    #[inline(always)]
    pub fn is_scalable(&self) -> bool {
        ffi::FT_IS_SCALABLE(self.raw)
    }

    #[inline(always)]
    pub fn is_sfnt(&self) -> bool {
        ffi::FT_IS_SFNT(self.raw)
    }

    #[inline(always)]
    pub fn is_fixed_width(&self) -> bool {
        ffi::FT_IS_FIXED_WIDTH(self.raw)
    }

    #[inline(always)]
    pub fn has_fixed_sizes(&self) -> bool {
        ffi::FT_HAS_FIXED_SIZES(self.raw)
    }

    #[inline(always)]
    pub fn has_glyph_names(&self) -> bool {
        ffi::FT_HAS_GLYPH_NAMES(self.raw)
    }

    #[inline(always)]
    pub fn is_cid_keyed(&self) -> bool {
        ffi::FT_IS_CID_KEYED(self.raw)
    }

    #[inline(always)]
    pub fn is_tricky(&self) -> bool {
        ffi::FT_IS_TRICKY(self.raw)
    }

    #[inline(always)]
    pub fn has_color(&self) -> bool {
        ffi::FT_HAS_COLOR(self.raw)
    }

    #[inline(always)]
    pub fn raw(&self) -> &ffi::FT_FaceRec {
        unsafe {
            &*self.raw
        }
    }

    #[inline(always)]
    pub fn ascender(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).ascender
        }
    }

    #[inline(always)]
    pub fn descender(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).descender
        }
    }

    #[inline(always)]
    pub fn height(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).height
        }
    }

    #[inline(always)]
    pub fn max_advance_width(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).max_advance_width
        }
    }

    #[inline(always)]
    pub fn max_advance_height(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).max_advance_height
        }
    }

    #[inline(always)]
    pub fn underline_position(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).underline_position
        }
    }

    #[inline(always)]
    pub fn underline_thickness(&self) -> ffi::FT_Short {
        unsafe {
            (*self.raw).underline_thickness
        }
    }

    pub fn family_name(&self) -> Option<String> {

        let family_name_ptr = unsafe { (*self.raw).family_name };

        if family_name_ptr.is_null() {
            None
        } else {
            let family_name = unsafe { CStr::from_ptr(family_name_ptr).to_bytes() };
            match str::from_utf8(family_name) {
                Ok(string)  => Some(string.to_owned()),
                _           => None
            }
        }
    }

    pub fn style_name(&self) -> Option<String> {

        let style_name_ptr = unsafe { (*self.raw).style_name };

        if style_name_ptr.is_null() {
            None
        } else {
            let style_name = unsafe { CStr::from_ptr(style_name_ptr).to_bytes() };
            match str::from_utf8(style_name) {
                Ok(string)  => Some(string.to_owned()),
                _           => None
            }
        }
    }
}

impl ::std::fmt::Debug for Face {
    fn fmt(&self, form: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
        let name = self.style_name().unwrap_or("[unknown name]".to_string());
        try!(form.write_str("Font Face: "));
        form.write_str(&name[..])
    }
}

impl Drop for Face {
    fn drop(&mut self) {
        unsafe {
            match ffi::FT_Done_Face(self.raw) {
                ffi::FT_Err_Ok => {
                    if ffi::FT_Done_Library(self.library_raw) != ffi::FT_Err_Ok {
                        panic!("Failed to deref library");
                    }
                },
                _ => panic!("Failed to drop face"),
            }
        }
    }
}
<|endoftext|>


<commit_before>use ffi::types::{hid_t, hsize_t, hbool_t};
use ffi::h5p::{H5P_FILE_CREATE, H5P_FILE_ACCESS, H5Pcreate, H5Pset_userblock};
use ffi::h5f::{H5F_ACC_RDONLY, H5F_ACC_RDWR, H5F_ACC_EXCL, H5F_ACC_TRUNC,
               H5Fopen, H5Fcreate, H5Fclose, H5Fget_filesize, H5Fget_intent};
use ffi::drivers::{H5Pset_fapl_sec2, H5Pset_fapl_stdio, H5Pset_fapl_core};

use object::{Handle, Object};
use error::Result;
use plist::PropertyList;
use ffi::util::string_to_cstr;
use std::path::{Path, PathBuf};

use libc::{size_t, c_uint};

#[derive(Clone)]
pub struct File {
    handle: Handle,
}

impl Object for File {
    fn id(&self) -> hid_t {
        self.handle.id()
    }

    fn from_id(id: hid_t) -> File {
        File { handle: Handle::new(id) }
    }
}

impl File {
    pub fn size(&self) -> u64 {
        unsafe {
            let size: *mut hsize_t = &mut 0;
            h5lock_s!(H5Fget_filesize(self.id(), size));
            if *size > 0 { *size as u64 } else { 0 }
        }
    }

    pub fn is_read_only(&self) -> bool {
        unsafe {
            let mode: *mut c_uint = &mut 0;
            h5lock_s!(H5Fget_intent(self.id(), mode));
            *mode == H5F_ACC_RDONLY
        }
    }

    pub fn open<P: AsRef<Path>, S: Into<String>>(filename: P, mode: S) -> Result<File> {
        FileBuilder::new(filename).mode(mode).open()
    }

    fn fapl(&self) -> PropertyList {

    }
}

impl Drop for File {
    fn drop(&mut self) {
        if self.refcount() == 1 {
            h5lock!(H5Fclose(self.id()));
        }
    }
}

pub struct FileBuilder {
    filename: PathBuf,
    driver: String,
    mode: String,
    userblock: size_t,
    filebacked: bool,
    increment: size_t,
}

impl FileBuilder {
    pub fn new<P: AsRef<Path>>(filename: P) -> FileBuilder {
        FileBuilder {
            filename: filename.as_ref().to_path_buf(),
            driver: "sec2".to_string(),
            mode: "r".to_string(),
            userblock: 0,
            filebacked: false,
            increment: 64 * 1024 * 1024,
        }
    }

    pub fn driver<S: Into<String>>(&mut self, driver: S) -> &mut FileBuilder {
        self.driver = driver.into(); self
    }

    pub fn mode<S: Into<String>>(&mut self, mode: S) -> &mut FileBuilder {
        self.mode = mode.into(); self
    }

    pub fn userblock(&mut self, userblock: size_t) -> &mut FileBuilder {
        self.userblock = userblock; self
    }

    pub fn filebacked(&mut self, filebacked: bool) -> &mut FileBuilder {
        self.filebacked = filebacked; self
    }

    pub fn increment(&mut self, increment: size_t) -> &mut FileBuilder {
        self.increment = increment; self
    }

    fn make_fapl(&self) -> Result<PropertyList> {
        let fapl = PropertyList::from_id(h5try!(H5Pcreate(*H5P_FILE_ACCESS)));
        match self.driver.as_ref() {
            "sec2"  => h5try!(H5Pset_fapl_sec2(fapl.id())),
            "stdio" => h5try!(H5Pset_fapl_stdio(fapl.id())),
            "core"  => h5try!(H5Pset_fapl_core(fapl.id(), self.increment,
                                               self.filebacked as hbool_t)),
            _       => fail!(format!("Invalid file driver: {}", self.driver)),
        };
        Ok(fapl)
    }

    fn open_file(&self, write: bool) -> Result<File> {
        ensure!(self.userblock == 0, "Cannot specify userblock when opening a file");
        h5lock_s!({
            let fapl = try!(self.make_fapl());
            let flags = if write { H5F_ACC_RDWR } else { H5F_ACC_RDONLY };
            match self.filename.to_str() {
                Some(filename) => {
                    let c_filename = string_to_cstr(filename);
                    let file = File::from_id(h5try!(H5Fopen(c_filename, flags, fapl.id())));
                    ensure!(file.is_valid(), "Invalid id for opened file");
                    Ok(file)
                },
                None          => fail!("Invalid UTF-8 in file name: {:?}", self.filename)
            }
        })
    }

    fn create_file(&self, exclusive: bool) -> Result<File> {
        h5lock_s!({
            let fcpl = PropertyList::from_id(h5try!(H5Pcreate(*H5P_FILE_CREATE)));
            h5try!(H5Pset_userblock(fcpl.id(), self.userblock));
            let fapl = try!(self.make_fapl());
            let flags = if exclusive { H5F_ACC_EXCL } else { H5F_ACC_TRUNC };
            match self.filename.to_str() {
                Some(filename) => {
                    let c_filename = string_to_cstr(filename);
                    let file = File::from_id(h5try!(H5Fcreate(c_filename, flags,
                                                              fcpl.id(), fapl.id())));
                    ensure!(file.is_valid(), "Invalid id for created file");
                    Ok(file)
                },
                None          => fail!("Invalid UTF-8 in file name: {:?}", self.filename)
            }
        })
    }

    pub fn open(&self) -> Result<File> {
        match self.mode.as_ref() {
            "r"        => self.open_file(false),
            "r+"       => self.open_file(true),
            "w"        => self.create_file(false),
            "w-" | "x" => self.create_file(true),
            "a"        => match self.open_file(true) {
                            Ok(file) => Ok(file),
                            _        => self.create_file(true),
                          },
            _          => fail!("Invalid file access mode, expected r|r+|w|w-|x|a"),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::File;
    use std::path::PathBuf;
    use tempdir::TempDir;

    fn with_tmpdir<F: Fn(PathBuf)>(f: F) {
        let dir = TempDir::new_in(".", "tmp").unwrap();
        let path = dir.path().to_path_buf();
        f(path);
    }

    #[test]
    pub fn test_invalid_mode() {
        with_tmpdir(|dir| {
            assert_err!(File::open(&dir, "foo"), "Invalid file access mode");
        })
    }

    #[test]
    pub fn test_is_read_only() {
        with_tmpdir(|dir| {
            let path = dir.join("foo.h5");
            assert!(!File::open(&path, "w").unwrap().is_read_only());
            assert!(File::open(&path, "r").unwrap().is_read_only());
            assert!(!File::open(&path, "r+").unwrap().is_read_only());
            assert!(!File::open(&path, "a").unwrap().is_read_only());
        });
        with_tmpdir(|dir| {
            assert!(!File::open(dir.join("foo.h"), "a").unwrap().is_read_only());
        });
        with_tmpdir(|dir| {
            assert!(!File::open(dir.join("foo.h"), "x").unwrap().is_read_only());
        });
    }

    #[test]
    pub fn test_invalid_filename() {
        with_tmpdir(|dir| {
            assert_err!(File::open(&dir, "r"), "unable to open file");
            assert_err!(File::open(&dir, "r+"), "unable to open file");
            assert_err!(File::open(&dir, "x"), "unable to create file");
            assert_err!(File::open(&dir, "w-"), "unable to create file");
            assert_err!(File::open(&dir, "w"), "unable to create file");
            assert_err!(File::open(&dir, "a"), "unable to create file");
        });
    }
}
<commit_msg>Add File::fapl()<commit_after>use ffi::types::{hid_t, hsize_t, hbool_t};
use ffi::h5i::H5I_INVALID_HID;
use ffi::h5p::{H5P_FILE_CREATE, H5P_FILE_ACCESS, H5Pcreate, H5Pset_userblock};
use ffi::h5f::{H5F_ACC_RDONLY, H5F_ACC_RDWR, H5F_ACC_EXCL, H5F_ACC_TRUNC,
               H5Fopen, H5Fcreate, H5Fclose, H5Fget_filesize, H5Fget_intent,
               H5Fget_access_plist};
use ffi::drivers::{H5Pset_fapl_sec2, H5Pset_fapl_stdio, H5Pset_fapl_core};

use object::{Handle, Object};
use error::Result;
use plist::PropertyList;
use ffi::util::string_to_cstr;
use std::path::{Path, PathBuf};

use libc::{size_t, c_uint};

#[derive(Clone)]
pub struct File {
    handle: Handle,
}

impl Object for File {
    fn id(&self) -> hid_t {
        self.handle.id()
    }

    fn from_id(id: hid_t) -> File {
        File { handle: Handle::new(id) }
    }
}

impl File {
    pub fn size(&self) -> u64 {
        unsafe {
            let size: *mut hsize_t = &mut 0;
            h5lock_s!(H5Fget_filesize(self.id(), size));
            if *size > 0 { *size as u64 } else { 0 }
        }
    }

    pub fn is_read_only(&self) -> bool {
        unsafe {
            let mode: *mut c_uint = &mut 0;
            h5lock_s!(H5Fget_intent(self.id(), mode));
            *mode == H5F_ACC_RDONLY
        }
    }

    pub fn open<P: AsRef<Path>, S: Into<String>>(filename: P, mode: S) -> Result<File> {
        FileBuilder::new(filename).mode(mode).open()
    }

    fn fapl(&self) -> PropertyList {
        PropertyList::from_id(h5call!(H5Fget_access_plist(self.id())).unwrap_or(H5I_INVALID_HID))
    }
}

impl Drop for File {
    fn drop(&mut self) {
        if self.refcount() == 1 {
            h5lock!(H5Fclose(self.id()));
        }
    }
}

pub struct FileBuilder {
    filename: PathBuf,
    driver: String,
    mode: String,
    userblock: size_t,
    filebacked: bool,
    increment: size_t,
}

impl FileBuilder {
    pub fn new<P: AsRef<Path>>(filename: P) -> FileBuilder {
        FileBuilder {
            filename: filename.as_ref().to_path_buf(),
            driver: "sec2".to_string(),
            mode: "r".to_string(),
            userblock: 0,
            filebacked: false,
            increment: 64 * 1024 * 1024,
        }
    }

    pub fn driver<S: Into<String>>(&mut self, driver: S) -> &mut FileBuilder {
        self.driver = driver.into(); self
    }

    pub fn mode<S: Into<String>>(&mut self, mode: S) -> &mut FileBuilder {
        self.mode = mode.into(); self
    }

    pub fn userblock(&mut self, userblock: size_t) -> &mut FileBuilder {
        self.userblock = userblock; self
    }

    pub fn filebacked(&mut self, filebacked: bool) -> &mut FileBuilder {
        self.filebacked = filebacked; self
    }

    pub fn increment(&mut self, increment: size_t) -> &mut FileBuilder {
        self.increment = increment; self
    }

    fn make_fapl(&self) -> Result<PropertyList> {
        let fapl = PropertyList::from_id(h5try!(H5Pcreate(*H5P_FILE_ACCESS)));
        match self.driver.as_ref() {
            "sec2"  => h5try!(H5Pset_fapl_sec2(fapl.id())),
            "stdio" => h5try!(H5Pset_fapl_stdio(fapl.id())),
            "core"  => h5try!(H5Pset_fapl_core(fapl.id(), self.increment,
                                               self.filebacked as hbool_t)),
            _       => fail!(format!("Invalid file driver: {}", self.driver)),
        };
        Ok(fapl)
    }

    fn open_file(&self, write: bool) -> Result<File> {
        ensure!(self.userblock == 0, "Cannot specify userblock when opening a file");
        h5lock_s!({
            let fapl = try!(self.make_fapl());
            let flags = if write { H5F_ACC_RDWR } else { H5F_ACC_RDONLY };
            match self.filename.to_str() {
                Some(filename) => {
                    let c_filename = string_to_cstr(filename);
                    let file = File::from_id(h5try!(H5Fopen(c_filename, flags, fapl.id())));
                    ensure!(file.is_valid(), "Invalid id for opened file");
                    Ok(file)
                },
                None          => fail!("Invalid UTF-8 in file name: {:?}", self.filename)
            }
        })
    }

    fn create_file(&self, exclusive: bool) -> Result<File> {
        h5lock_s!({
            let fcpl = PropertyList::from_id(h5try!(H5Pcreate(*H5P_FILE_CREATE)));
            h5try!(H5Pset_userblock(fcpl.id(), self.userblock));
            let fapl = try!(self.make_fapl());
            let flags = if exclusive { H5F_ACC_EXCL } else { H5F_ACC_TRUNC };
            match self.filename.to_str() {
                Some(filename) => {
                    let c_filename = string_to_cstr(filename);
                    let file = File::from_id(h5try!(H5Fcreate(c_filename, flags,
                                                              fcpl.id(), fapl.id())));
                    ensure!(file.is_valid(), "Invalid id for created file");
                    Ok(file)
                },
                None          => fail!("Invalid UTF-8 in file name: {:?}", self.filename)
            }
        })
    }

    pub fn open(&self) -> Result<File> {
        match self.mode.as_ref() {
            "r"        => self.open_file(false),
            "r+"       => self.open_file(true),
            "w"        => self.create_file(false),
            "w-" | "x" => self.create_file(true),
            "a"        => match self.open_file(true) {
                            Ok(file) => Ok(file),
                            _        => self.create_file(true),
                          },
            _          => fail!("Invalid file access mode, expected r|r+|w|w-|x|a"),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::File;
    use std::path::PathBuf;
    use tempdir::TempDir;

    fn with_tmpdir<F: Fn(PathBuf)>(f: F) {
        let dir = TempDir::new_in(".", "tmp").unwrap();
        let path = dir.path().to_path_buf();
        f(path);
    }

    #[test]
    pub fn test_invalid_mode() {
        with_tmpdir(|dir| {
            assert_err!(File::open(&dir, "foo"), "Invalid file access mode");
        })
    }

    #[test]
    pub fn test_is_read_only() {
        with_tmpdir(|dir| {
            let path = dir.join("foo.h5");
            assert!(!File::open(&path, "w").unwrap().is_read_only());
            assert!(File::open(&path, "r").unwrap().is_read_only());
            assert!(!File::open(&path, "r+").unwrap().is_read_only());
            assert!(!File::open(&path, "a").unwrap().is_read_only());
        });
        with_tmpdir(|dir| {
            assert!(!File::open(dir.join("foo.h"), "a").unwrap().is_read_only());
        });
        with_tmpdir(|dir| {
            assert!(!File::open(dir.join("foo.h"), "x").unwrap().is_read_only());
        });
    }

    #[test]
    pub fn test_invalid_filename() {
        with_tmpdir(|dir| {
            assert_err!(File::open(&dir, "r"), "unable to open file");
            assert_err!(File::open(&dir, "r+"), "unable to open file");
            assert_err!(File::open(&dir, "x"), "unable to create file");
            assert_err!(File::open(&dir, "w-"), "unable to create file");
            assert_err!(File::open(&dir, "w"), "unable to create file");
            assert_err!(File::open(&dir, "a"), "unable to create file");
        });
    }
}
<|endoftext|>


<commit_before>use std::{env, io, mem, num, i32, usize, fs, slice, string};
use std::io::{Read, Write, BufRead, Error, ErrorKind};
use std::cmp::Ordering;
use std::collections::{HashMap,VecDeque};

#[derive(Copy, Clone)]
struct CooccurRec {
    word1: u32,
    word2: u32,
    val: f32,
}

static mut verbose: i32 = 2;

macro_rules! progress {
    ($v:expr, $fmt:expr) => (
        if verbose > $v { write!(io::stderr(), $fmt).unwrap(); }
    );
    ($v:expr, $fmt:expr, $($arg:tt)*) => (
        if verbose > $v { write!(io::stderr(), $fmt, $($arg)*).unwrap(); }
    );
}
macro_rules! log {
    ($v:expr, $fmt:expr) => (
        if verbose > $v { writeln!(io::stderr(), $fmt).unwrap(); }
    );
    ($v:expr, $fmt:expr, $($arg:tt)*) => (
        if verbose > $v { writeln!(io::stderr(), $fmt, $($arg)*).unwrap(); }
    );
}

enum GetWord {
    Word(String),
    EndLine,
    EndOfFile,
    IOError(io::Error),
    Utf8Error(std::string::FromUtf8Error),
}

fn get_word(fin: &io::Read) -> GetWord {
    static mut next_endline: bool = false;
    if next_endline {
        next_endline = false;
        return GetWord::EndLine
    }

    let mut eof: bool = false;
    let mut word: Vec<u8> = vec![];
    let mut byte: [u8; 1];
    loop {
        match fin.read_exact(&mut byte) {
            Ok(_) => match byte[0] {
                b' ' | b'\t' => { break; },
                b'\n' => {
                    next_endline = true;
                    break;
                },
                c => { word.push(c); }
            },
            Err(e) => if e.kind() == ErrorKind::UnexpectedEof {
                eof = true;
                break;
            } else {
                return GetWord::IOError(e);
            }
        }
    }
    if word.len() > 0 {
        match String::from_utf8(word) {
            Ok(x) => GetWord::Word(x),
            Err(e) => GetWord::Utf8Error(e),
        }
    } else if eof {
        GetWord::EndOfFile
    } else {
        get_word(fin)
    }
}

fn merge_files(file_head: &str, num: i32) {
    progress!(1, "Merging cooccurrence files: processed 0 lines.");
    let pq = VecDeque::<(CooccurRec, i32)>::new();
    let fid = Vec::<fs::File>::new();
    for i in 0..num {
        let mut data: CooccurRec;
        let f = fs::File::open(format!("{}_{:>4}.bin", file_head, i)).unwrap();
        f.read_exact(unsafe {
            slice::from_raw_parts_mut((&mut data as *mut CooccurRec) as *mut u8,
            mem::size_of::<CooccurRec>())}).unwrap();
        pq.push_back((data, i));
        fid.push(f);
    }
    // Pop op node, save it in old to see if the next entry is a duplicate
    let mut size = num as usize;
    let (mut old, mut i) = match pq.pop_front().unwrap();

}

fn get_cooccurrence(symmetric: bool, window_size: usize,
                    max_product: usize, overflow_length: usize,
                    vocab_file: &str, file_head: &str) -> i32 {
    macro_rules! min {
        ($a: expr, $b: expr) => { if $a <= $b { $a } else { $b } };
    }
    macro_rules! max {
        ($a: expr, $b: expr) => { if $a >= $b { $a } else { $b } };
    }

    log!(-1, "COUNTING COOCCURRENCES");
    log!(0, "window size: {}", window_size);
    log!(0, "context: {}", if symmetric { "symmetric" } else { "asymmetric" });
    log!(1, "max product: {}", max_product);
    log!(1, "overflow length: {}", overflow_length);

    let mut vocab_hash: HashMap<String, i64> = HashMap::new();
    log!(1, "Reading vocab from file \"{}\"...", vocab_file);
    {
        let file = io::BufReader::new(fs::File::open(vocab_file)
                                      .expect("Unable to open vocab file."));
        for (i, line) in file.lines().enumerate() {
            let line = line.expect("vocab file read error.");
            let vec: Vec<&str> = line.split_whitespace().collect();
            let word = vec[0];
            //let num = vec[1].parse::<u32>().expect("Parse error.");

            vocab_hash.insert(word.to_string(), i as i64);
        }
        log!(1, "loaded {} words.", vocab_hash.len());
    }
    let mut table: Vec<Vec<f32>> = Vec::with_capacity(vocab_hash.len());
    {
        let mut size = 1;
        table.push(vec![0.0_f32]);
        for a in 1..vocab_hash.len() {
            size += min!(max_product / a, vocab_hash.len());
            table.push(vec![0.0_f32 ; size]);
        }
    }
    // for each token in input stream, calculate a weighted cooccurrence sum within window_size.
    let value = |j, k| 1.0f32 / (j - k) as f32;
    let write_chunk = |cr: &Vec<CooccurRec>, n| {
        let mut f = fs::File::create(format!("{}_{:>04}.bin", file_head, n)).unwrap();
        let mut old = cr[0];

        for x in &cr[1..] {
            if x.word1 == old.word1 && x.word2 == old.word2 {
                old.val += x.val;
                continue;
            }
            f.write(unsafe {
                slice::from_raw_parts((&old as *const CooccurRec) as *const u8,
                mem::size_of::<CooccurRec>())}).unwrap();
            old = *x;
        }
        f.write(unsafe {
            slice::from_raw_parts((&old as *const CooccurRec) as *const u8,
            mem::size_of::<CooccurRec>())}).unwrap();
    };

    let mut fcounter = 1i32;
    let mut cr: Vec<CooccurRec> = Vec::with_capacity(overflow_length + 1);
    let mut history = vec![0i64 ; window_size as usize];
    let mut n_words = 0usize;
    let mut j = 0i64;
    let stdin = io::BufReader::new(io::stdin());
    progress!(1, "Processing token: 0");
    loop {
        let word = match get_word(&stdin) {
            GetWord::Word(w) => w,
            GetWord::EndLine => { j = 0; continue },
            GetWord::EndOfFile => break,
            GetWord::Utf8Error(_) => continue,
            GetWord::IOError(e) => panic!(format!("{:?}", e)),
        };
        j += 1;
        n_words += 1;
        if n_words % 100000 == 0 { progress!(1, "\x1b[19G{}", j); }
        if let Some(&w2) = vocab_hash.get(&word) {
            for k in (max!(j - window_size as i64, 0) .. j - 1).rev() {
                let w1 = history[k as usize % window_size];
                if (w1 as usize) < max_product / (w2 as usize) {  // Product is small enough to store in a full array
                    table[(w1 - 1) as usize][(w2 - 2) as usize] += value(j, k);
                    if symmetric { table[(w2 - 1) as usize][(w1 - 2) as usize] += value(j, k); }
                }
                else {  // Product is too big, data is likely to be sparse. Store these entries in a temporary buffer to be sorted, merged (accumulated), and written to file when it gets full.
                    cr.push(CooccurRec { word1: w1 as u32, word2: w2 as u32, val: value(j, k)});
                    if symmetric { cr.push(CooccurRec { word1: w2 as u32, word2: w1 as u32, val: value(j, k)}); }
                    if cr.len() >= overflow_length - window_size {
                        cr.sort_by(|lhs, rhs| match lhs.word1.cmp(&rhs.word1) {
                            Ordering::Equal => lhs.word2.cmp(&rhs.word2),
                            x => x,
                        });
                        write_chunk(&cr, fcounter);
                        fcounter += 1;
                        cr.clear();
                    }
                }
                history[j as usize % window_size] = w2;
                j += 1;
            }
        }
    }
    log!(1, "\x1b[0GProcessed {} tokens.", n_words);
    cr.sort_by(|lhs, rhs| match lhs.word1.cmp(&rhs.word1) {
        Ordering::Equal => lhs.word2.cmp(&rhs.word2),
        x => x,
    });
    write_chunk(&cr, fcounter);
    log!(1, "Writing cooccurrences to disk");
    let j = i32::MAX;
    let file = fs::File::create(format!("{}_0000.bin", file_head)).unwrap();
    for (x, &v) in table.iter().enumerate() {
        if ((0.75f64 * (vocab_hash.len() / x) as f64).ln() as i32) < j {
            progress!(1, ".")
        }
        for (y, &r) in v.iter().enumerate() {
            if r != 0f32 {
                file.write(unsafe {mem::transmute::<u32, &[u8; 4]>(x as u32)});
                file.write(unsafe {mem::transmute::<u32, &[u8; 4]>(y as u32)});
                file.write(unsafe {mem::transmute::<f32, &[u8; 4]>(r)});
            }
        }
    }
    log!(1, "{} files in total.", fcounter);
    merge_files(file_head, fcounter + 1);
    0
}

fn main() {
    let mut window_size = 15usize;
    let mut symmetric = true;
    let mut memory_limit = 3.0;
    let mut vocab_file = "vocab.txt".to_string();
    let mut file_head = "overflow".to_string();
    let mut max_product: Option<usize> = None;
    let mut overflow_length: Option<usize> = None;
    {
        let args: Vec<String> = env::args().collect();

        if args.len() == 1 {
            println!("Tool to calculate word-word cooccurrence statistics
This program is transportation of GloVe cooccur.
Original program (written in C) author is Jeffrey Pennington (jpennin@stanford.edu)
Transpoted to rust by Takumi Kato (takumi.kt@gmail.com)

Usage options:
\t-verbose <int>
\t\tSet verbosity: 0, 1, or 2 (default)
\t-symmetric <int>
\t\tIf <int> = 0, only use left context; if <int> = 1 (default), use left and right
\t-window-size <int>
\t\tNumber of context words to the left (and to the right, if symmetric = 1); default 15
\t-vocab-file <file>
\t\tFile containing vocabulary (truncated unigram counts, produced by 'vocab_count'); default vocab.txt
\t-memory <float>
\t\tSoft limit for memory consumption, in GB -- based on simple heuristic, so not extremely accurate; default 4.0
\t-max-product <int>
\t\tLimit the size of dense cooccurrence array by specifying the max product <int> of the frequency counts of the two cooccurring words.
\t\tThis value overrides that which is automatically produced by '-memory'. Typically only needs adjustment for use with very large corpora.
\toverflow-file <file>
\t\tFilename, excluding extension, for temporary file; default overflow

Example usage:
./cooccur -verbose 2 -symmetric 0 -window-size 10 -vocab-file vocab.txt -memory 8.0 -overflow-file tempoverflow < corpus.txt > cooccurrences.bin

");
            std::process::exit(0);
        }
        let mut is_skip = true;
        let mut it = args.into_iter().peekable();
        loop {
            let arg = match it.next() {
                Some(s) => s,
                None => break,
            };
            if is_skip {
                is_skip = false;
                continue
            }
            match arg.as_ref() {
                "-verbose" => {
                    verbose = it.peek().and_then(|v| v.parse().ok())
                        .and_then(|x| if 0 <= x && x <= 2 { Some(x) } else { None })
                        .expect("-verbose <int>, verbosity: 0, 1, or 2 (default)");
                    is_skip = true;
                },
                "-symmetric" => {
                    symmetric = it.peek().and_then(|v| v.parse().ok())
                        .and_then(|x| if 0 <= x && x <= 1 { Some(x) } else { None })
                        .expect("symmetric <int>, 0 (only use left context) or 1 (use left and right; default)") == 1;
                    is_skip = true;
                },
                "-window-size" => {
                    window_size = it.peek().and_then(|v| v.parse().ok()).expect("-window-size <int>");
                    is_skip = true;
                },
                "-memory" => {
                    memory_limit = it.peek().and_then(|v| v.parse().ok()).expect("-memory <float>");
                    is_skip = true;
                },
                "-vocab-file" => {
                    vocab_file = it.peek().expect("-vocab-file <file>").to_string();
                    is_skip = true;
                },
                "-overflow" => {
                    file_head = it.peek().expect("-overflow <file>").to_string();
                    is_skip = true;
                },
                "-max-product" => {
                    max_product = Some(it.peek().and_then(|v| v.parse().ok()).expect("-max-product <int>"));
                    is_skip = true;
                },
                "-overflow-length" => {
                    overflow_length = Some(it.peek().and_then(|v| v.parse().ok()).expect("-overflow-length <int>"));
                    is_skip = true;
                },
                &_ => {},
            }
            let rlimit = 0.85 * memory_limit * 1073741824_f64 / mem::size_of::<f64>() as f64;
            let mut n = 1e5_f64;
            while (rlimit - n * (n.ln() + 0.1544313298)).abs() > 1e-3 { n = rlimit / (n.ln() + 0.1544313298) }
            max_product = max_product.or_else(|| Some(n as usize));
            overflow_length = overflow_length.or_else(|| Some((rlimit / 6.0) as usize));
        }
        //std::process::exit(get_counts(verbose, max_vocab, min_count));
    }
}
<commit_msg>Update cooccur (still not working)<commit_after>use std::{env, io, mem, num, i32, usize, fs, slice, string};
use std::io::{Read, Write, BufRead, Error, ErrorKind};
use std::cmp::Ordering;
use std::collections::{HashMap,BinaryHeap};

#[derive(Copy, Clone, PartialEq)]
struct CooccurRec {
    word1: u32,
    word2: u32,
    val: f32,
}

#[derive(Copy, Clone, PartialEq)]
struct CRecId {
    crec: CooccurRec,
    id: usize,
}
impl CRecId {
    fn new(c: CooccurRec, n: usize) -> CRecId {
        CRecId { crec: c, id: n}
    }
}
impl Eq for CRecId {}
impl Ord for CRecId {
    fn cmp(&self, other: &CRecId) -> Ordering {
        // For using std::collections::BinaryHeap, ordering is reversed.
        match other.crec.word1.cmp(&self.crec.word1) {
            Ordering::Equal => other.crec.word2.cmp(&self.crec.word2),
            x => x,
        }
    }
}
impl PartialOrd for CRecId {
    fn partial_cmp(&self, other: &CRecId) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

static mut verbose: i32 = 2;

macro_rules! progress {
    ($v:expr, $fmt:expr) => (
        if verbose > $v { write!(io::stderr(), $fmt).unwrap(); }
    );
    ($v:expr, $fmt:expr, $($arg:tt)*) => (
        if verbose > $v { write!(io::stderr(), $fmt, $($arg)*).unwrap(); }
    );
}
macro_rules! log {
    ($v:expr, $fmt:expr) => (
        if verbose > $v { writeln!(io::stderr(), $fmt).unwrap(); }
    );
    ($v:expr, $fmt:expr, $($arg:tt)*) => (
        if verbose > $v { writeln!(io::stderr(), $fmt, $($arg)*).unwrap(); }
    );
}

enum GetWord {
    Word(String),
    EndLine,
    EndOfFile,
    IOError(io::Error),
    Utf8Error(std::string::FromUtf8Error),
}

fn get_word(fin: &io::Read) -> GetWord {
    static mut next_endline: bool = false;
    if next_endline {
        next_endline = false;
        return GetWord::EndLine
    }

    let mut eof: bool = false;
    let mut word: Vec<u8> = vec![];
    let mut byte: [u8; 1];
    loop {
        match fin.read_exact(&mut byte) {
            Ok(_) => match byte[0] {
                b' ' | b'\t' => { break; },
                b'\n' => {
                    next_endline = true;
                    break;
                },
                c => { word.push(c); }
            },
            Err(e) => if e.kind() == ErrorKind::UnexpectedEof {
                eof = true;
                break;
            } else {
                return GetWord::IOError(e);
            }
        }
    }
    if word.len() > 0 {
        match String::from_utf8(word) {
            Ok(x) => GetWord::Word(x),
            Err(e) => GetWord::Utf8Error(e),
        }
    } else if eof {
        GetWord::EndOfFile
    } else {
        get_word(fin)
    }
}

fn merge_files(file_head: &str, num: usize) {
    progress!(1, "Merging cooccurrence files: processed 0 lines.");
    let pq = BinaryHeap::<CRecId>::new();
    let fid = Vec::<fs::File>::new();
    for i in 0..num {
        let mut data: CooccurRec;
        let f = fs::File::open(format!("{}_{:>4}.bin", file_head, i)).unwrap();
        f.read_exact(unsafe {
            slice::from_raw_parts_mut((&mut data as *mut CooccurRec) as *mut u8,
            mem::size_of::<CooccurRec>())}).unwrap();
        pq.push(CRecId::new(data, i));
        fid.push(f);
    }
    // Pop top node, save it in old to see if the next entry is a duplicate
    let mut new = pq.pop().unwrap();
    //...
    //Repeatedly...
}

fn get_cooccurrence(symmetric: bool, window_size: usize,
                    max_product: usize, overflow_length: usize,
                    vocab_file: &str, file_head: &str) -> i32 {
    macro_rules! min {
        ($a: expr, $b: expr) => { if $a <= $b { $a } else { $b } };
    }
    macro_rules! max {
        ($a: expr, $b: expr) => { if $a >= $b { $a } else { $b } };
    }

    log!(-1, "COUNTING COOCCURRENCES");
    log!(0, "window size: {}", window_size);
    log!(0, "context: {}", if symmetric { "symmetric" } else { "asymmetric" });
    log!(1, "max product: {}", max_product);
    log!(1, "overflow length: {}", overflow_length);

    let mut vocab_hash: HashMap<String, i64> = HashMap::new();
    log!(1, "Reading vocab from file \"{}\"...", vocab_file);
    {
        let file = io::BufReader::new(fs::File::open(vocab_file).unwrap());
        for (i, line) in file.lines().enumerate() {
            let line = line.unwrap();
            let vec: Vec<&str> = line.split_whitespace().collect();
            let word = vec[0];

            vocab_hash.insert(word.to_string(), i as i64);
        }
        log!(1, "loaded {} words.", vocab_hash.len());
    }
    let mut table: Vec<Vec<f32>> = Vec::with_capacity(vocab_hash.len());
    {
        let mut size = 1;
        table.push(vec![0.0_f32]);
        for a in 1..vocab_hash.len() {
            size += min!(max_product / a, vocab_hash.len());
            table.push(vec![0.0_f32 ; size]);
        }
    }
    // for each token in input stream, calculate a weighted cooccurrence sum within window_size.
    let value = |j, k| 1.0f32 / (j - k) as f32;
    let write_chunk = |cr: &Vec<CooccurRec>, n| {
        let mut f = fs::File::create(format!("{}_{:>04}.bin", file_head, n)).unwrap();
        let mut old = cr[0];

        for x in &cr[1..] {
            if x.word1 == old.word1 && x.word2 == old.word2 {
                old.val += x.val;
                continue;
            }
            f.write(unsafe {
                slice::from_raw_parts((&old as *const CooccurRec) as *const u8,
                mem::size_of::<CooccurRec>())}).unwrap();
            old = *x;
        }
        f.write(unsafe {
            slice::from_raw_parts((&old as *const CooccurRec) as *const u8,
            mem::size_of::<CooccurRec>())}).unwrap();
    };

    let mut fcounter = 1i32;
    let mut cr: Vec<CooccurRec> = Vec::with_capacity(overflow_length + 1);
    let mut history = vec![0i64 ; window_size as usize];
    let mut n_words = 0usize;
    let mut j = 0i64;
    let stdin = io::BufReader::new(io::stdin());
    progress!(1, "Processing token: 0");
    loop {
        let word = match get_word(&stdin) {
            GetWord::Word(w) => w,
            GetWord::EndLine => { j = 0; continue },
            GetWord::EndOfFile => break,
            GetWord::Utf8Error(_) => continue,
            GetWord::IOError(e) => panic!(format!("{:?}", e)),
        };
        j += 1;
        n_words += 1;
        if n_words % 100000 == 0 { progress!(1, "\x1b[19G{}", j); }
        if let Some(&w2) = vocab_hash.get(&word) {
            for k in (max!(j - window_size as i64, 0) .. j - 1).rev() {
                let w1 = history[k as usize % window_size];
                if (w1 as usize) < max_product / (w2 as usize) {  // Product is small enough to store in a full array
                    table[(w1 - 1) as usize][(w2 - 2) as usize] += value(j, k);
                    if symmetric { table[(w2 - 1) as usize][(w1 - 2) as usize] += value(j, k); }
                }
                else {  // Product is too big, data is likely to be sparse. Store these entries in a temporary buffer to be sorted, merged (accumulated), and written to file when it gets full.
                    cr.push(CooccurRec { word1: w1 as u32, word2: w2 as u32, val: value(j, k)});
                    if symmetric { cr.push(CooccurRec { word1: w2 as u32, word2: w1 as u32, val: value(j, k)}); }
                    if cr.len() >= overflow_length - window_size {
                        cr.sort_by(|lhs, rhs| match lhs.word1.cmp(&rhs.word1) {
                            Ordering::Equal => lhs.word2.cmp(&rhs.word2),
                            x => x,
                        });
                        write_chunk(&cr, fcounter);
                        fcounter += 1;
                        cr.clear();
                    }
                }
                history[j as usize % window_size] = w2;
                j += 1;
            }
        }
    }
    log!(1, "\x1b[0GProcessed {} tokens.", n_words);
    cr.sort_by(|lhs, rhs| match lhs.word1.cmp(&rhs.word1) {
        Ordering::Equal => lhs.word2.cmp(&rhs.word2),
        x => x,
    });
    write_chunk(&cr, fcounter);
    log!(1, "Writing cooccurrences to disk");
    let j = i32::MAX;
    let file = fs::File::create(format!("{}_0000.bin", file_head)).unwrap();
    for (x, &v) in table.iter().enumerate() {
        if ((0.75f64 * (vocab_hash.len() / x) as f64).ln() as i32) < j {
            progress!(1, ".")
        }
        for (y, &r) in v.iter().enumerate() {
            if r != 0f32 {
                file.write(unsafe {mem::transmute::<u32, &[u8; 4]>(x as u32)});
                file.write(unsafe {mem::transmute::<u32, &[u8; 4]>(y as u32)});
                file.write(unsafe {mem::transmute::<f32, &[u8; 4]>(r)});
            }
        }
    }
    log!(1, "{} files in total.", fcounter);
    merge_files(file_head, fcounter + 1);
    0
}

fn main() {
    let mut window_size = 15usize;
    let mut symmetric = true;
    let mut memory_limit = 3.0;
    let mut vocab_file = "vocab.txt".to_string();
    let mut file_head = "overflow".to_string();
    let mut max_product: Option<usize> = None;
    let mut overflow_length: Option<usize> = None;
    {
        let args: Vec<String> = env::args().collect();

        if args.len() == 1 {
            println!("Tool to calculate word-word cooccurrence statistics
This program is transportation of GloVe cooccur.
Original program (written in C) author is Jeffrey Pennington (jpennin@stanford.edu)
Transpoted to rust by Takumi Kato (takumi.kt@gmail.com)

Usage options:
\t-verbose <int>
\t\tSet verbosity: 0, 1, or 2 (default)
\t-symmetric <int>
\t\tIf <int> = 0, only use left context; if <int> = 1 (default), use left and right
\t-window-size <int>
\t\tNumber of context words to the left (and to the right, if symmetric = 1); default 15
\t-vocab-file <file>
\t\tFile containing vocabulary (truncated unigram counts, produced by 'vocab_count'); default vocab.txt
\t-memory <float>
\t\tSoft limit for memory consumption, in GB -- based on simple heuristic, so not extremely accurate; default 4.0
\t-max-product <int>
\t\tLimit the size of dense cooccurrence array by specifying the max product <int> of the frequency counts of the two cooccurring words.
\t\tThis value overrides that which is automatically produced by '-memory'. Typically only needs adjustment for use with very large corpora.
\toverflow-file <file>
\t\tFilename, excluding extension, for temporary file; default overflow

Example usage:
./cooccur -verbose 2 -symmetric 0 -window-size 10 -vocab-file vocab.txt -memory 8.0 -overflow-file tempoverflow < corpus.txt > cooccurrences.bin

");
            std::process::exit(0);
        }
        let mut is_skip = true;
        let mut it = args.into_iter().peekable();
        loop {
            let arg = match it.next() {
                Some(s) => s,
                None => break,
            };
            if is_skip {
                is_skip = false;
                continue
            }
            match arg.as_ref() {
                "-verbose" => {
                    verbose = it.peek().and_then(|v| v.parse().ok())
                        .and_then(|x| if 0 <= x && x <= 2 { Some(x) } else { None })
                        .expect("-verbose <int>, verbosity: 0, 1, or 2 (default)");
                    is_skip = true;
                },
                "-symmetric" => {
                    symmetric = it.peek().and_then(|v| v.parse().ok())
                        .and_then(|x| if 0 <= x && x <= 1 { Some(x) } else { None })
                        .expect("symmetric <int>, 0 (only use left context) or 1 (use left and right; default)") == 1;
                    is_skip = true;
                },
                "-window-size" => {
                    window_size = it.peek().and_then(|v| v.parse().ok()).expect("-window-size <int>");
                    is_skip = true;
                },
                "-memory" => {
                    memory_limit = it.peek().and_then(|v| v.parse().ok()).expect("-memory <float>");
                    is_skip = true;
                },
                "-vocab-file" => {
                    vocab_file = it.peek().expect("-vocab-file <file>").to_string();
                    is_skip = true;
                },
                "-overflow" => {
                    file_head = it.peek().expect("-overflow <file>").to_string();
                    is_skip = true;
                },
                "-max-product" => {
                    max_product = Some(it.peek().and_then(|v| v.parse().ok()).expect("-max-product <int>"));
                    is_skip = true;
                },
                "-overflow-length" => {
                    overflow_length = Some(it.peek().and_then(|v| v.parse().ok()).expect("-overflow-length <int>"));
                    is_skip = true;
                },
                &_ => {},
            }
            let rlimit = 0.85 * memory_limit * 1073741824_f64 / mem::size_of::<f64>() as f64;
            let mut n = 1e5_f64;
            while (rlimit - n * (n.ln() + 0.1544313298)).abs() > 1e-3 { n = rlimit / (n.ln() + 0.1544313298) }
            max_product = max_product.or_else(|| Some(n as usize));
            overflow_length = overflow_length.or_else(|| Some((rlimit / 6.0) as usize));
        }
        //std::process::exit(get_counts(verbose, max_vocab, min_count));
    }
}
<|endoftext|>


<commit_before>// Used internally for evalulating expressions
// Note: for now, negative is falsy and positive (including zero) is truthy

use interpreter::lexer::Token;
use interpreter::lexer;
use std::collections::HashMap;
use std::collections::hashmap::{Occupied, Vacant};
use std::num;

#[deriving(Show)]
enum ExprToken<'a> {
	Op(Operator),
	Value(f32),
	Var(uint),
	LParen,
	RParen,
}

#[deriving(Show)]
enum Operator {
	Add,
	Sub,
	Mul,
	Div,
	Exp,
	Mod,
	Neg,
	Less,
	Greater,
	Equ,
	NotEqu,
	ApproxEqu,
	Not,
	And,
	Or,
	Xor,
	GreaterEqual,
	LessEqual,
}

fn precedence(op: Operator) -> int {
	match op {
		Equ | NotEqu | ApproxEqu => 1,
		Less | Greater | GreaterEqual | LessEqual => 2,
		And | Or | Xor => 0,
		Add => 3,
		Sub => 3,
		Mul => 4,
		Div => 4,
		Mod => 4,
		Neg => 6,
		Not => 6,
		Exp => 6,
	}
}

fn nargs(op: Operator) -> uint {
	match op {
		Neg | Not => 1,
		_ => 2,
	}
}

#[deriving(PartialEq)]
enum Associativity {
	Left,
	Right,
}

fn associativity(op: Operator) -> Associativity {
	match op {
		Exp => Right,
		_ => Left,
	}
}

#[deriving(Show)]
pub struct Expression<'a> {
	rpn: Vec<ExprToken<'a>>, // reverse polish notation
	var_values: Vec<f32>, // values of vars by id
	vars: HashMap<&'a str, uint>, // map of var name -> var id
	value: Option<f32>, // holds the last value of the expression, if it's been evaluated and vars have not been updated
}

impl<'a> Expression<'a> {
	// converts a token slice from the lexer into an expression that can be evaluated
	pub fn new<'a>(tok: Vec<Token<'a>>) -> Expression {
		let mut vars: HashMap<&'a str, uint> = HashMap::new();
		let out: Vec<ExprToken<'a>> = tok.iter().filter_map(|&t| {
			match t {
				// Try to parse constants as f32. subject to maybe change but probably not.
				lexer::Const(v) => {
					if let Some(x) = from_str::<f32>(v) {
						Some(Value(x))
					} else {
						fail!("invalid const, regex must be broken!");
					}
				},
				lexer::Operator(v) => {
					Some(Op(match v {
						"+" => Add,
						"-" => Sub,
						"*" => Mul,
						"/" => Div,
						"^" => Exp,
						"%" => Mod,
						"==" => Equ,
						"!=" => NotEqu,
						"~=" => ApproxEqu,
						"<" => Less,
						">" => Greater,
						"<=" => LessEqual,
						">=" => GreaterEqual,
						"!" => Not,
						"&&" => And,
						"||" => Or,
						"^^" => Xor,
						_ => fail!("unexpected operator"),
					}))
				},
				lexer::Paren(v) => {
					if v == "(" {
						Some(LParen)
					} else if v == ")" {
						Some(RParen)
					} else {
						None
					}
				},
				// An identifier in the context of an expression is always a variable (TODO FALSE!!)
				lexer::Ident(v) => {
					let nv = vars.len();
					match vars.entry(v) {
						Occupied(entry) => {
							Some(Var(*entry.get()))
						},
						Vacant(entry) => {
							entry.set(nv);
							Some(Var(nv))
						},
					}
				},
				// Discard whitesapce
				lexer::Whitespace(_) | lexer::Newline => {
					None
				},
				_ => {
					fail!("not expression!");
				}
			}
		}).collect();

		// Handle special case with unary minus
		// If an subtraction operator is preceded by another operator, left paren, or the start of the
		// expression, make it a negation operator.
		let out = out.iter().enumerate().map(|(i, &v)| {
			match v {
				Op(Sub) => {
					if i == 0 || match out[i-1] { Op(_) | LParen => true, _ => false } {
						Op(Neg)
					} else {
						v
					}
				},
				_ => v,
			}
		}).collect();

		Expression {
			rpn: shunting_yard(out),
			var_values: Vec::from_elem(vars.len(), 0f32),
			vars: vars,
			value: None,
		}
	}

	// http://en.wikipedia.org/wiki/Reverse_Polish_notation
	pub fn eval(&mut self) -> f32 {
		// If the expression is constant, return it's value
		if let Some(v) = self.value {
			return v;
		}

		let mut stack: Vec<f32> = Vec::new();
		for &t in self.rpn.iter() {
			match t {
				Value(v) => {
					stack.push(v);
				},
				Op(op) => {
					let n = nargs(op);
					if stack.len() < n {
						fail!("invalid expression: not enough args for operator {}", op);
					}
					let args: Vec<f32> = range(0, n).map(|_| stack.pop().unwrap()).collect();
					stack.push(match op {
						Add => {
							args[1] + args[0]
						},
						Sub => {
							args[1] - args[0]
						},
						Mul => {
							args[1] * args[0]
						},
						Div => {
							args[1] / args[0]
						},
						Exp => {
							args[1].powf(args[0])
						},
						Mod => {
							let c = num::abs(args[1]/args[0]).fract()*num::abs(args[0]);
							if args[1] < 0f32 { -c } else { c }
						},
						Neg => {
							-args[0]
						},
						Not => {
							if args[0] > 0f32 { -1f32 } else { 1f32 }
						},
						Less => {
							if args[1] < args[0] { 1f32 } else { -1f32 }
						},
						Greater => {
							if args[1] > args[0] { 1f32 } else { -1f32 }
						},
						LessEqual => {
							if args[1] <= args[0] { 1f32 } else { -1f32 }
						},
						GreaterEqual => {
							if args[1] >= args[0] { 1f32 } else { -1f32 }
						},
						Equ => {
							if args[1] == args[0] { 1f32 } else { -1f32 }
						},
						NotEqu => {
							if args[1] != args[0] { 1f32 } else { -1f32 }
						},
						ApproxEqu => {
							if num::abs(args[1] - args[0]) < 0.0001 { 1f32 } else { -1f32 }
						},
						And => {
							if args[1] > 0f32 && args[0] > 0f32 { 1f32 } else { -1f32 }
						},
						Or => {
							if args[1] > 0f32 || args[0] > 0f32 { 1f32 } else { -1f32 }
						},
						Xor => {
							if (args[1] > 0f32) ^ (args[0] > 0f32) { 1f32 } else { -1f32 }
						},
					});
				},
				Var(id) => {
					stack.push(self.var_values[id]);
				},
				_ => fail!("unexpected token in expression"),
			};
		}
		match stack.len() {
			1 => {
				let val = stack.pop().unwrap();
				self.value = Some(val);
				val
			},
			0 => {
				fail!("zero values in expression");
			},
			_ => {
				fail!("to many values in expression");
			},
		}
	}

	// Returns a uint identifier which can be passed to set_var to quickly update the value of a
	// variable.
	pub fn var_id(&self, var: &'a str) -> Option<uint> {
		match self.vars.find(&var) {
			Some(v) => Some(*v),
			None => None,
		}
	}

	// Sets a variable to an id retrieved from var_id. Return value indicates whether it was set
	// sucessfully or not.
	pub fn set_var(&mut self, var_id: uint, value: f32) -> bool {
		if var_id > self.var_values.len() {
			return false;
		}
		self.var_values[var_id] = value;
		self.value = None; // Setting a var clears evaled value
		true
	}

	// Attempts to set a variable by name. If the variable does not exist, nothing is done.
	pub fn try_set_var(&mut self, var: &'a str, value: f32) {
		if let Some(id) = self.var_id(var) {
			self.set_var(id, value);
		}
	}
}

// http://en.wikipedia.org/wiki/Shunting-yard_algorithm
fn shunting_yard(tok: Vec<ExprToken>) -> Vec<ExprToken> {
	let mut out = Vec::new();
	let mut stack: Vec<ExprToken> = Vec::new();
	for &t in tok.iter() {
		match t {
			Value(_) | Var(_) => {
				out.push(t);
			}
			Op(op1) => {
				while stack.len() > 0 {
					let top = stack[stack.len()-1];
					match top {
						Op(op2) => {
							if (associativity(op1) == Left && precedence(op1) <= precedence(op2)) ||
								(precedence(op1) < precedence(op2)) {
								stack.pop();
								out.push(top);
							} else {
								break;
							}
						},
						_ => break,
					}
				}
				stack.push(t);
			},
			LParen => {
				stack.push(t);
			},
			RParen => {
				let mut foundleft = false;
				while stack.len() > 0 {
					let op = stack.pop().unwrap();
					match op {
						Op(_) => {
							out.push(op);
						},
						LParen => {
							foundleft = true;
							break;
						},
						_ => fail!("not sure how this can happen, but maybe"),
					}
				}
				if !foundleft {
					fail!("mismatched parens");
				}
			},
		};
	}
	while stack.len() > 0 {
		let top = stack[stack.len()-1];
		match top {
			Op(_) => {
				stack.pop();
				out.push(top);
			},
			LParen | RParen => fail!("mismatched parens"),
			_ => fail!("non operator on stack!"),
		}
	}
	out
}
<commit_msg>Remove useless lifetime parameter on ExprToken<commit_after>// Used internally for evalulating expressions
// Note: for now, negative is falsy and positive (including zero) is truthy

use interpreter::lexer::Token;
use interpreter::lexer;
use std::collections::HashMap;
use std::collections::hashmap::{Occupied, Vacant};
use std::num;

#[deriving(Show)]
enum ExprToken {
	Op(Operator),
	Value(f32),
	Var(uint),
	LParen,
	RParen,
}

#[deriving(Show)]
enum Operator {
	Add,
	Sub,
	Mul,
	Div,
	Exp,
	Mod,
	Neg,
	Less,
	Greater,
	Equ,
	NotEqu,
	ApproxEqu,
	Not,
	And,
	Or,
	Xor,
	GreaterEqual,
	LessEqual,
}

fn precedence(op: Operator) -> int {
	match op {
		Equ | NotEqu | ApproxEqu => 1,
		Less | Greater | GreaterEqual | LessEqual => 2,
		And | Or | Xor => 0,
		Add => 3,
		Sub => 3,
		Mul => 4,
		Div => 4,
		Mod => 4,
		Neg => 6,
		Not => 6,
		Exp => 6,
	}
}

fn nargs(op: Operator) -> uint {
	match op {
		Neg | Not => 1,
		_ => 2,
	}
}

#[deriving(PartialEq)]
enum Associativity {
	Left,
	Right,
}

fn associativity(op: Operator) -> Associativity {
	match op {
		Exp => Right,
		_ => Left,
	}
}

#[deriving(Show)]
pub struct Expression<'a> {
	rpn: Vec<ExprToken>, // reverse polish notation
	var_values: Vec<f32>, // values of vars by id
	vars: HashMap<&'a str, uint>, // map of var name -> var id
	value: Option<f32>, // holds the last value of the expression, if it's been evaluated and vars have not been updated
}

impl<'a> Expression<'a> {
	// converts a token slice from the lexer into an expression that can be evaluated
	pub fn new<'a>(tok: Vec<Token<'a>>) -> Expression {
		let mut vars: HashMap<&'a str, uint> = HashMap::new();
		let out: Vec<ExprToken> = tok.iter().filter_map(|&t| {
			match t {
				// Try to parse constants as f32. subject to maybe change but probably not.
				lexer::Const(v) => {
					if let Some(x) = from_str::<f32>(v) {
						Some(Value(x))
					} else {
						fail!("invalid const, regex must be broken!");
					}
				},
				lexer::Operator(v) => {
					Some(Op(match v {
						"+" => Add,
						"-" => Sub,
						"*" => Mul,
						"/" => Div,
						"^" => Exp,
						"%" => Mod,
						"==" => Equ,
						"!=" => NotEqu,
						"~=" => ApproxEqu,
						"<" => Less,
						">" => Greater,
						"<=" => LessEqual,
						">=" => GreaterEqual,
						"!" => Not,
						"&&" => And,
						"||" => Or,
						"^^" => Xor,
						_ => fail!("unexpected operator"),
					}))
				},
				lexer::Paren(v) => {
					if v == "(" {
						Some(LParen)
					} else if v == ")" {
						Some(RParen)
					} else {
						None
					}
				},
				// An identifier in the context of an expression is always a variable (TODO FALSE!!)
				lexer::Ident(v) => {
					let nv = vars.len();
					match vars.entry(v) {
						Occupied(entry) => {
							Some(Var(*entry.get()))
						},
						Vacant(entry) => {
							entry.set(nv);
							Some(Var(nv))
						},
					}
				},
				// Discard whitesapce
				lexer::Whitespace(_) | lexer::Newline => {
					None
				},
				_ => {
					fail!("not expression!");
				}
			}
		}).collect();

		// Handle special case with unary minus
		// If an subtraction operator is preceded by another operator, left paren, or the start of the
		// expression, make it a negation operator.
		let out = out.iter().enumerate().map(|(i, &v)| {
			match v {
				Op(Sub) => {
					if i == 0 || match out[i-1] { Op(_) | LParen => true, _ => false } {
						Op(Neg)
					} else {
						v
					}
				},
				_ => v,
			}
		}).collect();

		Expression {
			rpn: shunting_yard(out),
			var_values: Vec::from_elem(vars.len(), 0f32),
			vars: vars,
			value: None,
		}
	}

	// http://en.wikipedia.org/wiki/Reverse_Polish_notation
	pub fn eval(&mut self) -> f32 {
		// If the expression is constant, return it's value
		if let Some(v) = self.value {
			return v;
		}

		let mut stack: Vec<f32> = Vec::new();
		for &t in self.rpn.iter() {
			match t {
				Value(v) => {
					stack.push(v);
				},
				Op(op) => {
					let n = nargs(op);
					if stack.len() < n {
						fail!("invalid expression: not enough args for operator {}", op);
					}
					let args: Vec<f32> = range(0, n).map(|_| stack.pop().unwrap()).collect();
					stack.push(match op {
						Add => {
							args[1] + args[0]
						},
						Sub => {
							args[1] - args[0]
						},
						Mul => {
							args[1] * args[0]
						},
						Div => {
							args[1] / args[0]
						},
						Exp => {
							args[1].powf(args[0])
						},
						Mod => {
							let c = num::abs(args[1]/args[0]).fract()*num::abs(args[0]);
							if args[1] < 0f32 { -c } else { c }
						},
						Neg => {
							-args[0]
						},
						Not => {
							if args[0] > 0f32 { -1f32 } else { 1f32 }
						},
						Less => {
							if args[1] < args[0] { 1f32 } else { -1f32 }
						},
						Greater => {
							if args[1] > args[0] { 1f32 } else { -1f32 }
						},
						LessEqual => {
							if args[1] <= args[0] { 1f32 } else { -1f32 }
						},
						GreaterEqual => {
							if args[1] >= args[0] { 1f32 } else { -1f32 }
						},
						Equ => {
							if args[1] == args[0] { 1f32 } else { -1f32 }
						},
						NotEqu => {
							if args[1] != args[0] { 1f32 } else { -1f32 }
						},
						ApproxEqu => {
							if num::abs(args[1] - args[0]) < 0.0001 { 1f32 } else { -1f32 }
						},
						And => {
							if args[1] > 0f32 && args[0] > 0f32 { 1f32 } else { -1f32 }
						},
						Or => {
							if args[1] > 0f32 || args[0] > 0f32 { 1f32 } else { -1f32 }
						},
						Xor => {
							if (args[1] > 0f32) ^ (args[0] > 0f32) { 1f32 } else { -1f32 }
						},
					});
				},
				Var(id) => {
					stack.push(self.var_values[id]);
				},
				_ => fail!("unexpected token in expression"),
			};
		}
		match stack.len() {
			1 => {
				let val = stack.pop().unwrap();
				self.value = Some(val);
				val
			},
			0 => {
				fail!("zero values in expression");
			},
			_ => {
				fail!("to many values in expression");
			},
		}
	}

	// Returns a uint identifier which can be passed to set_var to quickly update the value of a
	// variable.
	pub fn var_id(&self, var: &'a str) -> Option<uint> {
		match self.vars.find(&var) {
			Some(v) => Some(*v),
			None => None,
		}
	}

	// Sets a variable to an id retrieved from var_id. Return value indicates whether it was set
	// sucessfully or not.
	pub fn set_var(&mut self, var_id: uint, value: f32) -> bool {
		if var_id > self.var_values.len() {
			return false;
		}
		self.var_values[var_id] = value;
		self.value = None; // Setting a var clears evaled value
		true
	}

	// Attempts to set a variable by name. If the variable does not exist, nothing is done.
	pub fn try_set_var(&mut self, var: &'a str, value: f32) {
		if let Some(id) = self.var_id(var) {
			self.set_var(id, value);
		}
	}
}

// http://en.wikipedia.org/wiki/Shunting-yard_algorithm
fn shunting_yard(tok: Vec<ExprToken>) -> Vec<ExprToken> {
	let mut out = Vec::new();
	let mut stack: Vec<ExprToken> = Vec::new();
	for &t in tok.iter() {
		match t {
			Value(_) | Var(_) => {
				out.push(t);
			}
			Op(op1) => {
				while stack.len() > 0 {
					let top = stack[stack.len()-1];
					match top {
						Op(op2) => {
							if (associativity(op1) == Left && precedence(op1) <= precedence(op2)) ||
								(precedence(op1) < precedence(op2)) {
								stack.pop();
								out.push(top);
							} else {
								break;
							}
						},
						_ => break,
					}
				}
				stack.push(t);
			},
			LParen => {
				stack.push(t);
			},
			RParen => {
				let mut foundleft = false;
				while stack.len() > 0 {
					let op = stack.pop().unwrap();
					match op {
						Op(_) => {
							out.push(op);
						},
						LParen => {
							foundleft = true;
							break;
						},
						_ => fail!("not sure how this can happen, but maybe"),
					}
				}
				if !foundleft {
					fail!("mismatched parens");
				}
			},
		};
	}
	while stack.len() > 0 {
		let top = stack[stack.len()-1];
		match top {
			Op(_) => {
				stack.pop();
				out.push(top);
			},
			LParen | RParen => fail!("mismatched parens"),
			_ => fail!("non operator on stack!"),
		}
	}
	out
}
<|endoftext|>