<commit_before>use std::ops::{Add, Sub};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::collections::VecDeque;
use std::cmp::Ordering::Equal;
use std::iter::repeat;
use std::fmt::{Debug, Formatter, Error};

#[derive(Clone)]
pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Vec<f32>,
}

impl Debug for WordVec {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        try!(self.word.fmt(fmt));
        try!(fmt.write_str(": "));
        try!(self.count.fmt(fmt));
        try!(fmt.write_str(", "));
        self.vec.iter().take(5).collect::<Vec<_>>().fmt(fmt)
    }
}

impl WordVec {

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| (x*x - y*y).abs().sqrt())
                .fold(0.0, |x, y| x + y)
    }

    fn new(word: String, num_words: usize) -> WordVec {
        WordVec {
            word: word,
            count: 0,
            vec: repeat(0.0).take(num_words).collect(),
        }
    }

    fn normalize(&mut self, word_freqs: &Vec<f32>) {
        let count = self.count as f32;
        for i in 0..word_freqs.len() {
            self.vec[i] = self.vec[i] / count;
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }

}

impl<'a> Add<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..newvec.vec.len() {
            newvec.vec[i] += other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} - {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..10_000 {
            newvec.vec[i] -= other.vec[i];
        }

        newvec
    }
}

pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    window_width: usize,
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct WordAcceptor<'a> {
    window: VecDeque<(usize, String)>,
    builder: &'a mut LanguageModelBuilder,
}

impl<'a> LanguageModelBuilder {
    pub fn new(words: Vec<String>) -> LanguageModelBuilder {

        let word_vecs = words.iter()
                             .map(|s| WordVec::new(s.clone(), words.len()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            window_width: 21,
            words: words,
            word_vecs: word_vecs,
        }
    }

    pub fn build(mut self) -> LanguageModel {
        let total_words = self.word_vecs.iter().fold(0.0, |a, b| a + b.count as f32);
        let word_freqs = self.word_vecs.iter()
                                       .map(|v| (v.count as f32 / total_words).max(1.0/total_words))
                                       .collect::<Vec<_>>();
        for vec in self.word_vecs.iter_mut() {
            vec.normalize(&word_freqs);
        }

        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs,
        }
    }

    pub fn new_file(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            window: VecDeque::new(),
            builder: self,
        }
    }

    fn get_vec(&mut self, word: &str) -> &mut WordVec {
        let i = self.words[word];
        &mut self.word_vecs[i]
    }

    fn add_word(&mut self, from: usize, to: usize) {
        self.word_vecs[from].inc(to);
    }

    fn word_seen(&mut self, word: &str) {
        self.get_vec(word).count += 1;
    }
}

impl<'a> WordAcceptor<'a> {
    pub fn add_word(&mut self, word: String) {
        let ww = self.builder.window_width;
        let idx_opt = self.builder.words.get(&word).map(|w| *w);

        if let Some(next_idx) = idx_opt {
            self.window.push_back((next_idx, word));
            if self.window.len() > ww {
                self.window.pop_front();
            }

            if self.window.len() == ww {
                let (center_idx, ref center_word) = self.window[ww / 2];
                self.builder.word_seen(center_word);
                for i in (0..ww).filter(|a| *a != ww / 2) {
                    self.builder.add_word(center_idx, self.window[i].0);
                }
            }
        }
    }
}

impl LanguageModel {
    pub fn get(&self, word: &str) -> Option<&WordVec> {
        if let Some(i) = self.words.get(word) {
            Some(&self.word_vecs[*i])
        } else {
            None
        }
    }

    pub fn nearest_words(&self, word: &WordVec) -> Vec<&WordVec> {
        let mut vec_refs = self.word_vecs.iter().filter_map(|w| {
            let dist = w.distance(word);
            if w.word != word.word && !dist.is_nan() {
                Some((w.distance(word), w))
            } else {
                None
            }
        }).collect::<Vec<_>>();
        vec_refs.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or_else(||
            panic!("{:?}, {:?}", a, b)));
        vec_refs.into_iter().map(|x| x.1).collect()
    }
}<commit_msg>Clean up some unused variables<commit_after>use std::ops::{Add, Sub};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::collections::VecDeque;
use std::iter::repeat;
use std::fmt::{Debug, Formatter, Error};

#[derive(Clone)]
pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Vec<f32>,
}

impl Debug for WordVec {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        try!(self.word.fmt(fmt));
        try!(fmt.write_str(": "));
        try!(self.count.fmt(fmt));
        try!(fmt.write_str(", "));
        self.vec.iter().take(5).collect::<Vec<_>>().fmt(fmt)
    }
}

impl WordVec {

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| (x*x - y*y).abs().sqrt())
                .fold(0.0, |x, y| x + y)
    }

    fn new(word: String, num_words: usize) -> WordVec {
        WordVec {
            word: word,
            count: 0,
            vec: repeat(0.0).take(num_words).collect(),
        }
    }

    fn normalize(&mut self) {
        let count = self.count as f32;
        for f in &mut self.vec {
            *f /= count;
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }

}

impl<'a> Add<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..newvec.vec.len() {
            newvec.vec[i] += other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} - {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..10_000 {
            newvec.vec[i] -= other.vec[i];
        }

        newvec
    }
}

pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    window_width: usize,
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct WordAcceptor<'a> {
    window: VecDeque<(usize, String)>,
    builder: &'a mut LanguageModelBuilder,
}

impl LanguageModelBuilder {
    pub fn new(window_radius: usize, words: Vec<String>) -> LanguageModelBuilder {

        let word_vecs = words.iter()
                             .map(|s| WordVec::new(s.clone(), words.len()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            window_width: 2 * window_radius + 1,
            words: words,
            word_vecs: word_vecs,
        }
    }

    pub fn build(mut self) -> LanguageModel {
        for vec in self.word_vecs.iter_mut() {
            vec.normalize();
        }

        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs,
        }
    }

    pub fn new_file<'a>(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            window: VecDeque::new(),
            builder: self,
        }
    }

    fn get_vec(&mut self, word: &str) -> &mut WordVec {
        let i = self.words[word];
        &mut self.word_vecs[i]
    }

    fn add_word(&mut self, from: usize, to: usize) {
        self.word_vecs[from].inc(to);
    }

    fn word_seen(&mut self, word: &str) {
        self.get_vec(word).count += 1;
    }
}

impl<'a> WordAcceptor<'a> {
    pub fn add_word(&mut self, word: String) {
        let ww = self.builder.window_width;
        let idx_opt = self.builder.words.get(&word).map(|w| *w);

        if let Some(next_idx) = idx_opt {
            self.window.push_back((next_idx, word));
            if self.window.len() > ww {
                self.window.pop_front();
            }

            if self.window.len() == ww {
                let (center_idx, ref center_word) = self.window[ww / 2];
                self.builder.word_seen(center_word);
                for i in (0..ww).filter(|a| *a != ww / 2) {
                    self.builder.add_word(center_idx, self.window[i].0);
                }
            }
        }
    }
}

impl LanguageModel {
    pub fn get(&self, word: &str) -> Option<&WordVec> {
        if let Some(i) = self.words.get(word) {
            Some(&self.word_vecs[*i])
        } else {
            None
        }
    }

    pub fn nearest_words(&self, word: &WordVec) -> Vec<&WordVec> {
        let mut vec_refs = self.word_vecs.iter().filter_map(|w| {
            let dist = w.distance(word);
            if w.word != word.word && !dist.is_nan() {
                Some((w.distance(word), w))
            } else {
                None
            }
        }).collect::<Vec<_>>();
        vec_refs.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or_else(||
            panic!("{:?}, {:?}", a, b)));
        vec_refs.into_iter().map(|x| x.1).collect()
    }
}<|endoftext|>


<commit_before>use libc::{c_char, c_uint};
use ffi::prelude::{LLVMValueRef, LLVMModuleRef};
use ffi::analysis::LLVMVerifierFailureAction;
use ffi::{analysis, core, linker, LLVMModule};
use ffi::transforms::pass_manager_builder as builder;
use ffi::bit_writer as writer;
use ffi::bit_reader as reader;
use cbox::{CBox, CSemiBox};
use std::ffi::CString;
use std::iter::{Iterator, IntoIterator};
use std::io::{Error, ErrorKind};
use std::io::Result as IoResult;
use std::{env, fmt, mem};
use std::marker::PhantomData;
use std::path::Path;
use std::process::Command;
use buffer::MemoryBuffer;
use context::{Context, GetContext};
use value::{Function, Value, GlobalValue};
use ty::Type;
use util;

/// Represents a single translation unit of code.
///
/// This is attached to the lifetime of the context that constructs it, but is owned by the `CSemiBox`.
pub struct Module;
native_ref!(&Module = LLVMModuleRef);
impl Module {
    /// Create a new module in the context given with the name given.
    ///
    /// The lifetime of the module will match the lifetime of the context
    /// you instance it in because the context contains it.
    ///
    /// ```rust
    /// use llvm::*;
    /// let context = Context::new();
    /// let context = context.as_semi();
    /// let module = Module::new("name", &context);
    /// println!("{:?}", module)
    /// ```
    pub fn new<'a>(name: &str, context: &'a Context) -> CSemiBox<'a, Module> {
        let c_name = CString::new(name).unwrap();
        unsafe { CSemiBox::new(core::LLVMModuleCreateWithNameInContext(c_name.as_ptr(), context.into())) }
    }
    /// Add an external global to the module with the given type and name.
    pub fn add_global<'a>(&'a self, name: &str, ty: &'a Type) -> &'a GlobalValue {
        util::with_cstr(name, |ptr| unsafe {
            core::LLVMAddGlobal(self.into(), ty.into(), ptr).into()
        })
    }
    /// Add a global in the given address space to the module with the given type and name.
    pub fn add_global_in_addr_space<'a>(&'a self, name: &str, ty: &'a Type, sp: AddressSpace) -> &'a GlobalValue {
        util::with_cstr(name, |ptr| unsafe {
            core::LLVMAddGlobalInAddressSpace(self.into(), ty.into(), ptr, sp as c_uint).into()
        })
    }
    /// Add a constant global to the module with the given type, name and value.
    pub fn add_global_constant<'a>(&'a self, name: &str, val: &'a Value) -> &'a GlobalValue {
        util::with_cstr(name, |ptr| unsafe {
            let global = core::LLVMAddGlobal(self.into(), val.get_type().into(), ptr);
            core::LLVMSetInitializer (global.into(), val.into());
            global.into()
        })
    }
    /// Get the global with the name given, or `None` if no global with that name exists.
    pub fn get_global<'a>(&'a self, name: &str) -> Option<&'a GlobalValue> {
        util::with_cstr(name, |ptr| unsafe {
            let ptr = core::LLVMGetNamedGlobal(self.into(), ptr);
            util::ptr_to_null(ptr)
        })
    }
    /// Parse this bitcode file into a module, or return an error string.
    pub fn parse_bitcode<'a>(context: &'a Context, path: &str) -> Result<CSemiBox<'a, Module>, CBox<str>> {
        unsafe {
            let mut out = mem::uninitialized();
            let mut err = mem::uninitialized();
            let buf = try!(MemoryBuffer::new_from_file(path));
            if reader::LLVMParseBitcodeInContext(context.into(), buf.as_ptr(), &mut out, &mut err) == 1 {
                Err(CBox::new(err))
            } else {
                Ok(CSemiBox::new(out))
            }
        }
    }
    /// Write this module's bitcode to the path given.
    pub fn write_bitcode(&self, path: &str) -> IoResult<()> {
        util::with_cstr(path, |cpath| unsafe {
            if writer::LLVMWriteBitcodeToFile(self.into(), cpath) != 0 {
                Err(Error::new(ErrorKind::Other, &format!("could not write to {}", path) as &str))
            } else {
                Ok(())
            }
        })
    }
    /// Add a function to the module with the name given.
    pub fn add_function<'a>(&'a self, name: &str, sig: &'a Type) -> &'a mut Function {
        let c_name = CString::new(name).unwrap();
        unsafe { core::LLVMAddFunction(self.into(), c_name.as_ptr(), sig.into()) }.into()
    }
    /// Returns the function with the name given if it exists.
    pub fn get_function<'a>(&'a self, name: &str) -> Option<&'a Function> {
        let c_name = CString::new(name).unwrap();
        unsafe {
            let ty = core::LLVMGetNamedFunction(self.into(), c_name.as_ptr());
            util::ptr_to_null(ty)
        }
    }
    /// Returns the type with the name given if it exists.
    pub fn get_type<'a>(&'a self, name: &str) -> Option<&'a Type> {
        let c_name = CString::new(name).unwrap();
        unsafe {
            let ty = core::LLVMGetTypeByName(self.into(), c_name.as_ptr());
            util::ptr_to_null(ty)
        }
    }
    /// Clone this module.
    pub fn clone<'a>(&'a self) -> CSemiBox<'a, Module> {
        CSemiBox::new(unsafe { core::LLVMCloneModule(self.into()) })
    }

    /// Optimize this module with the given optimization level and size level.
    pub fn optimize(&self, opt_level: usize, size_level: usize) {
        unsafe {
            let builder = builder::LLVMPassManagerBuilderCreate();
            builder::LLVMPassManagerBuilderSetOptLevel(builder, opt_level as c_uint);
            builder::LLVMPassManagerBuilderSetSizeLevel(builder, size_level as c_uint);
            let pass_manager = core::LLVMCreatePassManager();
            builder::LLVMPassManagerBuilderPopulateModulePassManager(builder, pass_manager);
            builder::LLVMPassManagerBuilderDispose(builder);
            core::LLVMRunPassManager(pass_manager, self.into());
        }
    }

    /// Returns the target data of this module represented as a string
    pub fn get_target(&self) -> &str {
        unsafe {
            let target = core::LLVMGetTarget(self.into());
            util::to_str(target as *mut c_char)
        }
    }

    /// Set the target data of this module
    pub fn set_target(&self, target: &str) {
        let c_target = CString::new(target).unwrap();
        unsafe { core::LLVMSetTarget(self.into(), c_target.as_ptr()) }
    }

    /// Verify that the module is safe to run.
    pub fn verify(&self) -> Result<(), CBox<str>> {
        unsafe {
            let mut error = mem::uninitialized();
            let action = LLVMVerifierFailureAction::LLVMReturnStatusAction;
            if analysis::LLVMVerifyModule(self.into(), action, &mut error) == 1 {
                Err(CBox::new(error))
            } else {
                Ok(())
            }
        }
    }

    /// Compile the module into an object file at the given location.
    ///
    /// Note that this uses the LLVM tool `llc` to do this, which may or may not be
    /// installed on the user's machine.
    pub fn compile(&self, path: &Path, opt_level: usize) -> IoResult<()> {
        let dir = env::temp_dir();
        let path = path.to_str().unwrap();
        let mod_path = dir.join("module.bc");
        let mod_path = mod_path.to_str().unwrap();
        try!(self.write_bitcode(mod_path));
        Command::new("llc")
            .arg(&format!("-O={}", opt_level))
            .arg("-filetype=obj")
            .arg("-o").arg(path)
            .arg(mod_path)
            .spawn()
            .map(|_| ())
    }

    /// Link a module into this module
    pub fn link(&self, src: &Module) -> Result<(), CBox<str>> {
        unsafe {
            let dest = self.into();
            let src = src.into();
            let mut message = mem::uninitialized();
            if linker::LLVMLinkModules(dest, src, linker::LLVMLinkerMode::LLVMLinkerPreserveSource, &mut message) == 1 {
                Err(CBox::new(message))
            } else {
                Ok(())
            }
        }
    }

    /// Link a module into this module
    pub fn link_destroy(&self, src: CSemiBox<Module>) -> Result<(), CBox<str>> {
        unsafe {
            let dest = self.into();
            let src = src.as_ptr();
            let mut message = mem::uninitialized();
            if linker::LLVMLinkModules(dest, src, linker::LLVMLinkerMode::LLVMLinkerDestroySource, &mut message) == 1 {
                Err(CBox::new(message))
            } else {
                Ok(())
            }
        }
    }
}
impl<'a> IntoIterator for &'a Module {
    type Item = &'a Function;
    type IntoIter = Functions<'a>;
    /// Iterate through the functions in the module
    fn into_iter(self) -> Functions<'a> {
        Functions {
            value: unsafe { core::LLVMGetFirstFunction(self.into()) },
            marker: PhantomData
        }
    }
}
get_context!(Module, LLVMGetModuleContext);
to_str!(Module, LLVMPrintModuleToString);
dispose!(Module, LLVMModule, core::LLVMDisposeModule);
#[derive(Copy, Clone)]
/// An iterator through the functions contained in a module.
pub struct Functions<'a> {
    value: LLVMValueRef,
    marker: PhantomData<&'a ()>
}
impl<'a> Iterator for Functions<'a> {
    type Item = &'a Function;
    fn next(&mut self) -> Option<&'a Function> {
        if self.value.is_null() {
            None
        } else {
            let c_next = unsafe { core::LLVMGetNextFunction(self.value) };
            self.value = c_next;
            Some(self.value.into())
        }
    }
}

#[repr(C)]
#[derive(Copy, Clone)]
pub enum AddressSpace {
    Generic = 0,
    Global = 1,
    Shared = 3,
    Const = 4,
    Local = 5,
}
<commit_msg>Remove unnecessary mutability<commit_after>use libc::{c_char, c_uint};
use ffi::prelude::{LLVMValueRef, LLVMModuleRef};
use ffi::analysis::LLVMVerifierFailureAction;
use ffi::{analysis, core, linker, LLVMModule};
use ffi::transforms::pass_manager_builder as builder;
use ffi::bit_writer as writer;
use ffi::bit_reader as reader;
use cbox::{CBox, CSemiBox};
use std::ffi::CString;
use std::iter::{Iterator, IntoIterator};
use std::io::{Error, ErrorKind};
use std::io::Result as IoResult;
use std::{env, fmt, mem};
use std::marker::PhantomData;
use std::path::Path;
use std::process::Command;
use buffer::MemoryBuffer;
use context::{Context, GetContext};
use value::{Function, Value, GlobalValue};
use ty::Type;
use util;

/// Represents a single translation unit of code.
///
/// This is attached to the lifetime of the context that constructs it, but is owned by the `CSemiBox`.
pub struct Module;
native_ref!(&Module = LLVMModuleRef);
impl Module {
    /// Create a new module in the context given with the name given.
    ///
    /// The lifetime of the module will match the lifetime of the context
    /// you instance it in because the context contains it.
    ///
    /// ```rust
    /// use llvm::*;
    /// let context = Context::new();
    /// let context = context.as_semi();
    /// let module = Module::new("name", &context);
    /// println!("{:?}", module)
    /// ```
    pub fn new<'a>(name: &str, context: &'a Context) -> CSemiBox<'a, Module> {
        let c_name = CString::new(name).unwrap();
        unsafe { CSemiBox::new(core::LLVMModuleCreateWithNameInContext(c_name.as_ptr(), context.into())) }
    }
    /// Add an external global to the module with the given type and name.
    pub fn add_global<'a>(&'a self, name: &str, ty: &'a Type) -> &'a GlobalValue {
        util::with_cstr(name, |ptr| unsafe {
            core::LLVMAddGlobal(self.into(), ty.into(), ptr).into()
        })
    }
    /// Add a global in the given address space to the module with the given type and name.
    pub fn add_global_in_addr_space<'a>(&'a self, name: &str, ty: &'a Type, sp: AddressSpace) -> &'a GlobalValue {
        util::with_cstr(name, |ptr| unsafe {
            core::LLVMAddGlobalInAddressSpace(self.into(), ty.into(), ptr, sp as c_uint).into()
        })
    }
    /// Add a constant global to the module with the given type, name and value.
    pub fn add_global_constant<'a>(&'a self, name: &str, val: &'a Value) -> &'a GlobalValue {
        util::with_cstr(name, |ptr| unsafe {
            let global = core::LLVMAddGlobal(self.into(), val.get_type().into(), ptr);
            core::LLVMSetInitializer (global.into(), val.into());
            global.into()
        })
    }
    /// Get the global with the name given, or `None` if no global with that name exists.
    pub fn get_global<'a>(&'a self, name: &str) -> Option<&'a GlobalValue> {
        util::with_cstr(name, |ptr| unsafe {
            let ptr = core::LLVMGetNamedGlobal(self.into(), ptr);
            util::ptr_to_null(ptr)
        })
    }
    /// Parse this bitcode file into a module, or return an error string.
    pub fn parse_bitcode<'a>(context: &'a Context, path: &str) -> Result<CSemiBox<'a, Module>, CBox<str>> {
        unsafe {
            let mut out = mem::uninitialized();
            let mut err = mem::uninitialized();
            let buf = try!(MemoryBuffer::new_from_file(path));
            if reader::LLVMParseBitcodeInContext(context.into(), buf.as_ptr(), &mut out, &mut err) == 1 {
                Err(CBox::new(err))
            } else {
                Ok(CSemiBox::new(out))
            }
        }
    }
    /// Write this module's bitcode to the path given.
    pub fn write_bitcode(&self, path: &str) -> IoResult<()> {
        util::with_cstr(path, |cpath| unsafe {
            if writer::LLVMWriteBitcodeToFile(self.into(), cpath) != 0 {
                Err(Error::new(ErrorKind::Other, &format!("could not write to {}", path) as &str))
            } else {
                Ok(())
            }
        })
    }
    /// Add a function to the module with the name given.
    pub fn add_function<'a>(&'a self, name: &str, sig: &'a Type) -> &'a Function {
        let c_name = CString::new(name).unwrap();
        unsafe { core::LLVMAddFunction(self.into(), c_name.as_ptr(), sig.into()) }.into()
    }
    /// Returns the function with the name given if it exists.
    pub fn get_function<'a>(&'a self, name: &str) -> Option<&'a Function> {
        let c_name = CString::new(name).unwrap();
        unsafe {
            let ty = core::LLVMGetNamedFunction(self.into(), c_name.as_ptr());
            util::ptr_to_null(ty)
        }
    }
    /// Returns the type with the name given if it exists.
    pub fn get_type<'a>(&'a self, name: &str) -> Option<&'a Type> {
        let c_name = CString::new(name).unwrap();
        unsafe {
            let ty = core::LLVMGetTypeByName(self.into(), c_name.as_ptr());
            util::ptr_to_null(ty)
        }
    }
    /// Clone this module.
    pub fn clone<'a>(&'a self) -> CSemiBox<'a, Module> {
        CSemiBox::new(unsafe { core::LLVMCloneModule(self.into()) })
    }

    /// Optimize this module with the given optimization level and size level.
    pub fn optimize(&self, opt_level: usize, size_level: usize) {
        unsafe {
            let builder = builder::LLVMPassManagerBuilderCreate();
            builder::LLVMPassManagerBuilderSetOptLevel(builder, opt_level as c_uint);
            builder::LLVMPassManagerBuilderSetSizeLevel(builder, size_level as c_uint);
            let pass_manager = core::LLVMCreatePassManager();
            builder::LLVMPassManagerBuilderPopulateModulePassManager(builder, pass_manager);
            builder::LLVMPassManagerBuilderDispose(builder);
            core::LLVMRunPassManager(pass_manager, self.into());
        }
    }

    /// Returns the target data of this module represented as a string
    pub fn get_target(&self) -> &str {
        unsafe {
            let target = core::LLVMGetTarget(self.into());
            util::to_str(target as *mut c_char)
        }
    }

    /// Set the target data of this module
    pub fn set_target(&self, target: &str) {
        let c_target = CString::new(target).unwrap();
        unsafe { core::LLVMSetTarget(self.into(), c_target.as_ptr()) }
    }

    /// Verify that the module is safe to run.
    pub fn verify(&self) -> Result<(), CBox<str>> {
        unsafe {
            let mut error = mem::uninitialized();
            let action = LLVMVerifierFailureAction::LLVMReturnStatusAction;
            if analysis::LLVMVerifyModule(self.into(), action, &mut error) == 1 {
                Err(CBox::new(error))
            } else {
                Ok(())
            }
        }
    }

    /// Compile the module into an object file at the given location.
    ///
    /// Note that this uses the LLVM tool `llc` to do this, which may or may not be
    /// installed on the user's machine.
    pub fn compile(&self, path: &Path, opt_level: usize) -> IoResult<()> {
        let dir = env::temp_dir();
        let path = path.to_str().unwrap();
        let mod_path = dir.join("module.bc");
        let mod_path = mod_path.to_str().unwrap();
        try!(self.write_bitcode(mod_path));
        Command::new("llc")
            .arg(&format!("-O={}", opt_level))
            .arg("-filetype=obj")
            .arg("-o").arg(path)
            .arg(mod_path)
            .spawn()
            .map(|_| ())
    }

    /// Link a module into this module
    pub fn link(&self, src: &Module) -> Result<(), CBox<str>> {
        unsafe {
            let dest = self.into();
            let src = src.into();
            let mut message = mem::uninitialized();
            if linker::LLVMLinkModules(dest, src, linker::LLVMLinkerMode::LLVMLinkerPreserveSource, &mut message) == 1 {
                Err(CBox::new(message))
            } else {
                Ok(())
            }
        }
    }

    /// Link a module into this module
    pub fn link_destroy(&self, src: CSemiBox<Module>) -> Result<(), CBox<str>> {
        unsafe {
            let dest = self.into();
            let src = src.as_ptr();
            let mut message = mem::uninitialized();
            if linker::LLVMLinkModules(dest, src, linker::LLVMLinkerMode::LLVMLinkerDestroySource, &mut message) == 1 {
                Err(CBox::new(message))
            } else {
                Ok(())
            }
        }
    }
}
impl<'a> IntoIterator for &'a Module {
    type Item = &'a Function;
    type IntoIter = Functions<'a>;
    /// Iterate through the functions in the module
    fn into_iter(self) -> Functions<'a> {
        Functions {
            value: unsafe { core::LLVMGetFirstFunction(self.into()) },
            marker: PhantomData
        }
    }
}
get_context!(Module, LLVMGetModuleContext);
to_str!(Module, LLVMPrintModuleToString);
dispose!(Module, LLVMModule, core::LLVMDisposeModule);
#[derive(Copy, Clone)]
/// An iterator through the functions contained in a module.
pub struct Functions<'a> {
    value: LLVMValueRef,
    marker: PhantomData<&'a ()>
}
impl<'a> Iterator for Functions<'a> {
    type Item = &'a Function;
    fn next(&mut self) -> Option<&'a Function> {
        if self.value.is_null() {
            None
        } else {
            let c_next = unsafe { core::LLVMGetNextFunction(self.value) };
            self.value = c_next;
            Some(self.value.into())
        }
    }
}

#[repr(C)]
#[derive(Copy, Clone)]
pub enum AddressSpace {
    Generic = 0,
    Global = 1,
    Shared = 3,
    Const = 4,
    Local = 5,
}
<|endoftext|>


<commit_before>use codemap::Spanned;
use lexer::Token;
use std::iter::Peekable;
use std::slice::Iter;
use self::ParseErrorKind::*;

type ParseIter<'a> = Peekable<Iter<'a, Spanned<Token>>>;
type Parse = Result<Node, ParseErrorKind>;

#[derive(Debug, Clone, PartialEq)]
pub enum Node {
  Block(Vec<Node>),
  If{cond: Box<Node>},
  Stmt(Box<Node>),
  Break,
  Continue,
  Expr,
  Pass,

  BinExpr {
    lhs: Box<Node>,
    op: Token,
    rhs: Box<Node>,
  },

  UnExpr {
    val: Box<Node>,
    op: Token,
  },

  // Literals
  Null,
  Bool(bool),
  Float(f64),
  Int(i64),
  Str(String),
  Name(String),
  Table,
}

#[derive(Debug, Clone, PartialEq)]
pub enum Op {
  Right(u32),
  Left(u32),
  None,
}

#[derive(Debug, Clone, PartialEq)]
pub enum ParseErrorKind {
  UnexpectedToken(Token),
  UnexpectedEOF,
  UnknownBinaryOperator,
  UnknownUnaryOperator,
}

// Return true if the next token in `it` is `kind`
fn peek_token(it: &mut ParseIter, kind: Token) -> bool {
  if let Some(&c) = it.peek() {
    c.node == kind
  }
  else {
    false
  }
}

// Return true if the next token in `it` is `kind` *and* consume the token
fn use_token(it: &mut ParseIter, kind: Token) -> bool {
  if let Some(&c) = it.peek() {
    if c.node == kind {
      it.next();
    }
    c.node == kind
  }
  else {
    false
  }
}

// Panic if the next token in `it` is *not* `kind`
fn require_token(it: &mut ParseIter, kind: Token) -> Result<(), ParseErrorKind> {
  if let Some(&c) = it.peek() {
    it.next();

    if c.node == kind {
      return Ok(());
    }

    return Err(UnexpectedToken(c.node.clone()));
  }

  return Err(UnexpectedEOF);
}

fn op_precedence(op: &Token) -> Op {
  match *op {
    Token::Add | Token::Sub => Op::Left(10),
    Token::Div | Token::Mul => Op::Left(20),
    Token::Car => Op::Right(30),
    _ => Op::None,
  }
}

fn parse_binexpr(it: &mut ParseIter) -> Parse {
  let mut expr = parse_unexpr(it)?;

  // prevents this from breaking the LHS until we know we made it
  // otherwise, things like (2 + 3) * 4 get restructured into 2 + (3 * 4)
  let mut break_left = false;

  while let Some(&c) = it.peek() {
    let prec = op_precedence(&c.node);

    if let Op::None = prec {
      break;
    }

    it.next();

    let rhs = parse_unexpr(it)?;

    expr = match (break_left, expr.clone()) {
      (true, Node::BinExpr{lhs: cur_lhs, op: cur_op, rhs: cur_rhs}) => {
        let cur_prec = op_precedence(&cur_op);
        match (cur_prec, prec) {
          // these should never happen
          (_, Op::None) => {break}
          (Op::None, _) => {break}

          // left-to-right
          // there has to be a better way to handle this, no?
          (Op::Left(n), Op::Left(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: c.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Right(n), Op::Right(m)) if n > m => {
            Node::BinExpr {lhs: Box::new(expr), op: c.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Right(n), Op::Left(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: c.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Left(n), Op::Right(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: c.node.clone(), rhs: Box::new(rhs)}
          }

          // right-to-left
          _ => {
            Node::BinExpr {
              lhs: cur_lhs,
              op: cur_op,
              rhs: Box::new(Node::BinExpr {lhs: cur_rhs, op: c.node.clone(), rhs: Box::new(rhs)}),
            }
          }
        }
      }
      _ => {
        Node::BinExpr {lhs: Box::new(expr), op: c.node.clone(), rhs: Box::new(rhs)}
      }
    };

    break_left = true;
  };

  Ok(expr)
}

fn parse_unexpr(it: &mut ParseIter) -> Parse {
  parse_simple(it)
}

fn parse_simple(it: &mut ParseIter) -> Parse {
  parse_prefix(it)
}

fn parse_prefix(it: &mut ParseIter) -> Parse {
  if let Some(&c) = it.peek() {
    return match c.node {
      Token::Null => {it.next(); Ok(Node::Null)},
      Token::Bool(x) => {it.next(); Ok(Node::Bool(x))},
      Token::Float(x) => {it.next(); Ok(Node::Float(x))},
      Token::Int(x) => {it.next(); Ok(Node::Int(x))},
      Token::Str(ref x) => {it.next(); Ok(Node::Str(x.clone()))},
      Token::Name(ref x) => {it.next(); Ok(Node::Name(x.clone()))},
      Token::Table => {it.next(); Ok(Node::Table)},
      Token::Pal => {
        it.next();
        let out = parse_binexpr(it)?;
        require_token(it, Token::Par)?;
        Ok(out)
      }
      _ => Err(UnexpectedToken(c.node.clone())),
    }
  }

  Err(UnexpectedEOF)
}

fn parse_stmt(it: &mut ParseIter) -> Parse {
  if let Some(&c) = it.peek() {
    return match c.node {
      Token::Break => {
        it.next();
        Ok(Node::Break)
      }

      Token::Continue => {
        it.next();
        Ok(Node::Continue)
      }

      Token::Pass => {
        it.next();
        Ok(Node::Pass)
      }

      _ => {
        parse_binexpr(it).map(|expr| Node::Stmt(Box::new(expr)))
      }
    }
  }

  Err(UnexpectedEOF)
}

pub fn parse(tokens: Vec<Spanned<Token>>) -> Parse {
  let mut it: ParseIter = tokens.iter().peekable();
  let mut nodes: Vec<Node> = vec![];

  while !peek_token(&mut it, Token::EOF) {
    nodes.push(parse_stmt(&mut it)?);
    require_token(&mut it, Token::End)?;
  }

  Ok(Node::Block(nodes))
}
<commit_msg>parser: Update with parse_atom and parse_quark<commit_after>use codemap::Spanned;
use lexer::Token;
use std::iter::Peekable;
use std::slice::Iter;
use self::ParseErrorKind::*;

type ParseIter<'a> = Peekable<Iter<'a, Spanned<Token>>>;
type Parse = Result<Node, ParseErrorKind>;

#[derive(Debug, Clone, PartialEq)]
pub enum Node {
  Block(Vec<Node>),
  If{cond: Box<Node>},
  Stmt(Box<Node>),
  Break,
  Continue,
  Expr,
  Pass,

  BinExpr {
    lhs: Box<Node>,
    op: Token,
    rhs: Box<Node>,
  },

  UnExpr {
    val: Box<Node>,
    op: Token,
  },

  // Literals
  Null,
  Bool(bool),
  Float(f64),
  Int(i64),
  Str(String),
  Name(String),
  Table,
}

#[derive(Debug, Clone, PartialEq)]
pub enum Op {
  Right(u32),
  Left(u32),
  None,
}

#[derive(Debug, Clone, PartialEq)]
pub enum ParseErrorKind {
  UnexpectedToken(Token),
  UnexpectedEOF,
  UnknownBinaryOperator,
  UnknownUnaryOperator,
}

// Return true if the next token in `it` is `kind`
fn peek_token(it: &mut ParseIter, kind: Token) -> bool {
  if let Some(&c) = it.peek() {
    c.node == kind
  }
  else {
    false
  }
}

// Return true if the next token in `it` is `kind` *and* consume the token
fn use_token(it: &mut ParseIter, kind: Token) -> bool {
  if let Some(&c) = it.peek() {
    if c.node == kind {
      it.next();
    }
    c.node == kind
  }
  else {
    false
  }
}

// Panic if the next token in `it` is *not* `kind`
fn require_token(it: &mut ParseIter, kind: Token) -> Result<(), ParseErrorKind> {
  if let Some(&c) = it.peek() {
    it.next();

    if c.node == kind {
      return Ok(());
    }

    return Err(UnexpectedToken(c.node.clone()));
  }

  return Err(UnexpectedEOF);
}

fn op_precedence(op: &Token) -> Op {
  match *op {
    Token::Add | Token::Sub => Op::Left(10),
    Token::Div | Token::Mul => Op::Left(20),
    Token::Car => Op::Right(30),
    _ => Op::None,
  }
}

fn parse_binexpr(it: &mut ParseIter) -> Parse {
  let mut expr = parse_unexpr(it)?;

  // prevents this from breaking the LHS until we know we made it
  // otherwise, things like (2 + 3) * 4 get restructured into 2 + (3 * 4)
  let mut break_left = false;

  while let Some(&c) = it.peek() {
    let prec = op_precedence(&c.node);

    if let Op::None = prec {
      break;
    }

    it.next();

    let rhs = parse_unexpr(it)?;

    expr = match (break_left, expr.clone()) {
      (true, Node::BinExpr{lhs: cur_lhs, op: cur_op, rhs: cur_rhs}) => {
        let cur_prec = op_precedence(&cur_op);
        match (cur_prec, prec) {
          // these should never happen
          (_, Op::None) => {break}
          (Op::None, _) => {break}

          // left-to-right
          // there has to be a better way to handle this, no?
          (Op::Left(n), Op::Left(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: c.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Right(n), Op::Right(m)) if n > m => {
            Node::BinExpr {lhs: Box::new(expr), op: c.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Right(n), Op::Left(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: c.node.clone(), rhs: Box::new(rhs)}
          }
          (Op::Left(n), Op::Right(m)) if n >= m => {
            Node::BinExpr {lhs: Box::new(expr), op: c.node.clone(), rhs: Box::new(rhs)}
          }

          // right-to-left
          _ => {
            Node::BinExpr {
              lhs: cur_lhs,
              op: cur_op,
              rhs: Box::new(Node::BinExpr {lhs: cur_rhs, op: c.node.clone(), rhs: Box::new(rhs)}),
            }
          }
        }
      }
      _ => {
        Node::BinExpr {lhs: Box::new(expr), op: c.node.clone(), rhs: Box::new(rhs)}
      }
    };

    break_left = true;
  };

  Ok(expr)
}

fn parse_unexpr(it: &mut ParseIter) -> Parse {
  parse_simple(it)
}

fn parse_simple(it: &mut ParseIter) -> Parse {
  parse_atom(it)
}

fn parse_atom(it: &mut ParseIter) -> Parse {
  if let Some(&c) = it.peek() {
    return match c.node {
      Token::Pal => {
        it.next();
        let out = parse_binexpr(it)?;
        require_token(it, Token::Par)?;
        Ok(out)
      }
      _ => parse_quark(it),
    }
  }

  Err(UnexpectedEOF)
}

fn parse_quark(it: &mut ParseIter) -> Parse {
  if let Some(&c) = it.peek() {
    return match c.node {
      Token::Null => {it.next(); Ok(Node::Null)},
      Token::Bool(x) => {it.next(); Ok(Node::Bool(x))},
      Token::Float(x) => {it.next(); Ok(Node::Float(x))},
      Token::Int(x) => {it.next(); Ok(Node::Int(x))},
      Token::Str(ref x) => {it.next(); Ok(Node::Str(x.clone()))},
      Token::Name(ref x) => {it.next(); Ok(Node::Name(x.clone()))},
      Token::Table => {it.next(); Ok(Node::Table)},
      _ => Err(UnexpectedToken(c.node.clone())),
    }
  }

  Err(UnexpectedEOF)
}

fn parse_stmt(it: &mut ParseIter) -> Parse {
  if let Some(&c) = it.peek() {
    return match c.node {
      Token::Break => {
        it.next();
        Ok(Node::Break)
      }

      Token::Continue => {
        it.next();
        Ok(Node::Continue)
      }

      Token::Pass => {
        it.next();
        Ok(Node::Pass)
      }

      _ => {
        parse_binexpr(it).map(|expr| Node::Stmt(Box::new(expr)))
      }
    }
  }

  Err(UnexpectedEOF)
}

pub fn parse(tokens: Vec<Spanned<Token>>) -> Parse {
  let mut it: ParseIter = tokens.iter().peekable();
  let mut nodes: Vec<Node> = vec![];

  while !peek_token(&mut it, Token::EOF) {
    nodes.push(parse_stmt(&mut it)?);
    require_token(&mut it, Token::End)?;
  }

  Ok(Node::Block(nodes))
}
<|endoftext|>


<commit_before>use std::mem;

use num::bigint::{self, ToBigInt};
use num::traits::{Zero, One};

use encoding::{Encoding, DecoderTrap};
use encoding::all::ISO_8859_1;

use bertterm::BertTerm;
use error::{Result, BertError};

const BERT_MAGIC_NUMBER: u8 = 131;
const SMALL_INTEGER_EXT: u8 = 97;
const INTEGER_EXT: u8 = 98;
const FLOAT_EXT: u8 = 99;
const ATOM_EXT: u8 = 100;
const SMALL_ATOM_EXT: u8 = 115;
const SMALL_TUPLE_EXT: u8 = 104;
const LARGE_TUPLE_EXT: u8 = 105;
const NIL_EXT: u8 = 106;
const STRING_EXT: u8 = 107;
const LIST_EXT: u8 = 108;
const BINARY_EXT: u8 = 109;
const SMALL_BIG_EXT: u8 = 110;
const LARGE_BIG_EXT: u8 = 111;
const ATOM_UTF8_EXT: u8 = 118;
const SMALL_ATOM_UTF8_EXT: u8 = 119;
const NEW_FLOAT_EXT: u8 = 70;
const MAP_EXT: u8 = 116;

#[derive(Debug)]
pub struct Parser {
    contents: Vec<u8>,
    pos: usize,
}

impl Parser {
    pub fn new(contents: Vec<u8>) -> Parser {
        Parser { contents: contents, pos: 0 }
    }

    pub fn parse(&mut self) -> Result<BertTerm> {
        let () = self.magic_number()?;
        let term = self.bert_term()?;
        if self.eof() {
            return Ok(term);
        } else {
            return Err(BertError::ExtraData(self.pos));
        }
    }

    pub fn parse_bert2(&mut self) -> Result<Vec<BertTerm>> {
        let mut terms = Vec::with_capacity(32);
        while !self.eof() {
            let _ = self.parse_varint()?;
            let _ = self.magic_number()?;
            let t = self.bert_term()?;
            terms.push(t);
        }
        if self.eof() {
            return Ok(terms);
        } else {
            return Err(BertError::ExtraData(self.pos));
        }
    }

    // Parsers
    fn magic_number(&mut self) -> Result<()> {
        let offset = self.pos;
        let magic = self.eat_u8()?;
        if magic != BERT_MAGIC_NUMBER {
            return Err(BertError::InvalidMagicNumber(offset));
        } else {
            return Ok(());
        }
    }

    fn bert_term(&mut self) -> Result<BertTerm> {
        let offset = self.pos;
        match self.eat_u8()? {
            SMALL_INTEGER_EXT => { self.small_integer() }
            INTEGER_EXT => { self.integer() }
            FLOAT_EXT => { self.old_float() }
            NEW_FLOAT_EXT => { self.new_float() }
            ATOM_EXT => {
                let len = self.eat_u16_be()? as usize;
                self.atom(len)
            }
            SMALL_ATOM_EXT => {
                let len = self.eat_u8()? as usize;
                self.atom(len)
            }
            ATOM_UTF8_EXT => {
                let len = self.eat_u16_be()? as usize;
                self.atom_utf8(len)
            }
            SMALL_ATOM_UTF8_EXT => {
                let len = self.eat_u8()? as usize;
                self.atom_utf8(len)
            }
            SMALL_TUPLE_EXT => {
                let len = self.eat_u8()? as usize;
                self.tuple(len)
            }
            LARGE_TUPLE_EXT => {
                let len = self.eat_u32_be()? as usize;
                self.tuple(len)
            }
            NIL_EXT => { Ok(BertTerm::Nil) }
            LIST_EXT => { self.list() }
            STRING_EXT => { self.string() }
            BINARY_EXT => { self.binary() }
            SMALL_BIG_EXT => {
                let len = self.eat_u8()?;
                self.bigint(len as usize)
            }
            LARGE_BIG_EXT => {
                let len = self.eat_u32_be()?;
                self.bigint(len as usize)
            }
            MAP_EXT => {
                self.map()
            }
            tag => { Err(BertError::InvalidTag(offset, tag)) }
        }
    }

    fn small_integer(&mut self) -> Result<BertTerm> {
        let b = self.eat_u8()?;
        Ok(BertTerm::Int(b as i32))
    }

    fn integer(&mut self) -> Result<BertTerm> {
        let n = self.eat_i32_be()?;
        Ok(BertTerm::Int(n))
    }

    fn old_float(&mut self) -> Result<BertTerm> {
        let offset = self.pos;
        let mut s = String::new();
        while !self.eof() && self.peek()? != 0 {
            s.push(self.eat_char()?);
        }

        while !self.eof() && self.peek()? == 0 {
            let _ = self.eat_u8()?;
        }

        s.parse::<f64>()
            .map_err(|_| BertError::InvalidFloat(offset))
            .map(|f| BertTerm::Float(f))
    }

    fn new_float(&mut self) -> Result<BertTerm> {
        let raw_bytes = self.eat_u64_be()?;
        let f: f64 = unsafe { mem::transmute(raw_bytes) };
        Ok(BertTerm::Float(f))
    }

    fn atom(&mut self, len: usize) -> Result<BertTerm> {
        let offset = self.pos;
        let mut bytes: Vec<u8> = Vec::with_capacity(len);
        let mut is_ascii = true;
        for _ in 0 .. len {
            let b = self.eat_u8()?;
            is_ascii = is_ascii && (b < 128);
            bytes.push(b);
        }

        // Optimization: ASCII atoms represent the overwhelming
        // majority of use cases of atoms. When we read the bytes
        // of the atom, we record whether they are all ASCII
        // (i.e., small than 128); if it's the case, we don't
        // need to bother with latin-1 decoding. We use an unsafe
        // method because ASCII strings are guaranteed to be valid
        // UTF-8 strings.
        if is_ascii {
            let s = unsafe { String::from_utf8_unchecked(bytes) };
            Ok(BertTerm::Atom(s))
        } else {
            ISO_8859_1.decode(&bytes, DecoderTrap::Strict)
                .map(|s| BertTerm::Atom(s))
                .map_err(|_| BertError::InvalidLatin1Atom(offset))
        }
    }

    fn atom_utf8(&mut self, len: usize) -> Result<BertTerm> {
        let offset = self.pos;
        let mut buf = Vec::with_capacity(len);
        for _ in 0 .. len {
            buf.push(self.eat_u8()?);
        }
        String::from_utf8(buf)
            .map(|s| BertTerm::Atom(s))
            .map_err(|_| BertError::InvalidUTF8Atom(offset))
    }

    fn tuple(&mut self, len: usize) -> Result<BertTerm> {
        let mut terms = Vec::with_capacity(len);
        for _ in 0 .. len {
            terms.push(self.bert_term()?);
        }
        Ok(BertTerm::Tuple(terms))
    }

    fn string(&mut self) -> Result<BertTerm> {
        let len = self.eat_u16_be()?;
        let mut bytes = Vec::with_capacity(len as usize);
        for _ in 0 .. len {
            bytes.push(self.eat_u8()?);
        }
        Ok(BertTerm::String(bytes))
    }

    fn binary(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()?;
        let mut bytes = Vec::with_capacity(len as usize);
        for _ in 0 .. len {
            bytes.push(self.eat_u8()?);
        }
        Ok(BertTerm::Binary(bytes))
    }

    fn list(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()?;
        let mut terms = Vec::with_capacity(len as usize + 1);
        for _ in 0 .. len {
            terms.push(self.bert_term()?);
        }
        let tail = self.bert_term()?;
        match tail {
            BertTerm::Nil => (),
            last_term => { terms.push(last_term); }
        };
        Ok(BertTerm::List(terms))
    }

    fn bigint(&mut self, len: usize) -> Result<BertTerm> {
        let sign = self.eat_u8()?;
        let mut sum: bigint::BigInt = Zero::zero();
        let mut pos: bigint::BigInt = One::one();
        for _ in 0 .. len {
            let d = self.eat_u8()?;
            let t = &pos * &(d.to_bigint().unwrap());
            sum = sum + &t;
            pos = pos * (256).to_bigint().unwrap();
        }
        if sign == 1 {
            sum = -sum;
        }
        Ok(BertTerm::BigInt(sum))
    }

    // TODO(vfoley): ensure no duplicate keys
    fn map(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()? as usize;
        let mut keys = Vec::with_capacity(len);
        let mut vals = Vec::with_capacity(len);
        for _ in 0 .. len {
            keys.push(self.bert_term()?);
            vals.push(self.bert_term()?);
        }
        Ok(BertTerm::Map(keys, vals))
    }

    // Low-level parsing methods
    fn eof(&self) -> bool {
        self.pos >= self.contents.len()
    }

    fn peek(&self) -> Result<u8> {
        if self.eof() {
            return Err(BertError::EOF(self.pos));
        } else {
            return Ok(self.contents[self.pos]);
        }
    }

    fn eat_u8(&mut self) -> Result<u8> {
        let b = self.peek()?;
        self.pos += 1;
        return Ok(b);
    }

    fn eat_char(&mut self) -> Result<char> {
        let b = self.eat_u8()?;
        return Ok(b as char);
    }

    fn eat_u16_be(&mut self) -> Result<u16> {
        let b0 = self.eat_u8()? as u16;
        let b1 = self.eat_u8()? as u16;
        return Ok((b0 << 8) + b1)
    }

    fn eat_i32_be(&mut self) -> Result<i32> {
        let b0 = self.eat_u8()? as i32;
        let b1 = self.eat_u8()? as i32;
        let b2 = self.eat_u8()? as i32;
        let b3 = self.eat_u8()? as i32;
        return Ok((b0 << 24) + (b1 << 16) + (b2 << 8) + b3)
    }

    fn eat_u32_be(&mut self) -> Result<u32> {
        let b0 = self.eat_u8()? as u32;
        let b1 = self.eat_u8()? as u32;
        let b2 = self.eat_u8()? as u32;
        let b3 = self.eat_u8()? as u32;
        return Ok((b0 << 24) + (b1 << 16) + (b2 << 8) + b3)
    }

    fn eat_u64_be(&mut self) -> Result<u64> {
        let mut n: u64 = 0;
        n += (self.eat_u8()? as u64) << 56;
        n += (self.eat_u8()? as u64) << 48;
        n += (self.eat_u8()? as u64) << 40;
        n += (self.eat_u8()? as u64) << 32;
        n += (self.eat_u8()? as u64) << 24;
        n += (self.eat_u8()? as u64) << 16;
        n += (self.eat_u8()? as u64) << 8;
        n += self.eat_u8()? as u64;
        return Ok(n);
    }

    // https://developers.google.com/protocol-buffers/docs/encoding#varints
    fn parse_varint(&mut self) -> Result<u64> {
        let start_pos = self.pos;

        let mut bytes = Vec::with_capacity(mem::size_of::<u64>());
        let mut i = 0;

        while !self.eof() && i < bytes.capacity() {
            let b = self.eat_u8()?;
            bytes.push(b);
            if b & 0x80 == 0 {
                break;
            }
            i += 1;
        }

        if i >= bytes.capacity() {
            return Err(BertError::VarintTooLarge(start_pos));
        }

        let mut x: u64 = 0;
        for (i, byte) in bytes.iter().rev().enumerate() {
            x = (x << (7*i) as u64) | (*byte as u64 & 0x7f);
        }

        return Ok(x);
    }
}

#[test]
fn test_varint() {
    assert_eq!(1, {
        match Parser::new(vec![1]).parse_varint() {
            Ok(x) => x,
            Err(_) => u64::max_value()
        }
    });

    assert_eq!(300, {
        match Parser::new(vec![0b1010_1100, 0b0000_0010]).parse_varint() {
            Ok(x) => x,
            Err(_) => u64::max_value()
        }
    });

    assert!(Parser::new(vec![0xff, 0xff, 0xff, 0xff,
                             0xff, 0xff, 0xff, 0x7f]).parse_varint().is_ok());
    assert!(Parser::new(vec![0xff, 0xff, 0xff, 0xff,
                             0xff, 0xff, 0xff, 0x80]).parse_varint().is_err());
}
<commit_msg>Improve performance of basic parser methods<commit_after>use std::mem;

use num::bigint::{self, ToBigInt};
use num::traits::{Zero, One};

use encoding::{Encoding, DecoderTrap};
use encoding::all::ISO_8859_1;

use bertterm::BertTerm;
use error::{Result, BertError};

const BERT_MAGIC_NUMBER: u8 = 131;
const SMALL_INTEGER_EXT: u8 = 97;
const INTEGER_EXT: u8 = 98;
const FLOAT_EXT: u8 = 99;
const ATOM_EXT: u8 = 100;
const SMALL_ATOM_EXT: u8 = 115;
const SMALL_TUPLE_EXT: u8 = 104;
const LARGE_TUPLE_EXT: u8 = 105;
const NIL_EXT: u8 = 106;
const STRING_EXT: u8 = 107;
const LIST_EXT: u8 = 108;
const BINARY_EXT: u8 = 109;
const SMALL_BIG_EXT: u8 = 110;
const LARGE_BIG_EXT: u8 = 111;
const ATOM_UTF8_EXT: u8 = 118;
const SMALL_ATOM_UTF8_EXT: u8 = 119;
const NEW_FLOAT_EXT: u8 = 70;
const MAP_EXT: u8 = 116;

#[derive(Debug)]
pub struct Parser {
    contents: Vec<u8>,
    pos: usize,
}

impl Parser {
    pub fn new(contents: Vec<u8>) -> Parser {
        Parser { contents: contents, pos: 0 }
    }

    pub fn parse(&mut self) -> Result<BertTerm> {
        let () = self.magic_number()?;
        let term = self.bert_term()?;
        if self.eof() {
            return Ok(term);
        } else {
            return Err(BertError::ExtraData(self.pos));
        }
    }

    pub fn parse_bert2(&mut self) -> Result<Vec<BertTerm>> {
        let mut terms = Vec::with_capacity(32);
        while !self.eof() {
            let _ = self.parse_varint()?;
            let _ = self.magic_number()?;
            let t = self.bert_term()?;
            terms.push(t);
        }
        if self.eof() {
            return Ok(terms);
        } else {
            return Err(BertError::ExtraData(self.pos));
        }
    }

    // Parsers
    fn magic_number(&mut self) -> Result<()> {
        let offset = self.pos;
        let magic = self.eat_u8()?;
        if magic != BERT_MAGIC_NUMBER {
            return Err(BertError::InvalidMagicNumber(offset));
        } else {
            return Ok(());
        }
    }

    fn bert_term(&mut self) -> Result<BertTerm> {
        let offset = self.pos;
        match self.eat_u8()? {
            SMALL_INTEGER_EXT => { self.small_integer() }
            INTEGER_EXT => { self.integer() }
            FLOAT_EXT => { self.old_float() }
            NEW_FLOAT_EXT => { self.new_float() }
            ATOM_EXT => {
                let len = self.eat_u16_be()? as usize;
                self.atom(len)
            }
            SMALL_ATOM_EXT => {
                let len = self.eat_u8()? as usize;
                self.atom(len)
            }
            ATOM_UTF8_EXT => {
                let len = self.eat_u16_be()? as usize;
                self.atom_utf8(len)
            }
            SMALL_ATOM_UTF8_EXT => {
                let len = self.eat_u8()? as usize;
                self.atom_utf8(len)
            }
            SMALL_TUPLE_EXT => {
                let len = self.eat_u8()? as usize;
                self.tuple(len)
            }
            LARGE_TUPLE_EXT => {
                let len = self.eat_u32_be()? as usize;
                self.tuple(len)
            }
            NIL_EXT => { Ok(BertTerm::Nil) }
            LIST_EXT => { self.list() }
            STRING_EXT => { self.string() }
            BINARY_EXT => { self.binary() }
            SMALL_BIG_EXT => {
                let len = self.eat_u8()?;
                self.bigint(len as usize)
            }
            LARGE_BIG_EXT => {
                let len = self.eat_u32_be()?;
                self.bigint(len as usize)
            }
            MAP_EXT => {
                self.map()
            }
            tag => { Err(BertError::InvalidTag(offset, tag)) }
        }
    }

    fn small_integer(&mut self) -> Result<BertTerm> {
        let b = self.eat_u8()?;
        Ok(BertTerm::Int(b as i32))
    }

    fn integer(&mut self) -> Result<BertTerm> {
        let n = self.eat_i32_be()?;
        Ok(BertTerm::Int(n))
    }

    fn old_float(&mut self) -> Result<BertTerm> {
        let offset = self.pos;
        let mut s = String::new();
        while !self.eof() && self.peek()? != 0 {
            s.push(self.eat_char()?);
        }

        while !self.eof() && self.peek()? == 0 {
            let _ = self.eat_u8()?;
        }

        s.parse::<f64>()
            .map_err(|_| BertError::InvalidFloat(offset))
            .map(|f| BertTerm::Float(f))
    }

    fn new_float(&mut self) -> Result<BertTerm> {
        let raw_bytes = self.eat_u64_be()?;
        let f: f64 = f64::from_bits(raw_bytes);
        Ok(BertTerm::Float(f))
    }

    fn atom(&mut self, len: usize) -> Result<BertTerm> {
        if !self.can_read(len) {
            return Err(BertError::EOF(self.pos));
        }

        let offset = self.pos;
        let mut is_ascii = true;
        let mut bytes: Vec<u8> = Vec::with_capacity(len);
        unsafe { bytes.set_len(len); }
        for (i, byte) in bytes.iter_mut().enumerate() {
            *byte = self.peek_u8_unchecked(i);
            is_ascii = is_ascii && (*byte < 128);
        }

        self.pos += len;

        // Optimization: ASCII atoms represent the overwhelming
        // majority of use cases of atoms. When we read the bytes
        // of the atom, we record whether they are all ASCII
        // (i.e., small than 128); if it's the case, we don't
        // need to bother with latin-1 decoding. We use an unsafe
        // method because ASCII strings are guaranteed to be valid
        // UTF-8 strings.
        if is_ascii {
            let s = unsafe { String::from_utf8_unchecked(bytes) };
            Ok(BertTerm::Atom(s))
        } else {
            ISO_8859_1.decode(&bytes, DecoderTrap::Strict)
                .map(|s| BertTerm::Atom(s))
                .map_err(|_| BertError::InvalidLatin1Atom(offset))
        }
    }

    fn atom_utf8(&mut self, len: usize) -> Result<BertTerm> {
        if !self.can_read(len) {
            return Err(BertError::EOF(self.pos));
        }

        let offset = self.pos;
        let mut buf = Vec::with_capacity(len);
        unsafe { buf.set_len(len); }
        for (i, byte) in buf.iter_mut().enumerate() {
            *byte = self.peek_u8_unchecked(i);
        }
        self.pos += len;
        String::from_utf8(buf)
            .map(|s| BertTerm::Atom(s))
            .map_err(|_| BertError::InvalidUTF8Atom(offset))
    }

    fn tuple(&mut self, len: usize) -> Result<BertTerm> {
        let mut terms = Vec::with_capacity(len);
        for _ in 0 .. len {
            terms.push(self.bert_term()?);
        }
        Ok(BertTerm::Tuple(terms))
    }

    fn string(&mut self) -> Result<BertTerm> {
        let len = self.eat_u16_be()? as usize;
        if !self.can_read(len) {
            return Err(BertError::EOF(self.pos));
        }

        let mut bytes = Vec::with_capacity(len);
        unsafe { bytes.set_len(len); }
        for (i, byte) in bytes.iter_mut().enumerate() {
            *byte = self.peek_u8_unchecked(i);
        }

        self.pos += len;
        Ok(BertTerm::String(bytes))
    }

    fn binary(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()? as usize;
        if !self.can_read(len) {
            return Err(BertError::EOF(self.pos));
        }

        let mut bytes = Vec::with_capacity(len);
        unsafe { bytes.set_len(len); }
        for (i, byte) in bytes.iter_mut().enumerate() {
            *byte = self.peek_u8_unchecked(i);
        }

        self.pos += len;
        Ok(BertTerm::Binary(bytes))
    }

    fn list(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()?;
        let mut terms = Vec::with_capacity(len as usize + 1);
        for _ in 0 .. len {
            terms.push(self.bert_term()?);
        }
        let tail = self.bert_term()?;
        match tail {
            BertTerm::Nil => (),
            last_term => { terms.push(last_term); }
        };
        Ok(BertTerm::List(terms))
    }

    fn bigint(&mut self, len: usize) -> Result<BertTerm> {
        let sign = self.eat_u8()?;
        let mut sum: bigint::BigInt = Zero::zero();
        let mut pos: bigint::BigInt = One::one();
        for _ in 0 .. len {
            let d = self.eat_u8()?;
            let t = &pos * &(d.to_bigint().unwrap());
            sum = sum + &t;
            pos = pos * (256).to_bigint().unwrap();
        }
        if sign == 1 {
            sum = -sum;
        }
        Ok(BertTerm::BigInt(sum))
    }

    // TODO(vfoley): ensure no duplicate keys
    fn map(&mut self) -> Result<BertTerm> {
        let len = self.eat_u32_be()? as usize;
        let mut keys = Vec::with_capacity(len);
        let mut vals = Vec::with_capacity(len);
        for _ in 0 .. len {
            keys.push(self.bert_term()?);
            vals.push(self.bert_term()?);
        }
        Ok(BertTerm::Map(keys, vals))
    }

    // Low-level parsing methods
    fn eof(&self) -> bool {
        self.pos >= self.contents.len()
    }

    fn peek(&self) -> Result<u8> {
        if self.eof() {
            return Err(BertError::EOF(self.pos));
        } else {
            return Ok(self.peek_u8_unchecked(0));
        }
    }

    fn can_read(&self, n: usize) -> bool {
        (self.pos + n) <= self.contents.len()
    }

    fn peek_u8_unchecked(&self, offset: usize) -> u8 {
        unsafe { *self.contents.get_unchecked(self.pos + offset) }
    }

    fn eat_u8(&mut self) -> Result<u8> {
        if !self.can_read(1) {
            return Err(BertError::EOF(self.pos));
        }
        let b = self.peek_u8_unchecked(0);
        self.pos += 1;
        return Ok(b);
    }

    fn eat_char(&mut self) -> Result<char> {
        let b = self.eat_u8()?;
        return Ok(b as char);
    }

    fn eat_u16_be(&mut self) -> Result<u16> {
        if !self.can_read(2) {
            return Err(BertError::EOF(self.pos));
        }
        let b0 = self.peek_u8_unchecked(0) as u16;
        let b1 = self.peek_u8_unchecked(1) as u16;
        self.pos += 2;
        return Ok((b0 << 8) + b1)
    }

    fn eat_i32_be(&mut self) -> Result<i32> {
        if !self.can_read(4) {
            return Err(BertError::EOF(self.pos));
        }
        let b0 = self.peek_u8_unchecked(0) as i32;
        let b1 = self.peek_u8_unchecked(1) as i32;
        let b2 = self.peek_u8_unchecked(2) as i32;
        let b3 = self.peek_u8_unchecked(3) as i32;
        self.pos += 4;
        return Ok((b0 << 24) + (b1 << 16) + (b2 << 8) + b3)
    }

    fn eat_u32_be(&mut self) -> Result<u32> {
        if !self.can_read(4) {
            return Err(BertError::EOF(self.pos));
        }
        let b0 = self.peek_u8_unchecked(0) as u32;
        let b1 = self.peek_u8_unchecked(1) as u32;
        let b2 = self.peek_u8_unchecked(2) as u32;
        let b3 = self.peek_u8_unchecked(3) as u32;
        self.pos += 4;
        return Ok((b0 << 24) + (b1 << 16) + (b2 << 8) + b3)
    }

    fn eat_u64_be(&mut self) -> Result<u64> {
        if !self.can_read(8) {
            return Err(BertError::EOF(self.pos));
        }
        let b0 = self.peek_u8_unchecked(0) as u64;
        let b1 = self.peek_u8_unchecked(1) as u64;
        let b2 = self.peek_u8_unchecked(2) as u64;
        let b3 = self.peek_u8_unchecked(3) as u64;
        let b4 = self.peek_u8_unchecked(4) as u64;
        let b5 = self.peek_u8_unchecked(5) as u64;
        let b6 = self.peek_u8_unchecked(6) as u64;
        let b7 = self.peek_u8_unchecked(7) as u64;
        self.pos += 8;
        return Ok((b0 << 56) + (b1 << 48) + (b2 << 40) + (b3 << 32) +
                  (b4 << 24) + (b5 << 16) + (b6 << 8) + b7);
    }

    // https://developers.google.com/protocol-buffers/docs/encoding#varints
    fn parse_varint(&mut self) -> Result<u64> {
        let start_pos = self.pos;

        let mut bytes = Vec::with_capacity(mem::size_of::<u64>());
        let mut i = 0;

        while !self.eof() && i < bytes.capacity() {
            let b = self.eat_u8()?;
            bytes.push(b);
            if b & 0x80 == 0 {
                break;
            }
            i += 1;
        }

        if i >= bytes.capacity() {
            return Err(BertError::VarintTooLarge(start_pos));
        }

        let mut x: u64 = 0;
        for (i, byte) in bytes.iter().rev().enumerate() {
            x = (x << (7*i) as u64) | (*byte as u64 & 0x7f);
        }

        return Ok(x);
    }
}

#[test]
fn test_varint() {
    assert_eq!(1, {
        match Parser::new(vec![1]).parse_varint() {
            Ok(x) => x,
            Err(_) => u64::max_value()
        }
    });

    assert_eq!(300, {
        match Parser::new(vec![0b1010_1100, 0b0000_0010]).parse_varint() {
            Ok(x) => x,
            Err(_) => u64::max_value()
        }
    });

    assert!(Parser::new(vec![0xff, 0xff, 0xff, 0xff,
                             0xff, 0xff, 0xff, 0x7f]).parse_varint().is_ok());
    assert!(Parser::new(vec![0xff, 0xff, 0xff, 0xff,
                             0xff, 0xff, 0xff, 0x80]).parse_varint().is_err());
}
<|endoftext|>


<commit_before>use syntax::ast::TokenTree;
use syntax::codemap::Span;
use syntax::ptr::P;

use syntax::ast::Expr;
use syntax::ext::base::{ExtCtxt, MacResult, MacEager};
use syntax::parse::parser::Parser;
use syntax::parse::token::Token;

pub fn expand<'cx>(cx: &'cx mut ExtCtxt, _: Span,
               tts: &[TokenTree]) -> Box<MacResult + 'cx> {
    let mut parser = cx.new_parser_from_tts(tts);
    let expr = parse_json(cx, &mut parser);
    if &parser.token != &Token::Eof {
        cx.span_fatal(parser.span, "expected end of `json!` macro invocation");
    }
    MacEager::expr(expr)
}

fn parse_json(cx: &ExtCtxt, parser: &mut Parser) -> P<Expr> {
    use syntax::ext::build::AstBuilder;
    use syntax::parse::token::{DelimToken, IdentStyle};

    macro_rules! comma_sep {
        () =>  {
            ::syntax::parse::common::SeqSep {
                sep: Some(Token::Comma),
                trailing_sep_allowed: true // we could be JSON pedants...
            }
        }
    }

    let orig_span = parser.span;

    match &parser.token {
        &Token::OpenDelim(DelimToken::Bracket) => {
            let _ = parser.bump();
            let r_bracket = Token::CloseDelim(DelimToken::Bracket);
            let exprs = parser.parse_seq_to_end(&r_bracket, comma_sep!(), |p| {
                Ok(parse_json(cx, p))
            }).ok().unwrap();
            let exprs = cx.expr_vec(orig_span, exprs);
            quote_expr!(cx, {
                use ::std::boxed::Box;
                let xs: Box<[_]> = Box::new($exprs);
                ::rustc_serialize::json::Json::Array(xs.into_vec())
            })
        },
        &Token::OpenDelim(DelimToken::Brace) => {
            let _ = parser.bump();
            let r_brace = Token::CloseDelim(DelimToken::Brace);
            let kvs = parser.parse_seq_to_end(&r_brace, comma_sep!(), |p| {
                let (istr, _) = p.parse_str().ok().unwrap();
                let s = &*istr;
                let _ = p.expect(&Token::Colon);
                let key = quote_expr!(cx, {
                    use ::std::borrow::ToOwned;
                    $s.to_owned()
                });
                Ok((key, parse_json(cx, p)))
            }).ok().unwrap();
            let mut insertions = vec![];
            // Can't use `quote_stmt!()` and interpolate a vector of
            // statements, seemingly.  Should consider filing a bug
            // upstream.
            for &(ref key, ref value) in kvs.iter() {
                insertions.push(quote_expr!(cx, {
                    _ob.insert($key, $value);
                }));
            }
            let expr = quote_expr!(cx, {
                let mut _ob = ::std::collections::BTreeMap::new();
                $insertions;
                ::rustc_serialize::json::Json::Object(_ob)
            });
            expr
        },
        &Token::OpenDelim(DelimToken::Paren) => {
            let expr = parser.parse_expr().unwrap();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        },
        &Token::Ident(id, IdentStyle::Plain) if id.name.as_str() == "null" => {
            let _ = parser.bump();
            quote_expr!(cx, { ::rustc_serialize::json::Json::Null })
        },
        _ => { // TODO: investigate can_begin_expr (maybe eliminate need for parens)?
            let expr = parser.parse_literal_maybe_minus().ok().unwrap();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        }
    }
}
<commit_msg>account for upstream Parser changes<commit_after>use syntax::ast::TokenTree;
use syntax::codemap::Span;
use syntax::ptr::P;

use syntax::ast::Expr;
use syntax::ext::base::{ExtCtxt, MacResult, MacEager};
use syntax::parse::parser::Parser;
use syntax::parse::token::Token;

pub fn expand<'cx>(cx: &'cx mut ExtCtxt, _: Span,
               tts: &[TokenTree]) -> Box<MacResult + 'cx> {
    let mut parser = cx.new_parser_from_tts(tts);
    let expr = parse_json(cx, &mut parser);
    if &parser.token != &Token::Eof {
        cx.span_fatal(parser.span, "expected end of `json!` macro invocation");
    }
    MacEager::expr(expr)
}

fn parse_json(cx: &ExtCtxt, parser: &mut Parser) -> P<Expr> {
    use syntax::ext::build::AstBuilder;
    use syntax::parse::token::{DelimToken, IdentStyle};

    macro_rules! comma_sep {
        () =>  {
            ::syntax::parse::common::SeqSep {
                sep: Some(Token::Comma),
                trailing_sep_allowed: true // we could be JSON pedants...
            }
        }
    }

    let orig_span = parser.span;

    match &parser.token {
        &Token::OpenDelim(DelimToken::Bracket) => {
            let _ = parser.bump();
            let r_bracket = Token::CloseDelim(DelimToken::Bracket);
            let exprs = parser.parse_seq_to_end(&r_bracket, comma_sep!(), |p| {
                Ok(parse_json(cx, p))
            }).ok().unwrap();
            let exprs = cx.expr_vec(orig_span, exprs);
            quote_expr!(cx, {
                use ::std::boxed::Box;
                let xs: Box<[_]> = Box::new($exprs);
                ::rustc_serialize::json::Json::Array(xs.into_vec())
            })
        },
        &Token::OpenDelim(DelimToken::Brace) => {
            let _ = parser.bump();
            let r_brace = Token::CloseDelim(DelimToken::Brace);
            let kvs = parser.parse_seq_to_end(&r_brace, comma_sep!(), |p| {
                let (istr, _) = p.parse_str().ok().unwrap();
                let s = &*istr;
                let _ = p.expect(&Token::Colon);
                let key = quote_expr!(cx, {
                    use ::std::borrow::ToOwned;
                    $s.to_owned()
                });
                Ok((key, parse_json(cx, p)))
            }).ok().unwrap();
            let mut insertions = vec![];
            // Can't use `quote_stmt!()` and interpolate a vector of
            // statements, seemingly.  Should consider filing a bug
            // upstream.
            for &(ref key, ref value) in kvs.iter() {
                insertions.push(quote_expr!(cx, {
                    _ob.insert($key, $value);
                }));
            }
            let expr = quote_expr!(cx, {
                let mut _ob = ::std::collections::BTreeMap::new();
                $insertions;
                ::rustc_serialize::json::Json::Object(_ob)
            });
            expr
        },
        &Token::OpenDelim(DelimToken::Paren) => {
            let expr = parser.parse_expr().unwrap();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        },
        &Token::Ident(id, IdentStyle::Plain) if id.name.as_str() == "null" => {
            let _ = parser.bump();
            quote_expr!(cx, { ::rustc_serialize::json::Json::Null })
        },
        _ => { // TODO: investigate can_begin_expr (maybe eliminate need for parens)?
            let expr = parser.parse_pat_literal_maybe_minus().ok().unwrap();
            quote_expr!(cx, {{
                use ::rustc_serialize::json::ToJson;
                ($expr).to_json()
            }})
        }
    }
}
<|endoftext|>


<commit_before>use std::collections::PriorityQueue;

#[deriving(PartialEq,Eq)]
struct GridItem {
    x: uint,
    y: uint,
    value: uint,
}

impl Ord for GridItem {
    fn cmp(&self, other: &GridItem) -> Ordering {
        self.value.cmp(&other.value)
    }
}

impl PartialOrd for GridItem {
    fn partial_cmp(&self, other: &GridItem) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

fn find_greatest_product(input_grid: Vec<Vec<uint>>) -> uint {
    let mut cell_queue = PriorityQueue::new();
    for i in input_grid.iter().enumerate() {
        let (y, row) = i;
        for j in row.iter().enumerate() {
            let (x, value) = j;
            cell_queue.push(GridItem {
                x: x,
                y: y,
                value: *value
            });
        }
    }

    let mut largest_product = 0u;
    for cell in cell_queue.iter() {
        // Discover which directions we can go from here
        let directions = vec![
            (0, 1), (1, 0), (0, -1), (-1, 0), // North, East, South, West
            (1, 1), (1, -1), (-1, 1), (-1, -1), // Angles
        ];
        for dir in directions.iter() {
            let mut product = 1u;
            for i in range(0, 4) {
                let (dir_x, dir_y) = *dir;
                let cell_x = cell.x + (i * dir_x);
                let cell_y = cell.y + (i * dir_y);
                if cell_x < 0 || cell_x >= input_grid.len() ||
                        cell_y < 0 || cell_y >= input_grid.len() {
                    continue;
                }

                // First vector is vector of rows, so Y comes first
                product *= input_grid[cell_y][cell_x];
            }

            if product > largest_product {
                largest_product = product;
            }
        }
    }
    largest_product
}

#[cfg(test)]
mod test {
    use super::find_greatest_product;

    #[test]
    fn expected_result() {
        let input_grid: Vec<Vec<uint>> = vec![
            vec![8, 2, 22, 97, 38, 15, 0, 40, 0, 75, 4, 5, 7, 78, 52, 12, 50, 77, 91, 8],
            vec![49, 49, 99, 40, 17, 81, 18, 57, 60, 87, 17, 40, 98, 43, 69, 48, 4, 56, 62, 0],
            vec![81, 49, 31, 73, 55, 79, 14, 29, 93, 71, 40, 67, 53, 88, 30, 3, 49, 13, 36, 65],
            vec![52, 70, 95, 23, 4, 60, 11, 42, 69, 24, 68, 56, 1, 32, 56, 71, 37, 2, 36, 91],
            vec![22, 31, 16, 71, 51, 67, 63, 89, 41, 92, 36, 54, 22, 40, 40, 28, 66, 33, 13, 80],
            vec![24, 47, 32, 60, 99, 3, 45, 2, 44, 75, 33, 53, 78, 36, 84, 20, 35, 17, 12, 50],
            vec![32, 98, 81, 28, 64, 23, 67, 10, 26, 38, 40, 67, 59, 54, 70, 66, 18, 38, 64, 70],
            vec![67, 26, 20, 68, 2, 62, 12, 20, 95, 63, 94, 39, 63, 8, 40, 91, 66, 49, 94, 21],
            vec![24, 55, 58, 5, 66, 73, 99, 26, 97, 17, 78, 78, 96, 83, 14, 88, 34, 89, 63, 72],
            vec![21, 36, 23, 9, 75, 0, 76, 44, 20, 45, 35, 14, 0, 61, 33, 97, 34, 31, 33, 95],
            vec![78, 17, 53, 28, 22, 75, 31, 67, 15, 94, 3, 80, 4, 62, 16, 14, 9, 53, 56, 92],
            vec![16, 39, 5, 42, 96, 35, 31, 47, 55, 58, 88, 24, 0, 17, 54, 24, 36, 29, 85, 57],
            vec![86, 56, 0, 48, 35, 71, 89, 7, 5, 44, 44, 37, 44, 60, 21, 58, 51, 54, 17, 58],
            vec![19, 80, 81, 68, 5, 94, 47, 69, 28, 73, 92, 13, 86, 52, 17, 77, 4, 89, 55, 40],
            vec![4, 52, 8, 83, 97, 35, 99, 16, 7, 97, 57, 32, 16, 26, 26, 79, 33, 27, 98, 66],
            vec![88, 36, 68, 87, 57, 62, 20, 72, 3, 46, 33, 67, 46, 55, 12, 32, 63, 93, 53, 69],
            vec![4, 42, 16, 73, 38, 25, 39, 11, 24, 94, 72, 18, 8, 46, 29, 32, 40, 62, 76, 36],
            vec![20, 69, 36, 41, 72, 30, 23, 88, 34, 62, 99, 69, 82, 67, 59, 85, 74, 4, 36, 16],
            vec![20, 73, 35, 29, 78, 31, 90, 1, 74, 31, 49, 71, 48, 86, 81, 16, 23, 57, 5, 54],
            vec![1, 70, 54, 71, 83, 51, 54, 69, 16, 92, 33, 48, 61, 43, 52, 1, 89, 19, 67, 48],
        ];
        assert_eq!(find_greatest_product(input_grid), 70600674u);
    }
}
<commit_msg>Correct potential uint/int squashing issue in problem 11.<commit_after>use std::collections::PriorityQueue;

#[deriving(PartialEq,Eq)]
struct GridItem {
    x: uint,
    y: uint,
    value: uint,
}

impl Ord for GridItem {
    fn cmp(&self, other: &GridItem) -> Ordering {
        self.value.cmp(&other.value)
    }
}

impl PartialOrd for GridItem {
    fn partial_cmp(&self, other: &GridItem) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

fn find_greatest_product(input_grid: Vec<Vec<uint>>) -> uint {
    let mut cell_queue = PriorityQueue::new();
    for i in input_grid.iter().enumerate() {
        let (y, row) = i;
        for j in row.iter().enumerate() {
            let (x, value) = j;
            cell_queue.push(GridItem {
                x: x,
                y: y,
                value: *value
            });
        }
    }

    let mut largest_product = 0u;
    for cell in cell_queue.iter() {
        // Discover which directions we can go from here
        let directions = vec![
            (0, 1), (1, 0), (0, -1), (-1, 0), // North, East, South, West
            (1, 1), (1, -1), (-1, 1), (-1, -1), // Angles
        ];
        for dir in directions.iter() {
            let mut product = 1u;
            for i in range(0, 4) {
                let (dir_x, dir_y) = *dir;
                let signed_cell_x: int = (cell.x as int) + (i * dir_x);
                let signed_cell_y: int = (cell.y as int) + (i * dir_y);
                let cell_x = signed_cell_x as uint;
                let cell_y = signed_cell_y as uint;
                if signed_cell_x < 0 || cell_x >= input_grid.len() ||
                        signed_cell_y < 0 || cell_y >= input_grid.len() {
                    continue;
                }

                // First vector is vector of rows, so Y comes first
                product *= input_grid[cell_y][cell_x];
            }

            if product > largest_product {
                largest_product = product;
            }
        }
    }
    largest_product
}

#[cfg(test)]
mod test {
    use super::find_greatest_product;

    #[test]
    fn expected_result() {
        let input_grid: Vec<Vec<uint>> = vec![
            vec![8, 2, 22, 97, 38, 15, 0, 40, 0, 75, 4, 5, 7, 78, 52, 12, 50, 77, 91, 8],
            vec![49, 49, 99, 40, 17, 81, 18, 57, 60, 87, 17, 40, 98, 43, 69, 48, 4, 56, 62, 0],
            vec![81, 49, 31, 73, 55, 79, 14, 29, 93, 71, 40, 67, 53, 88, 30, 3, 49, 13, 36, 65],
            vec![52, 70, 95, 23, 4, 60, 11, 42, 69, 24, 68, 56, 1, 32, 56, 71, 37, 2, 36, 91],
            vec![22, 31, 16, 71, 51, 67, 63, 89, 41, 92, 36, 54, 22, 40, 40, 28, 66, 33, 13, 80],
            vec![24, 47, 32, 60, 99, 3, 45, 2, 44, 75, 33, 53, 78, 36, 84, 20, 35, 17, 12, 50],
            vec![32, 98, 81, 28, 64, 23, 67, 10, 26, 38, 40, 67, 59, 54, 70, 66, 18, 38, 64, 70],
            vec![67, 26, 20, 68, 2, 62, 12, 20, 95, 63, 94, 39, 63, 8, 40, 91, 66, 49, 94, 21],
            vec![24, 55, 58, 5, 66, 73, 99, 26, 97, 17, 78, 78, 96, 83, 14, 88, 34, 89, 63, 72],
            vec![21, 36, 23, 9, 75, 0, 76, 44, 20, 45, 35, 14, 0, 61, 33, 97, 34, 31, 33, 95],
            vec![78, 17, 53, 28, 22, 75, 31, 67, 15, 94, 3, 80, 4, 62, 16, 14, 9, 53, 56, 92],
            vec![16, 39, 5, 42, 96, 35, 31, 47, 55, 58, 88, 24, 0, 17, 54, 24, 36, 29, 85, 57],
            vec![86, 56, 0, 48, 35, 71, 89, 7, 5, 44, 44, 37, 44, 60, 21, 58, 51, 54, 17, 58],
            vec![19, 80, 81, 68, 5, 94, 47, 69, 28, 73, 92, 13, 86, 52, 17, 77, 4, 89, 55, 40],
            vec![4, 52, 8, 83, 97, 35, 99, 16, 7, 97, 57, 32, 16, 26, 26, 79, 33, 27, 98, 66],
            vec![88, 36, 68, 87, 57, 62, 20, 72, 3, 46, 33, 67, 46, 55, 12, 32, 63, 93, 53, 69],
            vec![4, 42, 16, 73, 38, 25, 39, 11, 24, 94, 72, 18, 8, 46, 29, 32, 40, 62, 76, 36],
            vec![20, 69, 36, 41, 72, 30, 23, 88, 34, 62, 99, 69, 82, 67, 59, 85, 74, 4, 36, 16],
            vec![20, 73, 35, 29, 78, 31, 90, 1, 74, 31, 49, 71, 48, 86, 81, 16, 23, 57, 5, 54],
            vec![1, 70, 54, 71, 83, 51, 54, 69, 16, 92, 33, 48, 61, 43, 52, 1, 89, 19, 67, 48],
        ];
        assert_eq!(find_greatest_product(input_grid), 70600674u);
    }
}
<|endoftext|>


<commit_before>use std::{
    cmp::PartialEq,
    error::Error as StdError,
    fmt::{self, Display},
    io,
};

use crate::pagecache::{DiskPtr, PageView};

/// The top-level result type for dealing with
/// fallible operations. The errors tend to
/// be fail-stop, and nested results are used
/// in cases where the outer fail-stop error can
/// have try `?` used on it, exposing the inner
/// operation that is expected to fail under
/// normal operation. The philosophy behind this
/// is detailed [on the sled blog](https://sled.rs/errors).
pub type Result<T> = std::result::Result<T, Error>;

/// A compare and swap result.  If the CAS is successful,
/// the new `PagePtr` will be returned as `Ok`.  Otherwise,
/// the `Err` will contain a tuple of the current `PagePtr`
/// and the old value that could not be set atomically.
pub(crate) type CasResult<'a, R> =
    std::result::Result<PageView<'a>, Option<(PageView<'a>, R)>>;

/// An Error type encapsulating various issues that may come up
/// in the operation of a `Db`.
#[derive(Debug, Clone, Copy)]
pub enum Error {
    /// The underlying collection no longer exists.
    CollectionNotFound,
    /// The system has been used in an unsupported way.
    Unsupported(&'static str),
    /// An unexpected bug has happened. Please open an issue on github!
    ReportableBug(&'static str),
    /// A read or write error has happened when interacting with the file
    /// system.
    Io(io::ErrorKind, &'static str),
    /// Corruption has been detected in the storage file.
    Corruption {
        /// The file location that corrupted data was found at.
        at: Option<DiskPtr>,
    },
    // a failpoint has been triggered for testing purposes
    #[doc(hidden)]
    #[cfg(feature = "failpoints")]
    FailPoint,
}

impl Error {
    pub(crate) const fn corruption(at: Option<DiskPtr>) -> Error {
        Error::Corruption { at }
    }
}

impl Eq for Error {}

impl PartialEq for Error {
    fn eq(&self, other: &Self) -> bool {
        use self::Error::*;

        match *self {
            CollectionNotFound => CollectionNotFound == *other,
            Unsupported(ref l) => {
                if let Unsupported(ref r) = *other {
                    l == r
                } else {
                    false
                }
            }
            ReportableBug(ref l) => {
                if let ReportableBug(ref r) = *other {
                    l == r
                } else {
                    false
                }
            }
            #[cfg(feature = "failpoints")]
            FailPoint => {
                matches!(other, FailPoint)
            }
            Corruption { at: l, .. } => {
                if let Corruption { at: r, .. } = *other {
                    l == r
                } else {
                    false
                }
            }
            Io(_, _) => false,
        }
    }
}

impl From<io::Error> for Error {
    #[inline]
    fn from(io_error: io::Error) -> Self {
        Error::Io(io_error.kind(), "io error")
    }
}

impl From<Error> for io::Error {
    fn from(error: Error) -> io::Error {
        use self::Error::*;
        use std::io::ErrorKind;
        match error {
            Io(kind, reason) => io::Error::new(kind, reason),
            CollectionNotFound => io::Error::new(
                ErrorKind::NotFound,
                "collection not found"
            ),
            Unsupported(why) => io::Error::new(
                ErrorKind::InvalidInput,
                format!("operation not supported: {:?}", why),
            ),
            ReportableBug(what) => io::Error::new(
                ErrorKind::Other,
                format!(
                    "unexpected bug! please report this bug at <github.rs/spacejam/sled>: {:?}",
                    what
                ),
            ),
            Corruption { .. } => io::Error::new(
                ErrorKind::InvalidData,
                format!("corruption encountered: {:?}", error),
            ),
            #[cfg(feature = "failpoints")]
            FailPoint => io::Error::new(ErrorKind::Other, "failpoint"),
        }
    }
}

impl StdError for Error {}

impl Display for Error {
    fn fmt(
        &self,
        f: &mut fmt::Formatter<'_>,
    ) -> std::result::Result<(), fmt::Error> {
        use self::Error::*;

        match *self {
            CollectionNotFound => {
                write!(f, "Collection does not exist")
            }
            Unsupported(ref e) => write!(f, "Unsupported: {}", e),
            ReportableBug(ref e) => write!(
                f,
                "Unexpected bug has happened: {}. \
                 PLEASE REPORT THIS BUG!",
                e
            ),
            #[cfg(feature = "failpoints")]
            FailPoint => write!(f, "Fail point has been triggered."),
            Io(ref kind, ref reason) => {
                write!(f, "IO error: ({:?}, {})", kind, reason)
            }
            Corruption { at } => {
                write!(f, "Read corrupted data at file offset {:?}", at)
            }
        }
    }
}
<commit_msg>Fix infinite Eq loop<commit_after>use std::{
    cmp::PartialEq,
    error::Error as StdError,
    fmt::{self, Display},
    io,
};

use crate::pagecache::{DiskPtr, PageView};

/// The top-level result type for dealing with
/// fallible operations. The errors tend to
/// be fail-stop, and nested results are used
/// in cases where the outer fail-stop error can
/// have try `?` used on it, exposing the inner
/// operation that is expected to fail under
/// normal operation. The philosophy behind this
/// is detailed [on the sled blog](https://sled.rs/errors).
pub type Result<T> = std::result::Result<T, Error>;

/// A compare and swap result.  If the CAS is successful,
/// the new `PagePtr` will be returned as `Ok`.  Otherwise,
/// the `Err` will contain a tuple of the current `PagePtr`
/// and the old value that could not be set atomically.
pub(crate) type CasResult<'a, R> =
    std::result::Result<PageView<'a>, Option<(PageView<'a>, R)>>;

/// An Error type encapsulating various issues that may come up
/// in the operation of a `Db`.
#[derive(Debug, Clone, Copy)]
pub enum Error {
    /// The underlying collection no longer exists.
    CollectionNotFound,
    /// The system has been used in an unsupported way.
    Unsupported(&'static str),
    /// An unexpected bug has happened. Please open an issue on github!
    ReportableBug(&'static str),
    /// A read or write error has happened when interacting with the file
    /// system.
    Io(io::ErrorKind, &'static str),
    /// Corruption has been detected in the storage file.
    Corruption {
        /// The file location that corrupted data was found at.
        at: Option<DiskPtr>,
    },
    // a failpoint has been triggered for testing purposes
    #[doc(hidden)]
    #[cfg(feature = "failpoints")]
    FailPoint,
}

impl Error {
    pub(crate) const fn corruption(at: Option<DiskPtr>) -> Error {
        Error::Corruption { at }
    }
}

impl Eq for Error {}

impl PartialEq for Error {
    fn eq(&self, other: &Self) -> bool {
        use self::Error::*;

        match *self {
            CollectionNotFound => matches!(other, CollectionNotFound),
            Unsupported(ref l) => {
                if let Unsupported(ref r) = *other {
                    l == r
                } else {
                    false
                }
            }
            ReportableBug(ref l) => {
                if let ReportableBug(ref r) = *other {
                    l == r
                } else {
                    false
                }
            }
            #[cfg(feature = "failpoints")]
            FailPoint => {
                matches!(other, FailPoint)
            }
            Corruption { at: l, .. } => {
                if let Corruption { at: r, .. } = *other {
                    l == r
                } else {
                    false
                }
            }
            Io(_, _) => false,
        }
    }
}

impl From<io::Error> for Error {
    #[inline]
    fn from(io_error: io::Error) -> Self {
        Error::Io(io_error.kind(), "io error")
    }
}

impl From<Error> for io::Error {
    fn from(error: Error) -> io::Error {
        use self::Error::*;
        use std::io::ErrorKind;
        match error {
            Io(kind, reason) => io::Error::new(kind, reason),
            CollectionNotFound => io::Error::new(
                ErrorKind::NotFound,
                "collection not found"
            ),
            Unsupported(why) => io::Error::new(
                ErrorKind::InvalidInput,
                format!("operation not supported: {:?}", why),
            ),
            ReportableBug(what) => io::Error::new(
                ErrorKind::Other,
                format!(
                    "unexpected bug! please report this bug at <github.rs/spacejam/sled>: {:?}",
                    what
                ),
            ),
            Corruption { .. } => io::Error::new(
                ErrorKind::InvalidData,
                format!("corruption encountered: {:?}", error),
            ),
            #[cfg(feature = "failpoints")]
            FailPoint => io::Error::new(ErrorKind::Other, "failpoint"),
        }
    }
}

impl StdError for Error {}

impl Display for Error {
    fn fmt(
        &self,
        f: &mut fmt::Formatter<'_>,
    ) -> std::result::Result<(), fmt::Error> {
        use self::Error::*;

        match *self {
            CollectionNotFound => {
                write!(f, "Collection does not exist")
            }
            Unsupported(ref e) => write!(f, "Unsupported: {}", e),
            ReportableBug(ref e) => write!(
                f,
                "Unexpected bug has happened: {}. \
                 PLEASE REPORT THIS BUG!",
                e
            ),
            #[cfg(feature = "failpoints")]
            FailPoint => write!(f, "Fail point has been triggered."),
            Io(ref kind, ref reason) => {
                write!(f, "IO error: ({:?}, {})", kind, reason)
            }
            Corruption { at } => {
                write!(f, "Read corrupted data at file offset {:?}", at)
            }
        }
    }
}
<|endoftext|>