<commit_before>use tokenizer::Tokenizer;
use event::{Event, Rule};
use variable_mapper::VariableMapper;

pub fn nested<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut parents = Vec::new();
    nested_inner(&mut vm, &mut parents)
}

pub fn nested_inner<'a, I>(tokenizer: &mut I, parents: &mut Vec<String>) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut last = Event::End(Rule::SassRule);
    let mut current = String::from_str("");
    let mut properties = String::from_str("");
    let mut children = String::from_str("");

    while let Some(token) = tokenizer.next() {
       match token.clone() {
            Event::Start(_) => {
                match children.len() {
                    0 => children = nested_inner(tokenizer, parents),
                    _ => {
                        children.push_str("\n\n");
                        children.push_str(&nested_inner(tokenizer, parents)[..]);
                    },
                };
            },
            Event::Selector(name) => {
                parents.push((*name).to_string());
                current.push_str(&format!("{} ", name)[..]);
            },
            Event::Property(name, value) => {
                match last {
                    Event::Selector(_) => properties.push_str(&format!("{{\n  {}: {};", name, value)[..]),
                    _ => properties.push_str(&format!("\n  {}: {};", name, value)[..]),
                }
            },
            Event::End(_) => {
                match (properties.len(), children.len()) {
                    (0, 0) => current.push_str(" }\n"),
                    (_, 0) => {
                        current.push_str(&properties[..]);
                        current.push_str(" }");
                    },
                    (0, _) => {
                        current.push_str(&children[..]);
                    },
                    (_, _) => {
                        current.push_str(&properties[..]);
                        current.push_str(" }\n");
                        current.push_str(&parents.connect(" "));
                        current.push_str(" ");
                        current.push_str(&children[..]);
                    },
                }
                parents.pop();
                return current
            },
            Event::Variable(..) => unreachable!(),
        };
        last = token;
    }
    children
}

pub fn compressed<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut parents = Vec::new();
    compressed_inner(&mut vm, &mut parents)
}

fn compressed_inner<'a, I>(tokenizer: &mut I, parents: &mut Vec<String>) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut last = Event::End(Rule::SassRule);
    let mut current = String::from_str("");
    let mut properties = String::from_str("");
    let mut children = String::from_str("");

    while let Some(token) = tokenizer.next() {
         match token.clone() {
             Event::Start(_) => {
                 match children.len() {
                     0 => children = compressed_inner(tokenizer, parents),
                     _ => {
                         children.push_str(&compressed_inner(tokenizer, parents)[..]);
                     },
                 };
             },
             Event::Selector(name) => {
                 parents.push((*name).to_string());
                 match last {
                     Event::Selector(_) => current.push_str(&format!(" {}", name)[..]),
                     _ => current.push_str(&format!("{}", name)[..]),
                 }
             },
             Event::Property(name, value) => {
                 match last {
                     Event::Selector(..) => properties.push_str(&format!("{{{}:{}", name, value)[..]),
                     Event::Property(..) => properties.push_str(&format!(";{}:{}", name, value)[..]),
                     _ => properties.push_str(&format!("{}:{}", name, value)[..]),
                 }
             },
             Event::End(_) => {
                 match (properties.len(), children.len()) {
                     (0, 0) => current.push_str("}"),
                     (_, 0) => {
                         current.push_str(&properties[..]);
                         current.push_str("}");
                     },
                     (0, _) => {
                         current.push_str(" ");
                         current.push_str(&children[..]);
                     },
                     (_, _) => {
                         current.push_str(&properties[..]);
                         current.push_str("}");
                         current.push_str(&parents.connect(" "));
                         current.push_str(" ");
                         current.push_str(&children[..]);
                     },
                 }
                 parents.pop();
                 return current
             },
            Event::Variable(..) => unreachable!(),
        };
        last = token;
    }
    children
}

pub fn expanded<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut parents = Vec::new();
    expanded_inner(&mut vm, &mut parents)
}

fn expanded_inner<'a, I>(tokenizer: &mut I, parents: &mut Vec<String>) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut last = Event::End(Rule::SassRule);
    let mut current = String::from_str("");
    let mut properties = String::from_str("");
    let mut children = String::from_str("");

    while let Some(token) = tokenizer.next() {
        match token.clone() {
            Event::Start(_) => {
                match children.len() {
                    0 => children = expanded_inner(tokenizer, parents),
                    _ => {
                        children.push_str("\n\n");
                        children.push_str(&expanded_inner(tokenizer, parents)[..]);
                    },
                };
            },
            Event::Selector(name) => {
                parents.push((*name).to_string());
                match last {
                    Event::Selector(_) => current.push_str(&format!(" {}", name)[..]),
                    Event::End(_) => current.push_str(&format!("{}", name)[..]),
                    _ => current.push_str(&format!("{}", name)[..]),
                }
            },
            Event::Property(name, value) => {
                match last {
                    Event::Selector(_) => properties.push_str(&format!(" {{\n  {}: {};", name, value)[..]),
                    _ => properties.push_str(&format!("\n  {}: {};", name, value)[..]),
                }
            },
            Event::End(_) => {
                match (properties.len(), children.len()) {
                    (0, 0) => current.push_str("\n}"),
                    (_, 0) => {
                        current.push_str(&properties[..]);
                        current.push_str("\n}");
                    },
                    (0, _) => {
                        current.push_str(" ");
                        current.push_str(&children[..]);
                    },
                    (_, _) => {
                        current.push_str(&properties[..]);
                        current.push_str("\n}\n");
                        current.push_str(&parents.connect(" "));
                        current.push_str(" ");
                        current.push_str(&children[..]);
                    },
                }
                parents.pop();
                return current
            },
            Event::Variable(..) => unreachable!(),
        };

        last = token;
    }
    children
}

pub fn compact<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut parents = Vec::new();
    compact_inner(&mut vm, &mut parents)
}

fn compact_inner<'a, I>(tokenizer: &mut I, parents: &mut Vec<String>) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut last = Event::End(Rule::SassRule);
    let mut current = String::from_str("");
    let mut properties = String::from_str("");
    let mut children = String::from_str("");

    while let Some(token) = tokenizer.next() {
        match token.clone() {
            Event::Start(_) => {
                match children.len() {
                    0 => children = compact_inner(tokenizer, parents),
                    _ => {
                        children.push_str("\n");
                        children.push_str(&compact_inner(tokenizer, parents)[..]);
                    },
                };
            },
            Event::Selector(name) => {
                parents.push((*name).to_string());
                match last {
                    Event::Selector(_) => current.push_str(&format!(" {}", name)[..]),
                    Event::End(_) => current.push_str(&format!("{}", name)[..]),
                    _ => current.push_str(&format!("{}", name)[..]),
                }
            },
            Event::Property(name, value) => {
                match last {
                    Event::Selector(_) => properties.push_str(&format!(" {{ {}: {};", name, value)[..]),
                    _ => properties.push_str(&format!(" {}: {};", name, value)[..]),
                }
            },
            Event::End(_) => {
                match (properties.len(), children.len()) {
                    (0, 0) => current.push_str(" }"),
                    (_, 0) => {
                        current.push_str(&properties[..]);
                        current.push_str(" }\n");
                    },
                    (0, _) => {
                        current.push_str(" ");
                        current.push_str(&children[..]);
                    },
                    (_, _) => {
                        current.push_str(&properties[..]);
                        current.push_str(" }\n");
                        current.push_str(&parents.connect(" "));
                        current.push_str(" ");
                        current.push_str(&children[..]);
                    },
                }
                parents.pop();
                return current
            },
            Event::Variable(..) => unreachable!(),
        };
        last = token;
    }
    children
}

pub fn debug(tokenizer: &mut Tokenizer) -> String {
    let mut output = String::from_str("");
    let mut current = String::from_str("");

    while let Some(token) = tokenizer.next() {
        current.push_str(&format!("{:?}\n", token)[..]);
        match token {
            Event::End(_) => {
                output.push_str(&current[..]);
                current = String::from_str("");
            },
            _ => {},
        }
    }
    output
}
<commit_msg>Get rid of a whole bunch of slicing that just needed to be borrows<commit_after>use tokenizer::Tokenizer;
use event::{Event, Rule};
use variable_mapper::VariableMapper;

pub fn nested<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut parents = Vec::new();
    nested_inner(&mut vm, &mut parents)
}

pub fn nested_inner<'a, I>(tokenizer: &mut I, parents: &mut Vec<String>) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut last = Event::End(Rule::SassRule);
    let mut current = String::from_str("");
    let mut properties = String::from_str("");
    let mut children = String::from_str("");

    while let Some(token) = tokenizer.next() {
       match token.clone() {
            Event::Start(_) => {
                match children.len() {
                    0 => children = nested_inner(tokenizer, parents),
                    _ => {
                        children.push_str("\n\n");
                        children.push_str(&nested_inner(tokenizer, parents));
                    },
                };
            },
            Event::Selector(name) => {
                parents.push((*name).to_string());
                current.push_str(&format!("{} ", name));
            },
            Event::Property(name, value) => {
                match last {
                    Event::Selector(_) => properties.push_str(&format!("{{\n  {}: {};", name, value)),
                    _ => properties.push_str(&format!("\n  {}: {};", name, value)),
                }
            },
            Event::End(_) => {
                match (properties.len(), children.len()) {
                    (0, 0) => current.push_str(" }\n"),
                    (_, 0) => {
                        current.push_str(&properties);
                        current.push_str(" }");
                    },
                    (0, _) => {
                        current.push_str(&children);
                    },
                    (_, _) => {
                        current.push_str(&properties);
                        current.push_str(" }\n");
                        current.push_str(&parents.connect(" "));
                        current.push_str(" ");
                        current.push_str(&children);
                    },
                }
                parents.pop();
                return current
            },
            Event::Variable(..) => unreachable!(),
        };
        last = token;
    }
    children
}

pub fn compressed<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut parents = Vec::new();
    compressed_inner(&mut vm, &mut parents)
}

fn compressed_inner<'a, I>(tokenizer: &mut I, parents: &mut Vec<String>) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut last = Event::End(Rule::SassRule);
    let mut current = String::from_str("");
    let mut properties = String::from_str("");
    let mut children = String::from_str("");

    while let Some(token) = tokenizer.next() {
         match token.clone() {
             Event::Start(_) => {
                 match children.len() {
                     0 => children = compressed_inner(tokenizer, parents),
                     _ => {
                         children.push_str(&compressed_inner(tokenizer, parents));
                     },
                 };
             },
             Event::Selector(name) => {
                 parents.push((*name).to_string());
                 match last {
                     Event::Selector(_) => current.push_str(&format!(" {}", name)),
                     _ => current.push_str(&format!("{}", name)),
                 }
             },
             Event::Property(name, value) => {
                 match last {
                     Event::Selector(..) => properties.push_str(&format!("{{{}:{}", name, value)),
                     Event::Property(..) => properties.push_str(&format!(";{}:{}", name, value)),
                     _ => properties.push_str(&format!("{}:{}", name, value)),
                 }
             },
             Event::End(_) => {
                 match (properties.len(), children.len()) {
                     (0, 0) => current.push_str("}"),
                     (_, 0) => {
                         current.push_str(&properties);
                         current.push_str("}");
                     },
                     (0, _) => {
                         current.push_str(" ");
                         current.push_str(&children);
                     },
                     (_, _) => {
                         current.push_str(&properties);
                         current.push_str("}");
                         current.push_str(&parents.connect(" "));
                         current.push_str(" ");
                         current.push_str(&children);
                     },
                 }
                 parents.pop();
                 return current
             },
            Event::Variable(..) => unreachable!(),
        };
        last = token;
    }
    children
}

pub fn expanded<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut parents = Vec::new();
    expanded_inner(&mut vm, &mut parents)
}

fn expanded_inner<'a, I>(tokenizer: &mut I, parents: &mut Vec<String>) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut last = Event::End(Rule::SassRule);
    let mut current = String::from_str("");
    let mut properties = String::from_str("");
    let mut children = String::from_str("");

    while let Some(token) = tokenizer.next() {
        match token.clone() {
            Event::Start(_) => {
                match children.len() {
                    0 => children = expanded_inner(tokenizer, parents),
                    _ => {
                        children.push_str("\n\n");
                        children.push_str(&expanded_inner(tokenizer, parents));
                    },
                };
            },
            Event::Selector(name) => {
                parents.push((*name).to_string());
                match last {
                    Event::Selector(_) => current.push_str(&format!(" {}", name)),
                    Event::End(_) => current.push_str(&format!("{}", name)),
                    _ => current.push_str(&format!("{}", name)),
                }
            },
            Event::Property(name, value) => {
                match last {
                    Event::Selector(_) => properties.push_str(&format!(" {{\n  {}: {};", name, value)),
                    _ => properties.push_str(&format!("\n  {}: {};", name, value)),
                }
            },
            Event::End(_) => {
                match (properties.len(), children.len()) {
                    (0, 0) => current.push_str("\n}"),
                    (_, 0) => {
                        current.push_str(&properties);
                        current.push_str("\n}");
                    },
                    (0, _) => {
                        current.push_str(" ");
                        current.push_str(&children);
                    },
                    (_, _) => {
                        current.push_str(&properties);
                        current.push_str("\n}\n");
                        current.push_str(&parents.connect(" "));
                        current.push_str(" ");
                        current.push_str(&children);
                    },
                }
                parents.pop();
                return current
            },
            Event::Variable(..) => unreachable!(),
        };

        last = token;
    }
    children
}

pub fn compact<'a, I>(tokenizer: &mut I) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut vm = VariableMapper::new(tokenizer);
    let mut parents = Vec::new();
    compact_inner(&mut vm, &mut parents)
}

fn compact_inner<'a, I>(tokenizer: &mut I, parents: &mut Vec<String>) -> String
    where I: Iterator<Item = Event<'a>>
{
    let mut last = Event::End(Rule::SassRule);
    let mut current = String::from_str("");
    let mut properties = String::from_str("");
    let mut children = String::from_str("");

    while let Some(token) = tokenizer.next() {
        match token.clone() {
            Event::Start(_) => {
                match children.len() {
                    0 => children = compact_inner(tokenizer, parents),
                    _ => {
                        children.push_str("\n");
                        children.push_str(&compact_inner(tokenizer, parents));
                    },
                };
            },
            Event::Selector(name) => {
                parents.push((*name).to_string());
                match last {
                    Event::Selector(_) => current.push_str(&format!(" {}", name)),
                    Event::End(_) => current.push_str(&format!("{}", name)),
                    _ => current.push_str(&format!("{}", name)),
                }
            },
            Event::Property(name, value) => {
                match last {
                    Event::Selector(_) => properties.push_str(&format!(" {{ {}: {};", name, value)),
                    _ => properties.push_str(&format!(" {}: {};", name, value)),
                }
            },
            Event::End(_) => {
                match (properties.len(), children.len()) {
                    (0, 0) => current.push_str(" }"),
                    (_, 0) => {
                        current.push_str(&properties);
                        current.push_str(" }\n");
                    },
                    (0, _) => {
                        current.push_str(" ");
                        current.push_str(&children);
                    },
                    (_, _) => {
                        current.push_str(&properties);
                        current.push_str(" }\n");
                        current.push_str(&parents.connect(" "));
                        current.push_str(" ");
                        current.push_str(&children);
                    },
                }
                parents.pop();
                return current
            },
            Event::Variable(..) => unreachable!(),
        };
        last = token;
    }
    children
}

pub fn debug(tokenizer: &mut Tokenizer) -> String {
    let mut output = String::from_str("");
    let mut current = String::from_str("");

    while let Some(token) = tokenizer.next() {
        current.push_str(&format!("{:?}\n", token));
        match token {
            Event::End(_) => {
                output.push_str(&current);
                current = String::from_str("");
            },
            _ => {},
        }
    }
    output
}
<|endoftext|>


<commit_before>//!  Description of the data-structures for IA-32e paging mode.
use core::fmt;

#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub struct PAddr(u64);

#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub struct VAddr(usize);

impl PAddr {
    pub const fn as_u64(&self) -> u64 {
        self.0
    }
    pub const fn from_u64(v: u64) -> Self {
        PAddr(v)
    }
}

impl VAddr {
    pub const fn as_usize(&self) -> usize {
        self.0
    }
    pub const fn from_usize(v: usize) -> Self {
        VAddr(v)
    }
}

impl fmt::Binary for PAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::Display for PAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::LowerHex for PAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::Octal for PAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::UpperHex for PAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::Binary for VAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::Display for VAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::LowerHex for VAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::Octal for VAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::UpperHex for VAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

pub const BASE_PAGE_SIZE: u64 = 4096; // 4 KiB
pub const LARGE_PAGE_SIZE: u64 = 1024*1024*2; // 2 MiB
pub const HUGE_PAGE_SIZE: u64 = 1024*1024*1024; // 1 GiB
pub const CACHE_LINE_SIZE: usize = 64; // 64 Bytes

/// MAXPHYADDR, which is at most 52; (use CPUID for finding system value).
pub const MAXPHYADDR: u64 = 52;

/// Mask to find the physical address of an entry in a page-table.
const ADDRESS_MASK: u64 = ((1 << MAXPHYADDR) - 1) & !0xfff;

/// A PML4 table.
/// In practice this has only 4 entries but it still needs to be the size of a 4K page.
pub type PML4  = [PML4Entry; 512];

/// A page directory pointer table.
pub type PDPT  = [PDPTEntry; 512];

/// A page directory.
pub type PD    = [PDEntry; 512];

/// A page table.
pub type PT    = [PTEntry; 512];

/// Given virtual address calculate corresponding entry in PML4.
pub fn pml4_index(addr: VAddr) -> usize {
    (addr.as_usize() >> 39) & 0b111111111
}

/// Given virtual address calculate corresponding entry in PDPT.
#[inline]
pub fn pdpt_index(addr: VAddr) -> usize {
    (addr.as_usize() >> 30) & 0b111111111
}

/// Given virtual address calculate corresponding entry in PD.
#[inline]
pub fn pd_index(addr: VAddr) -> usize {
    (addr.as_usize() >> 21) & 0b111111111
}

/// Given virtual address calculate corresponding entry in PT.
#[inline]
pub fn pt_index(addr: VAddr) -> usize {
    (addr.as_usize() >> 12) & 0b111111111
}

/// PML4 Entry bits description.
bitflags! {
    #[derive(Debug)]
    flags PML4Entry: u64 {
        /// Present; must be 1 to reference a page-directory-pointer table
        const PML4_P       = 0b00000001,
        /// Read/write; if 0, writes may not be allowed to the 512-GByte region
        /// controlled by this entry (see Section 4.6)
        const PML4_RW      = 0b00000010,
        /// User/supervisor; if 0, user-mode accesses are not allowed
        /// to the 512-GByte region controlled by this entry.
        const PML4_US      = 0b00000100,
        /// Page-level write-through; indirectly determines the memory type used to
        /// access the page-directory-pointer table referenced by this entry.
        const PML4_PWT     = 0b00001000,
        /// Page-level cache disable; indirectly determines the memory type used to
        /// access the page-directory-pointer table referenced by this entry.
        const PML4_PCD     = 0b00010000,
        /// Accessed; indicates whether this entry has been used for linear-address translation.
        const PML4_A       = 0b00100000,
        /// If IA32_EFER.NXE = 1, execute-disable
        /// If 1, instruction fetches are not allowed from the 512-GByte region.
        const PML4_XD      = 1 << 63,
    }
}

impl PML4Entry {

    /// Creates a new PML4Entry.
    ///
    /// # Arguments
    ///
    ///  * `pdpt` - The physical address of the pdpt table.
    ///  * `flags`- Additional flags for the entry.
    pub fn new(pdpt: PAddr, flags: PML4Entry) -> PML4Entry {
        let pdpt_val = pdpt.as_u64();
        assert!(pdpt_val % BASE_PAGE_SIZE == 0);
        PML4Entry { bits: pdpt_val | flags.bits }
    }

    /// Retrieves the physical address in this entry.
    pub fn get_address(self) -> PAddr {
        PAddr::from_u64(self.bits & ADDRESS_MASK)
    }

    /// Convenience function to check if the present bit is set.
    pub fn is_present(self) -> bool {
        self.contains(PML4_P)
    }
}

/// PDPT Entry bits description.
bitflags! {
    #[derive(Debug)]
    flags PDPTEntry: u64 {
        /// Present; must be 1 to map a 1-GByte page or reference a page directory.
        const PDPT_P       = 0b00000001,
        /// Read/write; if 0, writes may not be allowed to the 1-GByte region controlled by this entry
        const PDPT_RW      = 0b00000010,
        /// User/supervisor; user-mode accesses are not allowed to the 1-GByte region controlled by this entry.
        const PDPT_US      = 0b00000100,
        /// Page-level write-through.
        const PDPT_PWT     = 0b00001000,
        /// Page-level cache disable.
        const PDPT_PCD     = 0b00010000,
        /// Accessed; if PDPT_PS set indicates whether software has accessed the 1-GByte page
        /// else indicates whether this entry has been used for linear-address translation
        const PDPT_A       = 0b00100000,
        /// Dirty; if PDPT_PS indicates whether software has written to the 1-GByte page referenced by this entry.
        /// else ignored.
        const PDPT_D       = 0b01000000,
        /// Page size; if set this entry maps a 1-GByte page; otherwise, this entry references a page directory.
        /// if not PDPT_PS this is ignored.
        const PDPT_PS      = 0b10000000,
        /// Global; if PDPT_PS && CR4.PGE = 1, determines whether the translation is global; ignored otherwise
        /// if not PDPT_PS this is ignored.
        const PDPT_G       = 1<<8,
        /// Indirectly determines the memory type used to access the 1-GByte page referenced by this entry.
        const PDPT_PAT     = 1<<12,
        /// If IA32_EFER.NXE = 1, execute-disable
        /// If 1, instruction fetches are not allowed from the 512-GByte region.
        const PDPT_XD      = 1 << 63,
    }
}

impl PDPTEntry {

    /// Creates a new PDPTEntry.
    ///
    /// # Arguments
    ///
    ///  * `pd` - The physical address of the page directory.
    ///  * `flags`- Additional flags for the entry.
    pub fn new(pd: PAddr, flags: PDPTEntry) -> PDPTEntry {
        let pd_val = pd.as_u64();
        assert!(pd_val % BASE_PAGE_SIZE == 0);
        PDPTEntry { bits: pd_val | flags.bits }
    }

    /// Retrieves the physical address in this entry.
    pub fn get_address(self) -> PAddr {
        PAddr::from_u64(self.bits & ADDRESS_MASK)
    }

    /// Convenience function to check if the present bit is set.
    pub fn is_present(self) -> bool {
        self.contains(PDPT_P)
    }
}

/// PD Entry bits description.
bitflags! {
    #[derive(Debug)]
    flags PDEntry: u64 {
        /// Present; must be 1 to map a 2-MByte page or reference a page table.
        const PD_P       = 0b00000001,
        /// Read/write; if 0, writes may not be allowed to the 2-MByte region controlled by this entry
        const PD_RW      = 0b00000010,
        /// User/supervisor; user-mode accesses are not allowed to the 2-MByte region controlled by this entry.
        const PD_US      = 0b00000100,
        /// Page-level write-through.
        const PD_PWT     = 0b00001000,
        /// Page-level cache disable.
        const PD_PCD     = 0b00010000,
        /// Accessed; if PD_PS set indicates whether software has accessed the 2-MByte page
        /// else indicates whether this entry has been used for linear-address translation
        const PD_A       = 0b00100000,
        /// Dirty; if PD_PS indicates whether software has written to the 2-MByte page referenced by this entry.
        /// else ignored.
        const PD_D       = 0b01000000,
        /// Page size; if set this entry maps a 2-MByte page; otherwise, this entry references a page directory.
        const PD_PS      = 0b10000000,
        /// Global; if PD_PS && CR4.PGE = 1, determines whether the translation is global; ignored otherwise
        /// if not PD_PS this is ignored.
        const PD_G       = 1<<8,
        /// Indirectly determines the memory type used to access the 2-MByte page referenced by this entry.
        /// if not PD_PS this is ignored.
        const PD_PAT     = 1<<12,
        /// If IA32_EFER.NXE = 1, execute-disable
        /// If 1, instruction fetches are not allowed from the 512-GByte region.
        const PD_XD      = 1 << 63,
    }
}

impl PDEntry {

    /// Creates a new PDEntry.
    ///
    /// # Arguments
    ///
    ///  * `pt` - The physical address of the page table.
    ///  * `flags`- Additional flags for the entry.
    pub fn new(pt: PAddr, flags: PDEntry) -> PDEntry {
        let pt_val = pt.as_u64();
        assert!(pt_val % BASE_PAGE_SIZE == 0);
        PDEntry { bits: pt_val | flags.bits }
    }

    /// Retrieves the physical address in this entry.
    pub fn get_address(self) -> PAddr {
        PAddr::from_u64(self.bits & ADDRESS_MASK)
    }

    /// Convenience function to check if the present bit is set.
    pub fn is_present(self) -> bool {
        self.contains(PD_P)
    }
}

/// PT Entry bits description.
bitflags! {
    #[derive(Debug)]
    flags PTEntry: u64 {
        /// Present; must be 1 to map a 4-KByte page.
        const PT_P       = 0b00000001,
        /// Read/write; if 0, writes may not be allowed to the 4-KByte region controlled by this entry
        const PT_RW      = 0b00000010,
        /// User/supervisor; user-mode accesses are not allowed to the 4-KByte region controlled by this entry.
        const PT_US      = 0b00000100,
        /// Page-level write-through.
        const PT_PWT     = 0b00001000,
        /// Page-level cache disable.
        const PT_PCD     = 0b00010000,
        /// Accessed; indicates whether software has accessed the 4-KByte page
        const PT_A       = 0b00100000,
        /// Dirty; indicates whether software has written to the 4-KByte page referenced by this entry.
        const PT_D       = 0b01000000,
        /// Global; if CR4.PGE = 1, determines whether the translation is global (see Section 4.10); ignored otherwise
        const PT_G       = 1<<8,
        /// If IA32_EFER.NXE = 1, execute-disable
        /// If 1, instruction fetches are not allowed from the 512-GByte region.
        const PT_XD      = 1 << 63,
    }
}

impl PTEntry {

    /// Creates a new PTEntry.
    ///
    /// # Arguments
    ///
    ///  * `page` - The physical address of the backing 4 KiB page.
    ///  * `flags`- Additional flags for the entry.
    pub fn new(page: PAddr, flags: PTEntry) -> PTEntry {
        let page_val = page.as_u64();
        assert!(page_val % BASE_PAGE_SIZE == 0);
        PTEntry{ bits: page_val | flags.bits }
    }

    /// Retrieves the physical address in this entry.
    pub fn get_address(self) -> PAddr {
        PAddr::from_u64(self.bits & ADDRESS_MASK)
    }

    /// Convenience function to check if the present bit is set.
    pub fn is_present(self) -> bool {
        self.contains(PT_P)
    }
}
<commit_msg>Add some memory type documentation<commit_after>//!  Description of the data-structures for IA-32e paging mode.
use core::fmt;

/// Represents a physical memory address
#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub struct PAddr(u64);

/// Represent a virtual (linear) memory address
#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub struct VAddr(usize);

impl PAddr {
    /// Convert to `u64`
    pub const fn as_u64(&self) -> u64 {
        self.0
    }
    /// Convert from `u64`
    pub const fn from_u64(v: u64) -> Self {
        PAddr(v)
    }
}

impl VAddr {
    /// Convert to `usize`
    pub const fn as_usize(&self) -> usize {
        self.0
    }
    /// Convert from `usize`
    pub const fn from_usize(v: usize) -> Self {
        VAddr(v)
    }
}

impl fmt::Binary for PAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::Display for PAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::LowerHex for PAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::Octal for PAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::UpperHex for PAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::Binary for VAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::Display for VAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::LowerHex for VAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::Octal for VAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

impl fmt::UpperHex for VAddr {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

pub const BASE_PAGE_SIZE: u64 = 4096; // 4 KiB
pub const LARGE_PAGE_SIZE: u64 = 1024*1024*2; // 2 MiB
pub const HUGE_PAGE_SIZE: u64 = 1024*1024*1024; // 1 GiB
pub const CACHE_LINE_SIZE: usize = 64; // 64 Bytes

/// MAXPHYADDR, which is at most 52; (use CPUID for finding system value).
pub const MAXPHYADDR: u64 = 52;

/// Mask to find the physical address of an entry in a page-table.
const ADDRESS_MASK: u64 = ((1 << MAXPHYADDR) - 1) & !0xfff;

/// A PML4 table.
/// In practice this has only 4 entries but it still needs to be the size of a 4K page.
pub type PML4  = [PML4Entry; 512];

/// A page directory pointer table.
pub type PDPT  = [PDPTEntry; 512];

/// A page directory.
pub type PD    = [PDEntry; 512];

/// A page table.
pub type PT    = [PTEntry; 512];

/// Given virtual address calculate corresponding entry in PML4.
pub fn pml4_index(addr: VAddr) -> usize {
    (addr.as_usize() >> 39) & 0b111111111
}

/// Given virtual address calculate corresponding entry in PDPT.
#[inline]
pub fn pdpt_index(addr: VAddr) -> usize {
    (addr.as_usize() >> 30) & 0b111111111
}

/// Given virtual address calculate corresponding entry in PD.
#[inline]
pub fn pd_index(addr: VAddr) -> usize {
    (addr.as_usize() >> 21) & 0b111111111
}

/// Given virtual address calculate corresponding entry in PT.
#[inline]
pub fn pt_index(addr: VAddr) -> usize {
    (addr.as_usize() >> 12) & 0b111111111
}

/// PML4 Entry bits description.
bitflags! {
    #[derive(Debug)]
    flags PML4Entry: u64 {
        /// Present; must be 1 to reference a page-directory-pointer table
        const PML4_P       = 0b00000001,
        /// Read/write; if 0, writes may not be allowed to the 512-GByte region
        /// controlled by this entry (see Section 4.6)
        const PML4_RW      = 0b00000010,
        /// User/supervisor; if 0, user-mode accesses are not allowed
        /// to the 512-GByte region controlled by this entry.
        const PML4_US      = 0b00000100,
        /// Page-level write-through; indirectly determines the memory type used to
        /// access the page-directory-pointer table referenced by this entry.
        const PML4_PWT     = 0b00001000,
        /// Page-level cache disable; indirectly determines the memory type used to
        /// access the page-directory-pointer table referenced by this entry.
        const PML4_PCD     = 0b00010000,
        /// Accessed; indicates whether this entry has been used for linear-address translation.
        const PML4_A       = 0b00100000,
        /// If IA32_EFER.NXE = 1, execute-disable
        /// If 1, instruction fetches are not allowed from the 512-GByte region.
        const PML4_XD      = 1 << 63,
    }
}

impl PML4Entry {

    /// Creates a new PML4Entry.
    ///
    /// # Arguments
    ///
    ///  * `pdpt` - The physical address of the pdpt table.
    ///  * `flags`- Additional flags for the entry.
    pub fn new(pdpt: PAddr, flags: PML4Entry) -> PML4Entry {
        let pdpt_val = pdpt.as_u64();
        assert!(pdpt_val % BASE_PAGE_SIZE == 0);
        PML4Entry { bits: pdpt_val | flags.bits }
    }

    /// Retrieves the physical address in this entry.
    pub fn get_address(self) -> PAddr {
        PAddr::from_u64(self.bits & ADDRESS_MASK)
    }

    /// Convenience function to check if the present bit is set.
    pub fn is_present(self) -> bool {
        self.contains(PML4_P)
    }
}

/// PDPT Entry bits description.
bitflags! {
    #[derive(Debug)]
    flags PDPTEntry: u64 {
        /// Present; must be 1 to map a 1-GByte page or reference a page directory.
        const PDPT_P       = 0b00000001,
        /// Read/write; if 0, writes may not be allowed to the 1-GByte region controlled by this entry
        const PDPT_RW      = 0b00000010,
        /// User/supervisor; user-mode accesses are not allowed to the 1-GByte region controlled by this entry.
        const PDPT_US      = 0b00000100,
        /// Page-level write-through.
        const PDPT_PWT     = 0b00001000,
        /// Page-level cache disable.
        const PDPT_PCD     = 0b00010000,
        /// Accessed; if PDPT_PS set indicates whether software has accessed the 1-GByte page
        /// else indicates whether this entry has been used for linear-address translation
        const PDPT_A       = 0b00100000,
        /// Dirty; if PDPT_PS indicates whether software has written to the 1-GByte page referenced by this entry.
        /// else ignored.
        const PDPT_D       = 0b01000000,
        /// Page size; if set this entry maps a 1-GByte page; otherwise, this entry references a page directory.
        /// if not PDPT_PS this is ignored.
        const PDPT_PS      = 0b10000000,
        /// Global; if PDPT_PS && CR4.PGE = 1, determines whether the translation is global; ignored otherwise
        /// if not PDPT_PS this is ignored.
        const PDPT_G       = 1<<8,
        /// Indirectly determines the memory type used to access the 1-GByte page referenced by this entry.
        const PDPT_PAT     = 1<<12,
        /// If IA32_EFER.NXE = 1, execute-disable
        /// If 1, instruction fetches are not allowed from the 512-GByte region.
        const PDPT_XD      = 1 << 63,
    }
}

impl PDPTEntry {

    /// Creates a new PDPTEntry.
    ///
    /// # Arguments
    ///
    ///  * `pd` - The physical address of the page directory.
    ///  * `flags`- Additional flags for the entry.
    pub fn new(pd: PAddr, flags: PDPTEntry) -> PDPTEntry {
        let pd_val = pd.as_u64();
        assert!(pd_val % BASE_PAGE_SIZE == 0);
        PDPTEntry { bits: pd_val | flags.bits }
    }

    /// Retrieves the physical address in this entry.
    pub fn get_address(self) -> PAddr {
        PAddr::from_u64(self.bits & ADDRESS_MASK)
    }

    /// Convenience function to check if the present bit is set.
    pub fn is_present(self) -> bool {
        self.contains(PDPT_P)
    }
}

/// PD Entry bits description.
bitflags! {
    #[derive(Debug)]
    flags PDEntry: u64 {
        /// Present; must be 1 to map a 2-MByte page or reference a page table.
        const PD_P       = 0b00000001,
        /// Read/write; if 0, writes may not be allowed to the 2-MByte region controlled by this entry
        const PD_RW      = 0b00000010,
        /// User/supervisor; user-mode accesses are not allowed to the 2-MByte region controlled by this entry.
        const PD_US      = 0b00000100,
        /// Page-level write-through.
        const PD_PWT     = 0b00001000,
        /// Page-level cache disable.
        const PD_PCD     = 0b00010000,
        /// Accessed; if PD_PS set indicates whether software has accessed the 2-MByte page
        /// else indicates whether this entry has been used for linear-address translation
        const PD_A       = 0b00100000,
        /// Dirty; if PD_PS indicates whether software has written to the 2-MByte page referenced by this entry.
        /// else ignored.
        const PD_D       = 0b01000000,
        /// Page size; if set this entry maps a 2-MByte page; otherwise, this entry references a page directory.
        const PD_PS      = 0b10000000,
        /// Global; if PD_PS && CR4.PGE = 1, determines whether the translation is global; ignored otherwise
        /// if not PD_PS this is ignored.
        const PD_G       = 1<<8,
        /// Indirectly determines the memory type used to access the 2-MByte page referenced by this entry.
        /// if not PD_PS this is ignored.
        const PD_PAT     = 1<<12,
        /// If IA32_EFER.NXE = 1, execute-disable
        /// If 1, instruction fetches are not allowed from the 512-GByte region.
        const PD_XD      = 1 << 63,
    }
}

impl PDEntry {

    /// Creates a new PDEntry.
    ///
    /// # Arguments
    ///
    ///  * `pt` - The physical address of the page table.
    ///  * `flags`- Additional flags for the entry.
    pub fn new(pt: PAddr, flags: PDEntry) -> PDEntry {
        let pt_val = pt.as_u64();
        assert!(pt_val % BASE_PAGE_SIZE == 0);
        PDEntry { bits: pt_val | flags.bits }
    }

    /// Retrieves the physical address in this entry.
    pub fn get_address(self) -> PAddr {
        PAddr::from_u64(self.bits & ADDRESS_MASK)
    }

    /// Convenience function to check if the present bit is set.
    pub fn is_present(self) -> bool {
        self.contains(PD_P)
    }
}

/// PT Entry bits description.
bitflags! {
    #[derive(Debug)]
    flags PTEntry: u64 {
        /// Present; must be 1 to map a 4-KByte page.
        const PT_P       = 0b00000001,
        /// Read/write; if 0, writes may not be allowed to the 4-KByte region controlled by this entry
        const PT_RW      = 0b00000010,
        /// User/supervisor; user-mode accesses are not allowed to the 4-KByte region controlled by this entry.
        const PT_US      = 0b00000100,
        /// Page-level write-through.
        const PT_PWT     = 0b00001000,
        /// Page-level cache disable.
        const PT_PCD     = 0b00010000,
        /// Accessed; indicates whether software has accessed the 4-KByte page
        const PT_A       = 0b00100000,
        /// Dirty; indicates whether software has written to the 4-KByte page referenced by this entry.
        const PT_D       = 0b01000000,
        /// Global; if CR4.PGE = 1, determines whether the translation is global (see Section 4.10); ignored otherwise
        const PT_G       = 1<<8,
        /// If IA32_EFER.NXE = 1, execute-disable
        /// If 1, instruction fetches are not allowed from the 512-GByte region.
        const PT_XD      = 1 << 63,
    }
}

impl PTEntry {

    /// Creates a new PTEntry.
    ///
    /// # Arguments
    ///
    ///  * `page` - The physical address of the backing 4 KiB page.
    ///  * `flags`- Additional flags for the entry.
    pub fn new(page: PAddr, flags: PTEntry) -> PTEntry {
        let page_val = page.as_u64();
        assert!(page_val % BASE_PAGE_SIZE == 0);
        PTEntry{ bits: page_val | flags.bits }
    }

    /// Retrieves the physical address in this entry.
    pub fn get_address(self) -> PAddr {
        PAddr::from_u64(self.bits & ADDRESS_MASK)
    }

    /// Convenience function to check if the present bit is set.
    pub fn is_present(self) -> bool {
        self.contains(PT_P)
    }
}
<|endoftext|>


<commit_before>use ast;

/* Grammar
    main {  }

*/

#[derive(Debug, Eq, PartialEq)]
enum Token {
    OpenBrace,
    CloseBrace,
    OpenParen,
    CloseParen,
    OpenBracket,
    CloseBracket,
    SingleQuote,
    DoubleQuote,
    Dot,
    Plus,
    Asterik,
    Question,
    Exclamation,
    Ambersand,
    Slash,
    Name(usize, usize)
}

fn tokenize(grammar : &String) -> Vec<Token> {
    let mut iterator = grammar.chars();
    let mut tokens = vec![];
    let mut name = (-1, -1);
    let mut i = 0;

    while let Some(item) = iterator.next() {
        match item {
            '{' => tokens.push(Token::OpenBrace),
            '}' => tokens.push(Token::CloseBrace),
            '(' => tokens.push(Token::OpenParen),
            ')' => tokens.push(Token::CloseParen),
            '[' => tokens.push(Token::OpenBracket),
            ']' => tokens.push(Token::CloseBracket),
            '\'' => tokens.push(Token::SingleQuote),
            '\"' => tokens.push(Token::DoubleQuote),
            '.' => tokens.push(Token::Dot),
            '+' => tokens.push(Token::Plus),
            '*' => tokens.push(Token::Asterik),
            '?' => tokens.push(Token::Question),
            '!' => tokens.push(Token::Exclamation),
            '&' => tokens.push(Token::Ambersand),
            '/' => tokens.push(Token::Slash),
            _ if item.is_alphanumeric() => {
                if name == (-1, -1) {
                    name = (i, i);
                } else {
                    name.1 += 1;
                }
            },
            _ if item.is_whitespace() && name != (-1, -1) => {
                tokens.push(Token::Name(name.0 as usize, name.1 as usize));
                name = (-1, -1);
            }
            _ => { }
        }
        i += 1;
    }
    tokens
}

fn parse(grammar : &String, tokens : Vec<Token>) -> Result<ast::Grammar, u8> {
    let mut iterator = tokens.iter();
    let mut grammar_object = ast::Grammar { rules: vec![], main: 0 };

    while let Some(token) = iterator.next() {
        if let &Token::Name(name_start, name_end) = token {
            if let Some(brace_token) = iterator.next() {
                if brace_token == &Token::OpenBrace {
                    let pattern = parse_pattern(grammar, &mut iterator);
                    match pattern {
                        Ok(p) => grammar_object.rules.push(p),
                        Err(x) => return Err(x)
                    }
                } else {
                    return Err(0);
                }
            } else {
                return Err(0);
            }
        } else {
            return Err(0);
        }
    }
    Ok(grammar_object)
}

fn parse_pattern<Iter>(grammar : &String, iterator : &mut Iter) -> Result<ast::Pattern, u8>
    where Iter : Iterator
{
    let mut result = None;

    while let Some(token) = iterator.next() {
        //match token {
        //    &Token::OpenParen =>
       // }
    }

    match result {
        Some(pattern) => Ok(pattern),
        None => Err(0)
    }
}

<commit_msg>Flesh out parser structure<commit_after>use std::iter::{Peekable};
use ast;

/* Grammar
    main {  }

*/

#[derive(Debug, Eq, PartialEq)]
enum Token {
    OpenBrace,
    CloseBrace,
    OpenParen,
    CloseParen,
    OpenBracket,
    CloseBracket,
    SingleQuote,
    DoubleQuote,
    Dot,
    Plus,
    Asterik,
    Question,
    Exclamation,
    Ambersand,
    Slash,
    Name(usize, usize)
}

fn tokenize(grammar : &String) -> Vec<Token> {
    let mut iterator = grammar.chars();
    let mut tokens = vec![];
    let mut name = (-1, -1);
    let mut i = 0;

    while let Some(item) = iterator.next() {
        match item {
            '{' => tokens.push(Token::OpenBrace),
            '}' => tokens.push(Token::CloseBrace),
            '(' => tokens.push(Token::OpenParen),
            ')' => tokens.push(Token::CloseParen),
            '[' => tokens.push(Token::OpenBracket),
            ']' => tokens.push(Token::CloseBracket),
            '\'' => tokens.push(Token::SingleQuote),
            '\"' => tokens.push(Token::DoubleQuote),
            '.' => tokens.push(Token::Dot),
            '+' => tokens.push(Token::Plus),
            '*' => tokens.push(Token::Asterik),
            '?' => tokens.push(Token::Question),
            '!' => tokens.push(Token::Exclamation),
            '&' => tokens.push(Token::Ambersand),
            '/' => tokens.push(Token::Slash),
            _ if item.is_alphanumeric() => {
                if name == (-1, -1) {
                    name = (i, i);
                } else {
                    name.1 += 1;
                }
            },
            _ if item.is_whitespace() && name != (-1, -1) => {
                tokens.push(Token::Name(name.0 as usize, name.1 as usize));
                name = (-1, -1);
            }
            _ => { }
        }
        i += 1;
    }
    tokens
}

fn parse(grammar : &String, tokens : Vec<Token>) -> Result<ast::Grammar, u8> {
    let mut iterator = tokens.iter().peekable();
    let mut grammar_object = ast::Grammar { rules: vec![], main: 0 };

    while let Some(token) = iterator.next() {
        if let &Token::Name(name_start, name_end) = token {
            if let Some(brace_token) = iterator.next() {
                if brace_token == &Token::OpenBrace {
                    let pattern = parse_pattern(grammar, &mut iterator, false);
                    match pattern {
                        Ok(p) => grammar_object.rules.push(p),
                        Err(x) => return Err(x)
                    }
                } else {
                    return Err(0);
                }
            } else {
                return Err(0);
            }
        } else {
            return Err(0);
        }
    }
    Ok(grammar_object)
}

fn parse_pattern<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, is_subpattern : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let mut sequence = vec![];

    while let Some(token) = iterator.next() {
        let mut initial_pattern = Err(0);
        match token {
            &Token::OpenParen => { initial_pattern = initial_pattern.or(parse_pattern(grammar, iterator, true)); },
            &Token::OpenBracket => { initial_pattern = initial_pattern.or(parse_char_class(grammar, iterator)); },
            &Token::DoubleQuote | &Token::SingleQuote => { initial_pattern = initial_pattern.or(parse_char_sequence(grammar, iterator)); },
            &Token::Dot => { initial_pattern = initial_pattern.or(Ok(ast::Pattern::CharAny)); },
            &Token::Name(le, ri) => { initial_pattern = initial_pattern.or(parse_variable(grammar, le, ri)); },
            &Token::Ambersand => { initial_pattern = initial_pattern.or(parse_lookahead(grammar, iterator, true)); },
            &Token::Exclamation => { initial_pattern = initial_pattern.or(parse_lookahead(grammar, iterator, false)); },
            _ => { }
        }

        if let Some(&suffix_token) = iterator.peek() {
            match suffix_token {
                &Token::Plus | &Token::Asterik | &Token::Question => {
                    initial_pattern = parse_suffix(grammar, iterator, initial_pattern);
                }
                _ => { }
            }
        }

        match initial_pattern {
            Ok(p) => sequence.push(p),
            Err(x) => return Err(x)
        }

        if let Some(&test_token) = iterator.peek() {
            match test_token {
                &Token::CloseParen => if is_subpattern {
                    iterator.next();
                } else {
                    return Err(0);
                },
                &Token::CloseBrace => if !is_subpattern {
                    iterator.next();
                    break;
                } else {
                    return Err(0);
                },
                &Token::Slash => {
                    let sub_pattern = parse_choice(grammar, iterator, &mut sequence);
                    match sub_pattern {
                        Ok(p) => sequence.push(p),
                        Err(x) => return Err(x)
                    }
                },
                _ => { }
            }
        }
    }

    let mut boxed_vec = vec![];
    while let Some(item) = sequence.pop() {
        boxed_vec.push(Box::new(item));
    }

    Ok(ast::Pattern::Sequence(boxed_vec))
}

fn parse_choice<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, patterns : &mut Vec<ast::Pattern>) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    Err(0)
}

fn parse_suffix<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, pattern : Result<ast::Pattern, u8>) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    Err(0)
}

fn parse_lookahead<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>, lookahead : bool) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    let sub_pattern = parse_pattern(grammar, iterator, false);
    match sub_pattern {
        Ok(p) => Ok(ast::Pattern::Lookahead(lookahead, Box::new(p))),
        Err(x) => Err(x)
    }
}

fn parse_char_class<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    Err(0)
}

fn parse_char_sequence<'a, Iter>(grammar : &String, iterator : &mut Peekable<Iter>) -> Result<ast::Pattern, u8>
    where Iter : Iterator<Item=&'a Token>
{
    Err(0)
}

fn parse_variable(grammar : &String, left : usize, right : usize) -> Result<ast::Pattern, u8>
{
    Err(0)
}

<|endoftext|>


<commit_before>use nom::{IResult, alphanumeric, line_ending, space};
use std::str;
use std::error::Error;

#[derive(Debug, PartialEq)]
pub struct SqlQuery {
    fields: Vec<String>,
    table: String,
}

/// Parse rule for a comma-separated list.
named!(csvlist<&[u8], Vec<&str> >,
       many0!(
           map_res!(
               chain!(
                   fieldname: alphanumeric ~
                   opt!(
                       chain!(
                           tag!(",") ~
                           opt!(space),
                           ||{}
                       )
                   ),
                   ||{ fieldname }),
               str::from_utf8)
           )
       );

/// Parse rule for a SQL selection query.
named!(selection<&[u8], SqlQuery>,
    chain!(
        tag!("select") ~
        space ~
        fields: csvlist ~  // XXX(malte): cover * case
        space ~
        tag!("from") ~
        space ~
        table: map_res!(alphanumeric, str::from_utf8) ~
        alt!(tag!(";") | line_ending),
        || {
            SqlQuery {
                table: String::from(table),
                fields: fields.iter().map(|s| String::from(*s)).collect()
            }
        }
    )
);

pub fn parse_query(input: &str) -> Result<SqlQuery, &str> {
    // we process all queries in lowercase to avoid having to deal with capitalization in the
    // parser.
    let q_lower = input.to_lowercase();

    // TODO(malte): appropriately pass through errors from nom
    match selection(&q_lower.into_bytes()) {
        IResult::Done(_, o) => Ok(o),
        IResult::Error(_) => Err("parse error"),
        IResult::Incomplete(_) => Err("incomplete query"),
    }
}

mod tests {
    use nom::IResult;
    use super::*;

    #[test]
    fn simple_select() {
        let qstring = "SELECT id, name FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users") } );
    }

    fn select_all() {
        let qstring = "SELECT * FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["ALL".into()],
                              table: String::from("users") } );
    }

    #[test]
    fn spaces_optional() {
        let qstring = "SELECT id,name FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users") } );
    }

    #[test]
    fn case_sensitivity() {
        let qstring = "select id, name from users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users") } );
    }

    #[test]
    fn termination() {
        let qstring_sem = "select id, name from users;";
        let qstring_linebreak = "select id, name from users\n";

        assert_eq!(parse_query(qstring_sem).unwrap(),
                   parse_query(qstring_linebreak).unwrap());
    }
}
<commit_msg>WHERE clause support (hacky), support * in fields<commit_after>use nom::{IResult, alphanumeric, eof, line_ending, space};
use std::str;

#[derive(Debug, PartialEq)]
pub struct SqlQuery {
    fields: Vec<String>,
    table: String,
    cond: Option<ConditionTree>,
}

#[derive(Debug, PartialEq)]
pub struct ConditionTree {
    field: String,
    expr: String,
}

/// Parse rule for a comma-separated list.
named!(csvlist<&[u8], Vec<&str> >,
       many0!(
           map_res!(
               chain!(
                   fieldname: alphanumeric ~
                   opt!(
                       chain!(
                           tag!(",") ~
                           opt!(space),
                           ||{}
                       )
                   ),
                   ||{ fieldname }
               ),
               str::from_utf8
           )
       )
);

/// Parse list of columns/fields.
/// XXX(malte): add support for named table notation
named!(fieldlist<&[u8], Vec<&str> >,
       alt!(
           tag!("*") => { |_| vec!["ALL".into()] }
         | csvlist
       )
);

named!(where_clause<&[u8], ConditionTree>,
    chain!(
        tag!("where") ~
        space ~
        field: map_res!(alphanumeric, str::from_utf8) ~
        space? ~
        tag!("=") ~
        space? ~
        expr: map_res!(tag_s!(b"?"), str::from_utf8),
        || {
            ConditionTree {
                field: String::from(field),
                expr: String::from(expr),
            }
        }
    )
);

/// Parse rule for a SQL selection query.
named!(selection<&[u8], SqlQuery>,
    chain!(
        tag!("select") ~
        space ~
        fields: fieldlist ~
        space ~
        tag!("from") ~
        space ~
        table: map_res!(alphanumeric, str::from_utf8) ~
        space? ~
        cond: opt!(where_clause) ~
        space? ~
        alt!(eof | tag!(";") | line_ending),  // N.B.: eof must come FIRST
        || {
            SqlQuery {
                table: String::from(table),
                fields: fields.iter().map(|s| String::from(*s)).collect(),
                cond: cond
            }
        }
    )
);

pub fn parse_query(input: &str) -> Result<SqlQuery, &str> {
    // we process all queries in lowercase to avoid having to deal with capitalization in the
    // parser.
    let q_lower = input.to_lowercase();

    // TODO(malte): appropriately pass through errors from nom
    match selection(&q_lower.into_bytes()) {
        IResult::Done(_, o) => Ok(o),
        IResult::Error(_) => Err("parse error"),
        IResult::Incomplete(_) => Err("incomplete query"),
    }
}

mod tests {
    use super::*;

    #[test]
    fn simple_select() {
        let qstring = "SELECT id, name FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn select_all() {
        let qstring = "SELECT * FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["ALL".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn spaces_optional() {
        let qstring = "SELECT id,name FROM users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn case_sensitivity() {
        let qstring = "select id, name from users;";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["id".into(), "name".into()],
                              table: String::from("users"),
                              cond: None } );
    }

    #[test]
    fn termination() {
        let qstring_sem = "select id, name from users;";
        let qstring_linebreak = "select id, name from users\n";

        assert_eq!(parse_query(qstring_sem).unwrap(),
                   parse_query(qstring_linebreak).unwrap());
    }

    #[test]
    fn where_clause() {
        let qstring = "select * from ContactInfo where email=?";

        assert_eq!(parse_query(qstring).unwrap(),
                   SqlQuery { fields: vec!["ALL".into(),],
                              table: String::from("ContactInfo").to_lowercase(),
                              cond: Some(ConditionTree {
                                  field: String::from("email"),
                                  expr: String::from("?") })
                            }
                   );
    }
}
<|endoftext|>


<commit_before>use super::models::{LanguageModel, WordVec};
use self::Token::*;
use super::mayberef::MaybeRef::{self, Val, Ref};
use std::fmt::{Display, Formatter, Error};

pub fn parse(expr: &str, model: &LanguageModel) -> Result<WordVec, String> {
    expression(&mut Tokens::from(expr.chars()), model).map(|w| w.take())
}

#[derive(PartialEq, Eq, Debug, Clone)]
enum Token {
    RParen,
    LParen,
    Plus,
    Minus,
    Word(String),
    Invalid(char),
}

impl Token {
    fn apply_addop(&self, lhs: WordVec, rhs: &WordVec) -> Result<WordVec, String> {
        match *self {
            Plus => Ok(lhs + rhs),
            Minus => Ok(lhs - rhs),
            _ => Err(format!("Internal error: tried calling add using {}", self)),
        }
    }
}

impl Display for Token {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        let invalid;
        fmt.write_str(match *self {
            RParen => ")",
            LParen => "(",
            Plus => "+",
            Minus => "-",
            Word(ref w) => &*w,
            Invalid(w) => {
                invalid = w.to_string();
                &*invalid
            },
        })
    }
}

fn get_word<'a>(model: &'a LanguageModel, word: &str) -> Result<&'a WordVec, String> {
    model.get(&*word).ok_or_else(||
        format!("'{}' is not present in the language model", &word)
    )
}

fn expression<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    let token = try!(tokens.next().ok_or("An expression may not be empty"));
    match token {
        LParen => {
            let vec = try!(expression(tokens, model));
            let rparen = try!(tokens.next().ok_or("Unbalanced parentheses"));
            if rparen != RParen {
                return Err(format!("Expected ')', found '{}'", rparen))
            }
            rhs(vec, tokens, model)
        },
        Word(word) => {
            let vec = try!(get_word(model, &*word));
            rhs(Ref(vec), tokens, model)
        },
        _ => Err(format!("An expression may not start with '{}'", token)),
    }
}

fn rhs<'a, I>(lhs: MaybeRef<'a, WordVec>, tokens: &mut Tokens<I>, model: &LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    match tokens.peek() {
        Some(&RParen) | None => return Ok(lhs),
        _ => {}
    }

    let token = tokens.next().unwrap();
    match token {
        Plus | Minus => {
            let lhs = lhs.take();
            let atom = try!(atom(tokens, model));
            let result = try!(token.apply_addop(lhs, &*atom));
            rhs(Val(result), tokens, model)
        }
        Word(_) | LParen => Err(format!("'{}' found in invalid position", token)),
        Invalid(s) => Err(format!("'{}' could not be tokenized", s)),
        RParen => unreachable!(),
    }
}

fn atom<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    {
        let token = try!(tokens.peek().ok_or("Invalid end of expression"));
        match *token {
            Plus | Minus | Invalid(_) | RParen => {
                return Err(format!("Invalid token {}", token));
            }
            _ => {},
        }
    }

    let token = tokens.next().unwrap();
    match token {
        LParen => {
            let expr = try!(expression(tokens, model));
            match tokens.next() {
                Some(RParen) => Ok(expr),
                Some(t) => Err(format!("Expected ')', found '{}'", t)),
                None => Err("Unclosed parentheses".to_string()),
            }
        }
        Word(word) =>  {
            let vec = try!(get_word(model, &*word));
            Ok(Ref(vec))
        }
        _ => unreachable!(),
    }
}

struct Tokens<I> {
    iter: I,
    next_char: Option<Option<char>>,
    next_token: Option<Option<Token>>
}

impl<I> Tokens<I> where I: Iterator<Item = char> {
    fn from(iter: I) -> Tokens<I> {
        Tokens {
            iter: iter,
            next_char: None,
            next_token: None,
        }
    }

    fn get_next_token(&mut self) -> Option<Token> {
        while let Some(' ') = self.peek_char() {
            self.take();
        }

        match self.peek_char() {
            Some('a'...'z') => return Some(self.word()),
            // Some('0'..'9') | Some('.') => return Some(self.number()),
            None => return None,
            _ => {}
        }

        Some(match self.take().unwrap() {
            '+' => Plus,
            '-' => Minus,
            '(' => LParen,
            ')' => RParen,
            c => Invalid(c),
        })
    }

    fn peek(&mut self) -> Option<&Token> {
        if let Some(ref t) = self.next_token {
            return t.as_ref();
        }

        self.next_token = Some(self.get_next_token());
        return self.next_token.as_ref().unwrap().as_ref();
    }

    fn peek_char(&mut self) -> Option<char> {
        match self.next_char {
            Some(x) => x,
            None => {
                let next = self.iter.next();
                self.next_char = Some(next);
                next
            }
        }
    }

    fn take(&mut self) -> Option<char> {
        self.peek_char();
        self.next_char.take().unwrap()
    }

    fn word(&mut self) -> Token {
        let mut token = String::new();
        while let Some('a'...'z') = self.peek_char() {
            token.push(self.take().unwrap());
        }

        Word(token)
    }

}

impl<I> Iterator for Tokens<I> where I: Iterator<Item = char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.peek();
        self.next_token.take().unwrap()
    }
}

mod test {
    use super::Tokens;
    use super::Token::*;
    use super::parse;
    use super::super::models::{WordVec, LanguageModel, LanguageModelBuilder};

    #[test]
    fn single_character() {
        let c = Tokens::from("a".chars()).next().unwrap();
        assert_eq!(c, Word("a".to_string()));
    }

    #[test]
    fn single_token() {
        let s = Tokens::from("abc".chars()).next().unwrap();
        assert_eq!(s, Word("abc".to_string()));
    }

    #[test]
    fn two_tokens() {
        let s = Tokens::from("abc def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Word("def".to_string())]);
    }

    #[test]
    fn plus_tokens() {
        let s = Tokens::from("   abc  +def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Plus, Word("def".to_string())]);
    }

    #[test]
    fn bracket_tokens() {
        let s = Tokens::from(" ()  (abc  +)def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![LParen, RParen, LParen, Word("abc".to_string()),
            Plus, RParen, Word("def".to_string())]);
    }

    fn get_model() -> LanguageModel {
        let words = vec!("a".to_string(), "b".to_string(), "c".to_string());
        let mut builder = LanguageModelBuilder::new(1, words);

        {
            let mut acc = builder.new_file();
            let text = "x x a a b a c b x x";
            for word in text.split(' ') {
                acc.add_word(word.to_string());
            }
        }

        builder.build()
    }

    #[test]
    fn test_addition() {
        let model = get_model();
        assert_eq!(parse("a - b + c", &model).unwrap(), parse("(a - b) + c", &model).unwrap());
        assert_eq!(parse("a + (b - c)", &model).unwrap(), parse("(a + b) - c", &model).unwrap());
        assert_eq!(parse("a + b - c)", &model).unwrap(), parse("a - c + b", &model).unwrap());
        assert!(parse("a + b", &model) != parse("a - b", &model));
    }

    #[test]
    fn test_division() {
        let model = get_model();
        let a = model.get("a").unwrap().clone();
        println!("{:?} == {:?}", a, a.clone() + &a);
        assert!(a == (a.clone() + &a) / 2);
    }
}<commit_msg>Allow division of vectors by an integer<commit_after>use super::models::{LanguageModel, WordVec};
use self::Token::*;
use super::mayberef::MaybeRef::{self, Val, Ref};
use std::fmt::{Display, Formatter, Error};

pub fn parse(expr: &str, model: &LanguageModel) -> Result<WordVec, String> {
    expression(&mut Tokens::from(expr.chars()), model).map(|w| w.take())
}

#[derive(PartialEq, Eq, Debug, Clone)]
enum Token {
    RParen,
    LParen,
    Plus,
    Minus,
    Divide,
    Word(String),
    Invalid(char),
    Number(i32),
}

impl Token {
    fn apply_addop(&self, lhs: WordVec, rhs: &WordVec) -> Result<WordVec, String> {
        match *self {
            Plus => Ok(lhs + rhs),
            Minus => Ok(lhs - rhs),
            _ => Err(format!("Internal error: tried calling add using {}", self)),
        }
    }
}

impl Display for Token {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        let string;
        fmt.write_str(match *self {
            RParen => ")",
            LParen => "(",
            Plus => "+",
            Minus => "-",
            Divide => "\\",
            Word(ref w) => &*w,
            Number(i) => {
                string = format!("{}", i);
                &*string
            }
            Invalid(w) => {
                string = w.to_string();
                &*string
            },
        })
    }
}

fn get_word<'a>(model: &'a LanguageModel, word: &str) -> Result<&'a WordVec, String> {
    model.get(&*word).ok_or_else(||
        format!("'{}' is not present in the language model", &word)
    )
}

fn expression<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    let atom = try!(atom(tokens, model));
    match tokens.peek() {
        Some(&Plus) | Some(&Minus) => rhs(atom, tokens, model),
        None => Ok(atom),
        _ => Err(format!("Invalid token '{:?}'", tokens.peek().unwrap()))
    }
    // let token = try!(tokens.next().ok_or("An expression may not be empty"));
    // match token {
    //     LParen => {
    //         let vec = try!(expression(tokens, model));
    //         let rparen = try!(tokens.next().ok_or("Unbalanced parentheses"));
    //         if rparen != RParen {
    //             return Err(format!("Expected ')', found '{}'", rparen))
    //         }
    //         rhs(vec, tokens, model)
    //     },
    //     Word(word) => {
    //         let vec = try!(get_word(model, &*word));
    //         rhs(Ref(vec), tokens, model)
    //     },
    //     _ => Err(format!("An expression may not start with '{}'", token)),
    // }
}

fn rhs<'a, I>(lhs: MaybeRef<'a, WordVec>, tokens: &mut Tokens<I>, model: &LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    match tokens.peek() {
        Some(&RParen) | None => return Ok(lhs),
        _ => {}
    }

    let token = tokens.next().unwrap();
    match token {
        Plus | Minus => {
            let lhs = lhs.take();
            let atom = try!(atom(tokens, model));
            let result = try!(token.apply_addop(lhs, &*atom));
            rhs(Val(result), tokens, model)
        }
        Word(_) | LParen => Err(format!("'{}' found in invalid position", token)),
        Invalid(s) => Err(format!("'{}' could not be tokenized", s)),
        Divide | Number(_) => Err("Not sure how we got here!".to_string()),
        RParen => unreachable!(),
    }
}

fn atom<'a, I>(tokens: &mut Tokens<I>, model: &'a LanguageModel)
    -> Result<MaybeRef<'a, WordVec>, String>
    where I: Iterator<Item = char>
{
    {
        let token = try!(tokens.peek().ok_or("Invalid end of expression"));
        match *token {
            Plus | Minus | Invalid(_) | RParen => {
                return Err(format!("Invalid token {}", token));
            }
            _ => {},
        }
    }

    let token = tokens.next().unwrap();
    let atom = match token {
        LParen => {
            let expr = try!(expression(tokens, model));
            match tokens.next() {
                Some(RParen) => expr,
                Some(t) => return Err(format!("Expected ')', found '{}'", t)),
                None => return Err("Unclosed parentheses".to_string()),
            }
        }
        Word(word) =>  {
            let vec = try!(get_word(model, &*word));
            Ref(vec)
        }
        _ => unreachable!(),
    };

    if tokens.peek() != Some(&Divide) {
        return Ok(atom);
    }

    tokens.next();

    let num = try!(tokens.next().ok_or(
        "An expression may not end with a division operator"));

    match num {
        Number(num) => Ok(Val(atom.take() / num)),
        _ => Err(format!("'{}' is not a valid numerator: must be an integer", num)),
    }

}

struct Tokens<I> {
    iter: I,
    next_char: Option<Option<char>>,
    next_token: Option<Option<Token>>
}

impl<I> Tokens<I> where I: Iterator<Item = char> {
    fn from(iter: I) -> Tokens<I> {
        Tokens {
            iter: iter,
            next_char: None,
            next_token: None,
        }
    }

    fn get_next_token(&mut self) -> Option<Token> {
        while let Some(' ') = self.peek_char() {
            self.take();
        }

        match self.peek_char() {
            Some('a'...'z') => return Some(self.word()),
            Some('0'...'9') => return Some(self.number()),
            // Some('0'..'9') | Some('.') => return Some(self.number()),
            None => return None,
            _ => {}
        }

        Some(match self.take().unwrap() {
            '+' => Plus,
            '-' => Minus,
            '(' => LParen,
            ')' => RParen,
            c => Invalid(c),
        })
    }

    fn peek(&mut self) -> Option<&Token> {
        if let Some(ref t) = self.next_token {
            return t.as_ref();
        }

        self.next_token = Some(self.get_next_token());
        return self.next_token.as_ref().unwrap().as_ref();
    }

    fn peek_char(&mut self) -> Option<char> {
        match self.next_char {
            Some(x) => x,
            None => {
                let next = self.iter.next();
                self.next_char = Some(next);
                next
            }
        }
    }

    fn take(&mut self) -> Option<char> {
        self.peek_char();
        self.next_char.take().unwrap()
    }

    fn word(&mut self) -> Token {
        let mut token = String::new();
        while let Some('a'...'z') = self.peek_char() {
            token.push(self.take().unwrap());
        }

        Word(token)
    }

    fn number(&mut self) -> Token {
        let mut token = String::new();
        while let Some('0'...'9') = self.peek_char() {
            token.push(self.take().unwrap());
        }

        Number(token.parse().unwrap())
    }

}

impl<I> Iterator for Tokens<I> where I: Iterator<Item = char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        self.peek();
        self.next_token.take().unwrap()
    }
}

mod test {
    use super::Tokens;
    use super::Token::*;
    use super::parse;
    use super::super::models::{WordVec, LanguageModel, LanguageModelBuilder};

    #[test]
    fn single_character() {
        let c = Tokens::from("a".chars()).next().unwrap();
        assert_eq!(c, Word("a".to_string()));
    }

    #[test]
    fn single_token() {
        let s = Tokens::from("abc".chars()).next().unwrap();
        assert_eq!(s, Word("abc".to_string()));
    }

    #[test]
    fn two_tokens() {
        let s = Tokens::from("abc def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Word("def".to_string())]);
    }

    #[test]
    fn plus_tokens() {
        let s = Tokens::from("   abc  +def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![Word("abc".to_string()), Plus, Word("def".to_string())]);
    }

    #[test]
    fn bracket_tokens() {
        let s = Tokens::from(" ()  (abc  +)def".chars()).collect::<Vec<_>>();
        assert_eq!(s, vec![LParen, RParen, LParen, Word("abc".to_string()),
            Plus, RParen, Word("def".to_string())]);
    }

    fn get_model() -> LanguageModel {
        let words = vec!("a".to_string(), "b".to_string(), "c".to_string());
        let mut builder = LanguageModelBuilder::new(1, words);

        {
            let mut acc = builder.new_file();
            let text = "x x a a b a c b x x";
            for word in text.split(' ') {
                acc.add_word(word.to_string());
            }
        }

        builder.build()
    }

    #[test]
    fn test_addition() {
        let model = get_model();
        assert_eq!(parse("a - b + c", &model).unwrap(), parse("(a - b) + c", &model).unwrap());
        assert_eq!(parse("a + (b - c)", &model).unwrap(), parse("(a + b) - c", &model).unwrap());
        assert_eq!(parse("a + b - c)", &model).unwrap(), parse("a - c + b", &model).unwrap());
        assert!(parse("a + b", &model) != parse("a - b", &model));
    }

    #[test]
    fn test_division() {
        let model = get_model();
        let a = model.get("a").unwrap().clone();
        println!("{:?} == {:?}", a, a.clone() + &a);
        assert!(a == (a.clone() + &a) / 2);
    }
}<|endoftext|>


<commit_before>use nom::*;

// http://rust.unhandledexpression.com/nom/macro.escaped!.html
// http://rust.unhandledexpression.com/nom/
// https://fr.wikipedia.org/wiki/Comma-separated_values
// https://github.com/Geal/nom/wiki/Making-a-new-parser-from-scratch
// https://github.com/Geal/nom
// http://rust.unhandledexpression.com/nom/macro.escaped!.html
named!(string_between_quotes, delimited!(char!('\"'), is_not!("\""), char!('\"')));

#[test]
fn check_file() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28";

    match string_between_quotes(f) {
        IResult::Done(_, out) => assert_eq!(out, b"nom"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}
<commit_msg>Big code part<commit_after>use nom::*;
use std::str;

// http://rust.unhandledexpression.com/nom/macro.escaped!.html
// http://rust.unhandledexpression.com/nom/
// https://fr.wikipedia.org/wiki/Comma-separated_values
// https://github.com/Geal/nom/wiki/Making-a-new-parser-from-scratch
// https://github.com/Geal/nom
// http://rust.unhandledexpression.com/nom/macro.escaped!.html
named!(string_between_quotes, delimited!(char!('\"'), is_not!("\""), char!('\"')));
named!(get_cell, take_until!(","));
named!(get_line, take_until!("\n"));
named!(consume_useless_chars, take_while!(is_whitespace));

fn is_whitespace(c: u8) -> bool {
    c as char == ' ' || c as char == '\t'
}

fn get_column_value<'a>(entry: &'a [u8]) -> IResult<&'a [u8], &'a [u8]> {
    match consume_useless_chars(entry) {
        IResult::Done(out, _) => {
            if out.len() < 1 {
                IResult::Done(b"", b"")
            } else if out[0] as char == '\"' {
                match string_between_quotes(out) {
                    IResult::Done(i, _) => {
                        match consume_useless_chars(i) {
                            IResult::Done(i, _) => {
                                match get_cell(i) {
                                    IResult::Done(i, _) => IResult::Done(i, out),
                                    // should return the input and keep the out
                                }
                            }
                            x => x,
                        }
                    }
                    x => x,
                }
            } else if out[0] as char == '\n' {
                IResult::Done(b"", b"")
            } else {
                get_cell(out)
            }
        },
        x => x,
    }
}

fn get_line_values<'a>(ret: &mut Vec<String>, entry: &'a [u8]) -> IResult<&'a [u8], &'a [u8]> {
    match get_column_value(entry) {
        IResult::Done(in_, out) => {
            if out.len() < 1 {
                IResult::Done(b"", b"")
            } else {
                if let Ok(s) = str::from_utf8(out) {
                    ret.push(s.to_owned());
                }
                get_line_values(ret, in_)
            }
        }
        ref n if n.is_incomplete() => {
            if let Some(out) = n.remaining_input() {
                if out.len() < 1 {
                    IResult::Done(b"", b"")
                } else {
                    if let Ok(s) = str::from_utf8(out) {
                        ret.push(s.to_owned());
                    }
                    IResult::Done(b"", b"")
                }
            } else {
                IResult::Done(b"", b"")
            }
        }
        x => x,
    }
}

fn get_lines_values(ret: &mut Vec<Vec<String>>, entry: &[u8]) {
    match get_line(entry) {
        IResult::Done(in_, out) => {
            let mut line = vec!();

            get_line_values(&mut line, out);
            if line.len() < 1 {
                return;
            }
            ret.push(line);
            get_lines_values(ret, in_)
        }
        _ => {}
    }
}

#[test]
fn check_file() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    match string_between_quotes(f) {
        IResult::Done(in_, out) => {
            assert_eq!(out, b"nom");
            assert_eq!(in_, b",age\ncarles,30\nlaure,28\n");
        },
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_line() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";

    match get_line(f) {
        IResult::Done(_, out) => assert_eq!(out, b"\"nom\",age"),
        IResult::Incomplete(x) => panic!("incomplete: {:?}", x),
        IResult::Error(e) => panic!("error: {:?}", e),
    }
}

#[test]
fn check_get_line() {
    let f = b"\"nom\",age\ncarles,30\nlaure,28\n";
    let mut cells = vec!();

    get_line_values(&mut cells, f);
    assert_eq!(cells, vec!("nom".to_owned(), "age".to_owned()));
}
<|endoftext|>