<commit_before>use std::fmt;

#[cfg(windows)]
mod mmap {
    use libc::{CreateFileW, CreateFileMappingW, MapViewOfFile, CloseHandle, UnmapViewOfFile,
               GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING, INVALID_HANDLE_VALUE, PAGE_READONLY,
               FILE_MAP_READ, HANDLE, TRUE};
    use libc::consts::os::extra::{FILE_FLAG_RANDOM_ACCESS};
    use libc::types::os::arch::extra::{LPCVOID};
    use std::ptr;
    struct Handle {
        hand: HANDLE,
    }
    impl Drop for Handle {
        fn drop(&mut self) {
            assert_eq!(unsafe { CloseHandle(self.hand) }, TRUE);
        }
    }
    struct MapView {
        data: LPCVOID,
    }
    impl Drop for MapView {
        fn drop(&mut self) {
            assert_eq!(unsafe { UnmapViewOfFile(self.data) }, TRUE);
        }
    }
    pub struct MapFile {
        view: MapView,
        map: Handle,
        file: Handle,
    }
    impl MapFile {
        pub fn data(&self) -> LPCVOID { self.view.data }
    }
    fn create_file(path: &Path) -> Result<Handle, &'static str> {
        let name = path.as_str().unwrap().to_utf16().append([0]);
        let file = unsafe {
            CreateFileW(name.as_ptr(), GENERIC_READ, FILE_SHARE_READ, ptr::mut_null(),
                        OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, ptr::mut_null())
        };
        if file.to_uint() == INVALID_HANDLE_VALUE as uint { return Err("Failed to open file"); }
        Ok(Handle{hand: file})
    }
    fn create_mapping(file: &Handle) -> Result<Handle, &'static str> {
        let map = unsafe {
            CreateFileMappingW(file.hand, ptr::mut_null(), PAGE_READONLY, 0, 0, ptr::null())
        };
        if map.is_null() { return Err("Failed to create file mapping"); }
        Ok(Handle{hand: map})
    }
    fn map_view(map: &Handle) -> Result<MapView, &'static str> {
        let view = unsafe { MapViewOfFile(map.hand, FILE_MAP_READ, 0, 0, 0) };
        if view.is_null() { return Err("Failed to map view of file"); }
        Ok(MapView{data: view as *_})
    }
    pub fn open(path: &Path) -> Result<MapFile, &'static str> {
        let file = try!(create_file(path));
        let map = try!(create_mapping(&file));
        let view = try!(map_view(&map));
        Ok(MapFile{file: file, map: map, view: view})
    }
}
#[cfg(unix)]
mod mmap {
    use libc::{close, mmap, munmap, O_RDONLY, PROT_READ, c_int, c_void};
    use libc::consts::os::posix88::{MAP_SHARED};
    use posix_open = libc::open;
    use std::ptr;
    use std::io::fs::stat;
    struct Handle {
        hand: c_int,
    }
    impl Drop for Handle {
        fn drop(&mut self) {
            assert_eq!(unsafe { close(self.hand) }, 0);
        }
    }
    struct Mapping {
        data: *c_void,
        size: u64,
    }
    impl Drop for Mapping {
        fn drop(&mut self) {
            assert_eq!(unsafe { munmap(self.data, self.size) }, 0);
        }
    }
    pub struct MapFile {
        map: Mapping,
        file: Handle,
    }
    impl MapFile {
        pub fn data(&self) -> * c_void { self.map.data }
    }
    fn open_file(path: &Path) -> Result<Handle, &'static str> {
        let name = path.to_c_str();
        let handle = unsafe {
             posix_open(name.unwrap(), O_RDONLY, 0)
        };
        if handle == -1 { return Err("Failed to open file") }
        Ok(Handle{hand: handle})
    }
    fn file_size(path: &Path) -> Result<u64, &'static str> {
        match path.stat() {
            Ok(stat) => Ok(stat.size),
            Err(_) => Err("Failed to get file size")
        }
    }
    fn map_file(file: &Handle, size: u64) -> Result<Mapping, &'static str> {
        let map = unsafe {
            mmap(ptr::null(), size as u64, PROT_READ, MAP_SHARED, file.hand, 0)
        };
        if map.to_uint() == -1 { return Err("Failed to map file"); }
        Ok(Mapping{data: map as *_, size: size})
    }
    pub fn open(path: &Path) -> Result<MapFile, &'static str> {
        let file = try!(open_file(path));
        let size = try!(file_size(path));
        let map = try!(map_file(&file, size));
        Ok(MapFile{file: file, map: map})
    }
}
pub struct File {
    map: mmap::MapFile,
    data: *u8,
    header: *Header,
    nodetable: *NodeData,
    stringtable: *u64,
}
impl File {
    pub fn open(path: &Path) -> Result<File, &'static str> {
        use std::mem::transmute;
        let map = try!(mmap::open(path));
        let data = map.data();
        let header: *Header = unsafe{ transmute(data) };
        if unsafe { (*header).magic } != 0x34474B50 {
            return Err("Not a valid NX PKG4 file");
        }
        unsafe {
            Ok(File{map: map, data: transmute(data), header: header,
                    nodetable: transmute(data.offset((*header).nodeoffset as int)),
                    stringtable: transmute(data.offset((*header).stringoffset as int))})
        }
    }
    pub fn header(&self) -> &Header {
        use std::mem::transmute;
        unsafe { transmute(self.header) }
    }
    pub fn root<'a>(&'a self) -> Node<'a> {
        unsafe { Node{data: &*self.nodetable, file: self,} }
    }
    fn get_str<'a>(&'a self, index: u32) -> &'a str {
        use std::str;
        use std::slice::raw;
        use std::mem::transmute;
        unsafe {
            let off = *self.stringtable.offset(index as int);
            let ptr = self.data.offset(off as int);
            let size: *u16 = transmute(ptr);
            raw::buf_as_slice(ptr.offset(2), (*size) as uint, |buf| {
                let bytes: &'a [u8] = transmute(buf);
                str::raw::from_utf8(bytes)
            })
        }
    }
}
#[packed]
struct Header {
    magic: u32,
    pub nodecount: u32,
    nodeoffset: u64,
    stringcount: u32,
    stringoffset: u64,
    bitmapcount: u32,
    bitmapoffset: u64,
    audiocount: u32,
    audiooffset: u64,
}
#[packed]
struct String {
    length: u16,
}
pub struct Node<'a> {
    data: &'a NodeData,
    file: &'a File,
}
impl <'a> Node<'a> {
    pub fn iter(&self) -> NodeIterator<'a> {
        unsafe {
            let data = self.file.nodetable.offset(self.data.children as int);
            NodeIterator{data: data, count: self.data.count, file: self.file,}
        }
    }
    pub fn name(&self) -> &'a str { self.file.get_str(self.data.name) }
    pub fn empty(&self) -> bool { self.data.count == 0 }
    pub fn get(&self, name: &'a str) -> Option<Node<'a>> {
        if self.empty() { return None; }
        unsafe {
            let mut data = self.file.nodetable.offset(self.data.children as int);
            let mut count = self.data.count;
            while count > 0 {
                let half = count / 2;
                let temp = data.offset(half as int);
                let other = self.file.get_str((*temp).name);
                if other < name {
                    data = temp.offset(1);
                    count -= half + 1;
                } else {
                    count = half;
                }
            }
            if name != self.file.get_str((*data).name) { return None; }
            Some(Node{data: &*data, file: self.file})
        }
    }
}
impl <'a> PartialEq for Node<'a> {
    fn eq(&self, other: &Node) -> bool { self.data as *_ == other.data as *_ }
}
impl <'a> fmt::Show for Node<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.name())
    }
}
struct NodeIterator<'a> {
    data: *NodeData,
    count: u16,
    file: &'a File,
}
impl <'a> Iterator<Node<'a>> for NodeIterator<'a> {
    fn next(&mut self) -> Option<Node<'a>> {
        match self.count {
            0 => None,
            _ => {
                self.count -= 1;
                let node = Node{data: unsafe { &*self.data }, file: self.file};
                self.data = unsafe { self.data.offset(1) };
                Some(node)
            }
        }
    }
}
#[packed]
struct NodeData {
    name: u32,
    children: u32,
    count: u16,
    dtype: u16,
    data: u64,
}
#[packed]
struct NodeInteger {
    value: i64,
}
#[packed]
struct NodeFloat {
    value: f64,
}
#[packed]
struct NodeString {
    index: u32,
    unused: u32,
}
#[packed]
struct NodeVector {
    x: i32,
    y: i32,
}
#[packed]
struct NodeBitmap {
    index: u32,
    width: u16,
    height: u16,
}
#[packed]
struct NodeAudio {
    index: u32,
    length: u32,
}
<commit_msg>This significantly improves performance. Not perfect yet though.<commit_after>use std::fmt;

#[cfg(windows)]
mod mmap {
    use libc::{CreateFileW, CreateFileMappingW, MapViewOfFile, CloseHandle, UnmapViewOfFile,
               GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING, INVALID_HANDLE_VALUE, PAGE_READONLY,
               FILE_MAP_READ, HANDLE, TRUE};
    use libc::consts::os::extra::{FILE_FLAG_RANDOM_ACCESS};
    use libc::types::os::arch::extra::{LPCVOID};
    use std::ptr;
    struct Handle {
        hand: HANDLE,
    }
    impl Drop for Handle {
        fn drop(&mut self) {
            assert_eq!(unsafe { CloseHandle(self.hand) }, TRUE);
        }
    }
    struct MapView {
        data: LPCVOID,
    }
    impl Drop for MapView {
        fn drop(&mut self) {
            assert_eq!(unsafe { UnmapViewOfFile(self.data) }, TRUE);
        }
    }
    pub struct MapFile {
        view: MapView,
        map: Handle,
        file: Handle,
    }
    impl MapFile {
        pub fn data(&self) -> LPCVOID { self.view.data }
    }
    fn create_file(path: &Path) -> Result<Handle, &'static str> {
        let name = path.as_str().unwrap().to_utf16().append([0]);
        let file = unsafe {
            CreateFileW(name.as_ptr(), GENERIC_READ, FILE_SHARE_READ, ptr::mut_null(),
                        OPEN_EXISTING, FILE_FLAG_RANDOM_ACCESS, ptr::mut_null())
        };
        if file.to_uint() == INVALID_HANDLE_VALUE as uint { return Err("Failed to open file"); }
        Ok(Handle{hand: file})
    }
    fn create_mapping(file: &Handle) -> Result<Handle, &'static str> {
        let map = unsafe {
            CreateFileMappingW(file.hand, ptr::mut_null(), PAGE_READONLY, 0, 0, ptr::null())
        };
        if map.is_null() { return Err("Failed to create file mapping"); }
        Ok(Handle{hand: map})
    }
    fn map_view(map: &Handle) -> Result<MapView, &'static str> {
        let view = unsafe { MapViewOfFile(map.hand, FILE_MAP_READ, 0, 0, 0) };
        if view.is_null() { return Err("Failed to map view of file"); }
        Ok(MapView{data: view as *_})
    }
    pub fn open(path: &Path) -> Result<MapFile, &'static str> {
        let file = try!(create_file(path));
        let map = try!(create_mapping(&file));
        let view = try!(map_view(&map));
        Ok(MapFile{file: file, map: map, view: view})
    }
}
#[cfg(unix)]
mod mmap {
    use libc::{close, mmap, munmap, O_RDONLY, PROT_READ, c_int, c_void};
    use libc::consts::os::posix88::{MAP_SHARED};
    use posix_open = libc::open;
    use std::ptr;
    use std::io::fs::stat;
    struct Handle {
        hand: c_int,
    }
    impl Drop for Handle {
        fn drop(&mut self) {
            assert_eq!(unsafe { close(self.hand) }, 0);
        }
    }
    struct Mapping {
        data: *c_void,
        size: u64,
    }
    impl Drop for Mapping {
        fn drop(&mut self) {
            assert_eq!(unsafe { munmap(self.data, self.size) }, 0);
        }
    }
    pub struct MapFile {
        map: Mapping,
        file: Handle,
    }
    impl MapFile {
        pub fn data(&self) -> * c_void { self.map.data }
    }
    fn open_file(path: &Path) -> Result<Handle, &'static str> {
        let name = path.to_c_str();
        let handle = unsafe {
             posix_open(name.unwrap(), O_RDONLY, 0)
        };
        if handle == -1 { return Err("Failed to open file") }
        Ok(Handle{hand: handle})
    }
    fn file_size(path: &Path) -> Result<u64, &'static str> {
        match path.stat() {
            Ok(stat) => Ok(stat.size),
            Err(_) => Err("Failed to get file size")
        }
    }
    fn map_file(file: &Handle, size: u64) -> Result<Mapping, &'static str> {
        let map = unsafe {
            mmap(ptr::null(), size as u64, PROT_READ, MAP_SHARED, file.hand, 0)
        };
        if map.to_uint() == -1 { return Err("Failed to map file"); }
        Ok(Mapping{data: map as *_, size: size})
    }
    pub fn open(path: &Path) -> Result<MapFile, &'static str> {
        let file = try!(open_file(path));
        let size = try!(file_size(path));
        let map = try!(map_file(&file, size));
        Ok(MapFile{file: file, map: map})
    }
}
pub struct File {
    map: mmap::MapFile,
    data: *u8,
    header: *Header,
    nodetable: *NodeData,
    stringtable: *u64,
}
impl File {
    pub fn open(path: &Path) -> Result<File, &'static str> {
        use std::mem::transmute;
        let map = try!(mmap::open(path));
        let data = map.data();
        let header: *Header = unsafe{ transmute(data) };
        if unsafe { (*header).magic } != 0x34474B50 {
            return Err("Not a valid NX PKG4 file");
        }
        unsafe {
            Ok(File{map: map, data: transmute(data), header: header,
                    nodetable: transmute(data.offset((*header).nodeoffset as int)),
                    stringtable: transmute(data.offset((*header).stringoffset as int))})
        }
    }
    pub fn header(&self) -> &Header {
        use std::mem::transmute;
        unsafe { transmute(self.header) }
    }
    pub fn root<'a>(&'a self) -> Node<'a> {
        unsafe { Node{data: &*self.nodetable, file: self,} }
    }
    fn get_str<'a>(&'a self, index: u32) -> &'a str {
        use std::str;
        use std::slice::raw;
        use std::mem::transmute;
        unsafe {
            let off = *self.stringtable.offset(index as int);
            let ptr = self.data.offset(off as int);
            let size: *u16 = transmute(ptr);
            raw::buf_as_slice(ptr.offset(2), (*size) as uint, |buf| {
                let bytes: &'a [u8] = transmute(buf);
                transmute(bytes)
            })
        }
    }
}
#[packed]
struct Header {
    magic: u32,
    pub nodecount: u32,
    nodeoffset: u64,
    stringcount: u32,
    stringoffset: u64,
    bitmapcount: u32,
    bitmapoffset: u64,
    audiocount: u32,
    audiooffset: u64,
}
#[packed]
struct String {
    length: u16,
}
pub struct Node<'a> {
    data: &'a NodeData,
    file: &'a File,
}
impl <'a> Node<'a> {
    pub fn iter(&self) -> NodeIterator<'a> {
        unsafe {
            let data = self.file.nodetable.offset(self.data.children as int);
            NodeIterator{data: data, count: self.data.count, file: self.file,}
        }
    }
    pub fn name(&self) -> &'a str { self.file.get_str(self.data.name) }
    pub fn empty(&self) -> bool { self.data.count == 0 }
    pub fn get(&self, name: &'a str) -> Option<Node<'a>> {
        if self.empty() { return None; }
        unsafe {
            let mut data = self.file.nodetable.offset(self.data.children as int);
            let mut count = self.data.count;
            while count > 0 {
                let half = count / 2;
                let temp = data.offset(half as int);
                let other = self.file.get_str((*temp).name);
                if other < name {
                    data = temp.offset(1);
                    count -= half + 1;
                } else {
                    count = half;
                }
            }
            if name != self.file.get_str((*data).name) { return None; }
            Some(Node{data: &*data, file: self.file})
        }
    }
}
impl <'a> PartialEq for Node<'a> {
    fn eq(&self, other: &Node) -> bool { self.data as *_ == other.data as *_ }
}
impl <'a> fmt::Show for Node<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", self.name())
    }
}
struct NodeIterator<'a> {
    data: *NodeData,
    count: u16,
    file: &'a File,
}
impl <'a> Iterator<Node<'a>> for NodeIterator<'a> {
    fn next(&mut self) -> Option<Node<'a>> {
        match self.count {
            0 => None,
            _ => {
                self.count -= 1;
                let node = Node{data: unsafe { &*self.data }, file: self.file};
                self.data = unsafe { self.data.offset(1) };
                Some(node)
            }
        }
    }
}
#[packed]
struct NodeData {
    name: u32,
    children: u32,
    count: u16,
    dtype: u16,
    data: u64,
}
#[packed]
struct NodeInteger {
    value: i64,
}
#[packed]
struct NodeFloat {
    value: f64,
}
#[packed]
struct NodeString {
    index: u32,
    unused: u32,
}
#[packed]
struct NodeVector {
    x: i32,
    y: i32,
}
#[packed]
struct NodeBitmap {
    index: u32,
    width: u16,
    height: u16,
}
#[packed]
struct NodeAudio {
    index: u32,
    length: u32,
}
<|endoftext|>


<commit_before>
use std;
import std::io;
import result;

type reader = obj {
    fn read_row() -> result::t<[str], str>;
};

fn mk_reader(f: std::io::reader, delim: char, quote: char, has_header: bool) -> reader {
    tag state {
        start;
        field([char]);
        escapedfield([char]);
        inquote([char]);
        escapeend([char]);
    };
    type readerstate = {
        f: std::io::reader,
        delim: char,
        quote: char,
        has_header: bool,
        mutable buf : [char],
        mutable offset : uint,
        mutable state : state,
    };
    obj reader(st: readerstate) {
        fn read_row() -> result::t<[str], str> {
            fn row_from_buf(st: readerstate, &row: [str]) -> bool {
                while st.offset < vec::len(st.buf) {
                    let c : char = st.buf[st.offset];
                    st.offset += 1u;
                    alt st.state {
                        start() {
                            if c == st.quote {
                                st.state = escapedfield([]);
                            } else if c == '\n' {
                                ret true;
                            } else if c == st.delim {
                                st.state = start;
                                row += [""];
                            } else {
                                st.state = field([c]);
                            }
                        }
                        field(x) {
                            if c == '\n' {
                                row += [str::from_chars(x)];
                                ret true;
                            } else if c == st.delim {
                                st.state = start;
                                row += [str::from_chars(x)];
                            } else {
                                st.state = field(x + [c]);
                            }
                        }
                        escapedfield(x) {
                            if c == st.quote {
                                st.state = inquote(x);
                            } else if c == st.delim {
                                st.state = start;
                                row += [str::from_chars(x)];
                            } else {
                                st.state = escapedfield(x + [c]);
                            }
                        }
                        inquote(x) {
                            if c == st.quote {
                                st.state = escapedfield(x + [st.quote]);
                            } else {
                                st.state = escapeend(x);
                            }
                        }
                        escapeend(x) {
                            // swallow odd chars, eg. space between field and "
                            io::println(#fmt("escapeend - %c", c));
                            if (c == st.delim) {
                                st.state = start;
                                row += [str::from_chars(x)];
                            } else if (c == '\n') {
                                row += [str::from_chars(x)];
                                ret true;
                            } else {
                                st.state = start;
                            }
                        }
                    }
                }
                ret false;
            }

            let row: [str] = [];
            st.state = start;
            while true {
                if st.offset >= vec::len(st.buf) {
                    st.offset = 0u;
                    st.buf = st.f.read_chars(1024u);
                    /* should probably use a result */
                    if vec::len(st.buf) == 0u {
                        ret result::err("EOF");
                    }
                }
                if (row_from_buf(st, row)) {
                    ret result::ok(row);
                }
            }
            ret result::err("EOF");
        }
    }

    let st = { f: f, delim: delim, quote: quote, has_header: has_header, mutable buf: [], mutable offset: 0u, mutable state: start };
    let r = reader(st);
    ret r;
}

fn main(args : [str])
{
    if (vec::len(args) != 2u) {
        ret;
    }
    let f : io::reader = result::get(io::file_reader(args[1]));
    let r = mk_reader(f, ',', '"', true);
    while true {
        let res = r.read_row();
        if result::failure(res) {
            break;
        }
        io::println("---------------");
        for field in result::get(res) {
            io::println("FIELD: " + field);
        }
    }
}

<commit_msg>fix quoting support<commit_after>
use std;
import std::io;
import result;

type reader = obj {
    fn read_row() -> result::t<[str], str>;
};

fn mk_reader(f: std::io::reader, delim: char, quote: char, has_header: bool) -> reader {
    tag state {
        start;
        field([char]);
        escapedfield([char]);
        inquote([char]);
    };
    type readerstate = {
        f: std::io::reader,
        delim: char,
        quote: char,
        has_header: bool,
        mutable buf : [char],
        mutable offset : uint,
        mutable state : state,
    };
    obj reader(st: readerstate) {
        fn read_row() -> result::t<[str], str> {
            fn row_from_buf(st: readerstate, &row: [str]) -> bool {
                while st.offset < vec::len(st.buf) {
                    let c : char = st.buf[st.offset];
                    st.offset += 1u;
                    /* bug: trailing commas don't generate a field */
                    alt st.state {
                        start() {
                            //io::println(#fmt("start - %c", c));
                            if c == st.quote {
                                st.state = escapedfield([]);
                            } else if c == '\n' {
                                ret true;
                            } else if c == st.delim {
                                st.state = start;
                                row += [""];
                            } else {
                                st.state = field([c]);
                            }
                        }
                        field(x) {
                            //io::println(#fmt("field - %c", c));
                            if c == '\n' {
                                row += [str::from_chars(x)];
                                ret true;
                            } else if c == st.delim {
                                st.state = start;
                                row += [str::from_chars(x)];
                            } else {
                                st.state = field(x + [c]);
                            }
                        }
                        escapedfield(x) {
                            //io::println(#fmt("escapefield - %c", c));
                            if c == st.quote {
                                st.state = inquote(x);
                            } else if c == st.delim {
                                st.state = start;
                                row += [str::from_chars(x)];
                            } else {
                                st.state = escapedfield(x + [c]);
                            }
                        }
                        inquote(x) {
                            //io::println(#fmt("inquote - %c", c));
                            if c == '\n' {
                                row += [str::from_chars(x)];
                                ret true;
                            } else if c == st.quote {
                                st.state = escapedfield(x + [st.quote]);
                            } else if c == st.delim {
                                st.state = start;
                                row += [str::from_chars(x)];
                            }
                            // swallow odd chars, eg. space between field and "
                        }
                    }
                }
                ret false;
            }

            let row: [str] = [];
            st.state = start;
            while true {
                if st.offset >= vec::len(st.buf) {
                    st.offset = 0u;
                    st.buf = st.f.read_chars(1024u);
                    /* should probably use a result */
                    if vec::len(st.buf) == 0u {
                        ret result::err("EOF");
                    }
                }
                if (row_from_buf(st, row)) {
                    ret result::ok(row);
                }
            }
            ret result::err("EOF");
        }
    }

    let st = { f: f, delim: delim, quote: quote, has_header: has_header, mutable buf: [], mutable offset: 0u, mutable state: start };
    let r = reader(st);
    ret r;
}

fn main(args : [str])
{
    if (vec::len(args) != 2u) {
        ret;
    }
    let f : io::reader = result::get(io::file_reader(args[1]));
    let r = mk_reader(f, ',', '"', true);
    while true {
        let res = r.read_row();
        if result::failure(res) {
            break;
        }
        io::println("---------------");
        for field in result::get(res) {
            io::println("FIELD: " + field);
        }
    }
}

<|endoftext|>


<commit_before>//! This modules implements "expand macro" functionality in the IDE

use crate::{db::RootDatabase, FilePosition};
use hir::db::AstDatabase;
use ra_db::SourceDatabase;
use rustc_hash::FxHashMap;

use ra_syntax::{
    algo::{find_node_at_offset, replace_descendants},
    ast::{self},
    AstNode, NodeOrToken, SyntaxKind, SyntaxNode, WalkEvent, T,
};

pub struct ExpandedMacro {
    pub name: String,
    pub expansion: String,
}

pub(crate) fn expand_macro(db: &RootDatabase, position: FilePosition) -> Option<ExpandedMacro> {
    let parse = db.parse(position.file_id);
    let file = parse.tree();
    let name_ref = find_node_at_offset::<ast::NameRef>(file.syntax(), position.offset)?;
    let mac = name_ref.syntax().ancestors().find_map(ast::MacroCall::cast)?;

    let source = hir::Source::new(position.file_id.into(), mac.syntax());
    let expanded = expand_macro_recur(db, source, source.with_value(&mac))?;

    // FIXME:
    // macro expansion may lose all white space information
    // But we hope someday we can use ra_fmt for that
    let expansion = insert_whitespaces(expanded);
    Some(ExpandedMacro { name: name_ref.text().to_string(), expansion })
}

fn expand_macro_recur(
    db: &RootDatabase,
    source: hir::Source<&SyntaxNode>,
    macro_call: hir::Source<&ast::MacroCall>,
) -> Option<SyntaxNode> {
    let analyzer = hir::SourceAnalyzer::new(db, source, None);
    let expansion = analyzer.expand(db, macro_call)?;
    let macro_file_id = expansion.file_id();
    let expanded: SyntaxNode = db.parse_or_expand(macro_file_id)?;

    let children = expanded.descendants().filter_map(ast::MacroCall::cast);
    let mut replaces = FxHashMap::default();

    for child in children.into_iter() {
        let node = hir::Source::new(macro_file_id, &child);
        let new_node = expand_macro_recur(db, source, node)?;

        replaces.insert(child.syntax().clone().into(), new_node.into());
    }

    Some(replace_descendants(&expanded, &replaces))
}

// FIXME: It would also be cool to share logic here and in the mbe tests,
// which are pretty unreadable at the moment.
fn insert_whitespaces(syn: SyntaxNode) -> String {
    use SyntaxKind::*;

    let mut res = String::new();
    let mut token_iter = syn
        .preorder_with_tokens()
        .filter_map(|event| {
            if let WalkEvent::Enter(NodeOrToken::Token(token)) = event {
                Some(token)
            } else {
                None
            }
        })
        .peekable();

    let mut indent = 0;
    let mut last: Option<SyntaxKind> = None;

    while let Some(token) = token_iter.next() {
        let mut is_next = |f: fn(SyntaxKind) -> bool, default| -> bool {
            token_iter.peek().map(|it| f(it.kind())).unwrap_or(default)
        };
        let is_last = |f: fn(SyntaxKind) -> bool, default| -> bool {
            last.map(|it| f(it)).unwrap_or(default)
        };

        res += &match token.kind() {
            k @ _ if is_text(k) && is_next(|it| !it.is_punct(), true) => {
                token.text().to_string() + " "
            }
            L_CURLY if is_next(|it| it != R_CURLY, true) => {
                indent += 1;
                let leading_space = if is_last(|it| is_text(it), false) { " " } else { "" };
                format!("{}{{\n{}", leading_space, "  ".repeat(indent))
            }
            R_CURLY if is_last(|it| it != L_CURLY, true) => {
                indent = indent.checked_sub(1).unwrap_or(0);
                format!("\n{}}}", "  ".repeat(indent))
            }
            R_CURLY => format!("}}\n{}", "  ".repeat(indent)),
            T![;] => format!(";\n{}", "  ".repeat(indent)),
            T![->] => " -> ".to_string(),
            T![=] => " = ".to_string(),
            T![=>] => " => ".to_string(),
            _ => token.text().to_string(),
        };

        last = Some(token.kind());
    }

    return res;

    fn is_text(k: SyntaxKind) -> bool {
        k.is_keyword() || k.is_literal() || k == IDENT
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::mock_analysis::analysis_and_position;
    use insta::assert_snapshot;

    fn check_expand_macro(fixture: &str) -> ExpandedMacro {
        let (analysis, pos) = analysis_and_position(fixture);
        analysis.expand_macro(pos).unwrap().unwrap()
    }

    #[test]
    fn macro_expand_recursive_expansion() {
        let res = check_expand_macro(
            r#"
        //- /lib.rs
        macro_rules! bar {
            () => { fn  b() {} }
        }
        macro_rules! foo {
            () => { bar!(); }
        }
        macro_rules! baz {
            () => { foo!(); }
        }
        f<|>oo!();
        "#,
        );

        assert_eq!(res.name, "foo");
        assert_snapshot!(res.expansion, @r###"
fn b(){}
"###);
    }

    #[test]
    fn macro_expand_multiple_lines() {
        let res = check_expand_macro(
            r#"
        //- /lib.rs
        macro_rules! foo {
            () => {
                fn some_thing() -> u32 {
                    let a = 0;
                    a + 10
                }
            }
        }
        f<|>oo!();
        "#,
        );

        assert_eq!(res.name, "foo");
        assert_snapshot!(res.expansion, @r###"
fn some_thing() -> u32 {
  let a = 0;
  a+10
}
"###);
    }

    #[test]
    fn macro_expand_match_ast() {
        let res = check_expand_macro(
            r#"
        //- /lib.rs
        macro_rules! match_ast {
            (match $node:ident { $($tt:tt)* }) => { match_ast!(match ($node) { $($tt)* }) };

            (match ($node:expr) {
                $( ast::$ast:ident($it:ident) => $res:block, )*
                _ => $catch_all:expr $(,)?
            }) => {{
                $( if let Some($it) = ast::$ast::cast($node.clone()) $res else )*
                { $catch_all }
            }};
        }

        fn main() {
            mat<|>ch_ast! {
                match container {
                    ast::TraitDef(it) => {},
                    ast::ImplBlock(it) => {},
                    _ => { continue },
                }
            }
        }
        "#,
        );

        assert_eq!(res.name, "match_ast");
        assert_snapshot!(res.expansion, @r###"
{
  if let Some(it) = ast::TraitDef::cast(container.clone()){}
  else if let Some(it) = ast::ImplBlock::cast(container.clone()){}
  else {
    {
      continue
    }
  }
}
"###);
    }
}
<commit_msg>More correct expand macro<commit_after>//! This modules implements "expand macro" functionality in the IDE

use crate::{db::RootDatabase, FilePosition};
use hir::db::AstDatabase;
use ra_db::SourceDatabase;
use rustc_hash::FxHashMap;

use ra_syntax::{
    algo::{find_node_at_offset, replace_descendants},
    ast::{self},
    AstNode, NodeOrToken, SyntaxKind, SyntaxNode, WalkEvent, T,
};

pub struct ExpandedMacro {
    pub name: String,
    pub expansion: String,
}

pub(crate) fn expand_macro(db: &RootDatabase, position: FilePosition) -> Option<ExpandedMacro> {
    let parse = db.parse(position.file_id);
    let file = parse.tree();
    let name_ref = find_node_at_offset::<ast::NameRef>(file.syntax(), position.offset)?;
    let mac = name_ref.syntax().ancestors().find_map(ast::MacroCall::cast)?;

    let source = hir::Source::new(position.file_id.into(), mac.syntax());
    let expanded = expand_macro_recur(db, source, source.with_value(&mac))?;

    // FIXME:
    // macro expansion may lose all white space information
    // But we hope someday we can use ra_fmt for that
    let expansion = insert_whitespaces(expanded);
    Some(ExpandedMacro { name: name_ref.text().to_string(), expansion })
}

fn expand_macro_recur(
    db: &RootDatabase,
    source: hir::Source<&SyntaxNode>,
    macro_call: hir::Source<&ast::MacroCall>,
) -> Option<SyntaxNode> {
    let analyzer = hir::SourceAnalyzer::new(db, source, None);
    let expansion = analyzer.expand(db, macro_call)?;
    let macro_file_id = expansion.file_id();
    let mut expanded: SyntaxNode = db.parse_or_expand(macro_file_id)?;

    let children = expanded.descendants().filter_map(ast::MacroCall::cast);
    let mut replaces = FxHashMap::default();

    for child in children.into_iter() {
        let node = hir::Source::new(macro_file_id, &child);
        let new_node = expand_macro_recur(db, source, node)?;

        // Replace the whole node if it is root
        // `replace_descendants` will not replace the parent node
        // but `SyntaxNode::descendants include itself
        if expanded == *child.syntax() {
            expanded = new_node;
        } else {
            replaces.insert(child.syntax().clone().into(), new_node.into());
        }
    }

    Some(replace_descendants(&expanded, &replaces))
}

// FIXME: It would also be cool to share logic here and in the mbe tests,
// which are pretty unreadable at the moment.
fn insert_whitespaces(syn: SyntaxNode) -> String {
    use SyntaxKind::*;

    let mut res = String::new();
    let mut token_iter = syn
        .preorder_with_tokens()
        .filter_map(|event| {
            if let WalkEvent::Enter(NodeOrToken::Token(token)) = event {
                Some(token)
            } else {
                None
            }
        })
        .peekable();

    let mut indent = 0;
    let mut last: Option<SyntaxKind> = None;

    while let Some(token) = token_iter.next() {
        let mut is_next = |f: fn(SyntaxKind) -> bool, default| -> bool {
            token_iter.peek().map(|it| f(it.kind())).unwrap_or(default)
        };
        let is_last = |f: fn(SyntaxKind) -> bool, default| -> bool {
            last.map(|it| f(it)).unwrap_or(default)
        };

        res += &match token.kind() {
            k @ _ if is_text(k) && is_next(|it| !it.is_punct(), true) => {
                token.text().to_string() + " "
            }
            L_CURLY if is_next(|it| it != R_CURLY, true) => {
                indent += 1;
                let leading_space = if is_last(|it| is_text(it), false) { " " } else { "" };
                format!("{}{{\n{}", leading_space, "  ".repeat(indent))
            }
            R_CURLY if is_last(|it| it != L_CURLY, true) => {
                indent = indent.checked_sub(1).unwrap_or(0);
                format!("\n{}}}", "  ".repeat(indent))
            }
            R_CURLY => format!("}}\n{}", "  ".repeat(indent)),
            T![;] => format!(";\n{}", "  ".repeat(indent)),
            T![->] => " -> ".to_string(),
            T![=] => " = ".to_string(),
            T![=>] => " => ".to_string(),
            _ => token.text().to_string(),
        };

        last = Some(token.kind());
    }

    return res;

    fn is_text(k: SyntaxKind) -> bool {
        k.is_keyword() || k.is_literal() || k == IDENT
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::mock_analysis::analysis_and_position;
    use insta::assert_snapshot;

    fn check_expand_macro(fixture: &str) -> ExpandedMacro {
        let (analysis, pos) = analysis_and_position(fixture);
        analysis.expand_macro(pos).unwrap().unwrap()
    }

    #[test]
    fn macro_expand_recursive_expansion() {
        let res = check_expand_macro(
            r#"
        //- /lib.rs
        macro_rules! bar {
            () => { fn  b() {} }
        }
        macro_rules! foo {
            () => { bar!(); }
        }
        macro_rules! baz {
            () => { foo!(); }
        }
        f<|>oo!();
        "#,
        );

        assert_eq!(res.name, "foo");
        assert_snapshot!(res.expansion, @r###"
fn b(){}
"###);
    }

    #[test]
    fn macro_expand_multiple_lines() {
        let res = check_expand_macro(
            r#"
        //- /lib.rs
        macro_rules! foo {
            () => {
                fn some_thing() -> u32 {
                    let a = 0;
                    a + 10
                }
            }
        }
        f<|>oo!();
        "#,
        );

        assert_eq!(res.name, "foo");
        assert_snapshot!(res.expansion, @r###"
fn some_thing() -> u32 {
  let a = 0;
  a+10
}
"###);
    }

    #[test]
    fn macro_expand_match_ast() {
        let res = check_expand_macro(
            r#"
        //- /lib.rs
        macro_rules! match_ast {
            (match $node:ident { $($tt:tt)* }) => { match_ast!(match ($node) { $($tt)* }) };

            (match ($node:expr) {
                $( ast::$ast:ident($it:ident) => $res:block, )*
                _ => $catch_all:expr $(,)?
            }) => {{
                $( if let Some($it) = ast::$ast::cast($node.clone()) $res else )*
                { $catch_all }
            }};
        }

        fn main() {
            mat<|>ch_ast! {
                match container {
                    ast::TraitDef(it) => {},
                    ast::ImplBlock(it) => {},
                    _ => { continue },
                }
            }
        }
        "#,
        );

        assert_eq!(res.name, "match_ast");
        assert_snapshot!(res.expansion, @r###"
{
  if let Some(it) = ast::TraitDef::cast(container.clone()){}
  else if let Some(it) = ast::ImplBlock::cast(container.clone()){}
  else {
    {
      continue
    }
  }
}
"###);
    }

    #[test]
    fn macro_expand_match_ast_inside_let_statement() {
        let res = check_expand_macro(
            r#"
        //- /lib.rs
        macro_rules! match_ast {
            (match $node:ident { $($tt:tt)* }) => { match_ast!(match ($node) { $($tt)* }) };
            (match ($node:expr) {}) => {{}};
        }

        fn main() {
            let p = f(|it| {
                let res = mat<|>ch_ast! { match c {}};
                Some(res)
            })?;
        }
        "#,
        );

        assert_eq!(res.name, "match_ast");
        assert_snapshot!(res.expansion, @r###"{}"###);
    }
}
<|endoftext|>


<commit_before>extern crate r2d2;

use std::comm;
use std::sync::{Mutex, Arc};
use std::sync::atomic::{AtomicBool, INIT_ATOMIC_BOOL, SeqCst};
use std::default::Default;

use r2d2::ErrorHandler;

mod config;

#[deriving(Show, PartialEq)]
struct FakeConnection;

struct OkManager;

impl r2d2::PoolManager<FakeConnection, ()> for OkManager {
    fn connect(&self) -> Result<FakeConnection, ()> {
        Ok(FakeConnection)
    }

    fn is_valid(&self, _: &mut FakeConnection) -> Result<(), ()> {
        Ok(())
    }

    fn has_broken(&self, _: &mut FakeConnection) -> bool {
        false
    }
}

struct NthConnectFailManager {
    n: Mutex<uint>,
}

impl r2d2::PoolManager<FakeConnection, ()> for NthConnectFailManager {
    fn connect(&self) -> Result<FakeConnection, ()> {
        let mut n = self.n.lock();
        if *n > 0 {
            *n -= 1;
            Ok(FakeConnection)
        } else {
            Err(())
        }
    }

    fn is_valid(&self, _: &mut FakeConnection) -> Result<(), ()> {
        Ok(())
    }

    fn has_broken(&self, _: &mut FakeConnection) -> bool {
        false
    }
}

#[test]
fn test_pool_size_ok() {
    let config = r2d2::Config {
        pool_size: 5,
        ..Default::default()
    };
    let manager = NthConnectFailManager { n: Mutex::new(5) };
    let pool = r2d2::Pool::new(config, manager, r2d2::NoopErrorHandler).unwrap();
    let mut conns = vec![];
    for _ in range(0, config.pool_size) {
        conns.push(pool.get().unwrap());
    }
}

#[test]
fn test_acquire_release() {
    let config = r2d2::Config {
        pool_size: 2,
        ..Default::default()
    };
    let pool = r2d2::Pool::new(config, OkManager, r2d2::NoopErrorHandler).unwrap();

    let conn1 = pool.get().unwrap();
    let conn2 = pool.get().unwrap();
    drop(conn1);
    let conn3 = pool.get().unwrap();
    drop(conn2);
    drop(conn3);
}

#[test]
fn test_is_send_sync() {
    fn is_send_sync<T: Send+Sync>() {}
    is_send_sync::<r2d2::Pool<FakeConnection, (), OkManager, r2d2::NoopErrorHandler>>();
}

#[test]
fn test_issue_2_unlocked_during_is_valid() {
    struct BlockingChecker {
        first: AtomicBool,
        s: Mutex<SyncSender<()>>,
        r: Mutex<Receiver<()>>,
    }

    impl r2d2::PoolManager<FakeConnection, ()> for BlockingChecker {
        fn connect(&self) -> Result<FakeConnection, ()> {
            Ok(FakeConnection)
        }

        fn is_valid(&self, _: &mut FakeConnection) -> Result<(), ()> {
            if self.first.compare_and_swap(true, false, SeqCst) {
                self.s.lock().send(());
                self.r.lock().recv();
            }
            Ok(())
        }

        fn has_broken(&self, _: &mut FakeConnection) -> bool {
            false
        }
    }

    let (s1, r1) = comm::sync_channel(0);
    let (s2, r2) = comm::sync_channel(0);

    let config = r2d2::Config {
        test_on_check_out: true,
        pool_size: 2,
        ..Default::default()
    };
    let manager = BlockingChecker {
        first: AtomicBool::new(true),
        s: Mutex::new(s1),
        r: Mutex::new(r2),
    };
    let pool = Arc::new(r2d2::Pool::new(config, manager, r2d2::NoopErrorHandler).unwrap());

    let p2 = pool.clone();
    spawn(move || {
        p2.get().unwrap();
    });

    r1.recv();
    // get call by other task has triggered the health check
    pool.get().unwrap();
    s2.send(());
}

#[test]
fn test_drop_on_broken() {
    static DROPPED: AtomicBool = INIT_ATOMIC_BOOL;
    DROPPED.store(false, SeqCst);

    struct Connection;

    impl Drop for Connection {
        fn drop(&mut self) {
            DROPPED.store(true, SeqCst);
        }
    }

    struct Handler;

    impl r2d2::PoolManager<Connection, ()> for Handler {
        fn connect(&self) -> Result<Connection, ()> {
            Ok(Connection)
        }

        fn is_valid(&self, _: &mut Connection) -> Result<(), ()> {
            Ok(())
        }

        fn has_broken(&self, _: &mut Connection) -> bool {
            true
        }
    }

    let pool = r2d2::Pool::new(Default::default(), Handler, r2d2::NoopErrorHandler).unwrap();

    drop(pool.get().unwrap());

    assert!(DROPPED.load(SeqCst));
}

// Just make sure that a boxed error handler works and doesn't self recurse or anything
#[test]
fn test_boxed_error_handler() {
    let handler: Box<ErrorHandler<()>> = box r2d2::NoopErrorHandler;
    handler.handle_error(());
    r2d2::Pool::new(Default::default(), OkManager, handler).unwrap();
}
<commit_msg>Fix deprecation warning<commit_after>extern crate r2d2;

use std::comm;
use std::sync::{Mutex, Arc};
use std::sync::atomic::{AtomicBool, INIT_ATOMIC_BOOL, SeqCst};
use std::default::Default;
use std::thread::Thread;

use r2d2::ErrorHandler;

mod config;

#[deriving(Show, PartialEq)]
struct FakeConnection;

struct OkManager;

impl r2d2::PoolManager<FakeConnection, ()> for OkManager {
    fn connect(&self) -> Result<FakeConnection, ()> {
        Ok(FakeConnection)
    }

    fn is_valid(&self, _: &mut FakeConnection) -> Result<(), ()> {
        Ok(())
    }

    fn has_broken(&self, _: &mut FakeConnection) -> bool {
        false
    }
}

struct NthConnectFailManager {
    n: Mutex<uint>,
}

impl r2d2::PoolManager<FakeConnection, ()> for NthConnectFailManager {
    fn connect(&self) -> Result<FakeConnection, ()> {
        let mut n = self.n.lock();
        if *n > 0 {
            *n -= 1;
            Ok(FakeConnection)
        } else {
            Err(())
        }
    }

    fn is_valid(&self, _: &mut FakeConnection) -> Result<(), ()> {
        Ok(())
    }

    fn has_broken(&self, _: &mut FakeConnection) -> bool {
        false
    }
}

#[test]
fn test_pool_size_ok() {
    let config = r2d2::Config {
        pool_size: 5,
        ..Default::default()
    };
    let manager = NthConnectFailManager { n: Mutex::new(5) };
    let pool = r2d2::Pool::new(config, manager, r2d2::NoopErrorHandler).unwrap();
    let mut conns = vec![];
    for _ in range(0, config.pool_size) {
        conns.push(pool.get().unwrap());
    }
}

#[test]
fn test_acquire_release() {
    let config = r2d2::Config {
        pool_size: 2,
        ..Default::default()
    };
    let pool = r2d2::Pool::new(config, OkManager, r2d2::NoopErrorHandler).unwrap();

    let conn1 = pool.get().unwrap();
    let conn2 = pool.get().unwrap();
    drop(conn1);
    let conn3 = pool.get().unwrap();
    drop(conn2);
    drop(conn3);
}

#[test]
fn test_is_send_sync() {
    fn is_send_sync<T: Send+Sync>() {}
    is_send_sync::<r2d2::Pool<FakeConnection, (), OkManager, r2d2::NoopErrorHandler>>();
}

#[test]
fn test_issue_2_unlocked_during_is_valid() {
    struct BlockingChecker {
        first: AtomicBool,
        s: Mutex<SyncSender<()>>,
        r: Mutex<Receiver<()>>,
    }

    impl r2d2::PoolManager<FakeConnection, ()> for BlockingChecker {
        fn connect(&self) -> Result<FakeConnection, ()> {
            Ok(FakeConnection)
        }

        fn is_valid(&self, _: &mut FakeConnection) -> Result<(), ()> {
            if self.first.compare_and_swap(true, false, SeqCst) {
                self.s.lock().send(());
                self.r.lock().recv();
            }
            Ok(())
        }

        fn has_broken(&self, _: &mut FakeConnection) -> bool {
            false
        }
    }

    let (s1, r1) = comm::sync_channel(0);
    let (s2, r2) = comm::sync_channel(0);

    let config = r2d2::Config {
        test_on_check_out: true,
        pool_size: 2,
        ..Default::default()
    };
    let manager = BlockingChecker {
        first: AtomicBool::new(true),
        s: Mutex::new(s1),
        r: Mutex::new(r2),
    };
    let pool = Arc::new(r2d2::Pool::new(config, manager, r2d2::NoopErrorHandler).unwrap());

    let p2 = pool.clone();
    let _t = Thread::spawn(move || {
        p2.get().unwrap();
    });

    r1.recv();
    // get call by other task has triggered the health check
    pool.get().unwrap();
    s2.send(());
}

#[test]
fn test_drop_on_broken() {
    static DROPPED: AtomicBool = INIT_ATOMIC_BOOL;
    DROPPED.store(false, SeqCst);

    struct Connection;

    impl Drop for Connection {
        fn drop(&mut self) {
            DROPPED.store(true, SeqCst);
        }
    }

    struct Handler;

    impl r2d2::PoolManager<Connection, ()> for Handler {
        fn connect(&self) -> Result<Connection, ()> {
            Ok(Connection)
        }

        fn is_valid(&self, _: &mut Connection) -> Result<(), ()> {
            Ok(())
        }

        fn has_broken(&self, _: &mut Connection) -> bool {
            true
        }
    }

    let pool = r2d2::Pool::new(Default::default(), Handler, r2d2::NoopErrorHandler).unwrap();

    drop(pool.get().unwrap());

    assert!(DROPPED.load(SeqCst));
}

// Just make sure that a boxed error handler works and doesn't self recurse or anything
#[test]
fn test_boxed_error_handler() {
    let handler: Box<ErrorHandler<()>> = box r2d2::NoopErrorHandler;
    handler.handle_error(());
    r2d2::Pool::new(Default::default(), OkManager, handler).unwrap();
}
<|endoftext|>


<commit_before>//! An unbounded set of streams

use core::fmt::{self, Debug};
use core::iter::FromIterator;
use core::pin::Pin;

use futures_core::stream::{Stream, FusedStream};
use futures_core::task::{Context, Poll};

use crate::stream::{StreamExt, StreamFuture, FuturesUnordered};

/// An unbounded set of streams
///
/// This "combinator" provides the ability to maintain a set of streams
/// and drive them all to completion.
///
/// Streams are pushed into this set and their realized values are
/// yielded as they become ready. Streams will only be polled when they
/// generate notifications. This allows to coordinate a large number of streams.
///
/// Note that you can create a ready-made `SelectAll` via the
/// `select_all` function in the `stream` module, or you can start with an
/// empty set with the `SelectAll::new` constructor.
#[must_use = "streams do nothing unless polled"]
pub struct SelectAll<St> {
    inner: FuturesUnordered<StreamFuture<St>>,
}

impl<St: Debug> Debug for SelectAll<St> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "SelectAll {{ ... }}")
    }
}

impl<St: Stream + Unpin> SelectAll<St> {
    /// Constructs a new, empty `SelectAll`
    ///
    /// The returned `SelectAll` does not contain any streams and, in this
    /// state, `SelectAll::poll` will return `Poll::Ready(None)`.
    pub fn new() -> SelectAll<St> {
        SelectAll { inner: FuturesUnordered::new() }
    }

    /// Returns the number of streams contained in the set.
    ///
    /// This represents the total number of in-flight streams.
    pub fn len(&self) -> usize {
        self.inner.len()
    }

    /// Returns `true` if the set contains no streams
    pub fn is_empty(&self) -> bool {
        self.inner.is_empty()
    }

    /// Push a stream into the set.
    ///
    /// This function submits the given stream to the set for managing. This
    /// function will not call `poll` on the submitted stream. The caller must
    /// ensure that `SelectAll::poll` is called in order to receive task
    /// notifications.
    pub fn push(&mut self, stream: St) {
        self.inner.push(stream.into_future());
    }
}

impl<St: Stream + Unpin> Default for SelectAll<St> {
    fn default() -> SelectAll<St> {
        SelectAll::new()
    }
}

impl<St: Stream + Unpin> Stream for SelectAll<St> {
    type Item = St::Item;

    fn poll_next(
        mut self: Pin<&mut Self>,
        cx: &mut Context<'_>,
    ) -> Poll<Option<Self::Item>> {
        loop {
            match ready!(self.inner.poll_next_unpin(cx)) {
                Some((Some(item), remaining)) => {
                    self.push(remaining);
                    return Poll::Ready(Some(item));
                }
                Some((None, _)) => {
                    // `FuturesUnordered` thinks it isn't terminated
                    // because it yielded a Some.
                    // We do not return, but poll `FuturesUnordered`
                    // in the next loop iteration.
                }
                None => return Poll::Ready(None),
            }
        }
    }
}

impl<St: Stream + Unpin> FusedStream for SelectAll<St> {
    fn is_terminated(&self) -> bool {
        self.inner.is_terminated()
    }
}

/// Convert a list of streams into a `Stream` of results from the streams.
///
/// This essentially takes a list of streams (e.g. a vector, an iterator, etc.)
/// and bundles them together into a single stream.
/// The stream will yield items as they become available on the underlying
/// streams internally, in the order they become available.
///
/// Note that the returned set can also be used to dynamically push more
/// futures into the set as they become available.
///
/// This function is only available when the `std` or `alloc` feature of this
/// library is activated, and it is activated by default.
pub fn select_all<I>(streams: I) -> SelectAll<I::Item>
    where I: IntoIterator,
          I::Item: Stream + Unpin
{
    let mut set = SelectAll::new();

    for stream in streams {
        set.push(stream);
    }

    set
}

impl<St: Stream + Unpin> FromIterator<St> for SelectAll<St> {
    fn from_iter<T: IntoIterator<Item = St>>(iter: T) -> Self {
        select_all(iter)
    }
}
<commit_msg>impl Extend for SelectAll<commit_after>//! An unbounded set of streams

use core::fmt::{self, Debug};
use core::iter::FromIterator;
use core::pin::Pin;

use futures_core::stream::{Stream, FusedStream};
use futures_core::task::{Context, Poll};

use crate::stream::{StreamExt, StreamFuture, FuturesUnordered};

/// An unbounded set of streams
///
/// This "combinator" provides the ability to maintain a set of streams
/// and drive them all to completion.
///
/// Streams are pushed into this set and their realized values are
/// yielded as they become ready. Streams will only be polled when they
/// generate notifications. This allows to coordinate a large number of streams.
///
/// Note that you can create a ready-made `SelectAll` via the
/// `select_all` function in the `stream` module, or you can start with an
/// empty set with the `SelectAll::new` constructor.
#[must_use = "streams do nothing unless polled"]
pub struct SelectAll<St> {
    inner: FuturesUnordered<StreamFuture<St>>,
}

impl<St: Debug> Debug for SelectAll<St> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "SelectAll {{ ... }}")
    }
}

impl<St: Stream + Unpin> SelectAll<St> {
    /// Constructs a new, empty `SelectAll`
    ///
    /// The returned `SelectAll` does not contain any streams and, in this
    /// state, `SelectAll::poll` will return `Poll::Ready(None)`.
    pub fn new() -> SelectAll<St> {
        SelectAll { inner: FuturesUnordered::new() }
    }

    /// Returns the number of streams contained in the set.
    ///
    /// This represents the total number of in-flight streams.
    pub fn len(&self) -> usize {
        self.inner.len()
    }

    /// Returns `true` if the set contains no streams
    pub fn is_empty(&self) -> bool {
        self.inner.is_empty()
    }

    /// Push a stream into the set.
    ///
    /// This function submits the given stream to the set for managing. This
    /// function will not call `poll` on the submitted stream. The caller must
    /// ensure that `SelectAll::poll` is called in order to receive task
    /// notifications.
    pub fn push(&mut self, stream: St) {
        self.inner.push(stream.into_future());
    }
}

impl<St: Stream + Unpin> Default for SelectAll<St> {
    fn default() -> SelectAll<St> {
        SelectAll::new()
    }
}

impl<St: Stream + Unpin> Stream for SelectAll<St> {
    type Item = St::Item;

    fn poll_next(
        mut self: Pin<&mut Self>,
        cx: &mut Context<'_>,
    ) -> Poll<Option<Self::Item>> {
        loop {
            match ready!(self.inner.poll_next_unpin(cx)) {
                Some((Some(item), remaining)) => {
                    self.push(remaining);
                    return Poll::Ready(Some(item));
                }
                Some((None, _)) => {
                    // `FuturesUnordered` thinks it isn't terminated
                    // because it yielded a Some.
                    // We do not return, but poll `FuturesUnordered`
                    // in the next loop iteration.
                }
                None => return Poll::Ready(None),
            }
        }
    }
}

impl<St: Stream + Unpin> FusedStream for SelectAll<St> {
    fn is_terminated(&self) -> bool {
        self.inner.is_terminated()
    }
}

/// Convert a list of streams into a `Stream` of results from the streams.
///
/// This essentially takes a list of streams (e.g. a vector, an iterator, etc.)
/// and bundles them together into a single stream.
/// The stream will yield items as they become available on the underlying
/// streams internally, in the order they become available.
///
/// Note that the returned set can also be used to dynamically push more
/// futures into the set as they become available.
///
/// This function is only available when the `std` or `alloc` feature of this
/// library is activated, and it is activated by default.
pub fn select_all<I>(streams: I) -> SelectAll<I::Item>
    where I: IntoIterator,
          I::Item: Stream + Unpin
{
    let mut set = SelectAll::new();

    for stream in streams {
        set.push(stream);
    }

    set
}

impl<St: Stream + Unpin> FromIterator<St> for SelectAll<St> {
    fn from_iter<T: IntoIterator<Item = St>>(iter: T) -> Self {
        select_all(iter)
    }
}

impl<St: Stream + Unpin> Extend<St> for SelectAll<St> {
    fn extend<T: IntoIterator<Item = St>>(&mut self, iter: T) {
        for st in iter {
            self.push(st)
        }
    }
}
<|endoftext|>


<commit_before>// pest. The Elegant Parser
// Copyright (c) 2018 Dragoș Tiselice
//
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

extern crate pest;
extern crate pest_meta;

use std::collections::HashMap;

use pest::{Atomicity, Error, ParserState, Position};
use pest::iterators::Pairs;

use pest_meta::ast::{Expr, Rule, RuleType};

pub struct Vm {
    rules: HashMap<String, Rule>
}

impl Vm {
    pub fn new(rules: Vec<Rule>) -> Vm {
        let rules = rules.into_iter().map(|r| (r.name.clone(), r)).collect();
        Vm { rules }
    }

    pub fn parse<'i>(
        &self,
        rule: &str,
        input: &'i str
    ) -> Option<Result<Pairs<'i, &str>, Error<'i, &str>>> {
        if let Some(rule) = self.rules.get(rule) {
            let result = pest::state(input, |mut state, pos| {
                self.parse_rule(rule, pos, &mut state)
            });

            Some(result)
        } else {
            None
        }
    }

    fn parse_rule<'a, 'i>(
        &self,
        rule: &'a Rule,
        pos: Position<'i>,
        state: &mut ParserState<'i, &'a str>
    ) -> Result<Position<'i>, Position<'i>> {
        if &rule.name == "whitespace" || &rule.name == "comment" {
            match rule.ty {
                RuleType::Normal => state.rule(&rule.name, pos, |state, pos| {
                    state.atomic(Atomicity::Atomic, move |state| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::Silent => state.atomic(Atomicity::Atomic, move |state| {
                    self.parse_expr(&rule.expr, pos, state)
                }),
                RuleType::Atomic => state.rule(&rule.name, pos, |state, pos| {
                    state.atomic(Atomicity::Atomic, move |state| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::CompoundAtomic => state.atomic(Atomicity::CompoundAtomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::NonAtomic => state.atomic(Atomicity::Atomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                })
            }
        } else {
            match rule.ty {
                RuleType::Normal => state.rule(&rule.name, pos, |state, pos| {
                    self.parse_expr(&rule.expr, pos, state)
                }),
                RuleType::Silent => self.parse_expr(&rule.expr, pos, state),
                RuleType::Atomic => state.rule(&rule.name, pos, |state, pos| {
                    state.atomic(Atomicity::Atomic, move |state| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::CompoundAtomic => state.atomic(Atomicity::CompoundAtomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::NonAtomic => state.atomic(Atomicity::NonAtomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                })
            }
        }
    }

    fn parse_expr<'i>(
        &self,
        expr: &Expr,
        pos: Position<'i>,
        state: &mut ParserState<'i, &str>
    ) -> Result<Position<'i>, Position<'i>> {
        unimplemented!()
    }

    fn skip<'a, 'i>(
        &'a self,
        pos: Position<'i>,
        state: &mut ParserState<'i, &'a str>
    ) -> Result<Position<'i>, Position<'i>> {
        match (self.rules.get("whitespace"), self.rules.get("comment")) {
            (None, None) => Ok(pos),
            (Some(whitespace), None) => if state.atomicity == Atomicity::NonAtomic {
                pos.repeat(|pos| {
                    self.parse_rule(whitespace, pos, state)
                })
            } else {
                Ok(pos)
            },
            (None, Some(comment)) => if state.atomicity == Atomicity::NonAtomic {
                pos.repeat(|pos| {
                    self.parse_rule(comment, pos, state)
                })
            } else {
                Ok(pos)
            },
            (Some(whitespace), Some(comment)) => if state.atomicity == Atomicity::NonAtomic {
                state.sequence(move |state| {
                    pos.sequence(|pos| {
                        pos.repeat(|pos| {
                            self.parse_rule(whitespace, pos, state)
                        }).and_then(|pos| {
                            pos.repeat(|pos| {
                                state.sequence(move |state| {
                                    pos.sequence(|pos| {
                                        self.parse_rule(comment, pos, state).and_then(|pos| {
                                            pos.repeat(|pos| {
                                                self.parse_rule(whitespace, pos, state)
                                            })
                                        })
                                    })
                                })
                            })
                        })
                    })
                })
            } else {
                Ok(pos)
            }
        }
    }
}
<commit_msg>Added VM string unescaping.<commit_after>// pest. The Elegant Parser
// Copyright (c) 2018 Dragoș Tiselice
//
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. All files in the project carrying such notice may not be copied,
// modified, or distributed except according to those terms.

extern crate pest;
extern crate pest_meta;

use std::char;
use std::collections::HashMap;

use pest::{Atomicity, Error, ParserState, Position};
use pest::iterators::Pairs;

use pest_meta::ast::{Expr, Rule, RuleType};

pub struct Vm {
    rules: HashMap<String, Rule>
}

impl Vm {
    pub fn new(rules: Vec<Rule>) -> Vm {
        let rules = rules.into_iter().map(|r| (r.name.clone(), r)).collect();
        Vm { rules }
    }

    pub fn parse<'i>(
        &self,
        rule: &str,
        input: &'i str
    ) -> Option<Result<Pairs<'i, &str>, Error<'i, &str>>> {
        if let Some(rule) = self.rules.get(rule) {
            let result = pest::state(input, |mut state, pos| {
                self.parse_rule(rule, pos, &mut state)
            });

            Some(result)
        } else {
            None
        }
    }

    fn parse_rule<'a, 'i>(
        &self,
        rule: &'a Rule,
        pos: Position<'i>,
        state: &mut ParserState<'i, &'a str>
    ) -> Result<Position<'i>, Position<'i>> {
        if &rule.name == "whitespace" || &rule.name == "comment" {
            match rule.ty {
                RuleType::Normal => state.rule(&rule.name, pos, |state, pos| {
                    state.atomic(Atomicity::Atomic, move |state| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::Silent => state.atomic(Atomicity::Atomic, move |state| {
                    self.parse_expr(&rule.expr, pos, state)
                }),
                RuleType::Atomic => state.rule(&rule.name, pos, |state, pos| {
                    state.atomic(Atomicity::Atomic, move |state| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::CompoundAtomic => state.atomic(Atomicity::CompoundAtomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::NonAtomic => state.atomic(Atomicity::Atomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                })
            }
        } else {
            match rule.ty {
                RuleType::Normal => state.rule(&rule.name, pos, |state, pos| {
                    self.parse_expr(&rule.expr, pos, state)
                }),
                RuleType::Silent => self.parse_expr(&rule.expr, pos, state),
                RuleType::Atomic => state.rule(&rule.name, pos, |state, pos| {
                    state.atomic(Atomicity::Atomic, move |state| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::CompoundAtomic => state.atomic(Atomicity::CompoundAtomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                }),
                RuleType::NonAtomic => state.atomic(Atomicity::NonAtomic, move |state| {
                    state.rule(&rule.name, pos, |state, pos| {
                        self.parse_expr(&rule.expr, pos, state)
                    })
                })
            }
        }
    }

    fn parse_expr<'i>(
        &self,
        expr: &Expr,
        pos: Position<'i>,
        state: &mut ParserState<'i, &str>
    ) -> Result<Position<'i>, Position<'i>> {
        unimplemented!()
    }

    fn skip<'a, 'i>(
        &'a self,
        pos: Position<'i>,
        state: &mut ParserState<'i, &'a str>
    ) -> Result<Position<'i>, Position<'i>> {
        match (self.rules.get("whitespace"), self.rules.get("comment")) {
            (None, None) => Ok(pos),
            (Some(whitespace), None) => if state.atomicity == Atomicity::NonAtomic {
                pos.repeat(|pos| {
                    self.parse_rule(whitespace, pos, state)
                })
            } else {
                Ok(pos)
            },
            (None, Some(comment)) => if state.atomicity == Atomicity::NonAtomic {
                pos.repeat(|pos| {
                    self.parse_rule(comment, pos, state)
                })
            } else {
                Ok(pos)
            },
            (Some(whitespace), Some(comment)) => if state.atomicity == Atomicity::NonAtomic {
                state.sequence(move |state| {
                    pos.sequence(|pos| {
                        pos.repeat(|pos| {
                            self.parse_rule(whitespace, pos, state)
                        }).and_then(|pos| {
                            pos.repeat(|pos| {
                                state.sequence(move |state| {
                                    pos.sequence(|pos| {
                                        self.parse_rule(comment, pos, state).and_then(|pos| {
                                            pos.repeat(|pos| {
                                                self.parse_rule(whitespace, pos, state)
                                            })
                                        })
                                    })
                                })
                            })
                        })
                    })
                })
            } else {
                Ok(pos)
            }
        }
    }
}

fn unescape(string: &str) -> Option<String> {
    let mut result = String::new();
    let mut chars = string.chars();

    loop {
        match chars.next() {
            Some('\\') => {
                match chars.next()? {
                    '"' => result.push('"'),
                    '\\' => result.push('\\'),
                    'r' => result.push('\r'),
                    'n' => result.push('\n'),
                    't' => result.push('\t'),
                    '0' => result.push('\0'),
                    '\'' => result.push('\''),
                    'x' => {
                        let string: String = chars.clone().take(2).collect();

                        if string.len() != 2 {
                            return None;
                        }

                        for _ in 0..string.len() { chars.next()?; }

                        let value = u8::from_str_radix(&string, 16).ok()?;

                        result.push(char::from(value));
                    }
                    'u' => {
                        if chars.next()? != '{' {
                            return None;
                        }

                        let string: String = chars.clone().take_while(|c| *c != '}').collect();

                        if string.len() < 2 || 6 < string.len() {
                            return None;
                        }

                        for _ in 0..string.len() + 1 { chars.next()?; }

                        let value = u32::from_str_radix(&string, 16).ok()?;

                        result.push(char::from_u32(value)?);
                    }
                    _ => return None
                }
            }
            Some(c) => result.push(c),
            None => return Some(result)
        };
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn unescape_all() {
        let string = r"a\nb\x55c\u{111}d";

        assert_eq!(unescape(string), Some("a\nb\x55c\u{111}d".to_owned()));
    }

    #[test]
    fn unescape_empty_escape() {
        let string = r"\";

        assert_eq!(unescape(string), None);
    }

    #[test]
    fn unescape_wrong_escape() {
        let string = r"\w";

        assert_eq!(unescape(string), None);
    }

    #[test]
    fn unescape_wrong_byte() {
        let string = r"\xfg";

        assert_eq!(unescape(string), None);
    }

    #[test]
    fn unescape_short_byte() {
        let string = r"\xf";

        assert_eq!(unescape(string), None);
    }

    #[test]
    fn unescape_no_open_brace_unicode() {
        let string = r"\u11";

        assert_eq!(unescape(string), None);
    }

    #[test]
    fn unescape_no_close_brace_unicode() {
        let string = r"\u{11";

        assert_eq!(unescape(string), None);
    }

    #[test]
    fn unescape_short_unicode() {
        let string = r"\u{1}";

        assert_eq!(unescape(string), None);
    }

    #[test]
    fn unescape_long_unicode() {
        let string = r"\u{1111111}";

        assert_eq!(unescape(string), None);
    }
}
<|endoftext|>


<commit_before>#![cfg(feature = "cloudfront")]

extern crate rusoto_core;
extern crate rusoto_cloudfront;

use rusoto_cloudfront::{CloudFront, CloudFrontClient, ListDistributionsRequest};
use rusoto_core::Region;

#[test]
fn should_list_distirbutions() {
    let client = CloudFrontClient::simple(Region::UsEast1);
    let request = ListDistributionsRequest::default();

    let result = client.list_distributions(&request).sync();
    println!("{:#?}", result);
    assert!(result.is_ok());
}

<commit_msg>Correct typo, use env_logger in cloudfront test.<commit_after>#![cfg(feature = "cloudfront")]

extern crate rusoto_core;
extern crate rusoto_cloudfront;
extern crate env_logger;

use rusoto_cloudfront::{CloudFront, CloudFrontClient, ListDistributionsRequest};
use rusoto_core::Region;

#[test]
fn should_list_distributions() {
    let _ = env_logger::try_init();
    let client = CloudFrontClient::simple(Region::UsEast1);
    let request = ListDistributionsRequest::default();

    let result = client.list_distributions(&request).sync();
    println!("{:#?}", result);
    assert!(result.is_ok());
}

<|endoftext|>


<commit_before>mod expand_repeats;
mod expand_tokens;
mod extract_default_aliases;
mod extract_tokens;
mod flatten_grammar;
mod intern_symbols;
mod process_inlines;

use super::Error;
use std::{
    cmp::Ordering,
    collections::{hash_map, HashMap},
    mem,
};

use self::expand_repeats::expand_repeats;
pub(crate) use self::expand_tokens::expand_tokens;
use self::extract_default_aliases::extract_default_aliases;
use self::extract_tokens::extract_tokens;
use self::flatten_grammar::flatten_grammar;
use self::intern_symbols::intern_symbols;
use self::process_inlines::process_inlines;
use crate::error::Result;
use crate::generate::grammars::{
    ExternalToken, InlinedProductionMap, InputGrammar, LexicalGrammar, SyntaxGrammar, Variable,
};
use crate::generate::rules::{AliasMap, Rule, Symbol};

pub(crate) struct IntermediateGrammar<T, U> {
    variables: Vec<Variable>,
    extra_symbols: Vec<T>,
    expected_conflicts: Vec<Vec<Symbol>>,
    precedence_orderings: Vec<Vec<String>>,
    external_tokens: Vec<U>,
    variables_to_inline: Vec<Symbol>,
    supertype_symbols: Vec<Symbol>,
    word_token: Option<Symbol>,
}

pub(crate) type InternedGrammar = IntermediateGrammar<Rule, Variable>;

pub(crate) type ExtractedSyntaxGrammar = IntermediateGrammar<Symbol, ExternalToken>;

#[derive(Debug, PartialEq, Eq)]
pub(crate) struct ExtractedLexicalGrammar {
    pub variables: Vec<Variable>,
    pub separators: Vec<Rule>,
}

/// Transform an input grammar into separate components that are ready
/// for parse table construction.
pub(crate) fn prepare_grammar(
    input_grammar: &InputGrammar,
) -> Result<(
    SyntaxGrammar,
    LexicalGrammar,
    InlinedProductionMap,
    AliasMap,
)> {
    validate_precedence_orderings(&input_grammar.precedence_orderings)?;

    let interned_grammar = intern_symbols(input_grammar)?;
    let (syntax_grammar, lexical_grammar) = extract_tokens(interned_grammar)?;
    let syntax_grammar = expand_repeats(syntax_grammar);
    let mut syntax_grammar = flatten_grammar(syntax_grammar)?;
    let lexical_grammar = expand_tokens(lexical_grammar)?;
    let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
    let inlines = process_inlines(&syntax_grammar);
    Ok((syntax_grammar, lexical_grammar, inlines, default_aliases))
}

/// Make sure that there are no conflicting orderings. For any two precedence
/// names `a` and `b`, if `a` comes before `b` in some list, then it cannot come
// *after* `b` in any list.
fn validate_precedence_orderings(order_lists: &[Vec<String>]) -> Result<()> {
    let mut pairs = HashMap::new();
    for list in order_lists {
        for (i, mut name1) in list.iter().enumerate() {
            for mut name2 in list.iter().skip(i + 1) {
                if name2 == name1 {
                    continue;
                }
                let mut ordering = Ordering::Greater;
                if name1 > name2 {
                    ordering = Ordering::Less;
                    mem::swap(&mut name1, &mut name2);
                }
                match pairs.entry((name1, name2)) {
                    hash_map::Entry::Vacant(e) => {
                        e.insert(ordering);
                    }
                    hash_map::Entry::Occupied(e) => {
                        if e.get() != &ordering {
                            return Err(Error::new(format!(
                                "Conflicting orderings for precedences '{}' and '{}'",
                                name1, name2
                            )));
                        }
                    }
                }
            }
        }
    }
    Ok(())
}
<commit_msg>Add further static validation of named precedences<commit_after>mod expand_repeats;
mod expand_tokens;
mod extract_default_aliases;
mod extract_tokens;
mod flatten_grammar;
mod intern_symbols;
mod process_inlines;

use super::{rules::Precedence, Error};
use std::{
    cmp::Ordering,
    collections::{hash_map, HashMap, HashSet},
    mem,
};

use self::expand_repeats::expand_repeats;
pub(crate) use self::expand_tokens::expand_tokens;
use self::extract_default_aliases::extract_default_aliases;
use self::extract_tokens::extract_tokens;
use self::flatten_grammar::flatten_grammar;
use self::intern_symbols::intern_symbols;
use self::process_inlines::process_inlines;
use crate::error::Result;
use crate::generate::grammars::{
    ExternalToken, InlinedProductionMap, InputGrammar, LexicalGrammar, SyntaxGrammar, Variable,
};
use crate::generate::rules::{AliasMap, Rule, Symbol};

pub(crate) struct IntermediateGrammar<T, U> {
    variables: Vec<Variable>,
    extra_symbols: Vec<T>,
    expected_conflicts: Vec<Vec<Symbol>>,
    precedence_orderings: Vec<Vec<String>>,
    external_tokens: Vec<U>,
    variables_to_inline: Vec<Symbol>,
    supertype_symbols: Vec<Symbol>,
    word_token: Option<Symbol>,
}

pub(crate) type InternedGrammar = IntermediateGrammar<Rule, Variable>;

pub(crate) type ExtractedSyntaxGrammar = IntermediateGrammar<Symbol, ExternalToken>;

#[derive(Debug, PartialEq, Eq)]
pub(crate) struct ExtractedLexicalGrammar {
    pub variables: Vec<Variable>,
    pub separators: Vec<Rule>,
}

/// Transform an input grammar into separate components that are ready
/// for parse table construction.
pub(crate) fn prepare_grammar(
    input_grammar: &InputGrammar,
) -> Result<(
    SyntaxGrammar,
    LexicalGrammar,
    InlinedProductionMap,
    AliasMap,
)> {
    validate_named_precedences(input_grammar)?;

    let interned_grammar = intern_symbols(input_grammar)?;
    let (syntax_grammar, lexical_grammar) = extract_tokens(interned_grammar)?;
    let syntax_grammar = expand_repeats(syntax_grammar);
    let mut syntax_grammar = flatten_grammar(syntax_grammar)?;
    let lexical_grammar = expand_tokens(lexical_grammar)?;
    let default_aliases = extract_default_aliases(&mut syntax_grammar, &lexical_grammar);
    let inlines = process_inlines(&syntax_grammar);
    Ok((syntax_grammar, lexical_grammar, inlines, default_aliases))
}

/// Check that all of the named precedences used in the grammar are declared
/// within the `precedences` lists, and also that there are no conflicting
/// precedence orderings declared in those lists.
fn validate_named_precedences(grammar: &InputGrammar) -> Result<()> {
    // For any two precedence names `a` and `b`, if `a` comes before `b`
    // in some list, then it cannot come *after* `b` in any list.
    let mut pairs = HashMap::new();
    for list in &grammar.precedence_orderings {
        for (i, mut name1) in list.iter().enumerate() {
            for mut name2 in list.iter().skip(i + 1) {
                if name2 == name1 {
                    continue;
                }
                let mut ordering = Ordering::Greater;
                if name1 > name2 {
                    ordering = Ordering::Less;
                    mem::swap(&mut name1, &mut name2);
                }
                match pairs.entry((name1, name2)) {
                    hash_map::Entry::Vacant(e) => {
                        e.insert(ordering);
                    }
                    hash_map::Entry::Occupied(e) => {
                        if e.get() != &ordering {
                            return Err(Error::new(format!(
                                "Conflicting orderings for precedences '{}' and '{}'",
                                name1, name2
                            )));
                        }
                    }
                }
            }
        }
    }

    // Check that no rule contains a named precedence that is not present in
    // any of the `precedences` lists.
    fn validate(rule_name: &str, rule: &Rule, names: &HashSet<&String>) -> Result<()> {
        match rule {
            Rule::Repeat(rule) => validate(rule_name, rule, names),
            Rule::Seq(elements) | Rule::Choice(elements) => elements
                .iter()
                .map(|e| validate(rule_name, e, names))
                .collect(),
            Rule::Metadata { rule, params } => {
                if let Precedence::Name(n) = &params.precedence {
                    if !names.contains(n) {
                        return Err(Error::new(format!(
                            "Undeclared precedence '{}' in rule '{}'",
                            n, rule_name
                        )));
                    }
                }
                validate(rule_name, rule, names)?;
                Ok(())
            }
            _ => Ok(()),
        }
    }

    let precedence_names = grammar
        .precedence_orderings
        .iter()
        .flat_map(|l| l.iter())
        .collect::<HashSet<&String>>();
    for variable in &grammar.variables {
        validate(&variable.name, &variable.rule, &precedence_names)?;
    }

    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::generate::grammars::{InputGrammar, Variable, VariableType};

    #[test]
    fn test_validate_named_precedences_with_undeclared_precedence() {
        let grammar = InputGrammar {
            name: String::new(),
            word_token: None,
            extra_symbols: vec![],
            external_tokens: vec![],
            supertype_symbols: vec![],
            expected_conflicts: vec![],
            variables_to_inline: vec![],
            precedence_orderings: vec![
                vec!["a".to_string(), "b".to_string()],
                vec!["b".to_string(), "c".to_string(), "d".to_string()],
            ],
            variables: vec![
                Variable {
                    name: "v1".to_string(),
                    kind: VariableType::Named,
                    rule: Rule::Seq(vec![
                        Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
                    ]),
                },
                Variable {
                    name: "v2".to_string(),
                    kind: VariableType::Named,
                    rule: Rule::repeat(Rule::Choice(vec![
                        Rule::prec_left(Precedence::Name("omg".to_string()), Rule::string("y")),
                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
                    ])),
                },
            ],
        };

        let result = validate_named_precedences(&grammar);
        assert_eq!(
            result.unwrap_err().message(),
            "Undeclared precedence 'omg' in rule 'v2'",
        );
    }

    #[test]
    fn test_validate_named_precedences_with_conflicting_order() {
        let grammar = InputGrammar {
            name: String::new(),
            word_token: None,
            extra_symbols: vec![],
            external_tokens: vec![],
            supertype_symbols: vec![],
            expected_conflicts: vec![],
            variables_to_inline: vec![],
            precedence_orderings: vec![
                vec!["a".to_string(), "b".to_string()],
                vec!["b".to_string(), "c".to_string(), "a".to_string()],
            ],
            variables: vec![
                Variable {
                    name: "v1".to_string(),
                    kind: VariableType::Named,
                    rule: Rule::Seq(vec![
                        Rule::prec_left(Precedence::Name("b".to_string()), Rule::string("w")),
                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("x")),
                    ]),
                },
                Variable {
                    name: "v2".to_string(),
                    kind: VariableType::Named,
                    rule: Rule::repeat(Rule::Choice(vec![
                        Rule::prec_left(Precedence::Name("a".to_string()), Rule::string("y")),
                        Rule::prec(Precedence::Name("c".to_string()), Rule::string("z")),
                    ])),
                },
            ],
        };

        let result = validate_named_precedences(&grammar);
        assert_eq!(
            result.unwrap_err().message(),
            "Conflicting orderings for precedences 'a' and 'b'",
        );
    }
}
<|endoftext|>


<commit_before>use proc_macro2::{TokenStream, TokenTree};
use quote::quote;
use std::env;
use std::ffi::OsString;
use std::fs::{self, File};
use std::io::{prelude::*, BufReader, BufWriter};
use std::path::Path;

fn get_data_file(path_fragment: &str) -> BufReader<File> {
    println!("cargo:rerun-if-changed=../data/{}", path_fragment);

    let path = Path::new("../data").join(path_fragment);

    BufReader::new(File::open(path).expect("Something went wrong reading the file"))
}

fn ivd_sequences() -> TokenStream {
    let mut result = String::new();
    let ivd_seqs_file = get_data_file("Unicode/IVD/IVD_Sequences.txt");

    let lines: Vec<_> = ivd_seqs_file
        .lines()
        .map(Result::unwrap)
        .filter(|line| line.len() > 0 && !line.starts_with('#')).collect();
    let array_size = lines.len();

    let structs: TokenStream = lines
        .iter()
        .map(|line| {
            let parts: Vec<_> = line.split(';').map(|s| s.trim()).collect();
            let codepoint_parts: Vec<_> = parts[0].split(' ').collect();
            let base_codepoint = u32::from_str_radix(codepoint_parts[0].trim(), 16).unwrap();
            let variation_selector = u32::from_str_radix(codepoint_parts[1].trim(), 16).unwrap();
            let collection = parts[1].trim();
            let item = parts[2].trim();

            quote! {
                IdeographicVariationSequence {
                    base_codepoint: #base_codepoint,
                    variation_selector: #variation_selector,
                    collection: #collection,
                    item: #item
                },
            }
        })
        .collect();

    quote! {
        const IVD_SEQUENCES: [IdeographicVariationSequence; #array_size] = [
            #structs
        ];
    }
}

fn main() {
    let out_dir = env::var_os("OUT_DIR").unwrap();
    let dest_path = Path::new(&out_dir).join("compiled-data.rs");

    write!(
        BufWriter::new(File::create(dest_path).unwrap()),
        "{}",
        ivd_sequences()
    );
}
<commit_msg>Remove compiler warnings from build.rs<commit_after>use proc_macro2::TokenStream;
use quote::quote;
use std::env;
use std::fs::File;
use std::io::{prelude::*, BufReader, BufWriter};
use std::path::Path;

fn get_data_file(path_fragment: &str) -> BufReader<File> {
    println!("cargo:rerun-if-changed=../data/{}", path_fragment);

    let path = Path::new("../data").join(path_fragment);

    BufReader::new(File::open(path).expect("Something went wrong reading the file"))
}

fn ivd_sequences() -> TokenStream {
    let ivd_seqs_file = get_data_file("Unicode/IVD/IVD_Sequences.txt");

    let lines: Vec<_> = ivd_seqs_file
        .lines()
        .map(Result::unwrap)
        .filter(|line| line.len() > 0 && !line.starts_with('#'))
        .collect();
    let array_size = lines.len();

    let structs: TokenStream = lines
        .iter()
        .map(|line| {
            let parts: Vec<_> = line.split(';').map(|s| s.trim()).collect();
            let codepoint_parts: Vec<_> = parts[0].split(' ').collect();
            let base_codepoint = u32::from_str_radix(codepoint_parts[0].trim(), 16).unwrap();
            let variation_selector = u32::from_str_radix(codepoint_parts[1].trim(), 16).unwrap();
            let collection = parts[1].trim();
            let item = parts[2].trim();

            quote! {
                IdeographicVariationSequence {
                    base_codepoint: #base_codepoint,
                    variation_selector: #variation_selector,
                    collection: #collection,
                    item: #item
                },
            }
        })
        .collect();

    quote! {
        const IVD_SEQUENCES: [IdeographicVariationSequence; #array_size] = [
            #structs
        ];
    }
}

fn main() {
    let out_dir = env::var_os("OUT_DIR").unwrap();
    let dest_path = Path::new(&out_dir).join("compiled-data.rs");

    write!(
        BufWriter::new(File::create(dest_path).unwrap()),
        "{}",
        ivd_sequences()
    )
    .unwrap();
}
<|endoftext|>


<commit_before>/// This is an implementation of an echo server.

/// One process reads input and sends it to the other process, which outputs it.

#[macro_use]
extern crate session_types;
use session_types::*;

use std::thread::spawn;

type Srv = Offer<Eps, Recv<String, Var<Z>>>;
fn srv(c: Chan<(), Rec<Srv>>) {

    let mut c = c.enter();

    loop {
        c = offer!{ c,
            CLOSE => {
                println!("Closing server.");
                c.close();
                break
            },
            RECV => {
                let (c, s) = c.recv();
                println!("Received: {}", s);
                c.zero()
            }
        };
    }
}

type Cli = <Srv as HasDual>::Dual;
fn cli(c: Chan<(), Rec<Cli>>) {

    let mut stdin = std::io::stdin();
    let mut count = 0usize;

    let mut c = c.enter();
    let mut buf = "".to_string();
    loop {
        stdin.read_line(&mut buf).ok().unwrap();
        if !buf.is_empty() {
            buf.pop();
        }
        match &buf[..] {
            "q" => {
                let c = c.sel2().send(format!("{} lines sent", count));
                c.zero().sel1().close();
                println!("Client quitting");
                break;
            }
            _ => {
                c = c.sel2().send(buf.clone()).zero();
                buf.clear();
                count += 1;
            }
        }
    }
}

fn main() {
    let (c1, c2) = session_channel();
    spawn(move || srv(c1));
    cli(c2);
}
<commit_msg>fix unused_mutable warning<commit_after>/// This is an implementation of an echo server.

/// One process reads input and sends it to the other process, which outputs it.

#[macro_use]
extern crate session_types;
use session_types::*;

use std::thread::spawn;

type Srv = Offer<Eps, Recv<String, Var<Z>>>;
fn srv(c: Chan<(), Rec<Srv>>) {

    let mut c = c.enter();

    loop {
        c = offer!{ c,
            CLOSE => {
                println!("Closing server.");
                c.close();
                break
            },
            RECV => {
                let (c, s) = c.recv();
                println!("Received: {}", s);
                c.zero()
            }
        };
    }
}

type Cli = <Srv as HasDual>::Dual;
fn cli(c: Chan<(), Rec<Cli>>) {

    let stdin = std::io::stdin();
    let mut count = 0usize;

    let mut c = c.enter();
    let mut buf = "".to_string();
    loop {
        stdin.read_line(&mut buf).ok().unwrap();
        if !buf.is_empty() {
            buf.pop();
        }
        match &buf[..] {
            "q" => {
                let c = c.sel2().send(format!("{} lines sent", count));
                c.zero().sel1().close();
                println!("Client quitting");
                break;
            }
            _ => {
                c = c.sel2().send(buf.clone()).zero();
                buf.clear();
                count += 1;
            }
        }
    }
}

fn main() {
    let (c1, c2) = session_channel();
    spawn(move || srv(c1));
    cli(c2);
}
<|endoftext|>