<commit_before>// Copyright (c) 2014-2016 Robert Clipsham <robert@octarineparrot.com>
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Support for sending and receiving data link layer packets using the WinPcap library.

extern crate winapi;

use super::bindings::{bpf, winpcap};
use super::{DataLinkReceiver, DataLinkSender, MacAddr, NetworkInterface};

use ipnetwork::{ip_mask_to_prefix, IpNetwork};

use self::winapi::ctypes;
use std::cmp;
use std::collections::VecDeque;
use std::ffi::{CStr, CString};
use std::io;
use std::mem;
use std::slice;
use std::str::from_utf8_unchecked;
use std::sync::Arc;

struct WinPcapAdapter {
    adapter: winpcap::LPADAPTER,
}

impl Drop for WinPcapAdapter {
    fn drop(&mut self) {
        unsafe {
            winpcap::PacketCloseAdapter(self.adapter);
        }
    }
}

struct WinPcapPacket {
    packet: winpcap::LPPACKET,
}

impl Drop for WinPcapPacket {
    fn drop(&mut self) {
        unsafe {
            winpcap::PacketFreePacket(self.packet);
        }
    }
}

/// The WinPcap's specific configuration.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Config {
    /// The size of buffer to use when writing packets. Defaults to 4096.
    pub write_buffer_size: usize,

    /// The size of buffer to use when reading packets. Defaults to 4096.
    pub read_buffer_size: usize,
}

impl<'a> From<&'a super::Config> for Config {
    fn from(config: &super::Config) -> Config {
        Config {
            write_buffer_size: config.write_buffer_size,
            read_buffer_size: config.read_buffer_size,
        }
    }
}

impl Default for Config {
    fn default() -> Config {
        Config {
            write_buffer_size: 4096,
            read_buffer_size: 4096,
        }
    }
}

/// Create a datalink channel using the WinPcap library.
#[inline]
pub fn channel(network_interface: &NetworkInterface, config: Config) -> io::Result<super::Channel> {
    let mut read_buffer = Vec::new();
    read_buffer.resize(config.read_buffer_size, 0u8);

    let mut write_buffer = Vec::new();
    write_buffer.resize(config.write_buffer_size, 0u8);

    let adapter = unsafe {
        let net_if_str = CString::new(network_interface.name.as_bytes()).unwrap();
        winpcap::PacketOpenAdapter(net_if_str.as_ptr() as *mut ctypes::c_char)
    };
    if adapter.is_null() {
        return Err(io::Error::last_os_error());
    }

    let ret = unsafe { winpcap::PacketSetHwFilter(adapter, winpcap::NDIS_PACKET_TYPE_PROMISCUOUS) };
    if ret == 0 {
        return Err(io::Error::last_os_error());
    }

    // Set kernel buffer size
    let ret = unsafe { winpcap::PacketSetBuff(adapter, config.read_buffer_size as ctypes::c_int) };
    if ret == 0 {
        return Err(io::Error::last_os_error());
    }

    // Immediate mode
    let ret = unsafe { winpcap::PacketSetMinToCopy(adapter, 1) };
    if ret == 0 {
        return Err(io::Error::last_os_error());
    }

    let read_packet = unsafe { winpcap::PacketAllocatePacket() };
    if read_packet.is_null() {
        unsafe {
            winpcap::PacketCloseAdapter(adapter);
        }
        return Err(io::Error::last_os_error());
    }

    unsafe {
        winpcap::PacketInitPacket(
            read_packet,
            read_buffer.as_mut_ptr() as winpcap::PVOID,
            config.read_buffer_size as winpcap::UINT,
        )
    }

    let write_packet = unsafe { winpcap::PacketAllocatePacket() };
    if write_packet.is_null() {
        unsafe {
            winpcap::PacketFreePacket(read_packet);
            winpcap::PacketCloseAdapter(adapter);
        }
        return Err(io::Error::last_os_error());
    }

    unsafe {
        winpcap::PacketInitPacket(
            write_packet,
            write_buffer.as_mut_ptr() as winpcap::PVOID,
            config.write_buffer_size as winpcap::UINT,
        )
    }

    let adapter = Arc::new(WinPcapAdapter { adapter: adapter });
    let sender = Box::new(DataLinkSenderImpl {
        adapter: adapter.clone(),
        _write_buffer: write_buffer,
        packet: WinPcapPacket {
            packet: write_packet,
        },
    });
    let receiver = Box::new(DataLinkReceiverImpl {
        adapter: adapter,
        _read_buffer: read_buffer,
        packet: WinPcapPacket {
            packet: read_packet,
        },
        // Enough room for minimally sized packets without reallocating
        packets: VecDeque::with_capacity(unsafe { (*read_packet).Length } as usize / 64),
    });
    Ok(super::Channel::Ethernet(sender, receiver))
}

struct DataLinkSenderImpl {
    adapter: Arc<WinPcapAdapter>,
    _write_buffer: Vec<u8>,
    packet: WinPcapPacket,
}

impl DataLinkSender for DataLinkSenderImpl {
    #[inline]
    fn build_and_send(
        &mut self,
        num_packets: usize,
        packet_size: usize,
        func: &mut dyn FnMut(&mut [u8]),
    ) -> Option<io::Result<()>> {
        let len = num_packets * packet_size;
        if len >= unsafe { (*self.packet.packet).Length } as usize {
            None
        } else {
            let min = unsafe { cmp::min((*self.packet.packet).Length as usize, len) };
            let slice: &mut [u8] =
                unsafe { slice::from_raw_parts_mut((*self.packet.packet).Buffer as *mut u8, min) };
            for chunk in slice.chunks_mut(packet_size) {
                func(chunk);

                // Make sure the right length of packet is sent
                let old_len = unsafe { (*self.packet.packet).Length };
                unsafe {
                    (*self.packet.packet).Length = packet_size as u32;
                }

                let ret = unsafe {
                    winpcap::PacketSendPacket(self.adapter.adapter, self.packet.packet, 0)
                };

                unsafe {
                    (*self.packet.packet).Length = old_len;
                }

                if ret == 0 {
                    return Some(Err(io::Error::last_os_error()));
                }
            }
            Some(Ok(()))
        }
    }

    #[inline]
    fn send_to(&mut self, packet: &[u8], _dst: Option<NetworkInterface>) -> Option<io::Result<()>> {
        self.build_and_send(1, packet.len(), &mut |eh: &mut [u8]| {
            eh.copy_from_slice(packet);
        })
    }
}

unsafe impl Send for DataLinkSenderImpl {}
unsafe impl Sync for DataLinkSenderImpl {}

struct DataLinkReceiverImpl {
    adapter: Arc<WinPcapAdapter>,
    _read_buffer: Vec<u8>,
    packet: WinPcapPacket,
    packets: VecDeque<(usize, usize)>,
}

unsafe impl Send for DataLinkReceiverImpl {}
unsafe impl Sync for DataLinkReceiverImpl {}

impl DataLinkReceiver for DataLinkReceiverImpl {
    fn next(&mut self) -> io::Result<&[u8]> {
        // NOTE Most of the logic here is identical to FreeBSD/OS X
        while self.packets.is_empty() {
            let ret = unsafe {
                winpcap::PacketReceivePacket(self.adapter.adapter, self.packet.packet, 0)
            };
            let buflen = match ret {
                0 => return Err(io::Error::last_os_error()),
                _ => unsafe { (*self.packet.packet).ulBytesReceived as isize },
            };
            let mut ptr = unsafe { (*self.packet.packet).Buffer };
            let end = unsafe { (*self.packet.packet).Buffer.offset(buflen) };
            while ptr < end {
                unsafe {
                    let packet: *const bpf::bpf_hdr = mem::transmute(ptr);
                    let start = ptr as isize + (*packet).bh_hdrlen as isize
                        - (*self.packet.packet).Buffer as isize;
                    self.packets
                        .push_back((start as usize, (*packet).bh_caplen as usize));
                    let offset = (*packet).bh_hdrlen as isize + (*packet).bh_caplen as isize;
                    ptr = ptr.offset(bpf::BPF_WORDALIGN(offset));
                }
            }
        }
        let (start, len) = self.packets.pop_front().unwrap();
        let slice = unsafe {
            let data = (*self.packet.packet).Buffer as usize + start;
            slice::from_raw_parts(data as *const u8, len)
        };
        Ok(slice)
    }
}

/// Get a list of available network interfaces for the current machine.
pub fn interfaces() -> Vec<NetworkInterface> {
    // use super::bindings::winpcap;

    let mut adapters_size = 0u32;

    unsafe {
        let mut tmp: winpcap::IP_ADAPTER_INFO = mem::zeroed();
        // FIXME [windows] This only gets IPv4 addresses - should use
        // GetAdaptersAddresses
        winpcap::GetAdaptersInfo(&mut tmp, &mut adapters_size);
    }

    let mut vec_size = adapters_size / mem::size_of::<winpcap::IP_ADAPTER_INFO>() as u32;
    if adapters_size % mem::size_of::<winpcap::IP_ADAPTER_INFO>() as u32 != 0 {
        vec_size += 1;
    }
    let mut adapters = Vec::with_capacity(vec_size as usize);

    // FIXME [windows] Check return code
    unsafe {
        winpcap::GetAdaptersInfo(adapters.as_mut_ptr(), &mut adapters_size);
    }

    // Create a complete list of NetworkInterfaces for the machine
    let mut cursor = adapters.as_mut_ptr();
    let mut all_ifaces = Vec::with_capacity(vec_size as usize);
    while !cursor.is_null() {
        let mac = unsafe {
            MacAddr(
                (*cursor).Address[0],
                (*cursor).Address[1],
                (*cursor).Address[2],
                (*cursor).Address[3],
                (*cursor).Address[4],
                (*cursor).Address[5],
            )
        };
        let mut ip_cursor = unsafe { &mut (*cursor).IpAddressList as winpcap::PIP_ADDR_STRING };
        let mut ips = Vec::new();
        while !ip_cursor.is_null() {
            if let Ok(ip_network) = parse_ip_network(ip_cursor) {
                ips.push(ip_network);
            }
            ip_cursor = unsafe { (*ip_cursor).Next };
        }

        unsafe {
            let name_str_ptr = (*cursor).AdapterName.as_ptr() as *const i8;

            let bytes = CStr::from_ptr(name_str_ptr).to_bytes();
            let name_str = from_utf8_unchecked(bytes).to_owned();

            let description_str_ptr = (*cursor).Description.as_ptr() as *const i8;
            let bytes = CStr::from_ptr(description_str_ptr).to_bytes();
            let description_str = from_utf8_unchecked(bytes).to_owned();

            all_ifaces.push(NetworkInterface {
                name: name_str,
                description: description_str,
                index: (*cursor).Index,
                mac: Some(mac),
                ips: ips,
                // flags: (*cursor).Type, // FIXME [windows]
                flags: 0,
            });

            cursor = (*cursor).Next;
        }
    }

    let mut buf = vec![0u8; 4096];
    let mut buflen = buf.len() as u32;

    if unsafe { winpcap::PacketGetAdapterNames(buf.as_mut_ptr() as *mut i8, &mut buflen) } == 0 {
        buf.resize(buflen as usize, 0);

        // Second call should now work with the correct buffer size. If not, this may be
        // due to some privilege or other unforseen issue.
        if unsafe { winpcap::PacketGetAdapterNames(buf.as_mut_ptr() as *mut i8, &mut buflen) } == 0
        {
            panic!("Unable to get interface list despite increasing buffer size");
        }
    }

    let buf_str = unsafe { from_utf8_unchecked(&buf) };
    let iface_names = buf_str.split("\0\0").next();
    let mut vec = Vec::new();

    // Return only supported adapters
    match iface_names {
        Some(iface_names) => {
            for iface in iface_names.split('\0') {
                let name = iface.to_owned();
                let next = all_ifaces
                    .iter()
                    .filter(|x| name[..].ends_with(&x.name[..]))
                    .next();
                if next.is_some() {
                    let mut iface = next.unwrap().clone();
                    iface.name = name;
                    vec.push(iface);
                }
            }
        }
        None => (),
    };

    vec
}

fn parse_ip_network(ip_cursor: winpcap::PIP_ADDR_STRING) -> Result<IpNetwork, ()> {
    let ip_str_ptr = unsafe { &(*ip_cursor) }.IpAddress.String.as_ptr() as *const i8;
    let ip_bytes = unsafe { CStr::from_ptr(ip_str_ptr).to_bytes() };
    let ip_str = unsafe { from_utf8_unchecked(ip_bytes).to_owned() };
    let ip = ip_str.parse().map_err(|_| ())?;

    let mask_str_ptr = unsafe { &(*ip_cursor) }.IpMask.String.as_ptr() as *const i8;
    let mask_bytes = unsafe { CStr::from_ptr(mask_str_ptr).to_bytes() };
    let mask_str = unsafe { from_utf8_unchecked(mask_bytes).to_owned() };
    let mask = mask_str.parse().map_err(|_| ())?;

    let prefix = ip_mask_to_prefix(mask).map_err(|_| ())?;
    IpNetwork::new(ip, prefix).map_err(|_| ())
}
<commit_msg>Fix c_void pointer cannot be offset<commit_after>// Copyright (c) 2014-2016 Robert Clipsham <robert@octarineparrot.com>
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Support for sending and receiving data link layer packets using the WinPcap library.

extern crate winapi;

use super::bindings::{bpf, winpcap};
use super::{DataLinkReceiver, DataLinkSender, MacAddr, NetworkInterface};

use ipnetwork::{ip_mask_to_prefix, IpNetwork};

use self::winapi::ctypes;
use std::cmp;
use std::collections::VecDeque;
use std::ffi::{CStr, CString};
use std::io;
use std::mem;
use std::slice;
use std::str::from_utf8_unchecked;
use std::sync::Arc;
use self::winapi::ctypes::c_char;

struct WinPcapAdapter {
    adapter: winpcap::LPADAPTER,
}

impl Drop for WinPcapAdapter {
    fn drop(&mut self) {
        unsafe {
            winpcap::PacketCloseAdapter(self.adapter);
        }
    }
}

struct WinPcapPacket {
    packet: winpcap::LPPACKET,
}

impl Drop for WinPcapPacket {
    fn drop(&mut self) {
        unsafe {
            winpcap::PacketFreePacket(self.packet);
        }
    }
}

/// The WinPcap's specific configuration.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Config {
    /// The size of buffer to use when writing packets. Defaults to 4096.
    pub write_buffer_size: usize,

    /// The size of buffer to use when reading packets. Defaults to 4096.
    pub read_buffer_size: usize,
}

impl<'a> From<&'a super::Config> for Config {
    fn from(config: &super::Config) -> Config {
        Config {
            write_buffer_size: config.write_buffer_size,
            read_buffer_size: config.read_buffer_size,
        }
    }
}

impl Default for Config {
    fn default() -> Config {
        Config {
            write_buffer_size: 4096,
            read_buffer_size: 4096,
        }
    }
}

/// Create a datalink channel using the WinPcap library.
#[inline]
pub fn channel(network_interface: &NetworkInterface, config: Config) -> io::Result<super::Channel> {
    let mut read_buffer = Vec::new();
    read_buffer.resize(config.read_buffer_size, 0u8);

    let mut write_buffer = Vec::new();
    write_buffer.resize(config.write_buffer_size, 0u8);

    let adapter = unsafe {
        let net_if_str = CString::new(network_interface.name.as_bytes()).unwrap();
        winpcap::PacketOpenAdapter(net_if_str.as_ptr() as *mut ctypes::c_char)
    };
    if adapter.is_null() {
        return Err(io::Error::last_os_error());
    }

    let ret = unsafe { winpcap::PacketSetHwFilter(adapter, winpcap::NDIS_PACKET_TYPE_PROMISCUOUS) };
    if ret == 0 {
        return Err(io::Error::last_os_error());
    }

    // Set kernel buffer size
    let ret = unsafe { winpcap::PacketSetBuff(adapter, config.read_buffer_size as ctypes::c_int) };
    if ret == 0 {
        return Err(io::Error::last_os_error());
    }

    // Immediate mode
    let ret = unsafe { winpcap::PacketSetMinToCopy(adapter, 1) };
    if ret == 0 {
        return Err(io::Error::last_os_error());
    }

    let read_packet = unsafe { winpcap::PacketAllocatePacket() };
    if read_packet.is_null() {
        unsafe {
            winpcap::PacketCloseAdapter(adapter);
        }
        return Err(io::Error::last_os_error());
    }

    unsafe {
        winpcap::PacketInitPacket(
            read_packet,
            read_buffer.as_mut_ptr() as winpcap::PVOID,
            config.read_buffer_size as winpcap::UINT,
        )
    }

    let write_packet = unsafe { winpcap::PacketAllocatePacket() };
    if write_packet.is_null() {
        unsafe {
            winpcap::PacketFreePacket(read_packet);
            winpcap::PacketCloseAdapter(adapter);
        }
        return Err(io::Error::last_os_error());
    }

    unsafe {
        winpcap::PacketInitPacket(
            write_packet,
            write_buffer.as_mut_ptr() as winpcap::PVOID,
            config.write_buffer_size as winpcap::UINT,
        )
    }

    let adapter = Arc::new(WinPcapAdapter { adapter: adapter });
    let sender = Box::new(DataLinkSenderImpl {
        adapter: adapter.clone(),
        _write_buffer: write_buffer,
        packet: WinPcapPacket {
            packet: write_packet,
        },
    });
    let receiver = Box::new(DataLinkReceiverImpl {
        adapter: adapter,
        _read_buffer: read_buffer,
        packet: WinPcapPacket {
            packet: read_packet,
        },
        // Enough room for minimally sized packets without reallocating
        packets: VecDeque::with_capacity(unsafe { (*read_packet).Length } as usize / 64),
    });
    Ok(super::Channel::Ethernet(sender, receiver))
}

struct DataLinkSenderImpl {
    adapter: Arc<WinPcapAdapter>,
    _write_buffer: Vec<u8>,
    packet: WinPcapPacket,
}

impl DataLinkSender for DataLinkSenderImpl {
    #[inline]
    fn build_and_send(
        &mut self,
        num_packets: usize,
        packet_size: usize,
        func: &mut dyn FnMut(&mut [u8]),
    ) -> Option<io::Result<()>> {
        let len = num_packets * packet_size;
        if len >= unsafe { (*self.packet.packet).Length } as usize {
            None
        } else {
            let min = unsafe { cmp::min((*self.packet.packet).Length as usize, len) };
            let slice: &mut [u8] =
                unsafe { slice::from_raw_parts_mut((*self.packet.packet).Buffer as *mut u8, min) };
            for chunk in slice.chunks_mut(packet_size) {
                func(chunk);

                // Make sure the right length of packet is sent
                let old_len = unsafe { (*self.packet.packet).Length };
                unsafe {
                    (*self.packet.packet).Length = packet_size as u32;
                }

                let ret = unsafe {
                    winpcap::PacketSendPacket(self.adapter.adapter, self.packet.packet, 0)
                };

                unsafe {
                    (*self.packet.packet).Length = old_len;
                }

                if ret == 0 {
                    return Some(Err(io::Error::last_os_error()));
                }
            }
            Some(Ok(()))
        }
    }

    #[inline]
    fn send_to(&mut self, packet: &[u8], _dst: Option<NetworkInterface>) -> Option<io::Result<()>> {
        self.build_and_send(1, packet.len(), &mut |eh: &mut [u8]| {
            eh.copy_from_slice(packet);
        })
    }
}

unsafe impl Send for DataLinkSenderImpl {}
unsafe impl Sync for DataLinkSenderImpl {}

struct DataLinkReceiverImpl {
    adapter: Arc<WinPcapAdapter>,
    _read_buffer: Vec<u8>,
    packet: WinPcapPacket,
    packets: VecDeque<(usize, usize)>,
}

unsafe impl Send for DataLinkReceiverImpl {}
unsafe impl Sync for DataLinkReceiverImpl {}

impl DataLinkReceiver for DataLinkReceiverImpl {
    fn next(&mut self) -> io::Result<&[u8]> {
        // NOTE Most of the logic here is identical to FreeBSD/OS X
        while self.packets.is_empty() {
            let ret = unsafe {
                winpcap::PacketReceivePacket(self.adapter.adapter, self.packet.packet, 0)
            };
            let buflen = match ret {
                0 => return Err(io::Error::last_os_error()),
                _ => unsafe { (*self.packet.packet).ulBytesReceived as isize },
            };
            let mut ptr = unsafe { (*self.packet.packet).Buffer  as *mut c_char};
            let end = unsafe { ((*self.packet.packet).Buffer as *mut c_char).offset(buflen) };
            while ptr < end {
                unsafe {
                    let packet: *const bpf::bpf_hdr = mem::transmute(ptr);
                    let start = ptr as isize + (*packet).bh_hdrlen as isize
                        - (*self.packet.packet).Buffer as isize;
                    self.packets
                        .push_back((start as usize, (*packet).bh_caplen as usize));
                    let offset = (*packet).bh_hdrlen as isize + (*packet).bh_caplen as isize;
                    ptr = ptr.offset(bpf::BPF_WORDALIGN(offset));
                }
            }
        }
        let (start, len) = self.packets.pop_front().unwrap();
        let slice = unsafe {
            let data = (*self.packet.packet).Buffer as usize + start;
            slice::from_raw_parts(data as *const u8, len)
        };
        Ok(slice)
    }
}

/// Get a list of available network interfaces for the current machine.
pub fn interfaces() -> Vec<NetworkInterface> {
    // use super::bindings::winpcap;

    let mut adapters_size = 0u32;

    unsafe {
        let mut tmp: winpcap::IP_ADAPTER_INFO = mem::zeroed();
        // FIXME [windows] This only gets IPv4 addresses - should use
        // GetAdaptersAddresses
        winpcap::GetAdaptersInfo(&mut tmp, &mut adapters_size);
    }

    let mut vec_size = adapters_size / mem::size_of::<winpcap::IP_ADAPTER_INFO>() as u32;
    if adapters_size % mem::size_of::<winpcap::IP_ADAPTER_INFO>() as u32 != 0 {
        vec_size += 1;
    }
    let mut adapters = Vec::with_capacity(vec_size as usize);

    // FIXME [windows] Check return code
    unsafe {
        winpcap::GetAdaptersInfo(adapters.as_mut_ptr(), &mut adapters_size);
    }

    // Create a complete list of NetworkInterfaces for the machine
    let mut cursor = adapters.as_mut_ptr();
    let mut all_ifaces = Vec::with_capacity(vec_size as usize);
    while !cursor.is_null() {
        let mac = unsafe {
            MacAddr(
                (*cursor).Address[0],
                (*cursor).Address[1],
                (*cursor).Address[2],
                (*cursor).Address[3],
                (*cursor).Address[4],
                (*cursor).Address[5],
            )
        };
        let mut ip_cursor = unsafe { &mut (*cursor).IpAddressList as winpcap::PIP_ADDR_STRING };
        let mut ips = Vec::new();
        while !ip_cursor.is_null() {
            if let Ok(ip_network) = parse_ip_network(ip_cursor) {
                ips.push(ip_network);
            }
            ip_cursor = unsafe { (*ip_cursor).Next };
        }

        unsafe {
            let name_str_ptr = (*cursor).AdapterName.as_ptr() as *const i8;

            let bytes = CStr::from_ptr(name_str_ptr).to_bytes();
            let name_str = from_utf8_unchecked(bytes).to_owned();

            let description_str_ptr = (*cursor).Description.as_ptr() as *const i8;
            let bytes = CStr::from_ptr(description_str_ptr).to_bytes();
            let description_str = from_utf8_unchecked(bytes).to_owned();

            all_ifaces.push(NetworkInterface {
                name: name_str,
                description: description_str,
                index: (*cursor).Index,
                mac: Some(mac),
                ips: ips,
                // flags: (*cursor).Type, // FIXME [windows]
                flags: 0,
            });

            cursor = (*cursor).Next;
        }
    }

    let mut buf = vec![0u8; 4096];
    let mut buflen = buf.len() as u32;

    if unsafe { winpcap::PacketGetAdapterNames(buf.as_mut_ptr() as *mut i8, &mut buflen) } == 0 {
        buf.resize(buflen as usize, 0);

        // Second call should now work with the correct buffer size. If not, this may be
        // due to some privilege or other unforseen issue.
        if unsafe { winpcap::PacketGetAdapterNames(buf.as_mut_ptr() as *mut i8, &mut buflen) } == 0
        {
            panic!("Unable to get interface list despite increasing buffer size");
        }
    }

    let buf_str = unsafe { from_utf8_unchecked(&buf) };
    let iface_names = buf_str.split("\0\0").next();
    let mut vec = Vec::new();

    // Return only supported adapters
    match iface_names {
        Some(iface_names) => {
            for iface in iface_names.split('\0') {
                let name = iface.to_owned();
                let next = all_ifaces
                    .iter()
                    .filter(|x| name[..].ends_with(&x.name[..]))
                    .next();
                if next.is_some() {
                    let mut iface = next.unwrap().clone();
                    iface.name = name;
                    vec.push(iface);
                }
            }
        }
        None => (),
    };

    vec
}

fn parse_ip_network(ip_cursor: winpcap::PIP_ADDR_STRING) -> Result<IpNetwork, ()> {
    let ip_str_ptr = unsafe { &(*ip_cursor) }.IpAddress.String.as_ptr() as *const i8;
    let ip_bytes = unsafe { CStr::from_ptr(ip_str_ptr).to_bytes() };
    let ip_str = unsafe { from_utf8_unchecked(ip_bytes).to_owned() };
    let ip = ip_str.parse().map_err(|_| ())?;

    let mask_str_ptr = unsafe { &(*ip_cursor) }.IpMask.String.as_ptr() as *const i8;
    let mask_bytes = unsafe { CStr::from_ptr(mask_str_ptr).to_bytes() };
    let mask_str = unsafe { from_utf8_unchecked(mask_bytes).to_owned() };
    let mask = mask_str.parse().map_err(|_| ())?;

    let prefix = ip_mask_to_prefix(mask).map_err(|_| ())?;
    IpNetwork::new(ip, prefix).map_err(|_| ())
}
<|endoftext|>


<commit_before>use storage::file::FileHeaderSpec as FHS;
use storage::file::FileHeaderData as FHD;

pub fn get_spec() -> FHS {
    FHS::Map { keys: vec![ url_key(), tags_key() ] }
}

fn url_key() -> FHS {
    FHS::Key { name: String::from("URL"), value_type: Box::new(FHS::Text) }
}

fn tags_key() -> FHS {
    FHS::Key { name: String::from("TAGS"), value_type: Box::new(text_array()) }
}

fn text_array() -> FHS {
    FHS::Array { allowed_types: vec![FHS::Text] }
}

pub fn build_header(url: &String, tags: &Vec<String>) -> FHD {
    FHD::Map {
        keys: vec![
            FHD::Key {
                name: String::from("URL"),
                value: Box::new(FHD::Text(url.clone()))
            },
            FHD::Key {
                name: String::from("TAGS"),
                value: Box::new(FHD::Text(tags.connect(",")))
            }
        ]
    }
}

<commit_msg>Use real array instead of text array to save tags<commit_after>use storage::file::FileHeaderSpec as FHS;
use storage::file::FileHeaderData as FHD;

pub fn get_spec() -> FHS {
    FHS::Map { keys: vec![ url_key(), tags_key() ] }
}

fn url_key() -> FHS {
    FHS::Key { name: String::from("URL"), value_type: Box::new(FHS::Text) }
}

fn tags_key() -> FHS {
    FHS::Key { name: String::from("TAGS"), value_type: Box::new(text_array()) }
}

fn text_array() -> FHS {
    FHS::Array { allowed_types: vec![FHS::Text] }
}

pub fn build_header(url: &String, tags: &Vec<String>) -> FHD {
    FHD::Map {
        keys: vec![
            FHD::Key {
                name: String::from("URL"),
                value: Box::new(FHD::Text(url.clone()))
            },
            FHD::Key {
                name: String::from("TAGS"),
                value: Box::new(build_tag_array(tags))
            }
        ]
    }
}

fn build_tag_array(tags: &Vec<String>) -> FHD {
    let texttags = tags.into_iter().map(|t| FHD::Text(t.clone())).collect();
    FHD::Array { values: Box::new(texttags) }
}

<|endoftext|>


<commit_before>extern crate serde_json;

#[macro_use]
mod spec;
mod block;
mod view;
mod config;

use std::borrow::Borrow;
use std::ops::Deref;

use time::Timespec;

use ::crypto::{Hash, hash};
use ::messages::{Precommit, Message, ConfigMessage, ServiceTx, ConfigPropose, ConfigVote, AnyTx};

use ::storage::{StorageValue, Patch, Database, Fork, Error, Map, List};

pub use self::block::Block;
pub use self::view::View;

pub use self::config::{StoredConfiguration, ConsensusCfg};

pub use self::view::ConfigurationData;

pub trait Blockchain: Sized + Clone + Send + Sync + 'static
    where Self: Deref<Target = <Self as Blockchain>::Database>
{
    type View: View<<<Self as Blockchain>::Database as Database>::Fork, Transaction=Self::Transaction>;
    type Database: Database;
    type Transaction: Message + StorageValue;

    fn last_hash(&self) -> Result<Option<Hash>, Error> {
        self.view().heights().last()
    }

    fn last_block(&self) -> Result<Option<Block>, Error> {
        let view = self.view();
        Ok(match view.heights().last()? {
            Some(hash) => Some(view.blocks().get(&hash)?.unwrap()),
            None => None,
        })
    }

    fn verify_tx(tx: &Self::Transaction) -> bool;
    fn state_hash(fork: &Self::View) -> Result<Hash, Error>;
    fn execute(fork: &Self::View, tx: &Self::Transaction) -> Result<(), Error>;
    // FIXME make private
    fn execute_service_tx(view: &Self::View, tx: &ServiceTx) -> Result<(), Error> {
        match *tx {
            ServiceTx::ConfigChange(ref config_message) => {
                Ok(Self::execute_config_change(view, config_message))
            }
        }
    }

    // TODO use Iterator to avoid memory allocations?
    fn create_patch(&self,
                    height: u64,
                    round: u32,
                    time: Timespec,
                    txs: &[(Hash, AnyTx<Self::Transaction>)])
                    -> Result<(Hash, Vec<Hash>, Patch), Error> {
        // Get last hash
        let last_hash = self.last_hash()?.unwrap_or_else(|| hash(&[]));
        // Create fork
        let fork = self.view();
        // Save & execute transactions
        let mut tx_hashes = Vec::new();
        for &(hash, ref tx) in txs {

            match *tx {
                AnyTx::Application(ref tx) => Self::execute(&fork, tx)?,
                AnyTx::Service(ref tx) => Self::execute_service_tx(&fork, tx)?,
            }
            fork.transactions()
                .put(&hash, tx.clone())
                .unwrap();
            fork.block_txs(height)
                .append(hash)
                .unwrap();
            tx_hashes.push(hash);
        }
        // Get tx hash
        let tx_hash = fork.block_txs(height).root_hash()?;
        // Get state hash
        let state_hash = Self::state_hash(&fork)?;
        // Create block
        let block = Block::new(height, round, time, &last_hash, &tx_hash, &state_hash);
        // Eval block hash
        let block_hash = block.hash();
        // Update height
        // TODO: check that height == propose.height
        fork.heights().append(block_hash).is_ok();
        // Save block
        fork.blocks().put(&block_hash, block).is_ok();
        Ok((block_hash, tx_hashes, fork.changes()))
    }

    fn execute_config_change(view: &Self::View, config_message: &ConfigMessage) {
        match *config_message {
            ConfigMessage::ConfigPropose(ref config_propose_tx) => {
                Self::handle_config_propose(view, config_propose_tx);
            }
            ConfigMessage::ConfigVote(ref config_vote_tx) => {
                Self::handle_config_vote(view, config_vote_tx);
            }
        }
    }

    fn last_block_(view: &Self::View) -> Result<Option<Block>, Error> {
        Ok(match view.heights().last()? {
            Some(hash) => Some(view.blocks().get(&hash)?.unwrap()),
            None => None,
        })
    }
    fn get_height(view: &Self::View) -> u64 {
        if let Ok(Some(last_block)) = Self::last_block_(view){
            return last_block.height() + 1;
        }
        0
    }

    fn get_actual_configuration(view: &Self::View) -> Option<StoredConfiguration> {

        let h = Self::get_height(view);

        let heights = view.configs_heights();

        if let Ok(height_values) = heights.values() {

            if let Some(idx) = height_values.into_iter()
                .rposition(|r| u64::from(r) <= h) {
                if let Ok(Some(height)) = heights.get(idx as u64) {
                    return Self::get_configuration_at_height(view, height.into());
                }
            }
        }
        None
    }

    fn get_configuration_at_height(view: &Self::View, height: u64) -> Option<StoredConfiguration> {
        let configs = view.configs();
        if let Ok(config) = configs.get(&height.into()) {
            match StoredConfiguration::deserialize(&config.unwrap()) {
                Ok(configuration) => {
                    return Some(configuration);
                }
                Err(_) => {
                    error!("Can't parse found configuration at height: {}", height);
                }
            }
        }
        None
    }

    fn handle_config_propose(view: &Self::View, config_propose: &ConfigPropose) {

        if let Some(config) = Self::get_actual_configuration(view) {
            if !config.validators.contains(config_propose.from()) {
                error!("ConfigPropose from unknown validator: {:?}",
                       config_propose.from());
                return;
            }

            let hash = <ConfigPropose as Message>::hash(config_propose);
            if view.config_proposes().get(&hash).unwrap().is_some() {
                error!("Received config_propose has already been handled, msg={:?}",
                       config_propose);
                return;
            }

            trace!("Handle ConfigPropose");
            let _ = view.config_proposes().put(&hash, config_propose.clone());
        }
    }

    fn handle_config_vote(view: &Self::View, config_vote: &ConfigVote) {

        if let Some(config) = Self::get_actual_configuration(view) {

            if !config.validators.contains(config_vote.from()) {
                error!("ConfigVote from unknown validator: {:?}",
                       config_vote.from());
                return;
            }

            if view.config_proposes().get(config_vote.hash_propose()).unwrap().is_some() {
                error!("Received config_vote for unknown transaciton, msg={:?}",
                       config_vote);
                return;
            }

            if let Some(vote) = view.config_votes().get(config_vote.from()).unwrap() {
                if vote.seed() != config_vote.seed() - 1 {
                    error!("Received config_vote with wrong seed, msg={:?}",
                           config_vote);
                    return;
                }
            }

            let msg = config_vote.clone();
            let _ = view.config_votes().put(msg.from(), config_vote.clone());

            let mut votes_count = 0;
            for pub_key in config.validators.clone() {
                if let Some(vote) = view.config_votes().get(&pub_key).unwrap() {
                    if !vote.revoke() {
                        votes_count += 1;
                    }
                }
            }

            if votes_count >= 2 / 3 * config.validators.len() {
                if let Some(config_propose) = view.config_proposes()
                    .get(config_vote.hash_propose())
                    .unwrap() {
                    let height_bytecode = config_propose.actual_from_height().into();
                    view.configs().put(&height_bytecode, config_propose.config().to_vec()).unwrap();
                    view.configs_heights().append(height_bytecode).unwrap();
                    // TODO: clear storages
                }
            }
        }
    }

    fn commit<'a, I: Iterator<Item = &'a Precommit>>(&self,
                                                     block_hash: Hash,
                                                     patch: &Patch,
                                                     precommits: I)
                                                     -> Result<(), Error> {
        let patch = {
            let view = self.view();
            view.merge(patch);

            for precommit in precommits {
                view.precommits(&block_hash).append(precommit.clone())?;
            }

            view.changes()
        };

        self.merge(&patch)
    }

    fn view(&self) -> Self::View {
        Self::View::from_fork(self.borrow().fork())
    }

    fn merge(&self, patch: &Patch) -> Result<(), Error> {
        self.deref().merge(patch)
    }
}
<commit_msg>Configuration panic fixed<commit_after>extern crate serde_json;

#[macro_use]
mod spec;
mod block;
mod view;
mod config;

use std::borrow::Borrow;
use std::ops::Deref;

use time::Timespec;

use ::crypto::{Hash, hash};
use ::messages::{Precommit, Message, ConfigMessage, ServiceTx, ConfigPropose, ConfigVote, AnyTx};

use ::storage::{StorageValue, Patch, Database, Fork, Error, Map, List};

pub use self::block::Block;
pub use self::view::View;

pub use self::config::{StoredConfiguration, ConsensusCfg};

pub use self::view::ConfigurationData;

pub trait Blockchain: Sized + Clone + Send + Sync + 'static
    where Self: Deref<Target = <Self as Blockchain>::Database>
{
    type View: View<<<Self as Blockchain>::Database as Database>::Fork, Transaction=Self::Transaction>;
    type Database: Database;
    type Transaction: Message + StorageValue;

    fn last_hash(&self) -> Result<Option<Hash>, Error> {
        self.view().heights().last()
    }

    fn last_block(&self) -> Result<Option<Block>, Error> {
        let view = self.view();
        Ok(match view.heights().last()? {
            Some(hash) => Some(view.blocks().get(&hash)?.unwrap()),
            None => None,
        })
    }

    fn verify_tx(tx: &Self::Transaction) -> bool;
    fn state_hash(fork: &Self::View) -> Result<Hash, Error>;
    fn execute(fork: &Self::View, tx: &Self::Transaction) -> Result<(), Error>;
    // FIXME make private
    fn execute_service_tx(view: &Self::View, tx: &ServiceTx) -> Result<(), Error> {
        match *tx {
            ServiceTx::ConfigChange(ref config_message) => {
                Ok(Self::execute_config_change(view, config_message))
            }
        }
    }

    // TODO use Iterator to avoid memory allocations?
    fn create_patch(&self,
                    height: u64,
                    round: u32,
                    time: Timespec,
                    txs: &[(Hash, AnyTx<Self::Transaction>)])
                    -> Result<(Hash, Vec<Hash>, Patch), Error> {
        // Get last hash
        let last_hash = self.last_hash()?.unwrap_or_else(|| hash(&[]));
        // Create fork
        let fork = self.view();
        // Save & execute transactions
        let mut tx_hashes = Vec::new();
        for &(hash, ref tx) in txs {

            match *tx {
                AnyTx::Application(ref tx) => Self::execute(&fork, tx)?,
                AnyTx::Service(ref tx) => Self::execute_service_tx(&fork, tx)?,
            }
            fork.transactions()
                .put(&hash, tx.clone())
                .unwrap();
            fork.block_txs(height)
                .append(hash)
                .unwrap();
            tx_hashes.push(hash);
        }
        // Get tx hash
        let tx_hash = fork.block_txs(height).root_hash()?;
        // Get state hash
        let state_hash = Self::state_hash(&fork)?;
        // Create block
        let block = Block::new(height, round, time, &last_hash, &tx_hash, &state_hash);
        // Eval block hash
        let block_hash = block.hash();
        // Update height
        // TODO: check that height == propose.height
        fork.heights().append(block_hash).is_ok();
        // Save block
        fork.blocks().put(&block_hash, block).is_ok();
        Ok((block_hash, tx_hashes, fork.changes()))
    }

    fn execute_config_change(view: &Self::View, config_message: &ConfigMessage) {
        match *config_message {
            ConfigMessage::ConfigPropose(ref config_propose_tx) => {
                Self::handle_config_propose(view, config_propose_tx);
            }
            ConfigMessage::ConfigVote(ref config_vote_tx) => {
                Self::handle_config_vote(view, config_vote_tx);
            }
        }
    }

    fn last_block_(view: &Self::View) -> Result<Option<Block>, Error> {
        Ok(match view.heights().last()? {
            Some(hash) => Some(view.blocks().get(&hash)?.unwrap()),
            None => None,
        })
    }
    fn get_height(view: &Self::View) -> u64 {
        if let Ok(Some(last_block)) = Self::last_block_(view){
            return last_block.height() + 1;
        }
        0
    }

    fn get_actual_configuration(view: &Self::View) -> Option<StoredConfiguration> {

        let h = Self::get_height(view);

        let heights = view.configs_heights();

        if let Ok(height_values) = heights.values() {

            if let Some(idx) = height_values.into_iter()
                .rposition(|r| u64::from(r) <= h) {
                if let Ok(Some(height)) = heights.get(idx as u64) {
                    return Self::get_configuration_at_height(view, height.into());
                }
            }
        }
        None
    }

    fn get_configuration_at_height(view: &Self::View, height: u64) -> Option<StoredConfiguration> {
        let configs = view.configs();
        if let Ok(Some(config)) = configs.get(&height.into()) {
            match StoredConfiguration::deserialize(&config) {
                Ok(configuration) => {
                    return Some(configuration);
                }
                Err(_) => {
                    error!("Can't parse found configuration at height: {}", height);
                }
            }
        }
        None
    }

    fn handle_config_propose(view: &Self::View, config_propose: &ConfigPropose) {

        if let Some(config) = Self::get_actual_configuration(view) {
            if !config.validators.contains(config_propose.from()) {
                error!("ConfigPropose from unknown validator: {:?}",
                       config_propose.from());
                return;
            }

            let hash = <ConfigPropose as Message>::hash(config_propose);
            if view.config_proposes().get(&hash).unwrap().is_some() {
                error!("Received config_propose has already been handled, msg={:?}",
                       config_propose);
                return;
            }

            trace!("Handle ConfigPropose");
            let _ = view.config_proposes().put(&hash, config_propose.clone());
        }
    }

    fn handle_config_vote(view: &Self::View, config_vote: &ConfigVote) {

        if let Some(config) = Self::get_actual_configuration(view) {

            if !config.validators.contains(config_vote.from()) {
                error!("ConfigVote from unknown validator: {:?}",
                       config_vote.from());
                return;
            }

            if view.config_proposes().get(config_vote.hash_propose()).unwrap().is_some() {
                error!("Received config_vote for unknown transaciton, msg={:?}",
                       config_vote);
                return;
            }

            if let Some(vote) = view.config_votes().get(config_vote.from()).unwrap() {
                if vote.seed() != config_vote.seed() - 1 {
                    error!("Received config_vote with wrong seed, msg={:?}",
                           config_vote);
                    return;
                }
            }

            let msg = config_vote.clone();
            let _ = view.config_votes().put(msg.from(), config_vote.clone());

            let mut votes_count = 0;
            for pub_key in config.validators.clone() {
                if let Some(vote) = view.config_votes().get(&pub_key).unwrap() {
                    if !vote.revoke() {
                        votes_count += 1;
                    }
                }
            }

            if votes_count >= 2 / 3 * config.validators.len() {
                if let Some(config_propose) = view.config_proposes()
                    .get(config_vote.hash_propose())
                    .unwrap() {
                    let height_bytecode = config_propose.actual_from_height().into();
                    view.configs().put(&height_bytecode, config_propose.config().to_vec()).unwrap();
                    view.configs_heights().append(height_bytecode).unwrap();
                    // TODO: clear storages
                }
            }
        }
    }

    fn commit<'a, I: Iterator<Item = &'a Precommit>>(&self,
                                                     block_hash: Hash,
                                                     patch: &Patch,
                                                     precommits: I)
                                                     -> Result<(), Error> {
        let patch = {
            let view = self.view();
            view.merge(patch);

            for precommit in precommits {
                view.precommits(&block_hash).append(precommit.clone())?;
            }

            view.changes()
        };

        self.merge(&patch)
    }

    fn view(&self) -> Self::View {
        Self::View::from_fork(self.borrow().fork())
    }

    fn merge(&self, patch: &Patch) -> Result<(), Error> {
        self.deref().merge(patch)
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate docopt;
#[macro_use]
extern crate lazy_static;
extern crate libc;
#[cfg(feature = "re-pcre1")]
extern crate libpcre_sys;
extern crate memmap;
#[cfg(feature = "re-onig")]
extern crate onig;
#[cfg(any(
    feature = "re-rust",
    feature = "re-rust-bytes",
    feature = "re-rust-plugin",
  ))]
extern crate regex;
#[cfg(feature = "re-rust")]
extern crate regex_syntax;
extern crate rustc_serialize;

use std::str;

use docopt::Docopt;
use memmap::{Mmap, Protection};

mod ffi;

const USAGE: &'static str = "
Count the number of matches of <pattern> in <file>.

This compiles the pattern once and counts all successive non-overlapping
matches in <file>. <file> is memory mapped. Matching is done as if <file> were
a single string (it is not line oriented).

Since this tool includes compilation of the <pattern>, sufficiently large
haystacks should be used to amortize the cost of compilation. (e.g., >1MB.)

Usage:
    regex-run-one [options] [onig | pcre1 | pcre2 | re2 | rust | rust-bytes | tcl] <pattern> <file>

Options:
    -h, --help   Show this usage message.
";

#[derive(Debug, RustcDecodable)]
struct Args {
    arg_pattern: String,
    arg_file: String,
    cmd_onig: bool,
    cmd_pcre1: bool,
    cmd_pcre2: bool,
    cmd_re2: bool,
    cmd_rust: bool,
    cmd_rust_bytes: bool,
    cmd_tcl: bool,
}

fn main() {
    let args: Args = Docopt::new(USAGE).and_then(|d| d.decode())
                                       .unwrap_or_else(|e| e.exit());

    let mmap = Mmap::open_path(&args.arg_file, Protection::Read).unwrap();
    let haystack = unsafe { str::from_utf8(mmap.as_slice()).unwrap() };

    println!("{}", args.count(&haystack));
}

impl Args {
    fn count(&self, haystack: &str) -> usize {
        let pat = &self.arg_pattern;
        if self.cmd_onig {
            count_onig(pat, haystack)
        } else if self.cmd_pcre1 {
            count_pcre1(pat, haystack)
        } else if self.cmd_pcre2 {
            count_pcre2(pat, haystack)
        } else if self.cmd_re2 {
            count_re2(pat, haystack)
        } else if self.cmd_rust {
            count_rust(pat, haystack)
        } else if self.cmd_rust_bytes {
            count_rust_bytes(pat, haystack)
        } else if self.cmd_tcl {
            count_tcl(pat, haystack)
        } else {
            panic!("unreachable")
        }
    }
}

macro_rules! nada {
    ($feature:expr, $name:ident) => {
        #[cfg(not(feature = $feature))]
        fn $name(_pat: &str, _haystack: &str) -> usize {
            panic!("Support not enabled. Re-compile with '--features {}' \
                    to enable.", $feature)
        }
    }
}

nada!("re-onig", count_onig);
#[cfg(feature = "re-onig")]
fn count_onig(pat: &str, haystack: &str) -> usize {
    use ffi::onig::Regex;
    Regex::new(pat).unwrap().find_iter(haystack).count()
}

nada!("re-pcre1", count_pcre1);
#[cfg(feature = "re-pcre1")]
fn count_pcre1(pat: &str, haystack: &str) -> usize {
    use ffi::pcre1::Regex;
    Regex::new(pat).unwrap().find_iter(haystack).count()
}

nada!("re-pcre2", count_pcre2);
#[cfg(feature = "re-pcre2")]
fn count_pcre2(pat: &str, haystack: &str) -> usize {
    use ffi::pcre2::Regex;
    Regex::new(pat).unwrap().find_iter(haystack).count()
}

nada!("re-re2", count_re2);
#[cfg(feature = "re-re2")]
fn count_re2(pat: &str, haystack: &str) -> usize {
    use ffi::re2::Regex;
    Regex::new(pat).unwrap().find_iter(haystack).count()
}

nada!("re-rust", count_rust);
#[cfg(feature = "re-rust")]
fn count_rust(pat: &str, haystack: &str) -> usize {
    use regex::Regex;
    Regex::new(pat).unwrap().find_iter(haystack).count()
}

nada!("re-rust-bytes", count_rust_bytes);
#[cfg(feature = "re-rust-bytes")]
fn count_rust_bytes(pat: &str, haystack: &str) -> usize {
    use regex::bytes::Regex;
    Regex::new(pat).unwrap().find_iter(haystack.as_bytes()).count()
}

nada!("re-tcl", count_tcl);
#[cfg(feature = "re-tcl")]
fn count_tcl(pat: &str, haystack: &str) -> usize {
    use ffi::tcl::{Regex, Text};
    Regex::new(pat).unwrap().find_iter(&Text::new(haystack.to_owned())).count()
}
<commit_msg>Tweak benchmark harness.<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate docopt;
#[macro_use]
extern crate lazy_static;
extern crate libc;
#[cfg(feature = "re-pcre1")]
extern crate libpcre_sys;
extern crate memmap;
#[cfg(feature = "re-onig")]
extern crate onig;
#[cfg(any(
    feature = "re-rust",
    feature = "re-rust-bytes",
    feature = "re-rust-plugin",
  ))]
extern crate regex;
#[cfg(feature = "re-rust")]
extern crate regex_syntax;
extern crate rustc_serialize;

use std::str;

use docopt::Docopt;
use memmap::{Mmap, Protection};

mod ffi;

const USAGE: &'static str = "
Count the number of matches of <pattern> in <file>.

This compiles the pattern once and counts all successive non-overlapping
matches in <file>. <file> is memory mapped. Matching is done as if <file> were
a single string (it is not line oriented).

Since this tool includes compilation of the <pattern>, sufficiently large
haystacks should be used to amortize the cost of compilation. (e.g., >1MB.)

Usage:
    regex-run-one [options] [onig | pcre1 | pcre2 | re2 | rust | rust-bytes | tcl] <pattern> <file>
    regex-run-one [options] (-h | --help)

Options:
    -h, --help   Show this usage message.
";

#[derive(Debug, RustcDecodable)]
struct Args {
    arg_pattern: String,
    arg_file: String,
    cmd_onig: bool,
    cmd_pcre1: bool,
    cmd_pcre2: bool,
    cmd_re2: bool,
    cmd_rust: bool,
    cmd_rust_bytes: bool,
    cmd_tcl: bool,
}

fn main() {
    let args: Args = Docopt::new(USAGE).and_then(|d| d.decode())
                                       .unwrap_or_else(|e| e.exit());

    let mmap = Mmap::open_path(&args.arg_file, Protection::Read).unwrap();
    let haystack = unsafe { str::from_utf8(mmap.as_slice()).unwrap() };

    println!("{}", args.count(&haystack));
}

impl Args {
    fn count(&self, haystack: &str) -> usize {
        let pat = &self.arg_pattern;
        if self.cmd_onig {
            count_onig(pat, haystack)
        } else if self.cmd_pcre1 {
            count_pcre1(pat, haystack)
        } else if self.cmd_pcre2 {
            count_pcre2(pat, haystack)
        } else if self.cmd_re2 {
            count_re2(pat, haystack)
        } else if self.cmd_rust {
            count_rust(pat, haystack)
        } else if self.cmd_rust_bytes {
            count_rust_bytes(pat, haystack)
        } else if self.cmd_tcl {
            count_tcl(pat, haystack)
        } else {
            panic!("unreachable")
        }
    }
}

macro_rules! nada {
    ($feature:expr, $name:ident) => {
        #[cfg(not(feature = $feature))]
        fn $name(_pat: &str, _haystack: &str) -> usize {
            panic!("Support not enabled. Re-compile with '--features {}' \
                    to enable.", $feature)
        }
    }
}

nada!("re-onig", count_onig);
#[cfg(feature = "re-onig")]
fn count_onig(pat: &str, haystack: &str) -> usize {
    use ffi::onig::Regex;
    Regex::new(pat).unwrap().find_iter(haystack).count()
}

nada!("re-pcre1", count_pcre1);
#[cfg(feature = "re-pcre1")]
fn count_pcre1(pat: &str, haystack: &str) -> usize {
    use ffi::pcre1::Regex;
    Regex::new(pat).unwrap().find_iter(haystack).count()
}

nada!("re-pcre2", count_pcre2);
#[cfg(feature = "re-pcre2")]
fn count_pcre2(pat: &str, haystack: &str) -> usize {
    use ffi::pcre2::Regex;
    Regex::new(pat).unwrap().find_iter(haystack).count()
}

nada!("re-re2", count_re2);
#[cfg(feature = "re-re2")]
fn count_re2(pat: &str, haystack: &str) -> usize {
    use ffi::re2::Regex;
    Regex::new(pat).unwrap().find_iter(haystack).count()
}

nada!("re-rust", count_rust);
#[cfg(feature = "re-rust")]
fn count_rust(pat: &str, haystack: &str) -> usize {
    use regex::Regex;
    Regex::new(pat).unwrap().find_iter(haystack).count()
}

nada!("re-rust-bytes", count_rust_bytes);
#[cfg(feature = "re-rust-bytes")]
fn count_rust_bytes(pat: &str, haystack: &str) -> usize {
    use regex::bytes::Regex;
    Regex::new(pat).unwrap().find_iter(haystack.as_bytes()).count()
}

nada!("re-tcl", count_tcl);
#[cfg(feature = "re-tcl")]
fn count_tcl(pat: &str, haystack: &str) -> usize {
    use ffi::tcl::{Regex, Text};
    Regex::new(pat).unwrap().find_iter(&Text::new(haystack.to_owned())).count()
}
<|endoftext|>


<commit_before>use std::io::{Result, Write};

use analysis;
use analysis::symbols;
use analysis::namespaces::MAIN;
use case::CaseExt;
use env::Env;
use file_saver::save_to_file;
use library::*;
use library::Type as LType;
use nameutil;
use regex::{Captures, Regex};
use self::format::reformat_doc;
use stripper_lib::Type as SType;
use stripper_lib::{TypeStruct, write_file_name, write_item_doc};

mod format;

trait FunctionTraitType {
    fn functions(&self) -> &Vec<Function>;
    fn name(&self) -> &str;
}

macro_rules! impl_function_trait_type {
    ($ty:ident) => {
        impl FunctionTraitType for $ty {
            fn functions(&self) -> &Vec<Function> {
                &self.functions
            }

            fn name(&self) -> &str {
                &self.name
            }
        }
    }
}

impl_function_trait_type!(Class);
impl_function_trait_type!(Interface);

trait ToStripperType {
    fn convert(&self) -> TypeStruct;
    fn doc(&self) -> Option<&str>;
    fn doc_deprecated(&self) -> Option<&str>;
}

macro_rules! impl_to_stripper_type {
    ($ty:ident, $enum_var:ident) => {
        impl ToStripperType for $ty {
            fn convert(&self) -> TypeStruct {
                TypeStruct::new(SType::$enum_var, &self.name)
            }

            fn doc(&self) -> Option<&str> {
                self.doc.as_ref().map(|s| &s[..])
            }

            fn doc_deprecated(&self) -> Option<&str> {
                self.doc_deprecated.as_ref().map(|s| &s[..])
            }
        }
    }
}

impl_to_stripper_type!(Alias, Struct);
impl_to_stripper_type!(Constant, Const);
impl_to_stripper_type!(Member, Variant);
impl_to_stripper_type!(Enumeration, Enum);
impl_to_stripper_type!(Bitfield, Type);
impl_to_stripper_type!(Record, Struct);
impl_to_stripper_type!(Field, Variant);
impl_to_stripper_type!(Union, Struct);
impl_to_stripper_type!(Function, Fn);
impl_to_stripper_type!(Interface, Trait);
impl_to_stripper_type!(Class, Struct);
impl_to_stripper_type!(Namespace, Mod);
impl_to_stripper_type!(Parameter, Variant);

pub fn generate(env: &Env) {
    let path =  env.config.target_path.join("docs.md");
    println!("Generating documentation {:?}", path);
    save_to_file(&path, env.config.make_backup,
        |w| generate_doc(w, &env));
}

fn generate_doc(mut w: &mut Write, env: &Env) -> Result<()> {
    try!(write_file_name(w, None));

    for (tid, type_) in env.library.namespace_types(MAIN) {
        let obj = env.config.objects.get(&tid.full_name(&env.library))
            .and_then(|obj| if obj.status.ignored() { None } else { Some(obj) });
        match *type_ {
            LType::Class(..) => {
                if let Some(obj) = obj {
                    try!(create_object_doc(w, env, &analysis::object::class(env, obj).unwrap()));
                }
            }
            LType::Interface(..) => {
                if let Some(obj) = obj {
                    try!(create_object_doc(w, env,
                                           &analysis::object::interface(env, obj).unwrap()));
                }
            }
            LType::Record(ref rec) => {
                if obj.is_some() {
                    let symbols = env.symbols.borrow();
                    try!(create_record_doc(w, rec, &symbols));
                }
            }
            _ => {
                let symbols = env.symbols.borrow();
                try!(handle_type(&mut w, type_, &symbols));
            }
        }
    }

    Ok(())
}

fn handle_type(w: &mut Write, ty: &LType, symbols: &symbols::Info) -> Result<()> {
    match *ty {
        LType::Alias(ref a) => create_sub_doc(w, a, symbols),
        LType::Enumeration(ref e) => create_enum_doc(w, &e, symbols),
        LType::Function(ref f) => create_fn_doc(w, &f, None, symbols),
        _ => Ok(()),
    }
}

fn create_object_doc(w: &mut Write, env: &Env, info: &analysis::object::Info) -> Result<()> {
    let symbols = env.symbols.borrow();
    let ty = TypeStruct::new(SType::Struct, &info.name);
    let ty_ext = TypeStruct::new(SType::Trait, &format!("{}Ext", info.name));
    let has_trait = info.has_children;
    let class: &ToStripperType;
    let functions: &[Function];

    match *env.library.type_(info.type_id) {
        Type::Class(ref cl) => {
            class = cl;
            functions = &cl.functions;
        }
        Type::Interface(ref iface) => {
            class = iface;
            functions = &iface.functions;
        }
        _ => unreachable!(),
    }

    if class.doc().is_some() || class.doc_deprecated().is_some() {
        try!(write_item_doc(w, &ty, |w| {
            if let Some(doc) = class.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, &symbols)));
            }

            try!(writeln!(w, "\n# Implements\n"));
            let impl_self = if has_trait { Some(info.type_id) } else { None };
            let implements = impl_self.iter()
                .chain(env.class_hierarchy.supertypes(info.type_id))
                .filter(|&tid| !env.type_status(&tid.full_name(&env.library)).ignored())
                .map(|&tid| format!("[`{name}Ext`](trait.{name}Ext.html)",
                                    name = env.library.type_(tid).get_name()))
                .collect::<Vec<_>>();
            try!(writeln!(w, "{}", &implements.join(", ")));

            if let Some(doc) = class.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, &symbols)));
            }
            Ok(())
        }));
    }

    if has_trait {
        try!(write_item_doc(w, &ty_ext, |w| {
            try!(writeln!(w, "Trait containing all `{}` methods.", ty.name));
            let mut implementors = Some(info.type_id).into_iter()
                .chain(env.class_hierarchy.subtypes(info.type_id))
                .filter(|&tid| !env.type_status(&tid.full_name(&env.library)).ignored())
                .map(|tid| format!("[`{name}`](struct.{name}.html)",
                                    name = env.library.type_(tid).get_name()))
                .collect::<Vec<_>>();
            implementors.sort();

            try!(writeln!(w, "\n# Implementors\n"));
            try!(writeln!(w, "{}", implementors.join(", ")));
            Ok(())
        }));
    }

    let ty = TypeStruct { ty: SType::Impl, ..ty };

    for function in functions {
        let ty = if has_trait && function.parameters.iter().any(|p| p.instance_parameter) {
            ty_ext.clone()
        }
        else {
            ty.clone()
        };
        try!(create_fn_doc(w, &function, Some(Box::new(ty)), &symbols));
    }
    Ok(())
}

fn create_record_doc(w: &mut Write, record: &Record, symbols: &symbols::Info) -> Result<()> {
    let ty = record.convert();

    if record.doc().is_some() || record.doc_deprecated().is_some() {
        try!(write_item_doc(w, &ty, |w| {
            if let Some(doc) = record.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }

            if let Some(doc) = record.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            Ok(())
        }));
    }

    let ty = TypeStruct { ty: SType::Impl, ..ty };
    for function in &record.functions {
        try!(create_fn_doc(w, &function, Some(Box::new(ty.clone())), &symbols));
    }
    Ok(())
}

fn create_enum_doc(w: &mut Write, enum_: &Enumeration, symbols: &symbols::Info) -> Result<()> {
    let ty = enum_.convert();

    try!(write_item_doc(w, &ty, |w| {
        if let Some(doc) = enum_.doc() {
            try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
        }
        if let Some(doc) = enum_.doc_deprecated() {
            try!(writeln!(w, "\n# Deprecated\n"));
            try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
        }
        Ok(())
    }));

    for member in enum_.members.iter() {
        let mut sub_ty = TypeStruct { name: member.name.to_camel(), ..member.convert()};

        if member.doc().is_some() || member.doc_deprecated().is_some() {
            sub_ty.parent = Some(Box::new(ty.clone()));
            try!(write_item_doc(w, &sub_ty, |w| {
                if let Some(doc) = member.doc() {
                    try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
                }
                if let Some(doc) = member.doc_deprecated() {
                    try!(writeln!(w, "\n# Deprecated\n"));
                    try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
                }
                Ok(())
            }));
        }
    }
    Ok(())
}

lazy_static! {
    static ref PARAM_NAME: Regex = Regex::new(r"@(\w+)\b").unwrap();
}

fn fix_param_names(doc: &str, self_name: &Option<String>) -> String {
    PARAM_NAME.replace_all(doc, |caps: &Captures| {
        if let Some(ref self_name) = *self_name {
            if &caps[1] == self_name {
                return "@self".into()
            }
        }
        format!("@{}", nameutil::mangle_keywords(&caps[1]))
    })
}

fn create_fn_doc(w: &mut Write, fn_: &Function, parent: Option<Box<TypeStruct>>,
                 symbols: &symbols::Info) -> Result<()> {
    if fn_.doc().is_none() && fn_.doc_deprecated().is_none() && fn_.ret.doc().is_none() {
        if fn_.parameters.iter().all(|x| {
            x.doc().is_none()
        }) {
            return Ok(());
        }
    }
    let mut sub_ty = fn_.convert();
    sub_ty.parent = parent;

    let self_name: Option<String> = fn_.parameters.iter()
        .find(|p| p.instance_parameter)
        .map(|p| p.name.clone());

    write_item_doc(w, &sub_ty, |w| {
        if let Some(doc) = fn_.doc() {
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        };
        if let Some(doc) = fn_.doc_deprecated() {
            try!(writeln!(w, "\n# Deprecated\n"));
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        };

        for parameter in fn_.parameters.iter() {
            if parameter.instance_parameter || parameter.name.is_empty() {
                continue
            }
            if let Some(doc) = parameter.doc() {
                try!(writeln!(w, "## `{}`", nameutil::mangle_keywords(&parameter.name[..])));
                try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
            }
        }

        if let Some(doc) = fn_.ret.doc() {
            try!(writeln!(w, "\n# Returns\n"));
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        }
        Ok(())
    })
}

fn create_sub_doc<T: ToStripperType>(w: &mut Write, ty: &T, symbols: &symbols::Info) -> Result<()> {
    let sub_ty = ty.convert();

    if ty.doc().is_some() || ty.doc_deprecated().is_some() {
        try!(write_item_doc(w, &sub_ty, |w| {
            if let Some(doc) = ty.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            if let Some(doc) = ty.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            Ok(())
        }));
    }
    Ok(())
}
<commit_msg>Doc generator: don't document function type aliases<commit_after>use std::io::{Result, Write};

use analysis;
use analysis::symbols;
use analysis::namespaces::MAIN;
use case::CaseExt;
use env::Env;
use file_saver::save_to_file;
use library::*;
use library::Type as LType;
use nameutil;
use regex::{Captures, Regex};
use self::format::reformat_doc;
use stripper_lib::Type as SType;
use stripper_lib::{TypeStruct, write_file_name, write_item_doc};

mod format;

trait FunctionTraitType {
    fn functions(&self) -> &Vec<Function>;
    fn name(&self) -> &str;
}

macro_rules! impl_function_trait_type {
    ($ty:ident) => {
        impl FunctionTraitType for $ty {
            fn functions(&self) -> &Vec<Function> {
                &self.functions
            }

            fn name(&self) -> &str {
                &self.name
            }
        }
    }
}

impl_function_trait_type!(Class);
impl_function_trait_type!(Interface);

trait ToStripperType {
    fn convert(&self) -> TypeStruct;
    fn doc(&self) -> Option<&str>;
    fn doc_deprecated(&self) -> Option<&str>;
}

macro_rules! impl_to_stripper_type {
    ($ty:ident, $enum_var:ident) => {
        impl ToStripperType for $ty {
            fn convert(&self) -> TypeStruct {
                TypeStruct::new(SType::$enum_var, &self.name)
            }

            fn doc(&self) -> Option<&str> {
                self.doc.as_ref().map(|s| &s[..])
            }

            fn doc_deprecated(&self) -> Option<&str> {
                self.doc_deprecated.as_ref().map(|s| &s[..])
            }
        }
    }
}

impl_to_stripper_type!(Alias, Struct);
impl_to_stripper_type!(Constant, Const);
impl_to_stripper_type!(Member, Variant);
impl_to_stripper_type!(Enumeration, Enum);
impl_to_stripper_type!(Bitfield, Type);
impl_to_stripper_type!(Record, Struct);
impl_to_stripper_type!(Field, Variant);
impl_to_stripper_type!(Union, Struct);
impl_to_stripper_type!(Function, Fn);
impl_to_stripper_type!(Interface, Trait);
impl_to_stripper_type!(Class, Struct);
impl_to_stripper_type!(Namespace, Mod);
impl_to_stripper_type!(Parameter, Variant);

pub fn generate(env: &Env) {
    let path =  env.config.target_path.join("docs.md");
    println!("Generating documentation {:?}", path);
    save_to_file(&path, env.config.make_backup,
        |w| generate_doc(w, &env));
}

fn generate_doc(mut w: &mut Write, env: &Env) -> Result<()> {
    try!(write_file_name(w, None));

    for (tid, type_) in env.library.namespace_types(MAIN) {
        let obj = env.config.objects.get(&tid.full_name(&env.library))
            .and_then(|obj| if obj.status.ignored() { None } else { Some(obj) });
        match *type_ {
            LType::Class(..) => {
                if let Some(obj) = obj {
                    try!(create_object_doc(w, env, &analysis::object::class(env, obj).unwrap()));
                }
            }
            LType::Interface(..) => {
                if let Some(obj) = obj {
                    try!(create_object_doc(w, env,
                                           &analysis::object::interface(env, obj).unwrap()));
                }
            }
            LType::Record(ref rec) => {
                if obj.is_some() {
                    let symbols = env.symbols.borrow();
                    try!(create_record_doc(w, rec, &symbols));
                }
            }
            _ => {
                let symbols = env.symbols.borrow();
                try!(handle_type(&mut w, type_, &symbols));
            }
        }
    }

    Ok(())
}

fn handle_type(w: &mut Write, ty: &LType, symbols: &symbols::Info) -> Result<()> {
    match *ty {
        LType::Alias(ref a) => create_sub_doc(w, a, symbols),
        LType::Enumeration(ref e) => create_enum_doc(w, &e, symbols),
        _ => Ok(()),
    }
}

fn create_object_doc(w: &mut Write, env: &Env, info: &analysis::object::Info) -> Result<()> {
    let symbols = env.symbols.borrow();
    let ty = TypeStruct::new(SType::Struct, &info.name);
    let ty_ext = TypeStruct::new(SType::Trait, &format!("{}Ext", info.name));
    let has_trait = info.has_children;
    let class: &ToStripperType;
    let functions: &[Function];

    match *env.library.type_(info.type_id) {
        Type::Class(ref cl) => {
            class = cl;
            functions = &cl.functions;
        }
        Type::Interface(ref iface) => {
            class = iface;
            functions = &iface.functions;
        }
        _ => unreachable!(),
    }

    if class.doc().is_some() || class.doc_deprecated().is_some() {
        try!(write_item_doc(w, &ty, |w| {
            if let Some(doc) = class.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, &symbols)));
            }

            try!(writeln!(w, "\n# Implements\n"));
            let impl_self = if has_trait { Some(info.type_id) } else { None };
            let implements = impl_self.iter()
                .chain(env.class_hierarchy.supertypes(info.type_id))
                .filter(|&tid| !env.type_status(&tid.full_name(&env.library)).ignored())
                .map(|&tid| format!("[`{name}Ext`](trait.{name}Ext.html)",
                                    name = env.library.type_(tid).get_name()))
                .collect::<Vec<_>>();
            try!(writeln!(w, "{}", &implements.join(", ")));

            if let Some(doc) = class.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, &symbols)));
            }
            Ok(())
        }));
    }

    if has_trait {
        try!(write_item_doc(w, &ty_ext, |w| {
            try!(writeln!(w, "Trait containing all `{}` methods.", ty.name));
            let mut implementors = Some(info.type_id).into_iter()
                .chain(env.class_hierarchy.subtypes(info.type_id))
                .filter(|&tid| !env.type_status(&tid.full_name(&env.library)).ignored())
                .map(|tid| format!("[`{name}`](struct.{name}.html)",
                                    name = env.library.type_(tid).get_name()))
                .collect::<Vec<_>>();
            implementors.sort();

            try!(writeln!(w, "\n# Implementors\n"));
            try!(writeln!(w, "{}", implementors.join(", ")));
            Ok(())
        }));
    }

    let ty = TypeStruct { ty: SType::Impl, ..ty };

    for function in functions {
        let ty = if has_trait && function.parameters.iter().any(|p| p.instance_parameter) {
            ty_ext.clone()
        }
        else {
            ty.clone()
        };
        try!(create_fn_doc(w, &function, Some(Box::new(ty)), &symbols));
    }
    Ok(())
}

fn create_record_doc(w: &mut Write, record: &Record, symbols: &symbols::Info) -> Result<()> {
    let ty = record.convert();

    if record.doc().is_some() || record.doc_deprecated().is_some() {
        try!(write_item_doc(w, &ty, |w| {
            if let Some(doc) = record.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }

            if let Some(doc) = record.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            Ok(())
        }));
    }

    let ty = TypeStruct { ty: SType::Impl, ..ty };
    for function in &record.functions {
        try!(create_fn_doc(w, &function, Some(Box::new(ty.clone())), &symbols));
    }
    Ok(())
}

fn create_enum_doc(w: &mut Write, enum_: &Enumeration, symbols: &symbols::Info) -> Result<()> {
    let ty = enum_.convert();

    try!(write_item_doc(w, &ty, |w| {
        if let Some(doc) = enum_.doc() {
            try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
        }
        if let Some(doc) = enum_.doc_deprecated() {
            try!(writeln!(w, "\n# Deprecated\n"));
            try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
        }
        Ok(())
    }));

    for member in enum_.members.iter() {
        let mut sub_ty = TypeStruct { name: member.name.to_camel(), ..member.convert()};

        if member.doc().is_some() || member.doc_deprecated().is_some() {
            sub_ty.parent = Some(Box::new(ty.clone()));
            try!(write_item_doc(w, &sub_ty, |w| {
                if let Some(doc) = member.doc() {
                    try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
                }
                if let Some(doc) = member.doc_deprecated() {
                    try!(writeln!(w, "\n# Deprecated\n"));
                    try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
                }
                Ok(())
            }));
        }
    }
    Ok(())
}

lazy_static! {
    static ref PARAM_NAME: Regex = Regex::new(r"@(\w+)\b").unwrap();
}

fn fix_param_names(doc: &str, self_name: &Option<String>) -> String {
    PARAM_NAME.replace_all(doc, |caps: &Captures| {
        if let Some(ref self_name) = *self_name {
            if &caps[1] == self_name {
                return "@self".into()
            }
        }
        format!("@{}", nameutil::mangle_keywords(&caps[1]))
    })
}

fn create_fn_doc(w: &mut Write, fn_: &Function, parent: Option<Box<TypeStruct>>,
                 symbols: &symbols::Info) -> Result<()> {
    if fn_.doc().is_none() && fn_.doc_deprecated().is_none() && fn_.ret.doc().is_none() {
        if fn_.parameters.iter().all(|x| {
            x.doc().is_none()
        }) {
            return Ok(());
        }
    }
    let mut sub_ty = fn_.convert();
    sub_ty.parent = parent;

    let self_name: Option<String> = fn_.parameters.iter()
        .find(|p| p.instance_parameter)
        .map(|p| p.name.clone());

    write_item_doc(w, &sub_ty, |w| {
        if let Some(doc) = fn_.doc() {
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        };
        if let Some(doc) = fn_.doc_deprecated() {
            try!(writeln!(w, "\n# Deprecated\n"));
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        };

        for parameter in fn_.parameters.iter() {
            if parameter.instance_parameter || parameter.name.is_empty() {
                continue
            }
            if let Some(doc) = parameter.doc() {
                try!(writeln!(w, "## `{}`", nameutil::mangle_keywords(&parameter.name[..])));
                try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
            }
        }

        if let Some(doc) = fn_.ret.doc() {
            try!(writeln!(w, "\n# Returns\n"));
            try!(writeln!(w, "{}", reformat_doc(&fix_param_names(doc, &self_name), symbols)));
        }
        Ok(())
    })
}

fn create_sub_doc<T: ToStripperType>(w: &mut Write, ty: &T, symbols: &symbols::Info) -> Result<()> {
    let sub_ty = ty.convert();

    if ty.doc().is_some() || ty.doc_deprecated().is_some() {
        try!(write_item_doc(w, &sub_ty, |w| {
            if let Some(doc) = ty.doc() {
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            if let Some(doc) = ty.doc_deprecated() {
                try!(writeln!(w, "\n# Deprecated\n"));
                try!(writeln!(w, "{}", reformat_doc(doc, symbols)));
            }
            Ok(())
        }));
    }
    Ok(())
}
<|endoftext|>


<commit_before>// Copyright 2018 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Exported interface to basic qcow functionality to be used from C.

use libc::{EINVAL, EIO, ENOSYS};
use std::ffi::CStr;
use std::fs::OpenOptions;
use std::io::{Seek, SeekFrom};
use std::os::raw::{c_char, c_int};

use disk::{ImageType, QcowFile};
use sys_util::{flock, FileSetLen, FlockOperation};

trait DiskFile: FileSetLen + Seek {}
impl<D: FileSetLen + Seek> DiskFile for D {}

#[no_mangle]
pub unsafe extern "C" fn create_qcow_with_size(path: *const c_char, virtual_size: u64) -> c_int {
    // NULL pointers are checked, but this will access any other invalid pointer passed from C
    // code. It's the caller's responsibility to pass a valid pointer.
    if path.is_null() {
        return -EINVAL;
    }
    let c_str = CStr::from_ptr(path);
    let file_path = match c_str.to_str() {
        Ok(s) => s,
        Err(_) => return -EINVAL,
    };

    let file = match OpenOptions::new()
        .create(true)
        .read(true)
        .write(true)
        .open(file_path)
    {
        Ok(f) => f,
        Err(_) => return -1,
    };

    match QcowFile::new(file, virtual_size) {
        Ok(_) => 0,
        Err(_) => -1,
    }
}

#[no_mangle]
pub unsafe extern "C" fn expand_disk_image(path: *const c_char, virtual_size: u64) -> c_int {
    // NULL pointers are checked, but this will access any other invalid pointer passed from C
    // code. It's the caller's responsibility to pass a valid pointer.
    if path.is_null() {
        return -EINVAL;
    }
    let c_str = CStr::from_ptr(path);
    let file_path = match c_str.to_str() {
        Ok(s) => s,
        Err(_) => return -EINVAL,
    };

    let raw_image = match OpenOptions::new().read(true).write(true).open(file_path) {
        Ok(f) => f,
        Err(_) => return -EIO,
    };

    // Lock the disk image to prevent other processes from using it.
    if let Err(_) = flock(&raw_image, FlockOperation::LockExclusive, true) {
        return -EIO;
    }

    let image_type = match disk::detect_image_type(&raw_image) {
        Ok(t) => t,
        Err(_) => return -EINVAL,
    };

    let mut disk_image: Box<dyn DiskFile> = match image_type {
        ImageType::Raw => Box::new(raw_image),
        ImageType::Qcow2 => match QcowFile::from(raw_image) {
            Ok(f) => Box::new(f),
            Err(_) => return -EINVAL,
        },
        _ => return -EINVAL,
    };

    // For safety against accidentally shrinking the disk image due to a
    // programming error elsewhere, verify that the new size is larger than
    // the current size.  This is safe against races due to the exclusive
    // flock() taken above - this is only an advisory lock, but it is also
    // acquired by other instances of this function as well as crosvm
    // itself when running a VM, so this should be safe in all cases that
    // can access a disk image in normal operation.
    let current_size = match disk_image.seek(SeekFrom::End(0)) {
        Ok(len) => len,
        Err(_) => return -EIO,
    };
    if current_size >= virtual_size {
        return 0;
    }

    match disk_image.set_len(virtual_size) {
        Ok(_) => 0,
        Err(_) => -ENOSYS,
    }
}
<commit_msg>qcow_utils: use DiskFile trait from disk crate<commit_after>// Copyright 2018 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Exported interface to basic qcow functionality to be used from C.

use libc::{EINVAL, EIO, ENOSYS};
use std::ffi::CStr;
use std::fs::OpenOptions;
use std::os::raw::{c_char, c_int};

use disk::{DiskFile, ImageType, QcowFile};
use sys_util::{flock, FlockOperation};

#[no_mangle]
pub unsafe extern "C" fn create_qcow_with_size(path: *const c_char, virtual_size: u64) -> c_int {
    // NULL pointers are checked, but this will access any other invalid pointer passed from C
    // code. It's the caller's responsibility to pass a valid pointer.
    if path.is_null() {
        return -EINVAL;
    }
    let c_str = CStr::from_ptr(path);
    let file_path = match c_str.to_str() {
        Ok(s) => s,
        Err(_) => return -EINVAL,
    };

    let file = match OpenOptions::new()
        .create(true)
        .read(true)
        .write(true)
        .open(file_path)
    {
        Ok(f) => f,
        Err(_) => return -1,
    };

    match QcowFile::new(file, virtual_size) {
        Ok(_) => 0,
        Err(_) => -1,
    }
}

#[no_mangle]
pub unsafe extern "C" fn expand_disk_image(path: *const c_char, virtual_size: u64) -> c_int {
    // NULL pointers are checked, but this will access any other invalid pointer passed from C
    // code. It's the caller's responsibility to pass a valid pointer.
    if path.is_null() {
        return -EINVAL;
    }
    let c_str = CStr::from_ptr(path);
    let file_path = match c_str.to_str() {
        Ok(s) => s,
        Err(_) => return -EINVAL,
    };

    let raw_image = match OpenOptions::new().read(true).write(true).open(file_path) {
        Ok(f) => f,
        Err(_) => return -EIO,
    };

    // Lock the disk image to prevent other processes from using it.
    if let Err(_) = flock(&raw_image, FlockOperation::LockExclusive, true) {
        return -EIO;
    }

    let image_type = match disk::detect_image_type(&raw_image) {
        Ok(t) => t,
        Err(_) => return -EINVAL,
    };

    let disk_image: Box<dyn DiskFile> = match image_type {
        ImageType::Raw => Box::new(raw_image),
        ImageType::Qcow2 => match QcowFile::from(raw_image) {
            Ok(f) => Box::new(f),
            Err(_) => return -EINVAL,
        },
        _ => return -EINVAL,
    };

    // For safety against accidentally shrinking the disk image due to a
    // programming error elsewhere, verify that the new size is larger than
    // the current size.  This is safe against races due to the exclusive
    // flock() taken above - this is only an advisory lock, but it is also
    // acquired by other instances of this function as well as crosvm
    // itself when running a VM, so this should be safe in all cases that
    // can access a disk image in normal operation.
    let current_size = match disk_image.get_len() {
        Ok(len) => len,
        Err(_) => return -EIO,
    };
    if current_size >= virtual_size {
        return 0;
    }

    match disk_image.set_len(virtual_size) {
        Ok(_) => 0,
        Err(_) => -ENOSYS,
    }
}
<|endoftext|>


<commit_before>use mio_uds::UnixStream;
use libc::{self,c_char,uint32_t,int32_t,pid_t};
use std::io;
use std::ffi::CString;
use std::iter::repeat;
use std::ptr::null_mut;
use std::process::Command;
use std::os::unix::process::CommandExt;
use std::os::unix::io::{AsRawFd,FromRawFd};
use nix::unistd::*;
use nix::fcntl::{fcntl,FcntlArg,FdFlag,FD_CLOEXEC};
use env_logger::LogBuilder;

use sozu::network::{ProxyOrder,ServerMessage,http,tls};
use sozu::channel::Channel;
use sozu_command::data::ProxyType;
use sozu_command::config::ProxyConfig;

use command::Proxy;

pub fn start_workers(tag: &str, ls: &ProxyConfig) -> Option<Vec<Proxy>> {
  match ls.proxy_type {
    ProxyType::HTTP => {
      //FIXME: make safer
      if ls.to_http().is_some() {
        let mut http_proxies = Vec::new();
        for index in 1..ls.worker_count.unwrap_or(1) {
          let (pid, command) = start_worker_process(ls, tag, &index.to_string());
          let l =  Proxy::new(tag.to_string(), index as u8, pid, ls.proxy_type, ls.address.clone(), ls.port, command);
          http_proxies.push(l);
        }

        let (pid, command) = start_worker_process(ls, tag, &0.to_string());
        let l =  Proxy::new(tag.to_string(), 0, pid, ls.proxy_type, ls.address.clone(), ls.port, command);
        http_proxies.push(l);

        Some(http_proxies)
      } else {
        None
      }
    },
    ProxyType::HTTPS => {
      if ls.to_tls().is_some() {
        let mut tls_proxies = Vec::new();
        for index in 1..ls.worker_count.unwrap_or(1) {
          let (pid, command) = start_worker_process(ls, tag, &index.to_string());
          let l =  Proxy::new(tag.to_string(), index as u8, pid, ls.proxy_type, ls.address.clone(), ls.port, command);
          tls_proxies.push(l);
        }

        let (pid, command) = start_worker_process(ls, tag, &0.to_string());
        let l =  Proxy::new(tag.to_string(), 0, pid, ls.proxy_type, ls.address.clone(), ls.port, command);
        tls_proxies.push(l);

        Some(tls_proxies)
      } else {
        None
      }
    },
    _ => unimplemented!()
  }
}

fn generate_channels() -> io::Result<(Channel<ProxyOrder,ServerMessage>, Channel<ServerMessage,ProxyOrder>)> {
  let (command,proxy) = try!(UnixStream::pair());
  //FIXME: configurable buffer size
  let proxy_channel   = Channel::new(proxy, 10000, 20000);
  let command_channel = Channel::new(command, 10000, 20000);
  Ok((command_channel, proxy_channel))
}

pub fn begin_worker_process(fd: i32, id: &str, tag: &str, channel_buffer_size: usize, mut builder: LogBuilder) {
  let mut command: Channel<ServerMessage,ProxyConfig> = Channel::new(
    unsafe { UnixStream::from_raw_fd(fd) },
    channel_buffer_size,
    channel_buffer_size * 2
  );

  command.set_nonblocking(false);

  let proxy_config = command.read_message().expect("worker could not read configuration from socket");
  println!("got message: {:?}", proxy_config);

  //FIXME: hardcoded log level
  builder.parse("info");
  builder.init().unwrap();

  command.set_nonblocking(true);
  let command: Channel<ServerMessage,ProxyOrder> = command.into();

  let t = format!("{}-{}", tag, id);

  match proxy_config.proxy_type {
    ProxyType::HTTP => {
      if let Some(config) = proxy_config.to_http() {
        http::start(t, config, command);
      }
    },
    ProxyType::HTTPS => {
      if let Some(config) = proxy_config.to_tls() {
        tls::start(t, config, command);
      }
    },
    _ => unimplemented!()
  }

  info!("proxy ended");
}

pub fn start_worker_process(config: &ProxyConfig, tag: &str, id: &str) -> (pid_t, Channel<ProxyOrder,ServerMessage>) {
  println!("parent({})", unsafe { libc::getpid() });

  let (server, client) = UnixStream::pair().unwrap();

  // FD_CLOEXEC is set by default on every fd in Rust standard lib,
  // so we need to remove the flag on the client, otherwise
  // it won't be accessible
  let cl_flags = fcntl(client.as_raw_fd(), FcntlArg::F_GETFD).unwrap();
  let mut new_cl_flags = FdFlag::from_bits(cl_flags).unwrap();
  new_cl_flags.remove(FD_CLOEXEC);
  fcntl(client.as_raw_fd(), FcntlArg::F_SETFD(new_cl_flags));

  let channel_buffer_size = config.channel_buffer_size.unwrap_or(10000);
  let channel_max_buffer_size = channel_buffer_size * 2;

  let mut command: Channel<ProxyConfig,ServerMessage> = Channel::new(
    server,
    channel_buffer_size,
    channel_max_buffer_size
  );
  command.set_nonblocking(false);

  let path = unsafe { get_executable_path() };

  println!("launching worker");
  //FIXME: remove the expect, return a result?
  match fork().expect("fork failed") {
    ForkResult::Parent{ child } => {
      println!("worker launched: {}", child);
      command.write_message(config);
      command.set_nonblocking(true);

      let command: Channel<ProxyOrder,ServerMessage> = command.into();
      return (child, command);
    }
    ForkResult::Child => {
      println!("child({}):\twill spawn a child", unsafe { libc::getpid() });
      Command::new(path.to_str().unwrap())
        .arg("worker")
        .arg("--fd")
        .arg(client.as_raw_fd().to_string())
        .arg("--tag")
        .arg(tag)
        .arg("--id")
        .arg(id)
        .exec();

      unreachable!();
    }
  }
}

#[cfg(target_os = "linux")]
pub unsafe fn get_executable_path() -> CString {
  let capacity = 2000;
  let mut temp:Vec<u8> = Vec::with_capacity(capacity);
  temp.extend(repeat(0).take(capacity));
  let mut pathbuf = CString::from_vec_unchecked(temp);
  let ptr = pathbuf.into_raw();

  let proc_path = CString::new("/proc/self/exe").unwrap();
  let sz = libc::readlink( proc_path.as_ptr(), ptr, 1999);
  let path = CString::from_raw(ptr);
  path
}

#[cfg(target_os = "macos")]
extern {
  pub fn _NSGetExecutablePath(buf: *mut c_char, size: *mut uint32_t) -> int32_t;
}

#[cfg(target_os = "macos")]
pub unsafe fn get_executable_path() -> CString {
  let capacity = 2000;
  let mut temp:Vec<u8> = Vec::with_capacity(capacity);
  temp.extend(repeat(0).take(capacity));
  let pathbuf = CString::from_vec_unchecked(temp);
  let ptr = pathbuf.into_raw();

  let mut size:uint32_t = capacity as u32;
  if _NSGetExecutablePath(ptr, &mut size) == 0 {

    let mut temp2:Vec<u8> = Vec::with_capacity(capacity);
    temp2.extend(repeat(0).take(capacity));
    let pathbuf2 = CString::from_vec_unchecked(temp2);
    let ptr2 = pathbuf2.into_raw();

    if libc::realpath(ptr, ptr2) != null_mut() {
      let path = CString::from_raw(ptr2);
      path
    } else {
      panic!();
    }
  } else {
    panic!("buffer too small");
  }
}
<commit_msg>worker channel: pass channel buffer size to worker spawn command<commit_after>use mio_uds::UnixStream;
use libc::{self,c_char,uint32_t,int32_t,pid_t};
use std::io;
use std::ffi::CString;
use std::iter::repeat;
use std::ptr::null_mut;
use std::process::Command;
use std::os::unix::process::CommandExt;
use std::os::unix::io::{AsRawFd,FromRawFd};
use nix::unistd::*;
use nix::fcntl::{fcntl,FcntlArg,FdFlag,FD_CLOEXEC};
use env_logger::LogBuilder;

use sozu::network::{ProxyOrder,ServerMessage,http,tls};
use sozu::channel::Channel;
use sozu_command::data::ProxyType;
use sozu_command::config::ProxyConfig;

use command::Proxy;

pub fn start_workers(tag: &str, ls: &ProxyConfig) -> Option<Vec<Proxy>> {
  match ls.proxy_type {
    ProxyType::HTTP => {
      //FIXME: make safer
      if ls.to_http().is_some() {
        let mut http_proxies = Vec::new();
        for index in 1..ls.worker_count.unwrap_or(1) {
          let (pid, command) = start_worker_process(ls, tag, &index.to_string());
          let l =  Proxy::new(tag.to_string(), index as u8, pid, ls.proxy_type, ls.address.clone(), ls.port, command);
          http_proxies.push(l);
        }

        let (pid, command) = start_worker_process(ls, tag, &0.to_string());
        let l =  Proxy::new(tag.to_string(), 0, pid, ls.proxy_type, ls.address.clone(), ls.port, command);
        http_proxies.push(l);

        Some(http_proxies)
      } else {
        None
      }
    },
    ProxyType::HTTPS => {
      if ls.to_tls().is_some() {
        let mut tls_proxies = Vec::new();
        for index in 1..ls.worker_count.unwrap_or(1) {
          let (pid, command) = start_worker_process(ls, tag, &index.to_string());
          let l =  Proxy::new(tag.to_string(), index as u8, pid, ls.proxy_type, ls.address.clone(), ls.port, command);
          tls_proxies.push(l);
        }

        let (pid, command) = start_worker_process(ls, tag, &0.to_string());
        let l =  Proxy::new(tag.to_string(), 0, pid, ls.proxy_type, ls.address.clone(), ls.port, command);
        tls_proxies.push(l);

        Some(tls_proxies)
      } else {
        None
      }
    },
    _ => unimplemented!()
  }
}

fn generate_channels() -> io::Result<(Channel<ProxyOrder,ServerMessage>, Channel<ServerMessage,ProxyOrder>)> {
  let (command,proxy) = try!(UnixStream::pair());
  //FIXME: configurable buffer size
  let proxy_channel   = Channel::new(proxy, 10000, 20000);
  let command_channel = Channel::new(command, 10000, 20000);
  Ok((command_channel, proxy_channel))
}

pub fn begin_worker_process(fd: i32, id: &str, tag: &str, channel_buffer_size: usize, mut builder: LogBuilder) {
  let mut command: Channel<ServerMessage,ProxyConfig> = Channel::new(
    unsafe { UnixStream::from_raw_fd(fd) },
    channel_buffer_size,
    channel_buffer_size * 2
  );

  command.set_nonblocking(false);

  let proxy_config = command.read_message().expect("worker could not read configuration from socket");
  println!("got message: {:?}", proxy_config);

  //FIXME: hardcoded log level
  builder.parse("info");
  builder.init().unwrap();

  command.set_nonblocking(true);
  let command: Channel<ServerMessage,ProxyOrder> = command.into();

  let t = format!("{}-{}", tag, id);

  match proxy_config.proxy_type {
    ProxyType::HTTP => {
      if let Some(config) = proxy_config.to_http() {
        http::start(t, config, command);
      }
    },
    ProxyType::HTTPS => {
      if let Some(config) = proxy_config.to_tls() {
        tls::start(t, config, command);
      }
    },
    _ => unimplemented!()
  }

  info!("proxy ended");
}

pub fn start_worker_process(config: &ProxyConfig, tag: &str, id: &str) -> (pid_t, Channel<ProxyOrder,ServerMessage>) {
  println!("parent({})", unsafe { libc::getpid() });

  let (server, client) = UnixStream::pair().unwrap();

  // FD_CLOEXEC is set by default on every fd in Rust standard lib,
  // so we need to remove the flag on the client, otherwise
  // it won't be accessible
  let cl_flags = fcntl(client.as_raw_fd(), FcntlArg::F_GETFD).unwrap();
  let mut new_cl_flags = FdFlag::from_bits(cl_flags).unwrap();
  new_cl_flags.remove(FD_CLOEXEC);
  fcntl(client.as_raw_fd(), FcntlArg::F_SETFD(new_cl_flags));

  let channel_buffer_size = config.channel_buffer_size.unwrap_or(10000);
  let channel_max_buffer_size = channel_buffer_size * 2;

  let mut command: Channel<ProxyConfig,ServerMessage> = Channel::new(
    server,
    channel_buffer_size,
    channel_max_buffer_size
  );
  command.set_nonblocking(false);

  let path = unsafe { get_executable_path() };

  println!("launching worker");
  //FIXME: remove the expect, return a result?
  match fork().expect("fork failed") {
    ForkResult::Parent{ child } => {
      println!("worker launched: {}", child);
      command.write_message(config);
      command.set_nonblocking(true);

      let command: Channel<ProxyOrder,ServerMessage> = command.into();
      return (child, command);
    }
    ForkResult::Child => {
      println!("child({}):\twill spawn a child", unsafe { libc::getpid() });
      Command::new(path.to_str().unwrap())
        .arg("worker")
        .arg("--fd")
        .arg(client.as_raw_fd().to_string())
        .arg("--tag")
        .arg(tag)
        .arg("--id")
        .arg(id)
        .arg("--channel-buffer-size")
        .arg(channel_buffer_size.to_string())
        .exec();

      unreachable!();
    }
  }
}

#[cfg(target_os = "linux")]
pub unsafe fn get_executable_path() -> CString {
  let capacity = 2000;
  let mut temp:Vec<u8> = Vec::with_capacity(capacity);
  temp.extend(repeat(0).take(capacity));
  let mut pathbuf = CString::from_vec_unchecked(temp);
  let ptr = pathbuf.into_raw();

  let proc_path = CString::new("/proc/self/exe").unwrap();
  let sz = libc::readlink( proc_path.as_ptr(), ptr, 1999);
  let path = CString::from_raw(ptr);
  path
}

#[cfg(target_os = "macos")]
extern {
  pub fn _NSGetExecutablePath(buf: *mut c_char, size: *mut uint32_t) -> int32_t;
}

#[cfg(target_os = "macos")]
pub unsafe fn get_executable_path() -> CString {
  let capacity = 2000;
  let mut temp:Vec<u8> = Vec::with_capacity(capacity);
  temp.extend(repeat(0).take(capacity));
  let pathbuf = CString::from_vec_unchecked(temp);
  let ptr = pathbuf.into_raw();

  let mut size:uint32_t = capacity as u32;
  if _NSGetExecutablePath(ptr, &mut size) == 0 {

    let mut temp2:Vec<u8> = Vec::with_capacity(capacity);
    temp2.extend(repeat(0).take(capacity));
    let pathbuf2 = CString::from_vec_unchecked(temp2);
    let ptr2 = pathbuf2.into_raw();

    if libc::realpath(ptr, ptr2) != null_mut() {
      let path = CString::from_raw(ptr2);
      path
    } else {
      panic!();
    }
  } else {
    panic!("buffer too small");
  }
}
<|endoftext|>