<commit_before>// A variation on cares-epoll.rs
//
// Here we hide the use of epoll() behind a `Resolver` object.  We also
// transform the asynchronous c-ares interface into a synchronous one, by
// using a std::sync::mpsc::channel.
extern crate c_ares;
extern crate nix;

use nix::sys::epoll::{
    epoll_create,
    epoll_ctl,
    epoll_wait,
    EpollEvent,
    EpollEventKind,
    EpollOp,
    EPOLLIN,
    EPOLLOUT,
};
use std::collections::HashSet;
use std::error::Error;
use std::os::unix::io;
use std::sync::{
    Arc,
    Condvar,
    Mutex,
    mpsc,
};
use std::thread;

struct Resolver {
    ares_channel: Arc<Mutex<c_ares::Channel>>,
    keep_going: Arc<(Mutex<bool>, Condvar)>,
    fd_handle: Option<thread::JoinHandle<()>>,
}

// A thread that keeps going processing file descriptors for c-ares, until it
// is asked to stop.
fn fd_handling_thread(
    ares_channel: Arc<Mutex<c_ares::Channel>>,
    keep_going: Arc<(Mutex<bool>, Condvar)>) {
    let (ref lock, ref cvar) = *keep_going;
    let mut carry_on = lock.lock().unwrap();
    while *carry_on {
        process_ares_fds(ares_channel.clone());
        carry_on = cvar.wait(carry_on).unwrap();
    }
}

// Process file descriptors for c-ares, while it wants us to do so.
fn process_ares_fds(ares_channel: Arc<Mutex<c_ares::Channel>>) {
    // Create an epoll file descriptor so that we can listen for events.
    let epoll = epoll_create().ok().expect("Failed to create epoll");
    let mut tracked_fds = HashSet::<io::RawFd>::new();
    loop {
        // Ask c-ares what file descriptors we should be listening on, and map
        // those requests onto the epoll file descriptor.
        let mut active = false;
        let sockets = ares_channel.lock().unwrap().get_sock();
        for (fd, readable, writable) in &sockets {
            let mut interest = EpollEventKind::empty();
            if readable { interest = interest | EPOLLIN; }
            if writable { interest = interest | EPOLLOUT; }
            let event = EpollEvent {
                events: interest,
                data: fd as u64,
            };
            let op = if tracked_fds.insert(fd) {
                EpollOp::EpollCtlAdd
            } else {
                EpollOp::EpollCtlMod
            };
            epoll_ctl(epoll, op, fd, &event).ok().expect("epoll_ctl failed");
            active = true;
        }
        if !active { break }

        // Wait for something to happen.
        let empty_event = EpollEvent {
            events: EpollEventKind::empty(),
            data: 0,
        };
        let mut events = [empty_event; 2];
        let results = epoll_wait(epoll, &mut events, 500)
            .ok()
            .expect("epoll_wait failed");

        // Process whatever happened.
        match results {
            0 => {
                // No events - must be a timeout.  Tell c-ares about it.
                ares_channel.lock().unwrap().process_fd(
                    c_ares::INVALID_FD,
                    c_ares::INVALID_FD);
            },
            n => {
                // Sockets became readable or writable.  Tell c-ares about it.
                for event in &events[0..n] {
                    let active_fd = event.data as io::RawFd;
                    let readable_fd = if (event.events & EPOLLIN).is_empty() {
                        c_ares::INVALID_FD
                    } else {
                        active_fd
                    };
                    let writable_fd = if (event.events & EPOLLOUT).is_empty() {
                        c_ares::INVALID_FD
                    } else {
                        active_fd
                    };
                    ares_channel
                        .lock()
                        .unwrap()
                        .process_fd(readable_fd, writable_fd);
                }
            }
        }
    }
}

impl Resolver {
    // Create a new Resolver
    pub fn new() -> Resolver {
        // Create a c_ares::Channel.
        let mut options = c_ares::Options::new();
        options
            .set_timeout(500)
            .set_tries(3);
        let ares_channel = c_ares::Channel::new(options)
            .ok()
            .expect("Failed to create channel");
        let locked_channel = Arc::new(Mutex::new(ares_channel));

        // Create a thread to handle file descriptors.
        let keep_going = Arc::new((Mutex::new(true), Condvar::new()));
        let channel_clone = locked_channel.clone();
        let keep_going_clone = keep_going.clone();
        let fd_handle = thread::spawn(move || {
            fd_handling_thread(channel_clone, keep_going_clone)
        });

        Resolver {
            ares_channel: locked_channel,
            keep_going: keep_going,
            fd_handle: Some(fd_handle),
        }
    }

    fn wake_fd_thread(&self) {
        let (ref _lock, ref cvar) = *self.keep_going;
        cvar.notify_one();
    }

    // A blocking NS query.  Achieve this by having the callback send the
    // result to a std::sync::mpsc::channel, and waiting on that channel.
    pub fn query_ns(&self, name: &str)
        -> Result<c_ares::NSResults, c_ares::AresError> {
        let (tx, rx) = mpsc::channel();
        self.ares_channel.lock().unwrap().query_ns(name, move |result| {
            tx.send(result).unwrap();
        });
        self.wake_fd_thread();
        rx.recv().unwrap()
    }

    // A blocking PTR query.
    pub fn query_ptr(&self, name: &str)
        -> Result<c_ares::PTRResults, c_ares::AresError> {
        let (tx, rx) = mpsc::channel();
        self.ares_channel.lock().unwrap().query_ptr(name, move |result| {
            tx.send(result).unwrap();
        });
        self.wake_fd_thread();
        rx.recv().unwrap()
    }

    // A blocking TXT query.
    pub fn query_txt(&self, name: &str)
        -> Result<c_ares::TXTResults, c_ares::AresError> {
        let (tx, rx) = mpsc::channel();
        self.ares_channel.lock().unwrap().query_txt(name, move |result| {
            tx.send(result).unwrap();
        });
        self.wake_fd_thread();
        rx.recv().unwrap()
    }
}

impl Drop for Resolver {
    fn drop(&mut self) {
        // Kick the file descriptor thread to stop.
        let (ref lock, ref cvar) = *self.keep_going;
        {
            let mut carry_on = lock.lock().unwrap();
            *carry_on = false;
        }
        cvar.notify_one();

        // Wait for it to do so.
        for handle in self.fd_handle.take() {
            handle.join().unwrap();
        }
    }
}

fn print_ns_results(result: Result<c_ares::NSResults, c_ares::AresError>) {
    match result {
        Err(e) => {
            println!("NS lookup failed with error '{}'", e.description());
        }
        Ok(ns_results) => {
            println!("Successful NS lookup...");
            for ns_result in &ns_results {
                println!("{}", ns_result.name_server());
            }
        }
    }
}

fn print_ptr_results(result: Result<c_ares::PTRResults, c_ares::AresError>) {
    match result {
        Err(e) => {
            println!("PTR lookup failed with error '{}'", e.description());
        }
        Ok(ptr_results) => {
            println!("Successful PTR lookup...");
            for ptr_result in &ptr_results {
                println!("{}", ptr_result.cname());
            }
        }
    }
}

fn print_txt_results(result: Result<c_ares::TXTResults, c_ares::AresError>) {
    match result {
        Err(e) => {
            println!("TXT lookup failed with error '{}'", e.description());
        }
        Ok(txt_results) => {
            println!("Successful TXT lookup...");
            for txt_result in &txt_results {
                println!("{}", txt_result.text());
            }
        }
    }
}

fn main() {
    // Create a Resolver.  Then make some requests.
    let resolver = Resolver::new();
    let result = resolver.query_ns("google.com");
    println!("");
    print_ns_results(result);

    let result = resolver.query_ptr("14.210.58.216.in-addr.arpa");
    println!("");
    print_ptr_results(result);

    let result = resolver.query_txt("google.com");
    println!("");
    print_txt_results(result);
}
<commit_msg>Keep file descriptors open<commit_after>// A variation on cares-epoll.rs
//
// Here we hide the use of epoll() behind a `Resolver` object.  We also
// transform the asynchronous c-ares interface into a synchronous one, by
// using a std::sync::mpsc::channel.
extern crate c_ares;
extern crate nix;

use nix::sys::epoll::{
    epoll_create,
    epoll_ctl,
    epoll_wait,
    EpollEvent,
    EpollEventKind,
    EpollOp,
    EPOLLIN,
    EPOLLOUT,
};
use std::collections::HashSet;
use std::error::Error;
use std::os::unix::io;
use std::sync::{
    Arc,
    Condvar,
    Mutex,
    mpsc,
};
use std::thread;

struct Resolver {
    ares_channel: Arc<Mutex<c_ares::Channel>>,
    keep_going: Arc<(Mutex<bool>, Condvar)>,
    fd_handle: Option<thread::JoinHandle<()>>,
}

// A thread that keeps going processing file descriptors for c-ares, until it
// is asked to stop.
fn fd_handling_thread(
    ares_channel: Arc<Mutex<c_ares::Channel>>,
    keep_going: Arc<(Mutex<bool>, Condvar)>) {
    let (ref lock, ref cvar) = *keep_going;
    let mut carry_on = lock.lock().unwrap();
    while *carry_on {
        process_ares_fds(ares_channel.clone());
        carry_on = cvar.wait(carry_on).unwrap();
    }
}

// Process file descriptors for c-ares, while it wants us to do so.
fn process_ares_fds(ares_channel: Arc<Mutex<c_ares::Channel>>) {
    // Create an epoll file descriptor so that we can listen for events.
    let epoll = epoll_create().ok().expect("Failed to create epoll");
    let mut tracked_fds = HashSet::<io::RawFd>::new();
    loop {
        // Ask c-ares what file descriptors we should be listening on, and map
        // those requests onto the epoll file descriptor.
        let mut active = false;
        let sockets = ares_channel.lock().unwrap().get_sock();
        for (fd, readable, writable) in &sockets {
            let mut interest = EpollEventKind::empty();
            if readable { interest = interest | EPOLLIN; }
            if writable { interest = interest | EPOLLOUT; }
            let event = EpollEvent {
                events: interest,
                data: fd as u64,
            };
            let op = if tracked_fds.insert(fd) {
                EpollOp::EpollCtlAdd
            } else {
                EpollOp::EpollCtlMod
            };
            epoll_ctl(epoll, op, fd, &event).ok().expect("epoll_ctl failed");
            active = true;
        }
        if !active { break }

        // Wait for something to happen.
        let empty_event = EpollEvent {
            events: EpollEventKind::empty(),
            data: 0,
        };
        let mut events = [empty_event; 2];
        let results = epoll_wait(epoll, &mut events, 500)
            .ok()
            .expect("epoll_wait failed");

        // Process whatever happened.
        match results {
            0 => {
                // No events - must be a timeout.  Tell c-ares about it.
                ares_channel.lock().unwrap().process_fd(
                    c_ares::INVALID_FD,
                    c_ares::INVALID_FD);
            },
            n => {
                // Sockets became readable or writable.  Tell c-ares about it.
                for event in &events[0..n] {
                    let active_fd = event.data as io::RawFd;
                    let readable_fd = if (event.events & EPOLLIN).is_empty() {
                        c_ares::INVALID_FD
                    } else {
                        active_fd
                    };
                    let writable_fd = if (event.events & EPOLLOUT).is_empty() {
                        c_ares::INVALID_FD
                    } else {
                        active_fd
                    };
                    ares_channel
                        .lock()
                        .unwrap()
                        .process_fd(readable_fd, writable_fd);
                }
            }
        }
    }
}

impl Resolver {
    // Create a new Resolver
    pub fn new() -> Resolver {
        // Create a c_ares::Channel.
        let mut options = c_ares::Options::new();
        options
            .set_flags(c_ares::flags::STAYOPEN)
            .set_timeout(500)
            .set_tries(3);
        let ares_channel = c_ares::Channel::new(options)
            .ok()
            .expect("Failed to create channel");
        let locked_channel = Arc::new(Mutex::new(ares_channel));

        // Create a thread to handle file descriptors.
        let keep_going = Arc::new((Mutex::new(true), Condvar::new()));
        let channel_clone = locked_channel.clone();
        let keep_going_clone = keep_going.clone();
        let fd_handle = thread::spawn(move || {
            fd_handling_thread(channel_clone, keep_going_clone)
        });

        Resolver {
            ares_channel: locked_channel,
            keep_going: keep_going,
            fd_handle: Some(fd_handle),
        }
    }

    fn wake_fd_thread(&self) {
        let (ref _lock, ref cvar) = *self.keep_going;
        cvar.notify_one();
    }

    // A blocking NS query.  Achieve this by having the callback send the
    // result to a std::sync::mpsc::channel, and waiting on that channel.
    pub fn query_ns(&self, name: &str)
        -> Result<c_ares::NSResults, c_ares::AresError> {
        let (tx, rx) = mpsc::channel();
        self.ares_channel.lock().unwrap().query_ns(name, move |result| {
            tx.send(result).unwrap();
        });
        self.wake_fd_thread();
        rx.recv().unwrap()
    }

    // A blocking PTR query.
    pub fn query_ptr(&self, name: &str)
        -> Result<c_ares::PTRResults, c_ares::AresError> {
        let (tx, rx) = mpsc::channel();
        self.ares_channel.lock().unwrap().query_ptr(name, move |result| {
            tx.send(result).unwrap();
        });
        self.wake_fd_thread();
        rx.recv().unwrap()
    }

    // A blocking TXT query.
    pub fn query_txt(&self, name: &str)
        -> Result<c_ares::TXTResults, c_ares::AresError> {
        let (tx, rx) = mpsc::channel();
        self.ares_channel.lock().unwrap().query_txt(name, move |result| {
            tx.send(result).unwrap();
        });
        self.wake_fd_thread();
        rx.recv().unwrap()
    }
}

impl Drop for Resolver {
    fn drop(&mut self) {
        // Kick the file descriptor thread to stop.
        let (ref lock, ref cvar) = *self.keep_going;
        {
            let mut carry_on = lock.lock().unwrap();
            *carry_on = false;
        }
        cvar.notify_one();

        // Wait for it to do so.
        for handle in self.fd_handle.take() {
            handle.join().unwrap();
        }
    }
}

fn print_ns_results(result: Result<c_ares::NSResults, c_ares::AresError>) {
    match result {
        Err(e) => {
            println!("NS lookup failed with error '{}'", e.description());
        }
        Ok(ns_results) => {
            println!("Successful NS lookup...");
            for ns_result in &ns_results {
                println!("{}", ns_result.name_server());
            }
        }
    }
}

fn print_ptr_results(result: Result<c_ares::PTRResults, c_ares::AresError>) {
    match result {
        Err(e) => {
            println!("PTR lookup failed with error '{}'", e.description());
        }
        Ok(ptr_results) => {
            println!("Successful PTR lookup...");
            for ptr_result in &ptr_results {
                println!("{}", ptr_result.cname());
            }
        }
    }
}

fn print_txt_results(result: Result<c_ares::TXTResults, c_ares::AresError>) {
    match result {
        Err(e) => {
            println!("TXT lookup failed with error '{}'", e.description());
        }
        Ok(txt_results) => {
            println!("Successful TXT lookup...");
            for txt_result in &txt_results {
                println!("{}", txt_result.text());
            }
        }
    }
}

fn main() {
    // Create a Resolver.  Then make some requests.
    let resolver = Resolver::new();
    let result = resolver.query_ns("google.com");
    println!("");
    print_ns_results(result);

    let result = resolver.query_ptr("14.210.58.216.in-addr.arpa");
    println!("");
    print_ptr_results(result);

    let result = resolver.query_txt("google.com");
    println!("");
    print_txt_results(result);
}
<|endoftext|>


<commit_before>use filetime::{self, FileTime};

use std::cmp::{self, PartialEq, PartialOrd};
use std::fs;
use std::io;
use std::path::Path;

pub trait Timestamped {
    fn timestamp(&self) -> Timestamp;
}

#[derive(Copy, Clone, PartialEq, Eq, Ord, Debug)]
pub struct Timestamp { pub secs: u64, pub nsecs: u64 }

#[allow(non_snake_case)]
pub fn Timestamp(ms: u64) -> Timestamp  {
    Timestamp::new(ms / 1_000, (ms % 1_000) * 1_000_000)
}

impl Timestamp {
    pub fn new(secs: u64, ns: u64) -> Timestamp {
        Timestamp { secs: secs, nsecs: ns }
    }
    pub fn to_filetime(&self) -> FileTime {
        assert!(self.nsecs < ::std::u32::MAX as u64);
        FileTime::from_seconds_since_1970(self.secs, self.nsecs as u32)
    }
    pub fn to_ms(&self) -> u64 {
        self.secs * 1000 + self.nsecs / 1_000_000
    }
    pub fn set_file_times<P: AsRef<Path>>(&self, p: P) -> io::Result<()> {
        let t = self.to_filetime();
        filetime::set_file_times(p, t, t)
    }
}

impl PartialEq<u64> for Timestamp {
    fn eq(&self, other: &u64) -> bool {
        self.to_ms().eq(other)
    }
}

impl PartialEq<i64> for Timestamp {
    fn eq(&self, other: &i64) -> bool {
        if *other < 0 {
            false
        } else {
            let other = *other as u64;
            self.to_ms().eq(&other)
        }
    }
}

impl PartialOrd<u64> for Timestamp {
    fn partial_cmp(&self, other: &u64) -> Option<cmp::Ordering> {
        self.to_ms().partial_cmp(other)
    }
}

impl PartialOrd<Timestamp> for Timestamp {
    fn partial_cmp(&self, other: &Timestamp) -> Option<cmp::Ordering> {
        match self.secs.partial_cmp(&other.secs) {
            Some(cmp::Ordering::Equal) => self.nsecs.partial_cmp(&other.nsecs),
            otherwise => otherwise,
        }
    }
}

impl Timestamped for fs::Metadata {
    fn timestamp(&self) -> Timestamp {
        let ft = FileTime::from_last_modification_time( self );
        let s = ft.seconds_relative_to_1970();
        let ns = ft.nanoseconds();
        // println!("metadata mtime: {} ns: {}", s, ns);
        Timestamp::new(s as u64, ns as u64)
    }
}
<commit_msg>Added method to compute a human-readable timestamp.<commit_after>use filetime::{self, FileTime};

use std::cmp::{self, PartialEq, PartialOrd};
use std::fs;
use std::io;
use std::path::Path;

pub trait Timestamped {
    fn timestamp(&self) -> Timestamp;
}

#[derive(Copy, Clone, PartialEq, Eq, Ord, Debug)]
pub struct Timestamp { pub secs: u64, pub nsecs: u64 }

#[allow(non_snake_case)]
pub fn Timestamp(ms: u64) -> Timestamp  {
    Timestamp::new(ms / 1_000, (ms % 1_000) * 1_000_000)
}

impl Timestamp {
    pub fn new(secs: u64, ns: u64) -> Timestamp {
        Timestamp { secs: secs, nsecs: ns }
    }
    pub fn to_filetime(&self) -> FileTime {
        assert!(self.nsecs < ::std::u32::MAX as u64);
        FileTime::from_seconds_since_1970(self.secs, self.nsecs as u32)
    }
    pub fn to_ms(&self) -> u64 {
        self.secs * 1000 + self.nsecs / 1_000_000
    }
    pub fn set_file_times<P: AsRef<Path>>(&self, p: P) -> io::Result<()> {
        let t = self.to_filetime();
        filetime::set_file_times(p, t, t)
    }
    pub fn date_fulltime_badly(&self) -> String {
        // TODO: throw away this function if/when something like Joda
        // time is available as a Rust crate.

        // (Seconds since January 1, 1970).
        let mut remain = self.to_filetime().seconds_relative_to_1970();

        let mut year = None;
        for y in 1970.. {
            let secs_per_year = secs_per_year(y);
            if remain > secs_per_year {
                remain -= secs_per_year;
            } else {
                year = Some(y);
                break;
            }
        }
        let year = year.unwrap();

        let mut month = None;
        for i in 0..12 {
            let secs_per_month = if is_leap_year(year) {
                SECS_PER_DAY * DAYS_PER_MONTH_IN_LEAP[i]
            } else {
                SECS_PER_DAY * DAYS_PER_MONTH_IN_COMMON[i]
            };
            if remain > secs_per_month {
                remain -= secs_per_month;
            } else {
                month = Some(i+1); // We count months starting from 1 ...
                break;
            }
        }
        let month = month.unwrap();

        let day = remain / SECS_PER_DAY + 1; // ... and we count days starting from 1
        let remain = remain % SECS_PER_DAY;

        let hour = remain / SECS_PER_HOUR; // ... but we count hours from zero (military time)
        let remain = remain % SECS_PER_HOUR;

        let min = remain / SECS_PER_MIN; // ... and likewise count minutes from zero
        let remain = remain % SECS_PER_MIN;

        let sec = remain; // ... and likewise count seconds from zero
        let nsec = self.nsecs; // ... et cetera.

        format!("{YEAR:04}-{MONTH:02}-{DAY:02} {HOUR:02}:{MIN:02}:{SEC:02}.{NSEC} (GMT)",
                YEAR=year, MONTH=month, DAY=day, HOUR=hour, MIN=min, SEC=sec, NSEC=nsec)
    }
}

fn is_leap_year(gregorian_year: u64) -> bool {
    let year = gregorian_year;
    if !(year % 4 == 0) {
        false
    } else if !(year % 100 == 0) {
        true
    } else if !(year % 400 == 0) {
        false
    } else {
        true
    }
}

fn secs_per_year(gregorian_year: u64) -> u64 {
    if is_leap_year(gregorian_year) {
        SECS_PER_LEAP_YEAR
    } else {
        SECS_PER_COMMON_YEAR
    }
}

const SECS_PER_LEAP_YEAR: u64 = 366 * SECS_PER_DAY;
const SECS_PER_COMMON_YEAR: u64 = 365 * SECS_PER_DAY;
const DAYS_PER_MONTH_IN_LEAP: [u64; 12] = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
const DAYS_PER_MONTH_IN_COMMON: [u64; 12] = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
const SECS_PER_DAY: u64 = 24 * SECS_PER_HOUR;
const SECS_PER_HOUR: u64 = 60 * SECS_PER_MIN;
const SECS_PER_MIN: u64 = 60;

impl PartialEq<u64> for Timestamp {
    fn eq(&self, other: &u64) -> bool {
        self.to_ms().eq(other)
    }
}

impl PartialEq<i64> for Timestamp {
    fn eq(&self, other: &i64) -> bool {
        if *other < 0 {
            false
        } else {
            let other = *other as u64;
            self.to_ms().eq(&other)
        }
    }
}

impl PartialOrd<u64> for Timestamp {
    fn partial_cmp(&self, other: &u64) -> Option<cmp::Ordering> {
        self.to_ms().partial_cmp(other)
    }
}

impl PartialOrd<Timestamp> for Timestamp {
    fn partial_cmp(&self, other: &Timestamp) -> Option<cmp::Ordering> {
        match self.secs.partial_cmp(&other.secs) {
            Some(cmp::Ordering::Equal) => self.nsecs.partial_cmp(&other.nsecs),
            otherwise => otherwise,
        }
    }
}

impl Timestamped for fs::Metadata {
    fn timestamp(&self) -> Timestamp {
        let ft = FileTime::from_last_modification_time( self );
        let s = ft.seconds_relative_to_1970();
        let ns = ft.nanoseconds();
        // println!("metadata mtime: {} ns: {}", s, ns);
        Timestamp::new(s as u64, ns as u64)
    }
}
<|endoftext|>


<commit_before>fn tokenize(input: &str) -> Vec<&str> {
    let mut first_alphanumeric_index = 0;
    let mut last_was_alphanumeric = false;
    let mut result: Vec<&str> = Vec::new();
    for (index, character) in input.chars().enumerate() {
        let current_is_alphanumeric = character.is_alphanumeric();
        if current_is_alphanumeric {
            if last_was_alphanumeric {
                // Do nothing, keep working on the current word.
            } else {
                // Last was something else, we just started a new word.
                first_alphanumeric_index = index;
            }
        } else {
            // Current is something else
            if last_was_alphanumeric {
                // Push the word that just ended
                result.push(&input[first_alphanumeric_index..index]);
            }

            // Push current char
            result.push(&input[index..index + 1]);
        }

        last_was_alphanumeric = current_is_alphanumeric;
    }

    if last_was_alphanumeric {
        // Push the ending word
        result.push(&input[first_alphanumeric_index..]);
    }

    return result;
}

#[cfg(test)]
mod tests {
    use super::*;

    #[cfg(test)]
    use pretty_assertions::assert_eq;

    #[test]
    fn test_empty() {
        let no_strings: Vec<String> = Vec::new();
        assert_eq!(tokenize(""), no_strings);
    }

    #[test]
    fn test_words() {
        assert_eq!(tokenize("word"), ["word"]);
        assert_eq!(tokenize("Adam Bea"), ["Adam", " ", "Bea"]);
    }

    #[test]
    fn test_numbers() {
        assert_eq!(tokenize("123"), ["123"]);
        assert_eq!(tokenize("123 456"), ["123", " ", "456"]);
    }

    #[test]
    fn test_alphanumeric() {
        assert_eq!(tokenize("0xC0deCafe"), ["0xC0deCafe"]);
    }

    #[test]
    fn test_others() {
        assert_eq!(tokenize("+!,"), ["+", "!", ","]);
    }
}
<commit_msg>Add missing docs<commit_after>// FIXME: The doctest doesn't run on `cargo test`, why?
/// Splits string into a vector of words. A word is any sequence of alphanumeric
/// characters. Non-words get into the vector one and one.
///
/// ```rust
/// assert_eq!(tokenize("Adam, Bea"), ["Adam", ", ", " ", "Bea"]);
/// ```
fn tokenize(input: &str) -> Vec<&str> {
    let mut first_alphanumeric_index = 0;
    let mut last_was_alphanumeric = false;
    let mut result: Vec<&str> = Vec::new();
    for (index, character) in input.chars().enumerate() {
        let current_is_alphanumeric = character.is_alphanumeric();
        if current_is_alphanumeric {
            if last_was_alphanumeric {
                // Do nothing, keep working on the current word.
            } else {
                // Last was something else, we just started a new word.
                first_alphanumeric_index = index;
            }
        } else {
            // Current is something else
            if last_was_alphanumeric {
                // Push the word that just ended
                result.push(&input[first_alphanumeric_index..index]);
            }

            // Push current char
            result.push(&input[index..index + 1]);
        }

        last_was_alphanumeric = current_is_alphanumeric;
    }

    if last_was_alphanumeric {
        // Push the ending word
        result.push(&input[first_alphanumeric_index..]);
    }

    return result;
}

#[cfg(test)]
mod tests {
    use super::*;

    #[cfg(test)]
    use pretty_assertions::assert_eq;

    #[test]
    fn test_empty() {
        let no_strings: Vec<String> = Vec::new();
        assert_eq!(tokenize(""), no_strings);
    }

    #[test]
    fn test_words() {
        assert_eq!(tokenize("word"), ["word"]);
        assert_eq!(tokenize("Adam Bea"), ["Adam", " ", "Bea"]);
    }

    #[test]
    fn test_numbers() {
        assert_eq!(tokenize("123"), ["123"]);
        assert_eq!(tokenize("123 456"), ["123", " ", "456"]);
    }

    #[test]
    fn test_alphanumeric() {
        assert_eq!(tokenize("0xC0deCafe"), ["0xC0deCafe"]);
    }

    #[test]
    fn test_others() {
        assert_eq!(tokenize("+!,"), ["+", "!", ","]);
    }
}
<|endoftext|>


<commit_before>use std::io::Write;

use inflector::Inflector;

use super::{
    error_type_name, generate_field_name, rest_request_generator, rest_response_parser, FileWriter,
    GenerateProtocol, IoResult,
};
use botocore::{Operation, Shape, ShapeType};
use Service;

pub struct RestJsonGenerator;

impl GenerateProtocol for RestJsonGenerator {
    fn generate_method_signatures(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        for (operation_name, operation) in service.operations().iter() {
            let input_type = operation.input_shape();
            let output_type = operation.output_shape_or("()");

            // Retrieve the `Shape` for the input for this operation.
            let input_shape = &service.get_shape(input_type).unwrap();

            writeln!(
                writer,
                "
                {documentation}
                {method_signature} -> \
                      RusotoFuture<{output_type}, {error_type}>;
                ",
                documentation = generate_documentation(operation).unwrap_or_else(|| "".to_owned()),
                method_signature = generate_method_signature(operation, input_shape),
                error_type = error_type_name(service, operation_name),
                output_type = output_type
            )?
        }
        Ok(())
    }

    fn generate_method_impls(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        for (operation_name, operation) in service.operations().iter() {
            let input_type = operation.input_shape();
            let output_type = operation.output_shape_or("()");

            // Retrieve the `Shape` for the input for this operation.
            let input_shape = &service.get_shape(input_type).unwrap();

            let (request_uri, _) =
                rest_request_generator::parse_query_string(&operation.http.request_uri);

            writeln!(writer,"
                {documentation}
                {method_signature} -> RusotoFuture<{output_type}, {error_type}> {{
                    {request_uri_formatter}

                    let mut request = SignedRequest::new(\"{http_method}\", \"{endpoint_prefix}\", &self.region, &request_uri);
                    {default_headers}
                    {set_headers}
                    {modify_endpoint_prefix}
                    {load_payload}
                    {load_headers}
                    {load_params}

                    self.client.sign_and_dispatch(request, |response| {{
                        if {status_check} {{
                            Box::new(response.buffer().from_err().and_then(|response| {{
                                {parse_body}
                                {parse_headers}
                                {parse_status_code}
                                Ok(result)
                            }}))
                        }} else {{
                            Box::new(response.buffer().from_err().and_then(|response| {{
                                Err({error_type}::from_response(response))
                            }}))
                        }}
                    }})
                }}
                ",
                documentation = generate_documentation(operation).unwrap_or_else(|| "".to_owned()),
                method_signature = generate_method_signature(operation, input_shape),
                endpoint_prefix = service.signing_name(),
                modify_endpoint_prefix = generate_endpoint_modification(service).unwrap_or_else(|| "".to_owned()),
                http_method = operation.http.method,
                error_type = error_type_name(service, operation_name),
                status_check = http_code_expected(operation.http.response_code),
                parse_body = generate_body_parser(operation, service),
                parse_status_code = generate_status_code_parser(operation, service),
                output_type = output_type,
                load_headers = rest_request_generator::generate_headers(service, operation).unwrap_or_else(|| "".to_string()),
                parse_headers = rest_response_parser::generate_response_headers_parser(service, operation)
                    .unwrap_or_else(|| "".to_owned()),
                request_uri_formatter = rest_request_generator::generate_uri_formatter(
                    &request_uri,
                    service,
                    operation
                ).unwrap_or_else(|| "".to_string()),
                load_payload = generate_payload(service, input_shape).unwrap_or_else(|| "".to_string()),
                load_params = rest_request_generator::generate_params_loading_string(service, operation).unwrap_or_else(|| "".to_string()),
                default_headers = generate_default_headers(service),
                set_headers = generate_headers(service).unwrap_or_else(|| "".to_string()),
            )?
        }
        Ok(())
    }

    fn generate_prelude(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        writeln!(writer, "use serde_json;")?;

        // avoid unused imports when building services that don't use params
        if service_has_query_parameters(service) {
            writeln!(writer, "use rusoto_core::param::{{Params, ServiceParams}};")?;
        }

        writeln!(
            writer,
            "use rusoto_core::signature::SignedRequest;
                  use rusoto_core::proto;"
        )
    }

    fn serialize_trait(&self) -> Option<&'static str> {
        Some("Serialize")
    }

    fn deserialize_trait(&self) -> Option<&'static str> {
        Some("Deserialize")
    }

    fn timestamp_type(&self) -> &'static str {
        "f64"
    }
}

fn http_code_expected(code: Option<i32>) -> String {
    match code {
        Some(actual_code) => format!("response.status.as_u16() == {}", actual_code),
        // Some service definitions such as elastictranscoder don't specify
        // the response code, we'll assume 2xx is okay:
        None => "response.status.is_success()".to_string(),
    }
}

// Glacier needs a special header added, not included in botocore definition.  See
// http://docs.aws.amazon.com/amazonglacier/latest/dev/api-common-request-headers.html
fn generate_headers(service: &Service) -> Option<String> {
    if service.full_name() == "Amazon Glacier" {
        return Some("request.add_header(\"x-amz-glacier-version\", \"2012-06-01\");".to_string());
    }
    None
}

// SageMaker Runtime allows to overwrite content-type
fn generate_default_headers(service: &Service) -> String {
    if service.full_name() == "Amazon SageMaker Runtime" {
        return "if input.content_type.is_none() {
                    request.set_content_type(\"application/x-amz-json-1.1\".to_owned());
                }"
        .to_string();
    }
    "request.set_content_type(\"application/x-amz-json-1.1\".to_owned());".to_string()
}

// IoT has an endpoint_prefix and a signing_name that differ
fn generate_endpoint_modification(service: &Service) -> Option<String> {
    if service.signing_name() == service.endpoint_prefix() {
        None
    } else {
        Some(format!(
            "request.set_endpoint_prefix(\"{}\".to_string());",
            service.endpoint_prefix()
        ))
    }
}

// IoT defines a lot of empty (and therefore unnecessary) request shapes
// don't clutter method signatures with them
fn generate_method_signature(operation: &Operation, shape: &Shape) -> String {
    if shape.members.is_some() && !shape.members.as_ref().unwrap().is_empty() {
        format!(
            "fn {method_name}(&self, input: {input_type})",
            method_name = operation.name.to_snake_case(),
            input_type = operation.input_shape()
        )
    } else {
        format!(
            "fn {method_name}(&self)",
            method_name = operation.name.to_snake_case()
        )
    }
}

// Figure out what, if anything, should be sent as the body of the http request
fn generate_payload(service: &Service, input_shape: &Shape) -> Option<String> {
    let declare_payload = match input_shape.payload {
        // if the input shape explicitly specifies a payload field, use that
        Some(ref payload_member_name) => {
            Some(declared_payload(input_shape, payload_member_name, service))
        }

        // otherwise see if the input_shape itself should be serialized as the payload
        None => {
            // only use the input_shape if it has non-query, non-header members
            // (i.e., location unspecified)
            if input_shape
                .members
                .as_ref()
                .unwrap()
                .iter()
                .any(|(_, member)| member.location.is_none())
            {
                Some("let encoded = Some(serde_json::to_vec(&input).unwrap());".to_owned())
            } else {
                None
            }
        }
    };

    if declare_payload.is_some() {
        Some(declare_payload.unwrap() + "request.set_payload(encoded);")
    } else {
        None
    }
}

fn declared_payload(input_shape: &Shape, payload_member_name: &str, service: &Service) -> String {
    let payload_member_shape = &input_shape.members.as_ref().unwrap()[payload_member_name].shape;
    let payload_shape = &service
        .get_shape(payload_member_shape)
        .expect("Shape missing from service definition");

    let field_name = generate_field_name(payload_member_name);

    match payload_shape.shape_type {
        // if it's a String or a Blob, send it as the raw payload
        payload_type if payload_type == ShapeType::Blob || payload_type == ShapeType::String => {
            if input_shape.required(payload_member_name) {
                format!("let encoded = Some(input.{}.to_owned());", field_name)
            } else {
                format!(
                    "let encoded = if let Some(ref payload) = input.{} {{
                            Some(payload.to_owned())
                        }} else {{
                            None
                        }};",
                    field_name
                )
            }
        }

        // otherwise serialize the payload member as json and use that
        _ => format!(
            "let encoded = Some(serde_json::to_vec(&input.{}).unwrap());",
            field_name
        ),
    }
}

// Do any input operations in the entire service use query parameters?
fn service_has_query_parameters(service: &Service) -> bool {
    service
        .operations()
        .iter()
        .map(|(_, operation)| operation.input_shape())
        .map(|input_type| service.get_shape(input_type).unwrap())
        .any(|input_shape| input_shape.has_query_parameters())
}

fn generate_documentation(operation: &Operation) -> Option<String> {
    operation
        .documentation
        .as_ref()
        .map(|docs| ::doco::Item(docs).to_string())
}

/// Generate code to plumb the response status code into any fields
/// in the output shape that specify it
fn generate_status_code_parser(operation: &Operation, service: &Service) -> String {
    if operation.output.is_none() {
        return "".to_owned();
    }

    let shape_name = &operation.output.as_ref().unwrap().shape;
    let output_shape = &service
        .get_shape(shape_name)
        .expect("Shape missing from service definition");

    let mut status_code_parser = "".to_string();

    for (member_name, member) in output_shape.members.as_ref().unwrap() {
        if let Some(ref location) = member.location {
            if location == "statusCode" {
                if output_shape.required(member_name) {
                    status_code_parser += &format!(
                        "result.{} = Some(response.status.as_u16());",
                        member_name.to_snake_case()
                    );
                } else {
                    status_code_parser += &format!(
                        "result.{} = Some(response.status.as_u16() as i64);",
                        member_name.to_snake_case()
                    );
                }
            }
        }
    }

    status_code_parser
}

/// Generate code to parse the http response body, either as a JSON object
/// deserialized with serde, or as a raw payload that's assigned to one of
/// the fields in the result object.
///
/// Needs to determine whether or not other fields in the result object
/// will be set later (e.g. from headers), so the compiler won't spit out
/// warnings about unnecessary mutability
fn generate_body_parser(operation: &Operation, service: &Service) -> String {
    if operation.output.is_none() {
        return "let result = ::std::mem::drop(response);".to_string();
    }

    let shape_name = &operation.output.as_ref().unwrap().shape;
    let output_shape = &service
        .get_shape(shape_name)
        .expect("Shape missing from service definition");

    let mutable_result = output_shape
        .members
        .as_ref()
        .unwrap()
        .iter()
        .any(|(_, member)| member.location.is_some())
        || output_shape.payload.is_some();

    match output_shape.payload {
        None => json_body_parser(shape_name, mutable_result),
        Some(ref payload_member_name) => {
            let payload_member_shape =
                &output_shape.members.as_ref().unwrap()[payload_member_name].shape;
            let payload_shape = &service
                .get_shape(payload_member_shape)
                .expect("Shape missing from service definition");
            // is the shape required?
            let payload_shape_required = match output_shape.required {
                Some(ref s) => {
                    // if there's any required shape present the body payload parser will handle it
                    s.is_empty()
                }
                None => false,
            };
            match payload_shape.shape_type {
                payload_type
                    if payload_type == ShapeType::Blob || payload_type == ShapeType::String =>
                {
                    payload_body_parser(
                        payload_type,
                        shape_name,
                        payload_member_name,
                        mutable_result,
                        payload_shape_required,
                    )
                }
                _ => json_body_parser(shape_name, false),
            }
        }
    }
}

/// Take the raw http response body and assign it to the payload field
/// on the result object
fn payload_body_parser(
    payload_type: ShapeType,
    output_shape: &str,
    payload_member: &str,
    mutable_result: bool,
    payload_required: bool,
) -> String {
    let response_body = match payload_required {
        true => match payload_type {
            ShapeType::Blob => "response.body",
            _ => "String::from_utf8_lossy(response.body.as_ref())",
        },
        false => match payload_type {
            ShapeType::Blob => "Some(response.body)",
            _ => "Some(String::from_utf8_lossy(response.body.as_ref()).into_owned())",
        },
    };

    format!(
        "
        let {mutable} result = {output_shape}::default();
        result.{payload_member} = {response_body};
        ",
        output_shape = output_shape,
        payload_member = payload_member.to_snake_case(),
        response_body = response_body,
        mutable = if mutable_result { "mut" } else { "" }
    )
}

/// Parse the http response body as a JSON object with serde, and use that
/// as the result object
fn json_body_parser(output_shape: &str, mutable_result: bool) -> String {
    format!(
        "let {mutable} result = proto::json::ResponsePayload::new(&response).deserialize::<{output_shape}, _>()?;",
        output_shape = output_shape,
        mutable = if mutable_result { "mut" } else { "" }
    )
}
<commit_msg>Revert a clippy lint.<commit_after>use std::io::Write;

use inflector::Inflector;

use super::{
    error_type_name, generate_field_name, rest_request_generator, rest_response_parser, FileWriter,
    GenerateProtocol, IoResult,
};
use botocore::{Operation, Shape, ShapeType};
use Service;

pub struct RestJsonGenerator;

impl GenerateProtocol for RestJsonGenerator {
    fn generate_method_signatures(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        for (operation_name, operation) in service.operations().iter() {
            let input_type = operation.input_shape();
            let output_type = operation.output_shape_or("()");

            // Retrieve the `Shape` for the input for this operation.
            let input_shape = &service.get_shape(input_type).unwrap();

            writeln!(
                writer,
                "
                {documentation}
                {method_signature} -> \
                      RusotoFuture<{output_type}, {error_type}>;
                ",
                documentation = generate_documentation(operation).unwrap_or_else(|| "".to_owned()),
                method_signature = generate_method_signature(operation, input_shape),
                error_type = error_type_name(service, operation_name),
                output_type = output_type
            )?
        }
        Ok(())
    }

    fn generate_method_impls(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        for (operation_name, operation) in service.operations().iter() {
            let input_type = operation.input_shape();
            let output_type = operation.output_shape_or("()");

            // Retrieve the `Shape` for the input for this operation.
            let input_shape = &service.get_shape(input_type).unwrap();

            let (request_uri, _) =
                rest_request_generator::parse_query_string(&operation.http.request_uri);

            writeln!(writer,"
                {documentation}
                {method_signature} -> RusotoFuture<{output_type}, {error_type}> {{
                    {request_uri_formatter}

                    let mut request = SignedRequest::new(\"{http_method}\", \"{endpoint_prefix}\", &self.region, &request_uri);
                    {default_headers}
                    {set_headers}
                    {modify_endpoint_prefix}
                    {load_payload}
                    {load_headers}
                    {load_params}

                    self.client.sign_and_dispatch(request, |response| {{
                        if {status_check} {{
                            Box::new(response.buffer().from_err().and_then(|response| {{
                                {parse_body}
                                {parse_headers}
                                {parse_status_code}
                                Ok(result)
                            }}))
                        }} else {{
                            Box::new(response.buffer().from_err().and_then(|response| {{
                                Err({error_type}::from_response(response))
                            }}))
                        }}
                    }})
                }}
                ",
                documentation = generate_documentation(operation).unwrap_or_else(|| "".to_owned()),
                method_signature = generate_method_signature(operation, input_shape),
                endpoint_prefix = service.signing_name(),
                modify_endpoint_prefix = generate_endpoint_modification(service).unwrap_or_else(|| "".to_owned()),
                http_method = operation.http.method,
                error_type = error_type_name(service, operation_name),
                status_check = http_code_expected(operation.http.response_code),
                parse_body = generate_body_parser(operation, service),
                parse_status_code = generate_status_code_parser(operation, service),
                output_type = output_type,
                load_headers = rest_request_generator::generate_headers(service, operation).unwrap_or_else(|| "".to_string()),
                parse_headers = rest_response_parser::generate_response_headers_parser(service, operation)
                    .unwrap_or_else(|| "".to_owned()),
                request_uri_formatter = rest_request_generator::generate_uri_formatter(
                    &request_uri,
                    service,
                    operation
                ).unwrap_or_else(|| "".to_string()),
                load_payload = generate_payload(service, input_shape).unwrap_or_else(|| "".to_string()),
                load_params = rest_request_generator::generate_params_loading_string(service, operation).unwrap_or_else(|| "".to_string()),
                default_headers = generate_default_headers(service),
                set_headers = generate_headers(service).unwrap_or_else(|| "".to_string()),
            )?
        }
        Ok(())
    }

    fn generate_prelude(&self, writer: &mut FileWriter, service: &Service) -> IoResult {
        writeln!(writer, "use serde_json;")?;

        // avoid unused imports when building services that don't use params
        if service_has_query_parameters(service) {
            writeln!(writer, "use rusoto_core::param::{{Params, ServiceParams}};")?;
        }

        writeln!(
            writer,
            "use rusoto_core::signature::SignedRequest;
                  use rusoto_core::proto;"
        )
    }

    fn serialize_trait(&self) -> Option<&'static str> {
        Some("Serialize")
    }

    fn deserialize_trait(&self) -> Option<&'static str> {
        Some("Deserialize")
    }

    fn timestamp_type(&self) -> &'static str {
        "f64"
    }
}

fn http_code_expected(code: Option<i32>) -> String {
    match code {
        Some(actual_code) => format!("response.status.as_u16() == {}", actual_code),
        // Some service definitions such as elastictranscoder don't specify
        // the response code, we'll assume 2xx is okay:
        None => "response.status.is_success()".to_string(),
    }
}

// Glacier needs a special header added, not included in botocore definition.  See
// http://docs.aws.amazon.com/amazonglacier/latest/dev/api-common-request-headers.html
fn generate_headers(service: &Service) -> Option<String> {
    if service.full_name() == "Amazon Glacier" {
        return Some("request.add_header(\"x-amz-glacier-version\", \"2012-06-01\");".to_string());
    }
    None
}

// SageMaker Runtime allows to overwrite content-type
fn generate_default_headers(service: &Service) -> String {
    if service.full_name() == "Amazon SageMaker Runtime" {
        return "if input.content_type.is_none() {
                    request.set_content_type(\"application/x-amz-json-1.1\".to_owned());
                }"
        .to_string();
    }
    "request.set_content_type(\"application/x-amz-json-1.1\".to_owned());".to_string()
}

// IoT has an endpoint_prefix and a signing_name that differ
fn generate_endpoint_modification(service: &Service) -> Option<String> {
    if service.signing_name() == service.endpoint_prefix() {
        None
    } else {
        Some(format!(
            "request.set_endpoint_prefix(\"{}\".to_string());",
            service.endpoint_prefix()
        ))
    }
}

// IoT defines a lot of empty (and therefore unnecessary) request shapes
// don't clutter method signatures with them
fn generate_method_signature(operation: &Operation, shape: &Shape) -> String {
    if shape.members.is_some() && !shape.members.as_ref().unwrap().is_empty() {
        format!(
            "fn {method_name}(&self, input: {input_type})",
            method_name = operation.name.to_snake_case(),
            input_type = operation.input_shape()
        )
    } else {
        format!(
            "fn {method_name}(&self)",
            method_name = operation.name.to_snake_case()
        )
    }
}

// Figure out what, if anything, should be sent as the body of the http request
fn generate_payload(service: &Service, input_shape: &Shape) -> Option<String> {
    let declare_payload = match input_shape.payload {
        // if the input shape explicitly specifies a payload field, use that
        Some(ref payload_member_name) => {
            Some(declared_payload(input_shape, payload_member_name, service))
        }

        // otherwise see if the input_shape itself should be serialized as the payload
        None => {
            // only use the input_shape if it has non-query, non-header members
            // (i.e., location unspecified)
            if input_shape
                .members
                .as_ref()
                .unwrap()
                .iter()
                .any(|(_, member)| member.location.is_none())
            {
                Some("let encoded = Some(serde_json::to_vec(&input).unwrap());".to_owned())
            } else {
                None
            }
        }
    };

    if declare_payload.is_some() {
        Some(declare_payload.unwrap() + "request.set_payload(encoded);")
    } else {
        None
    }
}

fn declared_payload(input_shape: &Shape, payload_member_name: &str, service: &Service) -> String {
    let payload_member_shape = &input_shape.members.as_ref().unwrap()[payload_member_name].shape;
    let payload_shape = &service
        .get_shape(payload_member_shape)
        .expect("Shape missing from service definition");

    let field_name = generate_field_name(payload_member_name);

    match payload_shape.shape_type {
        // if it's a String or a Blob, send it as the raw payload
        payload_type if payload_type == ShapeType::Blob || payload_type == ShapeType::String => {
            if input_shape.required(payload_member_name) {
                format!("let encoded = Some(input.{}.to_owned());", field_name)
            } else {
                format!(
                    "let encoded = if let Some(ref payload) = input.{} {{
                            Some(payload.to_owned())
                        }} else {{
                            None
                        }};",
                    field_name
                )
            }
        }

        // otherwise serialize the payload member as json and use that
        _ => format!(
            "let encoded = Some(serde_json::to_vec(&input.{}).unwrap());",
            field_name
        ),
    }
}

// Do any input operations in the entire service use query parameters?
fn service_has_query_parameters(service: &Service) -> bool {
    service
        .operations()
        .iter()
        .map(|(_, operation)| operation.input_shape())
        .map(|input_type| service.get_shape(input_type).unwrap())
        .any(|input_shape| input_shape.has_query_parameters())
}

fn generate_documentation(operation: &Operation) -> Option<String> {
    operation
        .documentation
        .as_ref()
        .map(|docs| ::doco::Item(docs).to_string())
}

/// Generate code to plumb the response status code into any fields
/// in the output shape that specify it
fn generate_status_code_parser(operation: &Operation, service: &Service) -> String {
    if operation.output.is_none() {
        return "".to_owned();
    }

    let shape_name = &operation.output.as_ref().unwrap().shape;
    let output_shape = &service
        .get_shape(shape_name)
        .expect("Shape missing from service definition");

    let mut status_code_parser = "".to_string();

    for (member_name, member) in output_shape.members.as_ref().unwrap() {
        if let Some(ref location) = member.location {
            if location == "statusCode" {
                if output_shape.required(member_name) {
                    status_code_parser += &format!(
                        "result.{} = Some(response.status.as_u16());",
                        member_name.to_snake_case()
                    );
                } else {
                    status_code_parser += &format!(
                        "result.{} = Some(response.status.as_u16() as i64);",
                        member_name.to_snake_case()
                    );
                }
            }
        }
    }

    status_code_parser
}

/// Generate code to parse the http response body, either as a JSON object
/// deserialized with serde, or as a raw payload that's assigned to one of
/// the fields in the result object.
///
/// Needs to determine whether or not other fields in the result object
/// will be set later (e.g. from headers), so the compiler won't spit out
/// warnings about unnecessary mutability
fn generate_body_parser(operation: &Operation, service: &Service) -> String {
    if operation.output.is_none() {
        return "let result = ::std::mem::drop(response);".to_string();
    }

    let shape_name = &operation.output.as_ref().unwrap().shape;
    let output_shape = &service
        .get_shape(shape_name)
        .expect("Shape missing from service definition");

    let mutable_result = output_shape
        .members
        .as_ref()
        .unwrap()
        .iter()
        .any(|(_, member)| member.location.is_some())
        || output_shape.payload.is_some();

    match output_shape.payload {
        None => json_body_parser(shape_name, mutable_result),
        Some(ref payload_member_name) => {
            let payload_member_shape =
                &output_shape.members.as_ref().unwrap()[payload_member_name].shape;
            let payload_shape = &service
                .get_shape(payload_member_shape)
                .expect("Shape missing from service definition");
            // is the shape required?
            let payload_shape_required = match output_shape.required {
                Some(ref s) => {
                    // if there's any required shape present the body payload parser will handle it
                    // This can't be converted to `s.is_empty()`. TODO: find out why.
                    s.len() > 0
                }
                None => false,
            };
            match payload_shape.shape_type {
                payload_type
                    if payload_type == ShapeType::Blob || payload_type == ShapeType::String =>
                {
                    payload_body_parser(
                        payload_type,
                        shape_name,
                        payload_member_name,
                        mutable_result,
                        payload_shape_required,
                    )
                }
                _ => json_body_parser(shape_name, false),
            }
        }
    }
}

/// Take the raw http response body and assign it to the payload field
/// on the result object
fn payload_body_parser(
    payload_type: ShapeType,
    output_shape: &str,
    payload_member: &str,
    mutable_result: bool,
    payload_required: bool,
) -> String {
    let response_body = match payload_required {
        true => match payload_type {
            ShapeType::Blob => "response.body",
            _ => "String::from_utf8_lossy(response.body.as_ref())",
        },
        false => match payload_type {
            ShapeType::Blob => "Some(response.body)",
            _ => "Some(String::from_utf8_lossy(response.body.as_ref()).into_owned())",
        },
    };

    format!(
        "
        let {mutable} result = {output_shape}::default();
        result.{payload_member} = {response_body};
        ",
        output_shape = output_shape,
        payload_member = payload_member.to_snake_case(),
        response_body = response_body,
        mutable = if mutable_result { "mut" } else { "" }
    )
}

/// Parse the http response body as a JSON object with serde, and use that
/// as the result object
fn json_body_parser(output_shape: &str, mutable_result: bool) -> String {
    format!(
        "let {mutable} result = proto::json::ResponsePayload::new(&response).deserialize::<{output_shape}, _>()?;",
        output_shape = output_shape,
        mutable = if mutable_result { "mut" } else { "" }
    )
}
<|endoftext|>


<commit_before>use event::Event;
use sass::comment::SassComment;
use sass::rule::SassRule;
use sass::selector::SassSelector;
use sass::variable::SassVariable;
use top_level_event::TopLevelEvent;

use std::borrow::Cow::Borrowed;

fn is_ascii_whitespace(c: u8) -> bool {
   is_newline(c) || is_ascii_whitespace_no_nl(c)
}

fn is_ascii_whitespace_no_nl(c: u8) -> bool {
    c == b'\t' || c == 0x0b || c == 0x0c || c == b' '
}

fn is_newline(c: u8) -> bool {
    c == b'\n' || c == b'\r'
}

fn isnt_newline(c: u8) -> bool {
    !is_newline(c)
}

// unusual among "scan" functions in that it scans from the _back_ of the string
// TODO: should also scan unicode whitespace?
fn scan_trailing_whitespace(data: &str) -> usize {
    match data.as_bytes().iter().rev().position(|&c| !is_ascii_whitespace_no_nl(c)) {
        Some(i) => i,
        None => data.len()
    }
}

#[derive(Debug)]
pub struct Tokenizer<'a> {
    sass: &'a str,
    bytes: &'a [u8],
    offset: usize,
    state: State,
    sass_rule_stack: Vec<SassRule<'a>>,
    current_sass_rule_selectors_done: bool,
}

#[derive(PartialEq, Debug, Copy, Clone)]
enum State {
    OutsideRules,
    InVariable,
    InComment,
    InRule,
    InSelectors,
    InProperties,
    EndRule,
    Eof,
}

impl<'a> Tokenizer<'a> {
    pub fn new(sass: &'a str) -> Tokenizer<'a> {
        Tokenizer {
            sass: &sass,
            bytes: &sass.as_bytes(),
            offset: 0,
            state: State::OutsideRules,
            sass_rule_stack: Vec::new(),
            current_sass_rule_selectors_done: false,
        }
    }

    fn start_something(&mut self) -> Option<TopLevelEvent<'a>> {
        let mut current_sass_rule = SassRule::new();
        self.current_sass_rule_selectors_done = false;

        self.pick_something();
        while self.state != State::OutsideRules {
            if self.state == State::Eof { return None }

            // See if this works now that State is Copy
            // self.state = match self.state

            if self.state == State::InSelectors {
                let sel = self.next_selector();
                if sel.is_some() {
                    current_sass_rule.selectors.push(sel.unwrap());
                }
            } else if self.state == State::InProperties {
                let prop = self.next_property();
                if prop.is_some() {
                    current_sass_rule.children.push(prop.unwrap());
                }
            } else if self.state == State::EndRule {
                // consume '}', should probably be checking that's what we actually have
                self.offset += 1;

                match self.sass_rule_stack.pop() {
                    Some(mut rule) => {
                        rule.children.push(Event::ChildRule(current_sass_rule));
                        current_sass_rule = rule;
                        self.pick_something();
                    },
                    None => self.state = State::OutsideRules,
                }
            } else if self.state == State::InRule {
                self.sass_rule_stack.push(current_sass_rule);
                current_sass_rule = SassRule::new();
                self.current_sass_rule_selectors_done = false;
                self.pick_something();
            } else if self.state == State::InVariable {
                let var = self.next_variable();
                if var.is_some() {
                    return var
                } else {
                    // is this really what we should be doing here? reachable?
                    self.state = State::Eof;
                }
            } else if self.state == State::InComment {
                let comment = self.next_comment();
                if comment.is_some() {
                    if self.sass_rule_stack.len() == 0 &&
                       current_sass_rule.selectors.len() == 0 &&
                       current_sass_rule.children.len() == 0 {
                           return Some(TopLevelEvent::Comment(SassComment { comment: comment.unwrap() }))
                    } else {
                        current_sass_rule.children.push(comment.unwrap());
                        self.pick_something();
                    }
                } else {
                    // is this really what we should be doing here? reachable?
                    self.state = State::Eof;
                }
            } else {
                println!("i dont know what to do for {:?}", self.state);
                println!("current sass rule = {:?}", current_sass_rule);
                self.state = State::Eof;
            }
        }

        Some(TopLevelEvent::Rule(current_sass_rule))
    }

    fn pick_something(&mut self) {
        self.skip_leading_whitespace();

        if self.offset == self.sass.len() {
            self.state = State::Eof;
            return
        }

        let c = self.bytes[self.offset];

        if c == b'}' {
            self.state = State::EndRule;
            return
        }

        if c == b'$' {
            self.state = State::InVariable;
            return
        }

        if c == b'/' && (self.offset + 1) < self.sass.len() {
            let d = self.bytes[self.offset + 1];
            if d == b'*' {
                self.state = State::InComment;
                return
            }
        }

        if self.current_sass_rule_selectors_done {
            self.state = State::InProperties;
            return
        }

        self.state = State::InSelectors;
    }

    fn scan_while<F>(&mut self, data: &str, f: F) -> usize
            where F: Fn(u8) -> bool {
        match data.as_bytes().iter().position(|&c| !f(c)) {
            Some(i) => i,
            None => data.len()
        }
    }

    fn skip_leading_whitespace(&mut self) {
        let mut i = self.offset;
        let limit = self.sass.len();

        while i < limit {
            let c = self.bytes[i];
            if is_ascii_whitespace(c) {
                i += self.scan_while(&self.sass[i..self.sass.len()], is_ascii_whitespace);
            } else if c == b'/' && i + 1 < limit && self.bytes[i + 1] == b'/' {
                i += self.scan_while(&self.sass[i..self.sass.len()], isnt_newline);
            } else {
                self.offset = i;
                return
            }
        }
        self.offset = limit;
    }

    fn next_comment(&mut self) -> Option<Event<'a>> {
        let comment_body_beginning = self.offset;
        let mut i = comment_body_beginning + 2;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b'*' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            if self.bytes[i+1] == b'/' {
                self.offset = i + 2;
                return Some(Event::Comment(Borrowed(&self.sass[comment_body_beginning..i + 2])))
            } else {
                i += 1;
            }
        }
        None
    }

    fn next_variable(&mut self) -> Option<TopLevelEvent<'a>> {
        // TODO: can parts of this be deduplicated with properties?
        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                return Some(TopLevelEvent::Variable(SassVariable{
                    name: Borrowed(&self.sass[name_beginning..name_end]),
                    value: Borrowed(&self.sass[value_beginning..value_end]),
                }))
            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_property(&mut self) -> Option<Event<'a>> {
        self.skip_leading_whitespace();

        if self.offset == self.sass.len() {
            self.state = State::Eof;
            return None
        }

        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        let c = self.bytes[i];
        if c == b'}' {
            self.state = State::EndRule;
            return None
        }

        let d = self.bytes[i + 1];
        if c == b'/' && d == b'*' {
            self.state = State::InComment;
            return None
        }

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' || c == b'{') {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            // Inefficient since we already skipped the whitespace and we'll have to
            // do it again but oh well
            let c = self.bytes[i];
            if c == b'{' {
                self.state = State::InRule;
                return None
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                if self.bytes[name_beginning] == b'$' {
                    return Some(Event::Variable(SassVariable {
                        name: Borrowed(&self.sass[name_beginning..name_end]),
                        value: Borrowed(&self.sass[value_beginning..value_end])
                    }))
                } else {
                    return Some(Event::Property(
                        Borrowed(&self.sass[name_beginning..name_end]),
                        Borrowed(&self.sass[value_beginning..value_end])
                    ))
                }
            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_selector(&mut self) -> Option<SassSelector<'a>> {
        self.skip_leading_whitespace();

        let beginning = self.offset;
        let mut i = beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b',' || c == b'{' || c == b':') {
                Some(pos) => { i += pos; },
                None => { i = limit; break; },
            }

            let c = self.bytes[i];

            if c == b':' {
                self.state = State::InProperties;
                return None
            }

            if c == b',' || c == b'{' {
                let n = scan_trailing_whitespace(&self.sass[beginning..i]);
                let end = i - n;
                if end > beginning {
                    if c == b'{' {
                        if self.current_sass_rule_selectors_done {
                            self.state = State::InRule;
                            return None
                        } else {
                            self.current_sass_rule_selectors_done = true;
                            self.state = State::InProperties;
                        }
                    }
                    self.offset = i + 1;
                    return Some(SassSelector::new(&self.sass[beginning..end]))
                } else {
                    // only whitespace between commas
                    self.offset += 1;
                    return self.next_selector()
                }
            }

            self.offset = i;
            if i > beginning {
                return Some(SassSelector::new(&self.sass[beginning..i]))
            }
            i += 1;
        }

        if i > beginning {
            self.offset = i;
            Some(SassSelector::new(&self.sass[beginning..i]))
        } else {
            self.state = State::Eof;
            None
        }
    }
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = TopLevelEvent<'a>;

    fn next(&mut self) -> Option<TopLevelEvent<'a>> {
        if self.offset < self.sass.len() {
            return self.start_something()
        }
        None
    }
}
<commit_msg>Parse import statements; don't actually import files yet<commit_after>use event::Event;
use sass::comment::SassComment;
use sass::rule::SassRule;
use sass::selector::SassSelector;
use sass::variable::SassVariable;
use top_level_event::TopLevelEvent;

use std::borrow::Cow::Borrowed;

fn is_ascii_whitespace(c: u8) -> bool {
   is_newline(c) || is_ascii_whitespace_no_nl(c)
}

fn is_ascii_whitespace_no_nl(c: u8) -> bool {
    c == b'\t' || c == 0x0b || c == 0x0c || c == b' '
}

fn is_newline(c: u8) -> bool {
    c == b'\n' || c == b'\r'
}

fn isnt_newline(c: u8) -> bool {
    !is_newline(c)
}

// unusual among "scan" functions in that it scans from the _back_ of the string
// TODO: should also scan unicode whitespace?
fn scan_trailing_whitespace(data: &str) -> usize {
    match data.as_bytes().iter().rev().position(|&c| !is_ascii_whitespace_no_nl(c)) {
        Some(i) => i,
        None => data.len()
    }
}

#[derive(Debug)]
pub struct Tokenizer<'a> {
    sass: &'a str,
    bytes: &'a [u8],
    offset: usize,
    state: State,
    sass_rule_stack: Vec<SassRule<'a>>,
    current_sass_rule_selectors_done: bool,
}

#[derive(PartialEq, Debug, Copy, Clone)]
enum State {
    OutsideRules,
    InVariable,
    InComment,
    InRule,
    InSelectors,
    InProperties,
    InImport,
    EndRule,
    Eof,
}

impl<'a> Tokenizer<'a> {
    pub fn new(sass: &'a str) -> Tokenizer<'a> {
        Tokenizer {
            sass: &sass,
            bytes: &sass.as_bytes(),
            offset: 0,
            state: State::OutsideRules,
            sass_rule_stack: Vec::new(),
            current_sass_rule_selectors_done: false,
        }
    }

    fn start_something(&mut self) -> Option<TopLevelEvent<'a>> {
        let mut current_sass_rule = SassRule::new();
        self.current_sass_rule_selectors_done = false;

        self.pick_something();
        while self.state != State::OutsideRules {
            if self.state == State::Eof { return None }

            // See if this works now that State is Copy
            // self.state = match self.state

            if self.state == State::InSelectors {
                let sel = self.next_selector();
                if sel.is_some() {
                    current_sass_rule.selectors.push(sel.unwrap());
                }
            } else if self.state == State::InProperties {
                let prop = self.next_property();
                if prop.is_some() {
                    current_sass_rule.children.push(prop.unwrap());
                }
            } else if self.state == State::EndRule {
                // consume '}', should probably be checking that's what we actually have
                self.offset += 1;

                match self.sass_rule_stack.pop() {
                    Some(mut rule) => {
                        rule.children.push(Event::ChildRule(current_sass_rule));
                        current_sass_rule = rule;
                        self.pick_something();
                    },
                    None => self.state = State::OutsideRules,
                }
            } else if self.state == State::InRule {
                self.sass_rule_stack.push(current_sass_rule);
                current_sass_rule = SassRule::new();
                self.current_sass_rule_selectors_done = false;
                self.pick_something();
            } else if self.state == State::InVariable {
                let var = self.next_variable();
                if var.is_some() {
                    return var
                } else {
                    // is this really what we should be doing here? reachable?
                    self.state = State::Eof;
                }
            } else if self.state == State::InComment {
                let comment = self.next_comment();
                if comment.is_some() {
                    if self.sass_rule_stack.len() == 0 &&
                       current_sass_rule.selectors.len() == 0 &&
                       current_sass_rule.children.len() == 0 {
                           return Some(TopLevelEvent::Comment(SassComment { comment: comment.unwrap() }))
                    } else {
                        current_sass_rule.children.push(comment.unwrap());
                        self.pick_something();
                    }
                } else {
                    // is this really what we should be doing here? reachable?
                    self.state = State::Eof;
                }
            } else if self.state == State::InImport {
                let imp = self.next_import_filename();
                if imp.is_some() {
                    println!("should import {:?}", imp.unwrap());
                }
                self.pick_something();
            } else {
                println!("i dont know what to do for {:?}", self.state);
                println!("current sass rule = {:?}", current_sass_rule);
                self.state = State::Eof;
            }
        }

        Some(TopLevelEvent::Rule(current_sass_rule))
    }

    fn pick_something(&mut self) {
        self.skip_leading_whitespace();

        if self.offset == self.sass.len() {
            self.state = State::Eof;
            return
        }

        let c = self.bytes[self.offset];

        if c == b'}' {
            self.state = State::EndRule;
            return
        }

        if c == b'$' {
            self.state = State::InVariable;
            return
        }

        if c == b'/' && (self.offset + 1) < self.sass.len() {
            let d = self.bytes[self.offset + 1];
            if d == b'*' {
                self.state = State::InComment;
                return
            }
        }

        if c == b'@' {
            self.state = State::InImport;
            return
        }

        if self.current_sass_rule_selectors_done {
            self.state = State::InProperties;
            return
        }

        self.state = State::InSelectors;
    }

    fn scan_while<F>(&mut self, data: &str, f: F) -> usize
            where F: Fn(u8) -> bool {
        match data.as_bytes().iter().position(|&c| !f(c)) {
            Some(i) => i,
            None => data.len()
        }
    }

    fn skip_leading_whitespace(&mut self) {
        let mut i = self.offset;
        let limit = self.sass.len();

        while i < limit {
            let c = self.bytes[i];
            if is_ascii_whitespace(c) {
                i += self.scan_while(&self.sass[i..self.sass.len()], is_ascii_whitespace);
            } else if c == b'/' && i + 1 < limit && self.bytes[i + 1] == b'/' {
                i += self.scan_while(&self.sass[i..self.sass.len()], isnt_newline);
            } else {
                self.offset = i;
                return
            }
        }
        self.offset = limit;
    }

    fn eat(&mut self, expected: &str) -> bool {
        for c in expected.as_bytes().iter() {
            if !self.eatch(c) {
                return false
            }
        }
        return true
    }

    fn eatch(&mut self, expected_char: &u8) -> bool {
        if self.bytes[self.offset] == *expected_char {
            self.offset += 1;
            true
        } else {
            false
        }
    }

    fn next_import_filename(&mut self) -> Option<&'a str> {
        if self.eat("@import \"") {
            let filename_beginning = self.offset;
            let mut i = self.offset;
            let limit = self.sass.len();

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b'"' ) {
                    Some(pos) => { i += pos; },
                    None => { break; },
                }
                let filename_end = i;
                self.offset = i + 1;
                if self.eatch(&b';') {
                    return Some(&self.sass[filename_beginning..filename_end])
                } else {
                    return None // UNEXPECTED CHAR
                }
            }
            self.offset = self.sass.len();
            None
        } else {
            None
        }
    }

    fn next_comment(&mut self) -> Option<Event<'a>> {
        let comment_body_beginning = self.offset;
        let mut i = comment_body_beginning + 2;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b'*' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            if self.bytes[i+1] == b'/' {
                self.offset = i + 2;
                return Some(Event::Comment(Borrowed(&self.sass[comment_body_beginning..i + 2])))
            } else {
                i += 1;
            }
        }
        None
    }

    fn next_variable(&mut self) -> Option<TopLevelEvent<'a>> {
        // TODO: can parts of this be deduplicated with properties?
        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' ) {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                return Some(TopLevelEvent::Variable(SassVariable{
                    name: Borrowed(&self.sass[name_beginning..name_end]),
                    value: Borrowed(&self.sass[value_beginning..value_end]),
                }))
            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_property(&mut self) -> Option<Event<'a>> {
        self.skip_leading_whitespace();

        if self.offset == self.sass.len() {
            self.state = State::Eof;
            return None
        }

        let name_beginning = self.offset;
        let mut i = name_beginning;
        let limit = self.sass.len();

        let c = self.bytes[i];
        if c == b'}' {
            self.state = State::EndRule;
            return None
        }

        if c == b'@' {
            self.state = State::InImport;
            return None
        }

        let d = self.bytes[i + 1];
        if c == b'/' && d == b'*' {
            self.state = State::InComment;
            return None
        }

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b':' || c == b'{') {
                Some(pos) => { i += pos; },
                None => { break; },
            }

            // Inefficient since we already skipped the whitespace and we'll have to
            // do it again but oh well
            let c = self.bytes[i];
            if c == b'{' {
                self.state = State::InRule;
                return None
            }

            let name_end = i;

            i += 1;
            self.offset = i;
            self.skip_leading_whitespace();

            let value_beginning = self.offset;
            i = value_beginning;

            while i < limit {
                match self.bytes[i..limit].iter().position(|&c| c == b';') {
                    Some(pos) => { i += pos; },
                    None => { i = limit; break; },
                }

                let value_end = i;
                self.offset = i + 1;

                self.skip_leading_whitespace();

                if self.bytes[name_beginning] == b'$' {
                    return Some(Event::Variable(SassVariable {
                        name: Borrowed(&self.sass[name_beginning..name_end]),
                        value: Borrowed(&self.sass[value_beginning..value_end])
                    }))
                } else {
                    return Some(Event::Property(
                        Borrowed(&self.sass[name_beginning..name_end]),
                        Borrowed(&self.sass[value_beginning..value_end])
                    ))
                }
            }
        }
        self.offset = self.sass.len();
        None
    }

    fn next_selector(&mut self) -> Option<SassSelector<'a>> {
        self.skip_leading_whitespace();

        let beginning = self.offset;
        let mut i = beginning;
        let limit = self.sass.len();

        while i < limit {
            match self.bytes[i..limit].iter().position(|&c| c == b',' || c == b'{' || c == b':') {
                Some(pos) => { i += pos; },
                None => { i = limit; break; },
            }

            let c = self.bytes[i];

            if c == b':' {
                self.state = State::InProperties;
                return None
            }

            if c == b',' || c == b'{' {
                let n = scan_trailing_whitespace(&self.sass[beginning..i]);
                let end = i - n;
                if end > beginning {
                    if c == b'{' {
                        if self.current_sass_rule_selectors_done {
                            self.state = State::InRule;
                            return None
                        } else {
                            self.current_sass_rule_selectors_done = true;
                            self.state = State::InProperties;
                        }
                    }
                    self.offset = i + 1;
                    return Some(SassSelector::new(&self.sass[beginning..end]))
                } else {
                    // only whitespace between commas
                    self.offset += 1;
                    return self.next_selector()
                }
            }

            self.offset = i;
            if i > beginning {
                return Some(SassSelector::new(&self.sass[beginning..i]))
            }
            i += 1;
        }

        if i > beginning {
            self.offset = i;
            Some(SassSelector::new(&self.sass[beginning..i]))
        } else {
            self.state = State::Eof;
            None
        }
    }
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = TopLevelEvent<'a>;

    fn next(&mut self) -> Option<TopLevelEvent<'a>> {
        if self.offset < self.sass.len() {
            return self.start_something()
        }
        None
    }
}
<|endoftext|>


<commit_before>//! Code to tokenize a `BufRead` instance into an iterator of `LineType`s.

use std::io::BufRead;

use types::LineType;

#[derive(Debug)]
pub struct Tokenizer<R: BufRead> {
    reader: R,
    current_line: String,
}

impl<R> Tokenizer<R> where R: BufRead {

    pub fn new(reader: R) -> Tokenizer<R> {
        Tokenizer {
            reader: reader,
            current_line: String::new(),
        }
    }

    pub fn next_token(&mut self) -> Option<LineType> {
        self.current_line.clear();
        let bytes_read = self.reader.read_line(&mut self.current_line).unwrap();
        match bytes_read {
            0 => None,
            _ => Some(LineType::from(&self.current_line[..])),
        }
    }

}
<commit_msg>Handle errors in Tokenizer::next_token<commit_after>//! Code to tokenize a `BufRead` instance into an iterator of `LineType`s.

use std::io::BufRead;

use types::LineType;

#[derive(Debug)]
pub struct Tokenizer<R: BufRead> {
    reader: R,
    current_line: String,
}

impl<R> Tokenizer<R> where R: BufRead {

    pub fn new(reader: R) -> Tokenizer<R> {
        Tokenizer {
            reader: reader,
            current_line: String::new(),
        }
    }

    pub fn next_token(&mut self) -> Option<LineType> {
        self.current_line.clear();
        let bytes_read = self.reader.read_line(&mut self.current_line);
        match bytes_read {
            Ok(0) => None,
            Err(e) => { warn!("Could not read line from token reader: {:?}", e); None},
            Ok(_) => Some(LineType::from(&self.current_line[..])),
        }
    }

}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    access_path::AccessPath,
    account_config::constants::{
        libra_root_address, type_tag_for_currency_code, CORE_CODE_ADDRESS,
    },
    event::EventHandle,
};
use anyhow::Result;
use move_core_types::{
    identifier::{IdentStr, Identifier},
    language_storage::{ResourceKey, StructTag},
    move_resource::MoveResource,
};
use serde::{Deserialize, Serialize};

/// Struct that represents a CurrencyInfo resource
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct CurrencyInfoResource {
    total_value: u128,
    preburn_value: u64,
    to_lbr_exchange_rate: u64,
    is_synthetic: bool,
    scaling_factor: u64,
    fractional_part: u64,
    currency_code: Identifier,
    can_mint: bool,
    mint_events: EventHandle,
    burn_events: EventHandle,
    preburn_events: EventHandle,
    cancel_burn_events: EventHandle,
    exchange_rate_update_events: EventHandle,
}

impl MoveResource for CurrencyInfoResource {
    const MODULE_NAME: &'static str = "Libra";
    const STRUCT_NAME: &'static str = "CurrencyInfo";
}

impl CurrencyInfoResource {
    pub fn currency_code(&self) -> &IdentStr {
        &self.currency_code
    }

    pub fn scaling_factor(&self) -> u64 {
        self.scaling_factor
    }

    pub fn fractional_part(&self) -> u64 {
        self.fractional_part
    }

    pub fn exchange_rate(&self) -> f32 {
        // Exchange rates are represented as 32|32 fixed-point numbers on-chain, so we divide by the scaling
        // factor (2^32) of the number to arrive at the floating point representation of the number.
        (self.to_lbr_exchange_rate as f32) / 2f32.powf(32f32)
    }

    pub fn convert_to_lbr(&self, amount: u64) -> u64 {
        (self.exchange_rate() * (amount as f32)) as u64
    }

    pub fn struct_tag_for(currency_code: Identifier) -> StructTag {
        StructTag {
            address: CORE_CODE_ADDRESS,
            module: CurrencyInfoResource::module_identifier(),
            name: CurrencyInfoResource::struct_identifier(),
            type_params: vec![type_tag_for_currency_code(currency_code)],
        }
    }

    pub fn resource_path_for(currency_code: Identifier) -> AccessPath {
        let resource_key = ResourceKey::new(
            libra_root_address(),
            CurrencyInfoResource::struct_tag_for(currency_code),
        );
        AccessPath::resource_access_path(&resource_key)
    }

    pub fn try_from_bytes(bytes: &[u8]) -> Result<Self> {
        lcs::from_bytes(bytes).map_err(Into::into)
    }

    pub fn mint_events(&self) -> &EventHandle {
        &self.mint_events
    }

    pub fn burn_events(&self) -> &EventHandle {
        &self.burn_events
    }

    pub fn preburn_events(&self) -> &EventHandle {
        &self.preburn_events
    }

    pub fn cancel_burn_events(&self) -> &EventHandle {
        &self.cancel_burn_events
    }

    pub fn exchange_rate_update_events(&self) -> &EventHandle {
        &self.exchange_rate_update_events
    }
}
<commit_msg>Expose preburn/total values of CurrencyInfoResource<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    access_path::AccessPath,
    account_config::constants::{
        libra_root_address, type_tag_for_currency_code, CORE_CODE_ADDRESS,
    },
    event::EventHandle,
};
use anyhow::Result;
use move_core_types::{
    identifier::{IdentStr, Identifier},
    language_storage::{ResourceKey, StructTag},
    move_resource::MoveResource,
};
use serde::{Deserialize, Serialize};

/// Struct that represents a CurrencyInfo resource
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct CurrencyInfoResource {
    total_value: u128,
    preburn_value: u64,
    to_lbr_exchange_rate: u64,
    is_synthetic: bool,
    scaling_factor: u64,
    fractional_part: u64,
    currency_code: Identifier,
    can_mint: bool,
    mint_events: EventHandle,
    burn_events: EventHandle,
    preburn_events: EventHandle,
    cancel_burn_events: EventHandle,
    exchange_rate_update_events: EventHandle,
}

impl MoveResource for CurrencyInfoResource {
    const MODULE_NAME: &'static str = "Libra";
    const STRUCT_NAME: &'static str = "CurrencyInfo";
}

impl CurrencyInfoResource {
    pub fn currency_code(&self) -> &IdentStr {
        &self.currency_code
    }

    pub fn scaling_factor(&self) -> u64 {
        self.scaling_factor
    }

    pub fn total_value(&self) -> u128 {
        self.total_value
    }

    pub fn preburn_value(&self) -> u64 {
        self.preburn_value
    }

    pub fn fractional_part(&self) -> u64 {
        self.fractional_part
    }

    pub fn exchange_rate(&self) -> f32 {
        // Exchange rates are represented as 32|32 fixed-point numbers on-chain, so we divide by the scaling
        // factor (2^32) of the number to arrive at the floating point representation of the number.
        (self.to_lbr_exchange_rate as f32) / 2f32.powf(32f32)
    }

    pub fn convert_to_lbr(&self, amount: u64) -> u64 {
        (self.exchange_rate() * (amount as f32)) as u64
    }

    pub fn struct_tag_for(currency_code: Identifier) -> StructTag {
        StructTag {
            address: CORE_CODE_ADDRESS,
            module: CurrencyInfoResource::module_identifier(),
            name: CurrencyInfoResource::struct_identifier(),
            type_params: vec![type_tag_for_currency_code(currency_code)],
        }
    }

    pub fn resource_path_for(currency_code: Identifier) -> AccessPath {
        let resource_key = ResourceKey::new(
            libra_root_address(),
            CurrencyInfoResource::struct_tag_for(currency_code),
        );
        AccessPath::resource_access_path(&resource_key)
    }

    pub fn try_from_bytes(bytes: &[u8]) -> Result<Self> {
        lcs::from_bytes(bytes).map_err(Into::into)
    }

    pub fn mint_events(&self) -> &EventHandle {
        &self.mint_events
    }

    pub fn burn_events(&self) -> &EventHandle {
        &self.burn_events
    }

    pub fn preburn_events(&self) -> &EventHandle {
        &self.preburn_events
    }

    pub fn cancel_burn_events(&self) -> &EventHandle {
        &self.cancel_burn_events
    }

    pub fn exchange_rate_update_events(&self) -> &EventHandle {
        &self.exchange_rate_update_events
    }
}
<|endoftext|>