<commit_before>//! redis-rs is a rust implementation of a Redis client library.  It exposes
//! a general purpose interface to Redis and also provides specific helpers for
//! commonly used functionality.
//!
//! # Basic Operation
//!
//! To open a connection you need to create a client and then to fetch a
//! connection from it.  In the future there will be a connection pool for
//! those, currently each connection is separate and not pooled.
//!
//! Queries are then performed through the `execute` and `query` methods.
//!
//! ```rust,no_run
//! extern crate redis;
//!
//! fn main() {
//!     let client = redis::Client::open("redis://127.0.0.1/").unwrap();
//!     let con = client.get_connection().unwrap();
//!     redis::cmd("SET").arg("my_key").arg(42i).execute(&con);
//!     assert_eq!(redis::cmd("GET").arg("my_key").query(&con), Ok(42i));
//! }
//! ```
//!
//! ## Type Conversions
//!
//! Because redis inherently is mostly type-less and the protocol is not
//! exactly friendly to developers, this library provides flexible support
//! for casting values to the intended results.  This is driven through the
//! `FromRedisValue` and `ToRedisArgs` traits.
//!
//! The `arg` method of the command will accept a wide range of types through
//! the `ToRedisArgs` trait and the `query` method of a command can convert the
//! value to what you expect the function to return through the `FromRedisValue`
//! trait.  This is quite flexible and allows vectors, tuples, hashsets, hashmaps
//! as well as optional values:
//!
//! ```rust,no_run
//! # use std::collections::{HashMap, HashSet};
//! # let client = redis::Client::open("redis://127.0.0.1/").unwrap();
//! # let con = client.get_connection().unwrap();
//! let count : i32 = redis::cmd("GET").arg("my_counter").query(&con).unwrap();
//! let count = redis::cmd("GET").arg("my_counter").query(&con).unwrap_or(0i32);
//! let k : Option<String> = redis::cmd("GET").arg("missing_key").query(&con).unwrap();
//! let name : String = redis::cmd("GET").arg("my_name").query(&con).unwrap();
//! let bin : Vec<u8> = redis::cmd("GET").arg("my_binary").query(&con).unwrap();
//! let map : HashMap<String, i32> = redis::cmd("HGETALL").arg("my_hash").query(&con).unwrap();
//! let keys : Vec<String> = redis::cmd("KEYS").query(&con).unwrap();
//! let mems : HashSet<i32> = redis::cmd("SMEMBERS").arg("s").query(&con).unwrap();
//! let (k1, k2) : (String, String) = redis::cmd("MGET").arg("k1").arg("k2").query(&con).unwrap();
//! ```
//!
//! ## Iteration Protocol
//!
//! In addition to sending a single query you iterators are also supported.  When
//! used with regular bulk responses they don't give you much over querying and
//! converting into a vector (both use a vector internally) but they can also
//! be used with `SCAN` like commands in which case iteration will send more
//! queries until the cursor is exhausted:
//!
//! ```rust,no_run
//! # let client = redis::Client::open("redis://127.0.0.1/").unwrap();
//! # let con = client.get_connection().unwrap();
//! let mut cmd = redis::cmd("SSCAN");
//! let mut iter : redis::Iter<int> = cmd.arg("my_set").cursor_arg(0).iter(&con).unwrap();
//! for x in iter {
//!     // do something with the item
//! }
//! ```
//!
//! As you can see the cursor argument needs to be defined with `cursor_arg` instead
//! of `arg` so that the library knows which argument needs updating as the
//! query is run for more items.
//!
//! ## Pipelining
//!
//! In addition to simple queries you can also send command pipelines.  This
//! is provided through the `pipe` function.  It works very similar to sending
//! individual commands but you can send more than one in one go.  This also
//! allows you to ignore individual results so that matching on the end result
//! is easier:
//!
//! ```rust,no_run
//! # let client = redis::Client::open("redis://127.0.0.1/").unwrap();
//! # let con = client.get_connection().unwrap();
//! let (k1, k2) : (i32, i32) = redis::pipe()
//!     .cmd("SET").arg("key_1").arg(42i).ignore()
//!     .cmd("SET").arg("key_2").arg(43i).ignore()
//!     .cmd("GET").arg("key_1")
//!     .cmd("GET").arg("key_2").query(&con).unwrap();
//! ```

#![crate_name = "redis"]
#![crate_type = "lib"]
#![license = "BSD"]
#![comment = "Bindings and wrapper functions for redis."]

#![deny(non_camel_case_types)]

#![feature(macro_rules)]
#![feature(default_type_params)]

#![experimental]

extern crate url;
extern crate serialize;

/* public api */
pub use parser::{parse_redis_value, Parser};
pub use client::Client;
pub use connection::Connection;
pub use cmd::{cmd, Cmd, pipe, Pipeline, Iter};

#[doc(hidden)]
pub use types::{
    /* low level values */
    Value,
        Nil,
        Int,
        Data,
        Bulk,
        Okay,
        Status,

    /* error and result types */
    Error,
    RedisResult,

    /* error kinds */
    ErrorKind,
        ResponseError,
        TypeError,
        ExecAbortError,
        BusyLoadingError,
        NoScriptError,
        ExtensionError,
        InternalIoError,

    /* utility types */
    InfoDict,

    /* conversion traits */
    FromRedisValue,
    ToRedisArgs,
};

pub mod macros;

mod parser;
mod client;
mod connection;
mod types;
mod cmd;
<commit_msg>Added atomic introduction docs<commit_after>//! redis-rs is a rust implementation of a Redis client library.  It exposes
//! a general purpose interface to Redis and also provides specific helpers for
//! commonly used functionality.
//!
//! # Basic Operation
//!
//! To open a connection you need to create a client and then to fetch a
//! connection from it.  In the future there will be a connection pool for
//! those, currently each connection is separate and not pooled.
//!
//! Queries are then performed through the `execute` and `query` methods.
//!
//! ```rust,no_run
//! extern crate redis;
//!
//! fn main() {
//!     let client = redis::Client::open("redis://127.0.0.1/").unwrap();
//!     let con = client.get_connection().unwrap();
//!     redis::cmd("SET").arg("my_key").arg(42i).execute(&con);
//!     assert_eq!(redis::cmd("GET").arg("my_key").query(&con), Ok(42i));
//! }
//! ```
//!
//! ## Type Conversions
//!
//! Because redis inherently is mostly type-less and the protocol is not
//! exactly friendly to developers, this library provides flexible support
//! for casting values to the intended results.  This is driven through the
//! `FromRedisValue` and `ToRedisArgs` traits.
//!
//! The `arg` method of the command will accept a wide range of types through
//! the `ToRedisArgs` trait and the `query` method of a command can convert the
//! value to what you expect the function to return through the `FromRedisValue`
//! trait.  This is quite flexible and allows vectors, tuples, hashsets, hashmaps
//! as well as optional values:
//!
//! ```rust,no_run
//! # use std::collections::{HashMap, HashSet};
//! # let client = redis::Client::open("redis://127.0.0.1/").unwrap();
//! # let con = client.get_connection().unwrap();
//! let count : i32 = redis::cmd("GET").arg("my_counter").query(&con).unwrap();
//! let count = redis::cmd("GET").arg("my_counter").query(&con).unwrap_or(0i32);
//! let k : Option<String> = redis::cmd("GET").arg("missing_key").query(&con).unwrap();
//! let name : String = redis::cmd("GET").arg("my_name").query(&con).unwrap();
//! let bin : Vec<u8> = redis::cmd("GET").arg("my_binary").query(&con).unwrap();
//! let map : HashMap<String, i32> = redis::cmd("HGETALL").arg("my_hash").query(&con).unwrap();
//! let keys : Vec<String> = redis::cmd("KEYS").query(&con).unwrap();
//! let mems : HashSet<i32> = redis::cmd("SMEMBERS").arg("s").query(&con).unwrap();
//! let (k1, k2) : (String, String) = redis::cmd("MGET").arg("k1").arg("k2").query(&con).unwrap();
//! ```
//!
//! ## Iteration Protocol
//!
//! In addition to sending a single query you iterators are also supported.  When
//! used with regular bulk responses they don't give you much over querying and
//! converting into a vector (both use a vector internally) but they can also
//! be used with `SCAN` like commands in which case iteration will send more
//! queries until the cursor is exhausted:
//!
//! ```rust,no_run
//! # let client = redis::Client::open("redis://127.0.0.1/").unwrap();
//! # let con = client.get_connection().unwrap();
//! let mut cmd = redis::cmd("SSCAN");
//! let mut iter : redis::Iter<int> = cmd.arg("my_set").cursor_arg(0).iter(&con).unwrap();
//! for x in iter {
//!     // do something with the item
//! }
//! ```
//!
//! As you can see the cursor argument needs to be defined with `cursor_arg` instead
//! of `arg` so that the library knows which argument needs updating as the
//! query is run for more items.
//!
//! ## Pipelining
//!
//! In addition to simple queries you can also send command pipelines.  This
//! is provided through the `pipe` function.  It works very similar to sending
//! individual commands but you can send more than one in one go.  This also
//! allows you to ignore individual results so that matching on the end result
//! is easier:
//!
//! ```rust,no_run
//! # let client = redis::Client::open("redis://127.0.0.1/").unwrap();
//! # let con = client.get_connection().unwrap();
//! let (k1, k2) : (i32, i32) = redis::pipe()
//!     .cmd("SET").arg("key_1").arg(42i).ignore()
//!     .cmd("SET").arg("key_2").arg(43i).ignore()
//!     .cmd("GET").arg("key_1")
//!     .cmd("GET").arg("key_2").query(&con).unwrap();
//! ```
//!
//! If you want the pipeline to be wrapped in a `MULTI`/`EXEC` block you can
//! easily do that by switching the pipeline into `atomic` mode.  From the
//! caller's point of view nothing changes, the pipeline itself will take
//! care of the rest for you:
//!
//! ```rust,no_run
//! # let client = redis::Client::open("redis://127.0.0.1/").unwrap();
//! # let con = client.get_connection().unwrap();
//! let (k1, k2) : (i32, i32) = redis::pipe()
//!     .atomic()
//!     .cmd("SET").arg("key_1").arg(42i).ignore()
//!     .cmd("SET").arg("key_2").arg(43i).ignore()
//!     .cmd("GET").arg("key_1")
//!     .cmd("GET").arg("key_2").query(&con).unwrap();
//! ```

#![crate_name = "redis"]
#![crate_type = "lib"]
#![license = "BSD"]
#![comment = "Bindings and wrapper functions for redis."]

#![deny(non_camel_case_types)]

#![feature(macro_rules)]
#![feature(default_type_params)]

#![experimental]

extern crate url;
extern crate serialize;

/* public api */
pub use parser::{parse_redis_value, Parser};
pub use client::Client;
pub use connection::Connection;
pub use cmd::{cmd, Cmd, pipe, Pipeline, Iter};

#[doc(hidden)]
pub use types::{
    /* low level values */
    Value,
        Nil,
        Int,
        Data,
        Bulk,
        Okay,
        Status,

    /* error and result types */
    Error,
    RedisResult,

    /* error kinds */
    ErrorKind,
        ResponseError,
        TypeError,
        ExecAbortError,
        BusyLoadingError,
        NoScriptError,
        ExtensionError,
        InternalIoError,

    /* utility types */
    InfoDict,

    /* conversion traits */
    FromRedisValue,
    ToRedisArgs,
};

pub mod macros;

mod parser;
mod client;
mod connection;
mod types;
mod cmd;
<|endoftext|>


<commit_before>use ::iter::Bytes;

pub unsafe fn parse_uri_batch_32<'a>(bytes: &mut Bytes<'a>) {
    while bytes.as_ref().len() >= 32 {
        let advance = match_url_char_32_avx(bytes.as_ref());
        bytes.advance(advance);

        if advance != 32 {
            break;
        }
    }
}

#[target_feature(enable = "avx2")]
#[inline]
#[allow(non_snake_case, overflowing_literals)]
unsafe fn match_url_char_32_avx(buf: &[u8]) -> usize {
    debug_assert!(buf.len() >= 32);

    #[cfg(target_arch = "x86")]
    use core::arch::x86::*;
    #[cfg(target_arch = "x86_64")]
    use core::arch::x86_64::*;

    let ptr = buf.as_ptr();

    let LSH: __m256i = _mm256_set1_epi8(0x0f);
    let URI: __m256i = _mm256_setr_epi8(
        0xb8, 0xfc, 0xf8, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc,
        0xfc, 0xfc, 0xfc, 0x7c, 0x54, 0x7c, 0xd4, 0x7c,
        0xb8, 0xfc, 0xf8, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc,
        0xfc, 0xfc, 0xfc, 0x7c, 0x54, 0x7c, 0xd4, 0x7c,
    );
    let ARF: __m256i = _mm256_setr_epi8(
        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
    );

    let data = _mm256_lddqu_si256(ptr as *const _);
    let rbms = _mm256_shuffle_epi8(URI, data);
    let cols = _mm256_and_si256(LSH, _mm256_srli_epi16(data, 4));
    let bits = _mm256_and_si256(_mm256_shuffle_epi8(ARF, cols), rbms);

    let v = _mm256_cmpeq_epi8(bits, _mm256_setzero_si256());
    let r = 0xffffffff_00000000 | _mm256_movemask_epi8(v) as u64;

    _tzcnt_u64(r) as usize
}

pub unsafe fn match_header_value_batch_32(bytes: &mut Bytes) {
    while bytes.as_ref().len() >= 32 {
        let advance = match_header_value_char_32_avx(bytes.as_ref());
        bytes.advance(advance);

        if advance != 32 {
            break;
        }
    }
}

#[target_feature(enable = "avx2")]
#[inline]
#[allow(non_snake_case)]
unsafe fn match_header_value_char_32_avx(buf: &[u8]) -> usize {
    debug_assert!(buf.len() >= 32);

    #[cfg(target_arch = "x86")]
    use core::arch::x86::*;
    #[cfg(target_arch = "x86_64")]
    use core::arch::x86_64::*;

    let ptr = buf.as_ptr();

    // %x09 %x20-%x7e %x80-%xff
    let TAB: __m256i = _mm256_set1_epi8(0x09);
    let DEL: __m256i = _mm256_set1_epi8(0x7f);
    let LOW: __m256i = _mm256_set1_epi8(0x1f);

    let dat = _mm256_lddqu_si256(ptr as *const _);
    let low = _mm256_cmpgt_epi8(dat, LOW);
    let tab = _mm256_cmpeq_epi8(dat, TAB);
    let del = _mm256_cmpeq_epi8(dat, DEL);
    let bit = _mm256_andnot_si256(del, _mm256_or_si256(low, tab));
    let rev = _mm256_cmpeq_epi8(bit, _mm256_setzero_si256());
    let res = 0xffffffff_00000000 | _mm256_movemask_epi8(rev) as u64;

    _tzcnt_u64(res) as usize
}

<commit_msg>disable avx2 on 32bit archs temporarily<commit_after>use ::iter::Bytes;

pub unsafe fn parse_uri_batch_32<'a>(bytes: &mut Bytes<'a>) {
    while bytes.as_ref().len() >= 32 {
        let advance = match_url_char_32_avx(bytes.as_ref());
        bytes.advance(advance);

        if advance != 32 {
            break;
        }
    }
}

#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2")]
#[inline]
#[allow(non_snake_case, overflowing_literals)]
unsafe fn match_url_char_32_avx(buf: &[u8]) -> usize {
    debug_assert!(buf.len() >= 32);

    /*
    #[cfg(target_arch = "x86")]
    use core::arch::x86::*;
    #[cfg(target_arch = "x86_64")]
    */
    use core::arch::x86_64::*;

    let ptr = buf.as_ptr();

    let LSH: __m256i = _mm256_set1_epi8(0x0f);
    let URI: __m256i = _mm256_setr_epi8(
        0xb8, 0xfc, 0xf8, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc,
        0xfc, 0xfc, 0xfc, 0x7c, 0x54, 0x7c, 0xd4, 0x7c,
        0xb8, 0xfc, 0xf8, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc,
        0xfc, 0xfc, 0xfc, 0x7c, 0x54, 0x7c, 0xd4, 0x7c,
    );
    let ARF: __m256i = _mm256_setr_epi8(
        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
        0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
    );

    let data = _mm256_lddqu_si256(ptr as *const _);
    let rbms = _mm256_shuffle_epi8(URI, data);
    let cols = _mm256_and_si256(LSH, _mm256_srli_epi16(data, 4));
    let bits = _mm256_and_si256(_mm256_shuffle_epi8(ARF, cols), rbms);

    let v = _mm256_cmpeq_epi8(bits, _mm256_setzero_si256());
    let r = 0xffffffff_00000000 | _mm256_movemask_epi8(v) as u64;

    _tzcnt_u64(r) as usize
}

#[cfg(target_arch = "x86")]
unsafe fn match_url_char_32_avx(_: &[u8]) -> usize {
    unreachable!("AVX2 detection should be disabled for x86");
}

pub unsafe fn match_header_value_batch_32(bytes: &mut Bytes) {
    while bytes.as_ref().len() >= 32 {
        let advance = match_header_value_char_32_avx(bytes.as_ref());
        bytes.advance(advance);

        if advance != 32 {
            break;
        }
    }
}

#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2")]
#[inline]
#[allow(non_snake_case)]
unsafe fn match_header_value_char_32_avx(buf: &[u8]) -> usize {
    debug_assert!(buf.len() >= 32);

    /*
    #[cfg(target_arch = "x86")]
    use core::arch::x86::*;
    #[cfg(target_arch = "x86_64")]
    */
    use core::arch::x86_64::*;

    let ptr = buf.as_ptr();

    // %x09 %x20-%x7e %x80-%xff
    let TAB: __m256i = _mm256_set1_epi8(0x09);
    let DEL: __m256i = _mm256_set1_epi8(0x7f);
    let LOW: __m256i = _mm256_set1_epi8(0x1f);

    let dat = _mm256_lddqu_si256(ptr as *const _);
    let low = _mm256_cmpgt_epi8(dat, LOW);
    let tab = _mm256_cmpeq_epi8(dat, TAB);
    let del = _mm256_cmpeq_epi8(dat, DEL);
    let bit = _mm256_andnot_si256(del, _mm256_or_si256(low, tab));
    let rev = _mm256_cmpeq_epi8(bit, _mm256_setzero_si256());
    let res = 0xffffffff_00000000 | _mm256_movemask_epi8(rev) as u64;

    _tzcnt_u64(res) as usize
}

#[cfg(target_arch = "x86")]
unsafe fn match_header_value_char_32_avx(_: &[u8]) -> usize {
    unreachable!("AVX2 detection should be disabled for x86");
}
<|endoftext|>


<commit_before>use {Errno, Result};
use libc::{self, c_int};
use std::os::unix::io::RawFd;
use std::ptr;

bitflags!(
    #[repr(C)]
    flags EpollFlags: u32 {
        const EPOLLIN = 0x001,
        const EPOLLPRI = 0x002,
        const EPOLLOUT = 0x004,
        const EPOLLRDNORM = 0x040,
        const EPOLLRDBAND = 0x080,
        const EPOLLWRNORM = 0x100,
        const EPOLLWRBAND = 0x200,
        const EPOLLMSG = 0x400,
        const EPOLLERR = 0x008,
        const EPOLLHUP = 0x010,
        const EPOLLRDHUP = 0x2000,
        const EPOLLEXCLUSIVE = 1 << 28,
        const EPOLLWAKEUP = 1 << 29,
        const EPOLLONESHOT = 1 << 30,
        const EPOLLET = 1 << 31
    }
);

#[derive(Clone, Copy)]
#[repr(C)]
pub enum EpollOp {
    EpollCtlAdd = 1,
    EpollCtlDel = 2,
    EpollCtlMod = 3
}

libc_bitflags!{
    flags EpollCreateFlags: c_int {
        EPOLL_CLOEXEC,
    }
}

#[derive(Clone, Copy)]
#[repr(C)]
pub struct EpollEvent {
    event: libc::epoll_event,
}

impl EpollEvent {
    pub fn new(events: EpollFlags, data: u64) -> Self {
        EpollEvent { event: libc::epoll_event { events: events.bits(), u64: data } }
    }

    pub empty() -> Self {
        EpollEvent::new(EpollFlags::from_bits(0), 0);
    }

    pub fn events(&self) -> EpollFlags {
        EpollFlags::from_bits(self.event.events).unwrap()
    }

    pub fn data(&self) -> u64 {
        self.event.u64
    }
}

impl<'a> Into<&'a mut EpollEvent> for Option<&'a mut EpollEvent> {
    #[inline]
    fn into(self) -> &'a mut EpollEvent {
        match self {
            Some(epoll_event) => epoll_event,
            None => unsafe { &mut *ptr::null_mut::<EpollEvent>() }
        }
    }
}

#[inline]
pub fn epoll_create() -> Result<RawFd> {
    let res = unsafe { libc::epoll_create(1024) };

    Errno::result(res)
}

#[inline]
pub fn epoll_create1(flags: EpollCreateFlags) -> Result<RawFd> {
    let res = unsafe { libc::epoll_create1(flags.bits()) };

    Errno::result(res)
}

#[inline]
pub fn epoll_ctl<'a, T>(epfd: RawFd, op: EpollOp, fd: RawFd, event: T) -> Result<()>
    where T: Into<&'a mut EpollEvent>
{
    let res = unsafe { libc::epoll_ctl(epfd, op as c_int, fd, &mut event.into().event) };
    Errno::result(res).map(drop)
}

#[inline]
pub fn epoll_wait(epfd: RawFd, events: &mut [EpollEvent], timeout_ms: isize) -> Result<usize> {
    let res = unsafe {
        libc::epoll_wait(epfd, events.as_mut_ptr() as *mut libc::epoll_event, events.len() as c_int, timeout_ms as c_int)
    };

    Errno::result(res).map(|r| r as usize)
}
<commit_msg>fix EpollEvent::empty()<commit_after>use {Errno, Result};
use libc::{self, c_int};
use std::os::unix::io::RawFd;
use std::ptr;

bitflags!(
    #[repr(C)]
    flags EpollFlags: u32 {
        const EPOLLIN = 0x001,
        const EPOLLPRI = 0x002,
        const EPOLLOUT = 0x004,
        const EPOLLRDNORM = 0x040,
        const EPOLLRDBAND = 0x080,
        const EPOLLWRNORM = 0x100,
        const EPOLLWRBAND = 0x200,
        const EPOLLMSG = 0x400,
        const EPOLLERR = 0x008,
        const EPOLLHUP = 0x010,
        const EPOLLRDHUP = 0x2000,
        const EPOLLEXCLUSIVE = 1 << 28,
        const EPOLLWAKEUP = 1 << 29,
        const EPOLLONESHOT = 1 << 30,
        const EPOLLET = 1 << 31
    }
);

#[derive(Clone, Copy)]
#[repr(C)]
pub enum EpollOp {
    EpollCtlAdd = 1,
    EpollCtlDel = 2,
    EpollCtlMod = 3
}

libc_bitflags!{
    flags EpollCreateFlags: c_int {
        EPOLL_CLOEXEC,
    }
}

#[derive(Clone, Copy)]
#[repr(C)]
pub struct EpollEvent {
    event: libc::epoll_event,
}

impl EpollEvent {
    pub fn new(events: EpollFlags, data: u64) -> Self {
        EpollEvent { event: libc::epoll_event { events: events.bits(), u64: data } }
    }

    pub fn empty() -> Self {
        EpollEvent::new(EpollFlags::empty(), 0)
    }

    pub fn events(&self) -> EpollFlags {
        EpollFlags::from_bits(self.event.events).unwrap()
    }

    pub fn data(&self) -> u64 {
        self.event.u64
    }
}

impl<'a> Into<&'a mut EpollEvent> for Option<&'a mut EpollEvent> {
    #[inline]
    fn into(self) -> &'a mut EpollEvent {
        match self {
            Some(epoll_event) => epoll_event,
            None => unsafe { &mut *ptr::null_mut::<EpollEvent>() }
        }
    }
}

#[inline]
pub fn epoll_create() -> Result<RawFd> {
    let res = unsafe { libc::epoll_create(1024) };

    Errno::result(res)
}

#[inline]
pub fn epoll_create1(flags: EpollCreateFlags) -> Result<RawFd> {
    let res = unsafe { libc::epoll_create1(flags.bits()) };

    Errno::result(res)
}

#[inline]
pub fn epoll_ctl<'a, T>(epfd: RawFd, op: EpollOp, fd: RawFd, event: T) -> Result<()>
    where T: Into<&'a mut EpollEvent>
{
    let res = unsafe { libc::epoll_ctl(epfd, op as c_int, fd, &mut event.into().event) };
    Errno::result(res).map(drop)
}

#[inline]
pub fn epoll_wait(epfd: RawFd, events: &mut [EpollEvent], timeout_ms: isize) -> Result<usize> {
    let res = unsafe {
        libc::epoll_wait(epfd, events.as_mut_ptr() as *mut libc::epoll_event, events.len() as c_int, timeout_ms as c_int)
    };

    Errno::result(res).map(|r| r as usize)
}
<|endoftext|>


<commit_before>use subframe;
use subframe::{Subframe, MAX_FIXED_ORDER};

pub fn fixed_restore_signal(order: usize,
                            residual: &[i32],
                            output: &mut [i32]) {
  debug_assert!(order <= MAX_FIXED_ORDER);

  let polynomial = [ &[][..]
                   , &[1][..]
                   , &[-1, 2][..]
                   , &[1, -3, 3][..]
                   , &[-1, 4, -6, 4][..]
                   ];

  let coefficients = polynomial[order];

  for i in 0..residual.len() {
    let offset     = i + order;
    let prediction = coefficients.iter()
                      .zip(&output[i..offset])
                      .fold(0, |result, (coefficient, signal)|
                            result + coefficient * signal);

    output[offset] = residual[i] + prediction;
  }
}

pub fn decode(subframe: &Subframe, output: &mut [i32]) {
  match subframe.data {
    subframe::Data::Constant(_)      => unimplemented!(),
    subframe::Data::Verbatim(_)      => unimplemented!(),
    subframe::Data::Fixed(ref fixed) => {
      let order = fixed.order as usize;

      for i in 0..order {
        output[i] = fixed.warmup[i];
      }

      fixed_restore_signal(order, &fixed.residual, output);
    }
    subframe::Data::LPC(_)           => unimplemented!(),
  }

  if subframe.wasted_bits > 0 {
    for value in output {
      *value <<= subframe.wasted_bits;
    }
  }
}
<commit_msg>Add restoration for FIR linear predictions<commit_after>use subframe;
use subframe::{Subframe, MAX_FIXED_ORDER, MAX_LPC_ORDER};

pub fn fixed_restore_signal(order: usize,
                            residual: &[i32],
                            output: &mut [i32]) {
  debug_assert!(order <= MAX_FIXED_ORDER);

  let polynomial = [ &[][..]
                   , &[1][..]
                   , &[-1, 2][..]
                   , &[1, -3, 3][..]
                   , &[-1, 4, -6, 4][..]
                   ];

  let coefficients = polynomial[order];

  for i in 0..residual.len() {
    let offset     = i + order;
    let prediction = coefficients.iter()
                      .zip(&output[i..offset])
                      .fold(0, |result, (coefficient, signal)|
                            result + coefficient * signal);

    output[offset] = residual[i] + prediction;
  }
}

pub fn lpc_restore_signal(quantization_level: i8,
                          coefficients: &[i32],
                          residual: &[i32],
                          output: &mut [i32]) {
  let order = coefficients.len();

  debug_assert!(order <= MAX_LPC_ORDER);

  for i in 0..residual.len() {
    let offset     = i + order;
    let prediction = coefficients.iter().rev()
                       .zip(&output[i..offset])
                       .fold(0, |result, (coefficient, signal)|
                             result + coefficient * signal);

    output[offset] = residual[i] + (prediction >> quantization_level);
  }
}

pub fn decode(subframe: &Subframe, output: &mut [i32]) {
  match subframe.data {
    subframe::Data::Constant(_)      => unimplemented!(),
    subframe::Data::Verbatim(_)      => unimplemented!(),
    subframe::Data::Fixed(ref fixed) => {
      let order = fixed.order as usize;

      for i in 0..order {
        output[i] = fixed.warmup[i];
      }

      fixed_restore_signal(order, &fixed.residual, output);
    }
    subframe::Data::LPC(_)           => unimplemented!(),
  }

  if subframe.wasted_bits > 0 {
    for value in output {
      *value <<= subframe.wasted_bits;
    }
  }
}
<|endoftext|>


<commit_before>use subframe::{self, Subframe, MAX_FIXED_ORDER, MAX_LPC_ORDER};

use std::ptr;

// Restore the original signal from a fixed linear prediction.
//
// Signal restoration is accomplished by summing up the residual and the
// predictor. With the linear prediction being "fixed", the polynomials will
// remain the same depending on the order value. And the max order is
// `MAX_FIXED_ORDER`, which is 4.
//
// This function also assumes that `output` already has the warm up values
// from the `Fixed` subframe in it.
pub fn fixed_restore_signal(order: usize,
                            block_size: usize,
                            output: &mut [i32]) {
  debug_assert!(order <= MAX_FIXED_ORDER);

  let polynomial = [ &[][..]
                   , &[1][..]
                   , &[-1, 2][..]
                   , &[1, -3, 3][..]
                   , &[-1, 4, -6, 4][..]
                   ];

  let coefficients = polynomial[order];
  let length       = block_size - order;

  for i in 0..length {
    let offset     = i + order;
    let prediction = coefficients.iter()
                      .zip(&output[i..offset])
                      .fold(0, |result, (coefficient, signal)|
                            result + coefficient * signal);

    output[offset] += prediction;
  }
}

// Restore the original signal from a FIR linear prediction.
//
// Signal restoration is accomplished by summing up the residual and the
// predictor. Figuring out the linear prediction for finite impulse response
// is a bit more involved because you have more to deal with, but the
// concept is very similar to the fixed version. Coefficients are passed in
// and reversed within function and the result of these reverse order
// coefficients and warm up values are summed and the quantization level
// will determine how much the bits gets shifted in order to figure the
// current predictor for it's corresponding residual value.
//
// The order doesn't get passed in explicitly because the coefficients
// length is assumed to be the value of order. And the max order is
// `MAX_LPC_ORDER`, which is 32. This function also assumes that `output`
// already has the warm up values from the `LPC` subframe in it.
pub fn lpc_restore_signal(quantization_level: i8,
                          block_size: usize,
                          coefficients: &[i32],
                          output: &mut [i32]) {
  let order  = coefficients.len();
  let length = block_size - order;

  debug_assert!(order <= MAX_LPC_ORDER);

  for i in 0..length {
    let offset     = i + order;
    let prediction = coefficients.iter().rev()
                       .zip(&output[i..offset])
                       .fold(0, |result, (coefficient, signal)|
                             result + coefficient * signal);

    output[offset] += prediction >> quantization_level;
  }
}

/// Decodes the current subframe.
///
/// * `Constant` - fills the length of `output` with the constant value
///   within the subframe.
/// * `Verbatim` - copies the data within the verbatim subframe over to
///   `output`.
/// * `Fixed` - restore the signal of the fixed linear prediction and put
///   the result into `output`.
/// * `LPC` - restore the signal of the finite impulse response linear
///   prediction and put the result into `output`.
pub fn decode(subframe: &Subframe, output: &mut [i32]) {
  match subframe.data {
    subframe::Data::Constant(constant)     => {
      for i in 0..output.len() {
        output[i] = constant
      }
    }
    subframe::Data::Verbatim(ref verbatim) => {
      let length = verbatim.len();

      unsafe {
        ptr::copy(verbatim.as_ptr(), output.as_mut_ptr(), length)
      }
    }
    subframe::Data::Fixed(ref fixed)       => {
      let order = fixed.order as usize;

      for i in 0..order {
        output[i] = fixed.warmup[i];
      }

      fixed_restore_signal(order, &fixed.residual, output);
    }
    subframe::Data::LPC(ref lpc)           => {
      let order        = lpc.order as usize;
      let coefficients = &lpc.qlp_coefficients[0..order];

      for i in 0..order {
        output[i] = lpc.warmup[i];
      }

      lpc_restore_signal(lpc.quantization_level, coefficients, &lpc.residual,
                         output);
    }
  }

  if subframe.wasted_bits > 0 {
    for value in output {
      *value <<= subframe.wasted_bits;
    }
  }
}

#[cfg(test)]
mod tests {
  use super::*;

  use subframe;
  use subframe::{
    Subframe, Fixed, LPC,
    EntropyCodingMethod, CodingMethod, PartitionedRice,
    PartitionedRiceContents,
  };

  #[test]
  fn test_fixed_restore_signal() {
    let residuals   = [ &[-19, -16, 17, -23, -7, 16, -16, -5, 3
                         , -8, -13, -15, -1][..]
                      , &[-6513][..]
                      ];
    let mut outputs = [ &mut [-729, -722, -667, 0, 0, 0, 0, 0, 0
                             , 0, 0, 0, 0, 0, 0, 0][..]
                      , &mut [21877, 27482, 0][..]
                      ];

    fixed_restore_signal(3, &residuals[0], &mut outputs[0]);
    fixed_restore_signal(2, &residuals[1], &mut outputs[1]);

    assert_eq!(&outputs[0], &[-729, -722, -667, -583, -486, -359, -225, -91
                             , 59, 209, 354, 497, 630, 740, 812, 845]);
    assert_eq!(&outputs[1], &[21877, 27482, 26574]);
  }

  #[test]
  fn test_lpc_restore_signal() {
    let residuals    = [ &[-2, -23, 14, 6, 3, -4, 12, -2, 10][..]
                       , &[3157][..]
                       ];
    let coefficients = [ &[1042, -399, -75, -269, 121, 166, -75][..]
                       , &[1757, -1199, 879, -836, 555, -255, 119][..]
                       ];
    let mut outputs = [ &mut [-796, -547, -285, -32, 199, 443, 670
                             , 0, 0, 0, 0, 0, 0, 0, 0, 0][..]
                      , &mut [-21363, -21951, -22649, -24364, -27297, -26870
                             ,-30017, 0][..]
                      ];

    lpc_restore_signal(9, &coefficients[0], &residuals[0], &mut outputs[0]);
    lpc_restore_signal(10, &coefficients[1], &residuals[1], &mut outputs[1]);

    assert_eq!(&outputs[0], &[-796, -547, -285, -32, 199, 443, 670, 875
                             , 1046, 1208, 1343, 1454, 1541, 1616, 1663
                             , 1701]);
    assert_eq!(&outputs[1], &[-21363, -21951, -22649, -24364, -27297, -26870
                             , -30017, -29718]);
  }

  #[test]
  fn test_decode() {
    let mut output = [0; 16];

    let constant = Subframe {
      data: subframe::Data::Constant(4),
      wasted_bits: 0,
    };

    let verbatim = Subframe {
      data: subframe::Data::Verbatim(vec![16, -3, 55, 49, -32, 6, 40, -90, 1
                                         ,0, 77, -12, 84, 10, -112, 136]),
      wasted_bits: 0,
    };

    let fixed = Subframe {
      data: subframe::Data::Fixed(Fixed {
        entropy_coding_method: EntropyCodingMethod {
          method_type: CodingMethod::PartitionedRice,
          data: PartitionedRice {
            order: 0,
            contents: PartitionedRiceContents {
              parameters: vec![],
              raw_bits: vec![],
            },
          },
        },
        order: 3,
        warmup: [-729, -722, -667, 0],
        residual: vec![-19, -16, 17, -23, -7, 16, -16, -5, 3 , -8, -13, -15
                      ,-1],
      }),
      wasted_bits: 0,
    };

    let lpc = Subframe {
      data: subframe::Data::LPC(LPC {
        entropy_coding_method: EntropyCodingMethod {
          method_type: CodingMethod::PartitionedRice,
          data: PartitionedRice {
            order: 0,
            contents: PartitionedRiceContents {
              parameters: vec![],
              raw_bits: vec![],
            },
          },
        },
        order: 7,
        qlp_coeff_precision: 0,
        quantization_level: 9,
        qlp_coefficients: [1042, -399, -75, -269, 121, 166, -75, 0, 0, 0, 0
                          ,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
                          ,0, 0, 0, 0],
        warmup: [-796, -547, -285, -32, 199, 443, 670, 0, 0, 0, 0, 0, 0, 0, 0
                ,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        residual: vec![-2, -23, 14, 6, 3, -4, 12, -2, 10],
      }),
      wasted_bits: 0,
    };

    decode(&constant, &mut output);
    assert_eq!(&output, &[4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]);

    decode(&verbatim, &mut output);
    assert_eq!(&output, &[16, -3, 55, 49, -32, 6, 40 , -90, 1, 0, 77, -12, 84
                         ,10 , -112, 136]);

    decode(&fixed, &mut output);
    assert_eq!(&output, &[-729, -722, -667, -583, -486, -359, -225, -91, 59
                         ,209, 354, 497, 630, 740, 812, 845]);

    decode(&lpc, &mut output);
    assert_eq!(&output, &[-796, -547, -285, -32, 199, 443, 670, 875, 1046
                         ,1208, 1343, 1454, 1541, 1616, 1663, 1701]);
  }

  #[test]
  fn test_wasted_bit_decode() {
    let mut output = [0; 4];

    let constant = Subframe {
      data: subframe::Data::Constant(1),
      wasted_bits: 10,
    };

    decode(&constant, &mut output);
    assert_eq!(&output, &[1024, 1024, 1024, 1024]);
  }
}
<commit_msg>Pass in block_size need for `*_restore_signal`<commit_after>use subframe::{self, Subframe, MAX_FIXED_ORDER, MAX_LPC_ORDER};

use std::ptr;

// Restore the original signal from a fixed linear prediction.
//
// Signal restoration is accomplished by summing up the residual and the
// predictor. With the linear prediction being "fixed", the polynomials will
// remain the same depending on the order value. And the max order is
// `MAX_FIXED_ORDER`, which is 4.
//
// This function also assumes that `output` already has the warm up values
// from the `Fixed` subframe in it.
pub fn fixed_restore_signal(order: usize,
                            block_size: usize,
                            output: &mut [i32]) {
  debug_assert!(order <= MAX_FIXED_ORDER);

  let polynomial = [ &[][..]
                   , &[1][..]
                   , &[-1, 2][..]
                   , &[1, -3, 3][..]
                   , &[-1, 4, -6, 4][..]
                   ];

  let coefficients = polynomial[order];
  let length       = block_size - order;

  for i in 0..length {
    let offset     = i + order;
    let prediction = coefficients.iter()
                      .zip(&output[i..offset])
                      .fold(0, |result, (coefficient, signal)|
                            result + coefficient * signal);

    output[offset] += prediction;
  }
}

// Restore the original signal from a FIR linear prediction.
//
// Signal restoration is accomplished by summing up the residual and the
// predictor. Figuring out the linear prediction for finite impulse response
// is a bit more involved because you have more to deal with, but the
// concept is very similar to the fixed version. Coefficients are passed in
// and reversed within function and the result of these reverse order
// coefficients and warm up values are summed and the quantization level
// will determine how much the bits gets shifted in order to figure the
// current predictor for it's corresponding residual value.
//
// The order doesn't get passed in explicitly because the coefficients
// length is assumed to be the value of order. And the max order is
// `MAX_LPC_ORDER`, which is 32. This function also assumes that `output`
// already has the warm up values from the `LPC` subframe in it.
pub fn lpc_restore_signal(quantization_level: i8,
                          block_size: usize,
                          coefficients: &[i32],
                          output: &mut [i32]) {
  let order  = coefficients.len();
  let length = block_size - order;

  debug_assert!(order <= MAX_LPC_ORDER);

  for i in 0..length {
    let offset     = i + order;
    let prediction = coefficients.iter().rev()
                       .zip(&output[i..offset])
                       .fold(0, |result, (coefficient, signal)|
                             result + coefficient * signal);

    output[offset] += prediction >> quantization_level;
  }
}

/// Decodes the current subframe.
///
/// * `Constant` - fills the length of `output` with the constant value
///   within the subframe.
/// * `Verbatim` - copies the data within the verbatim subframe over to
///   `output`.
/// * `Fixed` - restore the signal of the fixed linear prediction and put
///   the result into `output`.
/// * `LPC` - restore the signal of the finite impulse response linear
///   prediction and put the result into `output`.
pub fn decode(subframe: &Subframe, block_size: usize, output: &mut [i32]) {
  match subframe.data {
    subframe::Data::Constant(constant)     => {
      for i in 0..output.len() {
        output[i] = constant
      }
    }
    subframe::Data::Verbatim(ref verbatim) => {
      let length = verbatim.len();

      unsafe {
        ptr::copy(verbatim.as_ptr(), output.as_mut_ptr(), length)
      }
    }
    subframe::Data::Fixed(ref fixed)       => {
      let order = fixed.order as usize;

      for i in 0..order {
        output[i] = fixed.warmup[i];
      }

      fixed_restore_signal(order, block_size, output);
    }
    subframe::Data::LPC(ref lpc)           => {
      let order        = lpc.order as usize;
      let coefficients = &lpc.qlp_coefficients[0..order];

      for i in 0..order {
        output[i] = lpc.warmup[i];
      }

      lpc_restore_signal(lpc.quantization_level, block_size, coefficients,
                         output);
    }
  }

  if subframe.wasted_bits > 0 {
    for value in output {
      *value <<= subframe.wasted_bits;
    }
  }
}

#[cfg(test)]
mod tests {
  use super::*;

  use subframe;
  use subframe::{
    Subframe, Fixed, LPC,
    EntropyCodingMethod, CodingMethod, PartitionedRice,
    PartitionedRiceContents,
  };

  #[test]
  fn test_fixed_restore_signal() {
    let residuals   = [ &[-19, -16, 17, -23, -7, 16, -16, -5, 3
                         , -8, -13, -15, -1][..]
                      , &[-6513][..]
                      ];
    let mut outputs = [ &mut [-729, -722, -667, 0, 0, 0, 0, 0, 0
                             , 0, 0, 0, 0, 0, 0, 0][..]
                      , &mut [21877, 27482, 0][..]
                      ];

    fixed_restore_signal(3, &residuals[0], &mut outputs[0]);
    fixed_restore_signal(2, &residuals[1], &mut outputs[1]);

    assert_eq!(&outputs[0], &[-729, -722, -667, -583, -486, -359, -225, -91
                             , 59, 209, 354, 497, 630, 740, 812, 845]);
    assert_eq!(&outputs[1], &[21877, 27482, 26574]);
  }

  #[test]
  fn test_lpc_restore_signal() {
    let residuals    = [ &[-2, -23, 14, 6, 3, -4, 12, -2, 10][..]
                       , &[3157][..]
                       ];
    let coefficients = [ &[1042, -399, -75, -269, 121, 166, -75][..]
                       , &[1757, -1199, 879, -836, 555, -255, 119][..]
                       ];
    let mut outputs = [ &mut [-796, -547, -285, -32, 199, 443, 670
                             , 0, 0, 0, 0, 0, 0, 0, 0, 0][..]
                      , &mut [-21363, -21951, -22649, -24364, -27297, -26870
                             ,-30017, 0][..]
                      ];

    lpc_restore_signal(9, &coefficients[0], &residuals[0], &mut outputs[0]);
    lpc_restore_signal(10, &coefficients[1], &residuals[1], &mut outputs[1]);

    assert_eq!(&outputs[0], &[-796, -547, -285, -32, 199, 443, 670, 875
                             , 1046, 1208, 1343, 1454, 1541, 1616, 1663
                             , 1701]);
    assert_eq!(&outputs[1], &[-21363, -21951, -22649, -24364, -27297, -26870
                             , -30017, -29718]);
  }

  #[test]
  fn test_decode() {
    let mut output = [0; 16];

    let constant = Subframe {
      data: subframe::Data::Constant(4),
      wasted_bits: 0,
    };

    let verbatim = Subframe {
      data: subframe::Data::Verbatim(vec![16, -3, 55, 49, -32, 6, 40, -90, 1
                                         ,0, 77, -12, 84, 10, -112, 136]),
      wasted_bits: 0,
    };

    let fixed = Subframe {
      data: subframe::Data::Fixed(Fixed {
        entropy_coding_method: EntropyCodingMethod {
          method_type: CodingMethod::PartitionedRice,
          data: PartitionedRice {
            order: 0,
            contents: PartitionedRiceContents {
              parameters: vec![],
              raw_bits: vec![],
            },
          },
        },
        order: 3,
        warmup: [-729, -722, -667, 0],
        residual: vec![-19, -16, 17, -23, -7, 16, -16, -5, 3 , -8, -13, -15
                      ,-1],
      }),
      wasted_bits: 0,
    };

    let lpc = Subframe {
      data: subframe::Data::LPC(LPC {
        entropy_coding_method: EntropyCodingMethod {
          method_type: CodingMethod::PartitionedRice,
          data: PartitionedRice {
            order: 0,
            contents: PartitionedRiceContents {
              parameters: vec![],
              raw_bits: vec![],
            },
          },
        },
        order: 7,
        qlp_coeff_precision: 0,
        quantization_level: 9,
        qlp_coefficients: [1042, -399, -75, -269, 121, 166, -75, 0, 0, 0, 0
                          ,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
                          ,0, 0, 0, 0],
        warmup: [-796, -547, -285, -32, 199, 443, 670, 0, 0, 0, 0, 0, 0, 0, 0
                ,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
        residual: vec![-2, -23, 14, 6, 3, -4, 12, -2, 10],
      }),
      wasted_bits: 0,
    };

    decode(&constant, &mut output);
    assert_eq!(&output, &[4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]);

    decode(&verbatim, &mut output);
    assert_eq!(&output, &[16, -3, 55, 49, -32, 6, 40 , -90, 1, 0, 77, -12, 84
                         ,10 , -112, 136]);

    decode(&fixed, &mut output);
    assert_eq!(&output, &[-729, -722, -667, -583, -486, -359, -225, -91, 59
                         ,209, 354, 497, 630, 740, 812, 845]);

    decode(&lpc, &mut output);
    assert_eq!(&output, &[-796, -547, -285, -32, 199, 443, 670, 875, 1046
                         ,1208, 1343, 1454, 1541, 1616, 1663, 1701]);
  }

  #[test]
  fn test_wasted_bit_decode() {
    let mut output = [0; 4];

    let constant = Subframe {
      data: subframe::Data::Constant(1),
      wasted_bits: 10,
    };

    decode(&constant, &mut output);
    assert_eq!(&output, &[1024, 1024, 1024, 1024]);
  }
}
<|endoftext|>


<commit_before>use super::{Invocation, link};
use util::{CommandQueue, get_crate_root, CreateIfNotExists, Tool, };

use clang_driver;

use std::error::Error;
use std::path::{Path, PathBuf};
use std::process::{Command, };

fn build_flags(flags: Vec<String>) -> String {
  let mut out = String::new();
  for (i, flag) in flags.into_iter().enumerate() {
    if i == 0 {
      out = flag;
    } else {
      out.push(' ');
      out.push_str(&flag);
    }
  }

  out
}
impl Invocation {
  pub fn zlib_build_dir(&self) -> PathBuf {
    get_crate_root()
      .join("system/zlib-build")
  }
  pub fn build_zlib(&self, queue: &mut CommandQueue<Invocation>)
    -> Result<(), Box<Error>>
  {
    let src = get_crate_root()
      .join("system/zlib");

    if self.clobber_zlib_build {
      let f = move |sess: &mut &mut Invocation| {
        let build = sess.zlib_build_dir();
        if build.exists() {
          ::std::fs::remove_dir_all(&build)?;
          build.create_if_not_exists()?;
        }

        Ok(())
      };
      queue.enqueue_function(Some("clobber-zlib-build"), f);
    }

    let build_dir = self.zlib_build_dir()
      .create_if_not_exists()?;
    let install_dir = self.tc.sysroot_cache()
      .create_if_not_exists()?;

    let mut cflags = vec![self.c_cxx_linker_cflags()];

    if self.emit_wast {
      cflags.push("--emit-wast".into());
    }

    let cflags = build_flags(cflags);

    let mut conf = Command::new(src.join("configure"));
    conf.current_dir(&build_dir)
      .env("CC", self.cc())
      .env("CXX", self.cxx())
      .env("CFLAGS", &cflags)
      .env("CXXFLAGS", &cflags)
      .arg(format!("--prefix={}", install_dir.display()));

    {
      let cmd = queue
        .enqueue_simple_external(Some("configure zlib"),
                                 conf, None);

      cmd.prev_outputs = false;
      cmd.output_override = false;
    }

    let mut install = Command::new("make");
    install.current_dir(&build_dir)
      .arg("install");
    {
      let cmd = queue
        .enqueue_simple_external(Some("install zlib"),
                                 install, None);

      cmd.prev_outputs = false;
      cmd.output_override = false;
    }

    Ok(())
  }
}
<commit_msg>Make sure `zlib` is optimizated.<commit_after>use super::{Invocation, link};
use util::{CommandQueue, get_crate_root, CreateIfNotExists, Tool, };

use clang_driver;

use std::error::Error;
use std::path::{Path, PathBuf};
use std::process::{Command, };

fn build_flags(flags: Vec<String>) -> String {
  let mut out = String::new();
  for (i, flag) in flags.into_iter().enumerate() {
    if i == 0 {
      out = flag;
    } else {
      out.push(' ');
      out.push_str(&flag);
    }
  }

  out
}
impl Invocation {
  pub fn zlib_build_dir(&self) -> PathBuf {
    get_crate_root()
      .join("system/zlib-build")
  }
  pub fn build_zlib(&self, queue: &mut CommandQueue<Invocation>)
    -> Result<(), Box<Error>>
  {
    let src = get_crate_root()
      .join("system/zlib");

    if self.clobber_zlib_build {
      let f = move |sess: &mut &mut Invocation| {
        let build = sess.zlib_build_dir();
        if build.exists() {
          ::std::fs::remove_dir_all(&build)?;
          build.create_if_not_exists()?;
        }

        Ok(())
      };
      queue.enqueue_function(Some("clobber-zlib-build"), f);
    }

    let build_dir = self.zlib_build_dir()
      .create_if_not_exists()?;
    let install_dir = self.tc.sysroot_cache()
      .create_if_not_exists()?;

    let mut cflags = vec![self.c_cxx_linker_cflags(), "-Os".into(), ];

    if self.emit_wast {
      cflags.push("--emit-wast".into());
    }

    let cflags = build_flags(cflags);

    let mut conf = Command::new(src.join("configure"));
    conf.current_dir(&build_dir)
      .env("CC", self.cc())
      .env("CXX", self.cxx())
      .env("CFLAGS", &cflags)
      .env("CXXFLAGS", &cflags)
      .arg(format!("--prefix={}", install_dir.display()));

    {
      let cmd = queue
        .enqueue_simple_external(Some("configure zlib"),
                                 conf, None);

      cmd.prev_outputs = false;
      cmd.output_override = false;
    }

    let mut install = Command::new("make");
    install.current_dir(&build_dir)
      .arg("install");
    {
      let cmd = queue
        .enqueue_simple_external(Some("install zlib"),
                                 install, None);

      cmd.prev_outputs = false;
      cmd.output_override = false;
    }

    Ok(())
  }
}
<|endoftext|>


<commit_before>use token::{Lexeme, Token};
use error::{Result, SassError, ErrorKind};

use std::str::CharIndices;
use std::iter::Peekable;

pub struct Tokenizer<'a> {
    chars: Peekable<CharIndices<'a>>,
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = Result<Lexeme>;

    fn next(&mut self) -> Option<Result<Lexeme>> {
        match self.parse() {
            Ok(Some(t)) => Some(Ok(t)),
            Ok(None) => None,
            Err(e) => Some(Err(e)),
        }
    }
}

impl<'a> Tokenizer<'a> {
    pub fn new(text: &str) -> Tokenizer {
        Tokenizer {
            chars: text.char_indices().peekable(),
        }
    }

    // In the string parsing sense, not the parsing-out-semantics part yet.
    // Well, there's a tiny bit of semantics, depending on how you define "meaning".
    pub fn parse(&mut self) -> Result<Option<Lexeme>> {
        while let Some((char_offset, curr_char)) = self.chars.next() {
            // Skip leading whitespace
            if curr_char.is_whitespace() {
                continue;
            } else {
                let single_char_token = Token::from_char(curr_char);
                if single_char_token.is_some() && !self.hyphen_starting_shit(curr_char) {
                    // We already tested that single_char_token was Some.
                    return Ok(Some(Lexeme {
                        token: single_char_token.unwrap(),
                        offset: Some(char_offset),
                    }))
                } else {
                    if curr_char.is_numeric() || self.hyphen_starting_number(curr_char) {
                        return self.number(curr_char, char_offset)
                    } else {
                        return self.ident(curr_char, char_offset)
                    }
                }
            }
        }
        return Ok(None)
    }

    fn hyphen_starting_shit(&mut self, curr_char: char) -> bool {
        let peek_char = self.peek_char();
        curr_char == '-' && peek_char.is_some() && !peek_char.unwrap().is_whitespace()
    }

    fn hyphen_starting_number(&mut self, curr_char: char) -> bool {
        let peek_char = self.peek_char();
        curr_char == '-' && peek_char.is_some() && peek_char.unwrap().is_numeric()
    }

    fn peek_char(&mut self) -> Option<char> {
        match self.chars.peek() {
            Some(&(_, peek_char)) => Some(peek_char),
            None => None,
        }
    }

    fn ident(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        while let Some(peek_char) = self.peek_char() {
            // Stop when we reach a non-ident char (hyphens are special)
            if peek_char.is_whitespace() || (
                is_single_char_token(peek_char) && peek_char != '-'
            ) {
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }
        Ok(Some(Lexeme { token: Token::Ident(value), offset: Some(start) }))
    }

    fn number(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        while let Some(peek_char) = self.peek_char() {
            // Stop when we reach a non-numeric char
            // TODO: disallow two `.`s in one number
            if !peek_char.is_numeric() && peek_char != '.' {
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }

        let value = match value.parse() {
            Ok(v) => v,
            Err(_) => return Err(SassError {
                offset: start,
                kind: ErrorKind::TokenizerError,
                message: format!(
                    "Tried to parse `{}` into a f32 but failed.",
                    value,
                ),
            })
        };

        let mut unit = String::new();
        while let Some(peek_char) = self.peek_char() {
            // Get units; stop when we reach a space or non-percent operator
            if peek_char == '%' || (
                !peek_char.is_whitespace() && !is_single_char_token(peek_char)
            ) {
                unit.push(peek_char);
                self.chars.next();
            } else {
                break;
            }
        }

        let token = if unit.len() > 0 {
            Token::Number(value, Some(unit))
        } else {
            Token::Number(value, None)
        };

        Ok(Some(Lexeme { token: token, offset: Some(start) }))
    }
}

fn is_single_char_token(ch: char) -> bool {
    Token::from_char(ch).is_some()
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::{Lexeme, Token};
    use error::Result;

    fn expected_lexeme(expected_token: Token, expected_offset: usize) -> Option<Result<Lexeme>> {
        Some(Ok(Lexeme {
            token: expected_token,
            offset: Some(expected_offset),
        }))
    }

    fn expected_ident(expected_value: &str, expected_offset: usize) -> Option<Result<Lexeme>> {
        expected_lexeme(Token::Ident(expected_value.into()), expected_offset)
    }

    #[test]
    fn it_returns_none_for_empty_string() {
        let mut tokenizer = Tokenizer::new("");
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_returns_none_for_only_whitespace() {
        let mut tokenizer = Tokenizer::new("     ");
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_returns_words() {
        // Without regard to Sass word validity
        let mut tokenizer = Tokenizer::new(" \n  div   aoeu  ");
        assert_eq!(tokenizer.next(), expected_ident("div", 4));
        assert_eq!(tokenizer.next(), expected_ident("aoeu", 10));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_curly_braces() {
        // Without regard to matching
        let mut tokenizer = Tokenizer::new("{}}a{ blah}");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftCurlyBrace, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 1));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 2));
        assert_eq!(tokenizer.next(), expected_ident("a", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftCurlyBrace, 4));
        assert_eq!(tokenizer.next(), expected_ident("blah", 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 10));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_colon() {
        let mut tokenizer = Tokenizer::new(":invalid: property::");
        // This might be wrong. I have a note somewhere else that colons can start idents.
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 0));
        assert_eq!(tokenizer.next(), expected_ident("invalid", 1));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 8));
        assert_eq!(tokenizer.next(), expected_ident("property", 10));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 18));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 19));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_semicolon() {
        let mut tokenizer = Tokenizer::new(";;\na;\nb\n;");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 1));
        assert_eq!(tokenizer.next(), expected_ident("a", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 4));
        assert_eq!(tokenizer.next(), expected_ident("b", 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 8));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_numbers() {
        let mut tokenizer = Tokenizer::new("border: 0px 1.5 11em;");
        assert_eq!(tokenizer.next(), expected_ident("border", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 6));
        assert_eq!(tokenizer.next(), expected_lexeme(
            Token::Number(0.0, Some("px".into())), 8
        ));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(1.5, None), 12));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(11.0, Some("em".into())), 16));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 20));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_hyphen() {
        let mut tokenizer = Tokenizer::new("font-weight -webkit -3 - 4-5 a-1 -");
        assert_eq!(tokenizer.next(), expected_ident("font-weight", 0));
        assert_eq!(tokenizer.next(), expected_ident("-webkit", 12));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(-3.0, None), 20));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Minus, 23));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(4.0, None), 25));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(-5.0, None), 26));
        assert_eq!(tokenizer.next(), expected_ident("a-1", 29));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Minus, 33));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_parens() {
        let mut tokenizer = Tokenizer::new("() rgb)()(");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 1));
        assert_eq!(tokenizer.next(), expected_ident("rgb", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 7));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 8));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 9));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_slash() {
        let mut tokenizer = Tokenizer::new("/ / 3/4 / 8");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 2));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(3.0, None), 4));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 5));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(4.0, None), 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 8));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(8.0, None), 10));
        assert_eq!(tokenizer.next(), None);
    }
}
<commit_msg>Add a test for tokenizing commas in selectors<commit_after>use token::{Lexeme, Token};
use error::{Result, SassError, ErrorKind};

use std::str::CharIndices;
use std::iter::Peekable;

pub struct Tokenizer<'a> {
    chars: Peekable<CharIndices<'a>>,
}

impl<'a> Iterator for Tokenizer<'a> {
    type Item = Result<Lexeme>;

    fn next(&mut self) -> Option<Result<Lexeme>> {
        match self.parse() {
            Ok(Some(t)) => Some(Ok(t)),
            Ok(None) => None,
            Err(e) => Some(Err(e)),
        }
    }
}

impl<'a> Tokenizer<'a> {
    pub fn new(text: &str) -> Tokenizer {
        Tokenizer {
            chars: text.char_indices().peekable(),
        }
    }

    // In the string parsing sense, not the parsing-out-semantics part yet.
    // Well, there's a tiny bit of semantics, depending on how you define "meaning".
    pub fn parse(&mut self) -> Result<Option<Lexeme>> {
        while let Some((char_offset, curr_char)) = self.chars.next() {
            // Skip leading whitespace
            if curr_char.is_whitespace() {
                continue;
            } else {
                let single_char_token = Token::from_char(curr_char);
                if single_char_token.is_some() && !self.hyphen_starting_shit(curr_char) {
                    // We already tested that single_char_token was Some.
                    return Ok(Some(Lexeme {
                        token: single_char_token.unwrap(),
                        offset: Some(char_offset),
                    }))
                } else {
                    if curr_char.is_numeric() || self.hyphen_starting_number(curr_char) {
                        return self.number(curr_char, char_offset)
                    } else {
                        return self.ident(curr_char, char_offset)
                    }
                }
            }
        }
        return Ok(None)
    }

    fn hyphen_starting_shit(&mut self, curr_char: char) -> bool {
        let peek_char = self.peek_char();
        curr_char == '-' && peek_char.is_some() && !peek_char.unwrap().is_whitespace()
    }

    fn hyphen_starting_number(&mut self, curr_char: char) -> bool {
        let peek_char = self.peek_char();
        curr_char == '-' && peek_char.is_some() && peek_char.unwrap().is_numeric()
    }

    fn peek_char(&mut self) -> Option<char> {
        match self.chars.peek() {
            Some(&(_, peek_char)) => Some(peek_char),
            None => None,
        }
    }

    fn ident(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        while let Some(peek_char) = self.peek_char() {
            // Stop when we reach a non-ident char (hyphens are special)
            if peek_char.is_whitespace() || (
                is_single_char_token(peek_char) && peek_char != '-'
            ) {
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }
        Ok(Some(Lexeme { token: Token::Ident(value), offset: Some(start) }))
    }

    fn number(&mut self, curr_char: char, start: usize) -> Result<Option<Lexeme>> {
        let mut value = String::new();
        value.push(curr_char);

        while let Some(peek_char) = self.peek_char() {
            // Stop when we reach a non-numeric char
            // TODO: disallow two `.`s in one number
            if !peek_char.is_numeric() && peek_char != '.' {
                break;
            } else {
                value.push(peek_char);
                self.chars.next();
            }
        }

        let value = match value.parse() {
            Ok(v) => v,
            Err(_) => return Err(SassError {
                offset: start,
                kind: ErrorKind::TokenizerError,
                message: format!(
                    "Tried to parse `{}` into a f32 but failed.",
                    value,
                ),
            })
        };

        let mut unit = String::new();
        while let Some(peek_char) = self.peek_char() {
            // Get units; stop when we reach a space or non-percent operator
            if peek_char == '%' || (
                !peek_char.is_whitespace() && !is_single_char_token(peek_char)
            ) {
                unit.push(peek_char);
                self.chars.next();
            } else {
                break;
            }
        }

        let token = if unit.len() > 0 {
            Token::Number(value, Some(unit))
        } else {
            Token::Number(value, None)
        };

        Ok(Some(Lexeme { token: token, offset: Some(start) }))
    }
}

fn is_single_char_token(ch: char) -> bool {
    Token::from_char(ch).is_some()
}

#[cfg(test)]
mod tests {
    use super::*;
    use token::{Lexeme, Token};
    use error::Result;

    fn expected_lexeme(expected_token: Token, expected_offset: usize) -> Option<Result<Lexeme>> {
        Some(Ok(Lexeme {
            token: expected_token,
            offset: Some(expected_offset),
        }))
    }

    fn expected_ident(expected_value: &str, expected_offset: usize) -> Option<Result<Lexeme>> {
        expected_lexeme(Token::Ident(expected_value.into()), expected_offset)
    }

    #[test]
    fn it_returns_none_for_empty_string() {
        let mut tokenizer = Tokenizer::new("");
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_returns_none_for_only_whitespace() {
        let mut tokenizer = Tokenizer::new("     ");
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_returns_words() {
        // Without regard to Sass word validity
        let mut tokenizer = Tokenizer::new(" \n  div   aoeu  ");
        assert_eq!(tokenizer.next(), expected_ident("div", 4));
        assert_eq!(tokenizer.next(), expected_ident("aoeu", 10));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_curly_braces() {
        // Without regard to matching
        let mut tokenizer = Tokenizer::new("{}}a{ blah}");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftCurlyBrace, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 1));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 2));
        assert_eq!(tokenizer.next(), expected_ident("a", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftCurlyBrace, 4));
        assert_eq!(tokenizer.next(), expected_ident("blah", 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightCurlyBrace, 10));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_colon() {
        let mut tokenizer = Tokenizer::new(":invalid: property::");
        // This might be wrong. I have a note somewhere else that colons can start idents.
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 0));
        assert_eq!(tokenizer.next(), expected_ident("invalid", 1));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 8));
        assert_eq!(tokenizer.next(), expected_ident("property", 10));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 18));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 19));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_semicolon() {
        let mut tokenizer = Tokenizer::new(";;\na;\nb\n;");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 1));
        assert_eq!(tokenizer.next(), expected_ident("a", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 4));
        assert_eq!(tokenizer.next(), expected_ident("b", 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 8));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_numbers() {
        let mut tokenizer = Tokenizer::new("border: 0px 1.5 11em;");
        assert_eq!(tokenizer.next(), expected_ident("border", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Colon, 6));
        assert_eq!(tokenizer.next(), expected_lexeme(
            Token::Number(0.0, Some("px".into())), 8
        ));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(1.5, None), 12));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(11.0, Some("em".into())), 16));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Semicolon, 20));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_hyphen() {
        let mut tokenizer = Tokenizer::new("font-weight -webkit -3 - 4-5 a-1 -");
        assert_eq!(tokenizer.next(), expected_ident("font-weight", 0));
        assert_eq!(tokenizer.next(), expected_ident("-webkit", 12));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(-3.0, None), 20));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Minus, 23));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(4.0, None), 25));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(-5.0, None), 26));
        assert_eq!(tokenizer.next(), expected_ident("a-1", 29));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Minus, 33));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_parens() {
        let mut tokenizer = Tokenizer::new("() rgb)()(");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 1));
        assert_eq!(tokenizer.next(), expected_ident("rgb", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 7));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::RightParen, 8));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftParen, 9));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_slash() {
        let mut tokenizer = Tokenizer::new("/ / 3/4 / 8");
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 2));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(3.0, None), 4));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 5));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(4.0, None), 6));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Slash, 8));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Number(8.0, None), 10));
        assert_eq!(tokenizer.next(), None);
    }

    #[test]
    fn it_separates_comma() {
        let mut tokenizer = Tokenizer::new("a, b {");
        assert_eq!(tokenizer.next(), expected_ident("a", 0));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::Comma, 1));
        assert_eq!(tokenizer.next(), expected_ident("b", 3));
        assert_eq!(tokenizer.next(), expected_lexeme(Token::LeftCurlyBrace, 5));
        assert_eq!(tokenizer.next(), None);
    }
}
<|endoftext|>


<commit_before>use std::iter::Peekable;

#[deriving(PartialEq, Show)]
pub enum Token {
    Comma,
    Number(f64),
    ParenClose,
    ParenOpen,
    Word(String),
}

pub fn tokenize(input: &str) -> Tokens {
    Tokens {text: String::from_str(input)}
}

fn is_whitespace(c: char) -> bool {
    match c {
        '\n' | '\r' | '\t' | ' ' => true,
        _                        => false,
    }
}

fn is_numberlike(c: char) -> bool {
    match c {
        c if c.is_numeric() => true,
        '.' | '-' | '+'     => true,
        _                   => false,
    }
}

pub type PeekableTokens = Peekable<Token, Tokens>;

pub struct Tokens {
    text: String
}

impl Iterator<Token> for Tokens {
    fn next(&mut self) -> Option<Token> {
        let next_char = match self.pop_char() {
            Some(c) => c,
            None => return None,
        };

        match next_char {
            '\0' => None,
            '(' => Some(Token::ParenOpen),
            ')' => Some(Token::ParenClose),
            ',' => Some(Token::Comma),
            c if is_whitespace(c) => self.next(),
            c if is_numberlike(c) => {
                let mut number = c.to_string() + self.read_until_whitespace().as_slice();
                number = number.trim_left_matches('+').to_string();
                match number.parse::<f64>() {
                    Some(parsed_num) => Some(Token::Number(parsed_num)),
                    None => panic!("Could not parse number: {}", number),
                }
            }
            c => {
                let word = c.to_string() + self.read_until_whitespace().as_slice();
                Some(Token::Word(word))
            }
        }
    }
}

impl Tokens {
    fn pop_char(&mut self) -> Option<char> {
        if self.text.is_empty() {
            None
        } else {
            Some(self.text.remove(0))
        }
    }

    fn read_until_whitespace(&mut self) -> String {
        let popped_char = self.pop_char();
        if popped_char.is_none() {
            return "".to_string()
        }
        let next_char = popped_char.unwrap();

        match next_char {
            '\0' | '(' | ')' | ',' => {
                self.text.insert(0, next_char);
                "".to_string()
            }
            c if is_whitespace(c) => "".to_string(),
            _ => next_char.to_string() + self.read_until_whitespace().as_slice(),
        }
    }
}

#[test]
fn test_tokenizer_empty() {
    let test_str = "";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![]);
}

#[test]
fn test_tokenizer_1word() {
    let test_str = "hello";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("hello".to_string()),
    ]);
}

#[test]
fn test_tokenizer_2words() {
    let test_str = "hello world";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("hello".to_string()),
        Token::Word("world".to_string()),
    ]);
}

#[test]
fn test_tokenizer_1number() {
    let test_str = "4.2";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Number(4.2),
    ]);
}

#[test]
fn test_tokenizer_1number_plus() {
    let test_str = "+4.2";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Number(4.2),
    ]);
}

#[test]
fn test_tokenizer_2numbers() {
    let test_str = ".4 -2";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Number(0.4),
        Token::Number(-2.0),
    ]);
}

#[test]
fn test_tokenizer_point() {
    let test_str = "POINT (10 -20)";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("POINT".to_string()),
        Token::ParenOpen,
        Token::Number(10.0),
        Token::Number(-20.0),
        Token::ParenClose,
    ]);
}
<commit_msg>Simplify pop char logic<commit_after>use std::iter::Peekable;

#[deriving(PartialEq, Show)]
pub enum Token {
    Comma,
    Number(f64),
    ParenClose,
    ParenOpen,
    Word(String),
}

pub fn tokenize(input: &str) -> Tokens {
    Tokens {text: String::from_str(input)}
}

fn is_whitespace(c: char) -> bool {
    match c {
        '\n' | '\r' | '\t' | ' ' => true,
        _                        => false,
    }
}

fn is_numberlike(c: char) -> bool {
    match c {
        c if c.is_numeric() => true,
        '.' | '-' | '+'     => true,
        _                   => false,
    }
}

pub type PeekableTokens = Peekable<Token, Tokens>;

pub struct Tokens {
    text: String
}

impl Iterator<Token> for Tokens {
    fn next(&mut self) -> Option<Token> {
        let next_char = match self.pop_front() {
            Some(c) => c,
            None => return None,
        };

        match next_char {
            '\0' => None,
            '(' => Some(Token::ParenOpen),
            ')' => Some(Token::ParenClose),
            ',' => Some(Token::Comma),
            c if is_whitespace(c) => self.next(),
            c if is_numberlike(c) => {
                let mut number = c.to_string() + self.read_until_whitespace().as_slice();
                number = number.trim_left_matches('+').to_string();
                match number.parse::<f64>() {
                    Some(parsed_num) => Some(Token::Number(parsed_num)),
                    None => panic!("Could not parse number: {}", number),
                }
            }
            c => {
                let word = c.to_string() + self.read_until_whitespace().as_slice();
                Some(Token::Word(word))
            }
        }
    }
}

impl Tokens {
    fn pop_front(&mut self) -> Option<char> {
        match self.text.is_empty() {
            true => None,
            false => Some(self.text.remove(0))
        }
    }

    fn read_until_whitespace(&mut self) -> String {
        let popped_char = self.pop_front();
        if popped_char.is_none() {
            return "".to_string()
        }
        let next_char = popped_char.unwrap();

        match next_char {
            '\0' | '(' | ')' | ',' => {
                self.text.insert(0, next_char);
                "".to_string()
            }
            c if is_whitespace(c) => "".to_string(),
            _ => next_char.to_string() + self.read_until_whitespace().as_slice(),
        }
    }
}

#[test]
fn test_tokenizer_empty() {
    let test_str = "";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![]);
}

#[test]
fn test_tokenizer_1word() {
    let test_str = "hello";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("hello".to_string()),
    ]);
}

#[test]
fn test_tokenizer_2words() {
    let test_str = "hello world";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("hello".to_string()),
        Token::Word("world".to_string()),
    ]);
}

#[test]
fn test_tokenizer_1number() {
    let test_str = "4.2";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Number(4.2),
    ]);
}

#[test]
fn test_tokenizer_1number_plus() {
    let test_str = "+4.2";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Number(4.2),
    ]);
}

#[test]
fn test_tokenizer_2numbers() {
    let test_str = ".4 -2";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Number(0.4),
        Token::Number(-2.0),
    ]);
}

#[test]
fn test_tokenizer_point() {
    let test_str = "POINT (10 -20)";
    let tokens: Vec<Token> = tokenize(test_str).collect();
    assert_eq!(tokens, vec![
        Token::Word("POINT".to_string()),
        Token::ParenOpen,
        Token::Number(10.0),
        Token::Number(-20.0),
        Token::ParenClose,
    ]);
}
<|endoftext|>