<commit_before>#![feature(core, std_misc)]
#![cfg_attr(test, feature(io, net))]

extern crate "route-recognizer" as router;
extern crate conduit;

use std::collections::hash_map::{HashMap, Entry};
use std::error::Error;
use std::fmt;

use router::{Router, Match};
use conduit::{Method, Handler, Request, Response};

pub struct RouteBuilder {
    routers: HashMap<Method, Router<Box<Handler>>>,
}

// FIXME(#22655) these should not be necessary
unsafe impl Sync for RouteBuilder {}
unsafe impl Send for RouteBuilder {}

pub struct RouterError(String);

impl RouteBuilder {
    pub fn new() -> RouteBuilder {
        RouteBuilder { routers: HashMap::new() }
    }

    pub fn recognize<'a>(&'a self, method: &Method, path: &str)
                         -> Result<Match<&'a Box<Handler>>,
                                   RouterError>
    {
        match self.routers.get(method) {
            Some(router) => router.recognize(path),
            None => Err(format!("No router found for {:?}", method)),
        }.map_err(RouterError)
    }

    pub fn map<'a, H: Handler>(&'a mut self, method: Method, pattern: &str,
                               handler: H) -> &'a mut RouteBuilder {
        {
            let router = match self.routers.entry(method) {
                Entry::Occupied(e) => e.into_mut(),
                Entry::Vacant(e) => e.insert(Router::new()),
            };
            router.add(pattern, Box::new(handler));
        }
        self
    }

    pub fn get<'a, H: Handler>(&'a mut self, pattern: &str, handler: H)
                               -> &'a mut RouteBuilder {
        self.map(Method::Get, pattern, handler)
    }

    pub fn post<'a, H: Handler>(&'a mut self, pattern: &str, handler: H)
                                -> &'a mut RouteBuilder {
        self.map(Method::Post, pattern, handler)
    }

    pub fn put<'a, H: Handler>(&'a mut self, pattern: &str, handler: H)
                               -> &'a mut RouteBuilder {
        self.map(Method::Put, pattern, handler)
    }

    pub fn delete<'a, H: Handler>(&'a mut self, pattern: &str, handler: H)
                                  -> &'a mut RouteBuilder {
        self.map(Method::Delete, pattern, handler)
    }

    pub fn head<'a, H: Handler>(&'a mut self, pattern: &str, handler: H)
                                -> &'a mut RouteBuilder {
        self.map(Method::Head, pattern, handler)
    }
}

impl conduit::Handler for RouteBuilder {
    fn call(&self, request: &mut Request) -> Result<Response, Box<Error+Send>> {
        let m = {
            let method = request.method();
            let path = request.path();

            match self.recognize(&method, path) {
                Ok(m) => m,
                Err(e) => return Err(Box::new(e) as Box<Error+Send>)
            }
        };

        {
            let extensions = request.mut_extensions();
            extensions.insert(m.params.clone());
        }

        (*m.handler).call(request)
    }
}

impl Error for RouterError {
    fn description(&self) -> &str { self.0.as_slice() }
}

impl fmt::Display for RouterError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        fmt::Display::fmt(&self.0, f)
    }
}

pub trait RequestParams<'a> {
    fn params(self) -> &'a router::Params;
}

pub fn params<'a>(req: &'a Request) -> &'a router::Params {
    req.extensions().find::<router::Params>()
        .expect("Missing params")
}

impl<'a> RequestParams<'a> for &'a (Request + 'a) {
    fn params(self) -> &'a router::Params {
        params(self)
    }
}

#[cfg(test)]
mod tests {
    extern crate semver;
    use std::collections::HashMap;
    use std::io;
    use std::net::IpAddr;

    use {RouteBuilder, RequestParams};

    use conduit;
    use conduit::{Handler, Method, Scheme, Host, Headers, Extensions, TypeMap};

    struct RequestSentinel {
        method: Method,
        path: String,
        extensions: conduit::Extensions
    }

    impl RequestSentinel {
        fn new(method: Method, path: &'static str) -> RequestSentinel {
            RequestSentinel {
                path: path.to_string(),
                extensions: TypeMap::new(),
                method: method
            }
        }
    }

    impl conduit::Request for RequestSentinel {
        fn http_version(&self) -> semver::Version { unimplemented!() }
        fn conduit_version(&self) -> semver::Version { unimplemented!() }
        fn method(&self) -> Method { self.method }
        fn scheme(&self) -> Scheme { unimplemented!() }
        fn host<'a>(&'a self) -> Host<'a> { unimplemented!() }
        fn virtual_root<'a>(&'a self) -> Option<&'a str> { unimplemented!() }
        fn path<'a>(&'a self) -> &'a str {
            self.path.as_slice()
        }
        fn query_string<'a>(&'a self) -> Option<&'a str> { unimplemented!() }
        fn remote_ip(&self) -> IpAddr { unimplemented!() }
        fn content_length(&self) -> Option<u64> { unimplemented!() }
        fn headers<'a>(&'a self) -> &'a Headers { unimplemented!() }
        fn body<'a>(&'a mut self) -> &'a mut io::Read { unimplemented!() }
        fn extensions<'a>(&'a self) -> &'a Extensions {
            &self.extensions
        }
        fn mut_extensions<'a>(&'a mut self) -> &'a mut Extensions {
            &mut self.extensions
        }
    }

    #[test]
    fn basic_get() {
        let router = test_router();
        let mut req = RequestSentinel::new(Method::Get, "/posts/1");
        let mut res = router.call(&mut req).ok().expect("No response");

        assert_eq!(res.status, (200, "OK"));
        let mut s = String::new();
        res.body.read_to_string(&mut s).unwrap();
        assert_eq!(s, "1, Get".to_string());
    }

    #[test]
    fn basic_post() {
        let router = test_router();
        let mut req = RequestSentinel::new(Method::Post, "/posts/10");
        let mut res = router.call(&mut req).ok().expect("No response");

        assert_eq!(res.status, (200, "OK"));
        let mut s = String::new();
        res.body.read_to_string(&mut s).unwrap();
        assert_eq!(s, "10, Post".to_string());
    }

    #[test]
    fn nonexistent_route() {
        let router = test_router();
        let mut req = RequestSentinel::new(Method::Post, "/nonexistent");
        router.call(&mut req).err().expect("No response");
    }

    fn test_router() -> RouteBuilder {
        let mut router = RouteBuilder::new();
        router.post("/posts/:id", test_handler);
        router.get("/posts/:id", test_handler);
        router
    }

    fn test_handler(req: &mut conduit::Request) -> io::Result<conduit::Response> {
        let mut res = vec!();
        res.push(req.params()["id"].clone());
        res.push(format!("{:?}", req.method()));

        Ok(conduit::Response {
            status: (200, "OK"),
            headers: HashMap::new(),
            body: Box::new(io::Cursor::new(res.connect(", ").into_bytes()))
        })
    }
}
<commit_msg>Update to rust master<commit_after>#![feature(core, std_misc)]

extern crate "route-recognizer" as router;
extern crate conduit;

use std::collections::hash_map::{HashMap, Entry};
use std::error::Error;
use std::fmt;

use router::{Router, Match};
use conduit::{Method, Handler, Request, Response};

pub struct RouteBuilder {
    routers: HashMap<Method, Router<Box<Handler>>>,
}

// FIXME(#22655) these should not be necessary
unsafe impl Sync for RouteBuilder {}
unsafe impl Send for RouteBuilder {}

pub struct RouterError(String);

impl RouteBuilder {
    pub fn new() -> RouteBuilder {
        RouteBuilder { routers: HashMap::new() }
    }

    pub fn recognize<'a>(&'a self, method: &Method, path: &str)
                         -> Result<Match<&'a Box<Handler>>,
                                   RouterError>
    {
        match self.routers.get(method) {
            Some(router) => router.recognize(path),
            None => Err(format!("No router found for {:?}", method)),
        }.map_err(RouterError)
    }

    pub fn map<'a, H: Handler>(&'a mut self, method: Method, pattern: &str,
                               handler: H) -> &'a mut RouteBuilder {
        {
            let router = match self.routers.entry(method) {
                Entry::Occupied(e) => e.into_mut(),
                Entry::Vacant(e) => e.insert(Router::new()),
            };
            router.add(pattern, Box::new(handler));
        }
        self
    }

    pub fn get<'a, H: Handler>(&'a mut self, pattern: &str, handler: H)
                               -> &'a mut RouteBuilder {
        self.map(Method::Get, pattern, handler)
    }

    pub fn post<'a, H: Handler>(&'a mut self, pattern: &str, handler: H)
                                -> &'a mut RouteBuilder {
        self.map(Method::Post, pattern, handler)
    }

    pub fn put<'a, H: Handler>(&'a mut self, pattern: &str, handler: H)
                               -> &'a mut RouteBuilder {
        self.map(Method::Put, pattern, handler)
    }

    pub fn delete<'a, H: Handler>(&'a mut self, pattern: &str, handler: H)
                                  -> &'a mut RouteBuilder {
        self.map(Method::Delete, pattern, handler)
    }

    pub fn head<'a, H: Handler>(&'a mut self, pattern: &str, handler: H)
                                -> &'a mut RouteBuilder {
        self.map(Method::Head, pattern, handler)
    }
}

impl conduit::Handler for RouteBuilder {
    fn call(&self, request: &mut Request) -> Result<Response, Box<Error+Send>> {
        let m = {
            let method = request.method();
            let path = request.path();

            match self.recognize(&method, path) {
                Ok(m) => m,
                Err(e) => return Err(Box::new(e) as Box<Error+Send>)
            }
        };

        {
            let extensions = request.mut_extensions();
            extensions.insert(m.params.clone());
        }

        (*m.handler).call(request)
    }
}

impl Error for RouterError {
    fn description(&self) -> &str { self.0.as_slice() }
}

impl fmt::Display for RouterError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        fmt::Display::fmt(&self.0, f)
    }
}

pub trait RequestParams<'a> {
    fn params(self) -> &'a router::Params;
}

pub fn params<'a>(req: &'a Request) -> &'a router::Params {
    req.extensions().find::<router::Params>()
        .expect("Missing params")
}

impl<'a> RequestParams<'a> for &'a (Request + 'a) {
    fn params(self) -> &'a router::Params {
        params(self)
    }
}

#[cfg(test)]
mod tests {
    extern crate semver;
    use std::collections::HashMap;
    use std::io;
    use std::net::SocketAddr;

    use {RouteBuilder, RequestParams};

    use conduit;
    use conduit::{Handler, Method, Scheme, Host, Headers, Extensions, TypeMap};

    struct RequestSentinel {
        method: Method,
        path: String,
        extensions: conduit::Extensions
    }

    impl RequestSentinel {
        fn new(method: Method, path: &'static str) -> RequestSentinel {
            RequestSentinel {
                path: path.to_string(),
                extensions: TypeMap::new(),
                method: method
            }
        }
    }

    impl conduit::Request for RequestSentinel {
        fn http_version(&self) -> semver::Version { unimplemented!() }
        fn conduit_version(&self) -> semver::Version { unimplemented!() }
        fn method(&self) -> Method { self.method }
        fn scheme(&self) -> Scheme { unimplemented!() }
        fn host<'a>(&'a self) -> Host<'a> { unimplemented!() }
        fn virtual_root<'a>(&'a self) -> Option<&'a str> { unimplemented!() }
        fn path<'a>(&'a self) -> &'a str {
            self.path.as_slice()
        }
        fn query_string<'a>(&'a self) -> Option<&'a str> { unimplemented!() }
        fn remote_addr(&self) -> SocketAddr { unimplemented!() }
        fn content_length(&self) -> Option<u64> { unimplemented!() }
        fn headers<'a>(&'a self) -> &'a Headers { unimplemented!() }
        fn body<'a>(&'a mut self) -> &'a mut io::Read { unimplemented!() }
        fn extensions<'a>(&'a self) -> &'a Extensions {
            &self.extensions
        }
        fn mut_extensions<'a>(&'a mut self) -> &'a mut Extensions {
            &mut self.extensions
        }
    }

    #[test]
    fn basic_get() {
        let router = test_router();
        let mut req = RequestSentinel::new(Method::Get, "/posts/1");
        let mut res = router.call(&mut req).ok().expect("No response");

        assert_eq!(res.status, (200, "OK"));
        let mut s = String::new();
        res.body.read_to_string(&mut s).unwrap();
        assert_eq!(s, "1, Get".to_string());
    }

    #[test]
    fn basic_post() {
        let router = test_router();
        let mut req = RequestSentinel::new(Method::Post, "/posts/10");
        let mut res = router.call(&mut req).ok().expect("No response");

        assert_eq!(res.status, (200, "OK"));
        let mut s = String::new();
        res.body.read_to_string(&mut s).unwrap();
        assert_eq!(s, "10, Post".to_string());
    }

    #[test]
    fn nonexistent_route() {
        let router = test_router();
        let mut req = RequestSentinel::new(Method::Post, "/nonexistent");
        router.call(&mut req).err().expect("No response");
    }

    fn test_router() -> RouteBuilder {
        let mut router = RouteBuilder::new();
        router.post("/posts/:id", test_handler);
        router.get("/posts/:id", test_handler);
        router
    }

    fn test_handler(req: &mut conduit::Request) -> io::Result<conduit::Response> {
        let mut res = vec!();
        res.push(req.params()["id"].clone());
        res.push(format!("{:?}", req.method()));

        Ok(conduit::Response {
            status: (200, "OK"),
            headers: HashMap::new(),
            body: Box::new(io::Cursor::new(res.connect(", ").into_bytes()))
        })
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, edef <edef@edef.eu>
// See the LICENSE file included in this distribution.
#![feature(no_std)]
#![feature(asm, core)]
#![feature(libc)]
#![no_std]

#[macro_use]
extern crate core;

#[cfg(test)]
#[macro_use]
extern crate std;

pub use context::Context;
pub use stack::Stack;
#[cfg(feature = "os")]
pub use os::Stack as OsStack;

#[cfg(not(test))]
mod std { pub use core::*; }

mod context;
mod stack;

mod debug;

mod arch;

#[cfg(feature = "os")]
mod os;
<commit_msg>rejig lib.rs a little<commit_after>// Copyright (c) 2015, edef <edef@edef.eu>
// See the LICENSE file included in this distribution.
#![feature(no_std)]
#![feature(asm, core)]
#![feature(libc)]
#![no_std]

#[macro_use]
extern crate core;

#[cfg(test)]
#[macro_use]
extern crate std;

pub use context::Context;
pub use stack::Stack;

#[cfg(feature = "os")]
pub use os::Stack as OsStack;

mod context;
mod stack;

#[cfg(feature = "os")]
mod os;

mod arch;
mod debug;

#[cfg(not(test))]
mod std { pub use core::*; }
<|endoftext|>


<commit_before>//!
//! ### Namespaces
//!
//! The XPath specification assumes that the XML being processed has a
//! prefix defined for every namespace present in the document. If you
//! are processing XML that was parsed from text, this will be true by
//! construction.
//!
//! If you have programmatically created XML with namespaces but not
//! defined prefixes, some XPath behavior may be confusing:
//!
//! 1. The `name` method will not include a prefix, even if the
//! element or attribute has a namespace.
//! 2. The `namespace` axis will not include namespaces without
//! prefixes.
//!

#![cfg_attr(feature = "unstable", feature(core))]

#[macro_use]
extern crate peresil;
extern crate sxd_document;

use std::borrow::ToOwned;
use std::collections::HashMap;
use std::{iter,string};

use self::Value::{Boolean,Number,String};

use nodeset::{Nodeset,Node};
use parser::Parser;
use tokenizer::{Tokenizer,TokenDeabbreviator};

pub use function::Function;
pub use expression::Expression;

#[macro_use]
pub mod macros;
pub mod nodeset;
pub mod axis;
mod expression;
pub mod function;
mod node_test;
pub mod parser;
mod token;
pub mod tokenizer;

// TODO - this doesn't need to be public, figure out how to make it so
#[derive(PartialEq,Debug,Clone)]
pub enum LiteralValue {
    Boolean(bool),
    Number(f64),
    String(string::String),
}

impl LiteralValue {
    fn into_value<'d>(self) -> Value<'d> {
        match self {
            LiteralValue::Boolean(v) => Boolean(v),
            LiteralValue::Number(v)  => Number(v),
            LiteralValue::String(v)  => String(v),
        }
    }
}

#[derive(PartialEq,Debug,Clone)]
pub enum Value<'d> {
    Boolean(bool),
    Number(f64),
    String(string::String),
    Nodeset(Nodeset<'d>),
}

fn str_to_num(s: &str) -> f64 {
    s.trim().parse().unwrap_or(::std::f64::NAN)
}

impl<'d> Value<'d> {
    pub fn boolean(&self) -> bool {
        use Value::*;
        match *self {
            Boolean(val) => val,
            Number(n) => n != 0.0 && ! n.is_nan(),
            String(ref s) => ! s.is_empty(),
            Nodeset(ref nodeset) => nodeset.size() > 0,
        }
    }

    pub fn number(&self) -> f64 {
        use Value::*;
        match *self {
            Boolean(val) => if val { 1.0 } else { 0.0 },
            Number(val) => val,
            String(ref s) => str_to_num(s),
            Nodeset(..) => str_to_num(&self.string()),
        }
    }

    pub fn string(&self) -> string::String {
        use Value::*;
        match *self {
            Boolean(v) => v.to_string(),
            Number(n) => {
                if n.is_infinite() {
                    if n.signum() < 0.0 {
                        "-Infinity".to_owned()
                    } else {
                        "Infinity".to_owned()
                    }
                } else {
                    n.to_string()
                }
            },
            String(ref val) => val.clone(),
            Nodeset(ref ns) => match ns.document_order_first() {
                Some(n) => n.string_value(),
                None => "".to_owned(),
            },
        }
    }

    #[allow(dead_code)]
    fn into_literal_value(self) -> LiteralValue {
        use Value::*;
        match self {
            Boolean(v) => LiteralValue::Boolean(v),
            Number(v)  => LiteralValue::Number(v),
            String(v)  => LiteralValue::String(v),
            Nodeset(_) => panic!("Cannot convert a nodeset to a literal"),
        }
    }
}

type BoxFunc = Box<Function + 'static>;
pub type Functions = HashMap<string::String, BoxFunc>;
pub type Variables<'d> = HashMap<string::String, Value<'d>>;
pub type Namespaces = HashMap<string::String, string::String>;

#[derive(Copy,Clone)]
pub struct EvaluationContext<'a, 'd : 'a> {
    node: Node<'d>,
    functions: &'a Functions,
    variables: &'a Variables<'d>,
    namespaces: &'a Namespaces,
    position: usize,
    size: usize,
}

impl<'a, 'd> EvaluationContext<'a, 'd> {
    pub fn new<N>(node: N,
                  functions: &'a Functions,
                  variables: &'a Variables<'d>,
                  namespaces: &'a Namespaces)
                  -> EvaluationContext<'a, 'd>
        where N: Into<Node<'d>>
    {
        EvaluationContext {
            node: node.into(),
            functions: functions,
            variables: variables,
            namespaces: namespaces,
            position: 1,
            size: 1,
        }
    }

    fn new_context_for<N>(&self, node: N) -> EvaluationContext<'a, 'd>
        where N: Into<Node<'d>>
    {
        EvaluationContext {
            node: node.into(),
            .. *self
        }
    }

    fn position(&self) -> usize {
        self.position
    }

    fn size(&self) -> usize {
        self.size
    }

    fn function_for_name(&self, name: &str) -> Option<&'a BoxFunc> {
        self.functions.get(&name.to_owned())
    }

    fn value_of(&self, name: &str) -> Option<&Value<'d>> {
        self.variables.get(&name.to_owned())
    }

    fn namespace_for(&self, prefix: &str) -> Option<&str> {
        self.namespaces.get(prefix).map(|ns| &ns[..])
    }

    fn predicate_iter<'p>(&'p self, nodes: Nodeset<'d>)
                          -> EvaluationContextPredicateIter<'a, 'd, 'p>
    {
        let sz = nodes.size();
        EvaluationContextPredicateIter {
            parent: self,
            nodes: nodes.into_iter().enumerate(),
            size: sz,
        }
    }
}

struct EvaluationContextPredicateIter<'a : 'p, 'd : 'a + 'p, 'p> {
    parent: &'p EvaluationContext<'a, 'd>,
    nodes: iter::Enumerate<nodeset::IntoIter<'d>>,
    size: usize,
}

impl<'a, 'd, 'p> Iterator for EvaluationContextPredicateIter<'a, 'd, 'p> {
    type Item = EvaluationContext<'a, 'd>;

    fn next(&mut self) -> Option<EvaluationContext<'a, 'd>> {
        self.nodes.next().map(|(idx, node)| {
            EvaluationContext {
                node: node,
                position: idx + 1,
                size: self.size,
                .. *self.parent
            }
        })
    }
}

pub struct Factory {
    parser: Parser,
}

impl Factory {
    pub fn new() -> Factory {
        Factory { parser: Parser::new() }
    }

    pub fn build(&self, xpath: &str) -> parser::ParseResult {
        let tokenizer = Tokenizer::new(xpath);
        let deabbreviator = TokenDeabbreviator::new(tokenizer);

        self.parser.parse(deabbreviator)
    }
}

#[cfg(test)]
mod test {
    use std::borrow::ToOwned;

    use sxd_document::Package;

    use super::Value;

    #[test]
    fn number_of_string_is_ieee_754_number() {
        let v = Value::String("1.5".to_owned());
        assert_eq!(1.5, v.number());
    }

    #[test]
    fn number_of_string_with_negative_is_negative_number() {
        let v = Value::String("-1.5".to_owned());
        assert_eq!(-1.5, v.number());
    }

    #[test]
    fn number_of_string_with_surrounding_whitespace_is_number_without_whitespace() {
        let v = Value::String("\r\n1.5 \t".to_owned());
        assert_eq!(1.5, v.number());
    }

    #[test]
    fn number_of_garbage_string_is_nan() {
        let v = Value::String("I am not an IEEE 754 number".to_owned());
        assert!(v.number().is_nan());
    }

    #[test]
    fn number_of_boolean_true_is_1() {
        let v = Value::Boolean(true);
        assert_eq!(1.0, v.number());
    }

    #[test]
    fn number_of_boolean_false_is_0() {
        let v = Value::Boolean(false);
        assert_eq!(0.0, v.number());
    }

    #[test]
    fn number_of_nodeset_is_number_value_of_first_node_in_document_order() {
        let package = Package::new();
        let doc = package.as_document();

        let c1 = doc.create_comment("42.42");
        let c2 = doc.create_comment("1234");
        doc.root().append_child(c1);
        doc.root().append_child(c2);

        let v = Value::Nodeset(nodeset![c2, c1]);
        assert_eq!(42.42, v.number());
    }

    #[test]
    fn string_of_true_is_true() {
        let v = Value::Boolean(true);
        assert_eq!("true", v.string());
    }

    #[test]
    fn string_of_false_is_false() {
        let v = Value::Boolean(false);
        assert_eq!("false", v.string());
    }

    #[test]
    fn string_of_nan_is_nan() {
        let v = Value::Number(::std::f64::NAN);
        assert_eq!("NaN", v.string());
    }

    #[test]
    fn string_of_positive_zero_is_zero() {
        let v = Value::Number(0.0);
        assert_eq!("0", v.string());
    }

    #[test]
    fn string_of_negative_zero_is_zero() {
        let v = Value::Number(-0.0);
        assert_eq!("0", v.string());
    }

    #[test]
    fn string_of_positive_infinity_is_infinity() {
        let v = Value::Number(::std::f64::INFINITY);
        assert_eq!("Infinity", v.string());
    }

    #[test]
    fn string_of_negative_infinity_is_minus_infinity() {
        let v = Value::Number(::std::f64::NEG_INFINITY);
        assert_eq!("-Infinity", v.string());
    }

    #[test]
    fn string_of_integer_has_no_decimal() {
        let v = Value::Number(-42.0);
        assert_eq!("-42", v.string());
    }

    #[test]
    fn string_of_decimal_has_fractional_part() {
        let v = Value::Number(1.2);
        assert_eq!("1.2", v.string());
    }

    #[test]
    fn string_of_nodeset_is_string_value_of_first_node_in_document_order() {
        let package = Package::new();
        let doc = package.as_document();

        let c1 = doc.create_comment("comment 1");
        let c2 = doc.create_comment("comment 2");
        doc.root().append_child(c1);
        doc.root().append_child(c2);

        let v = Value::Nodeset(nodeset![c2, c1]);
        assert_eq!("comment 1", v.string());
    }
}
<commit_msg>Update feature flag for nightly<commit_after>//!
//! ### Namespaces
//!
//! The XPath specification assumes that the XML being processed has a
//! prefix defined for every namespace present in the document. If you
//! are processing XML that was parsed from text, this will be true by
//! construction.
//!
//! If you have programmatically created XML with namespaces but not
//! defined prefixes, some XPath behavior may be confusing:
//!
//! 1. The `name` method will not include a prefix, even if the
//! element or attribute has a namespace.
//! 2. The `namespace` axis will not include namespaces without
//! prefixes.
//!

#![cfg_attr(feature = "unstable", feature(iter_cmp))]

#[macro_use]
extern crate peresil;
extern crate sxd_document;

use std::borrow::ToOwned;
use std::collections::HashMap;
use std::{iter,string};

use self::Value::{Boolean,Number,String};

use nodeset::{Nodeset,Node};
use parser::Parser;
use tokenizer::{Tokenizer,TokenDeabbreviator};

pub use function::Function;
pub use expression::Expression;

#[macro_use]
pub mod macros;
pub mod nodeset;
pub mod axis;
mod expression;
pub mod function;
mod node_test;
pub mod parser;
mod token;
pub mod tokenizer;

// TODO - this doesn't need to be public, figure out how to make it so
#[derive(PartialEq,Debug,Clone)]
pub enum LiteralValue {
    Boolean(bool),
    Number(f64),
    String(string::String),
}

impl LiteralValue {
    fn into_value<'d>(self) -> Value<'d> {
        match self {
            LiteralValue::Boolean(v) => Boolean(v),
            LiteralValue::Number(v)  => Number(v),
            LiteralValue::String(v)  => String(v),
        }
    }
}

#[derive(PartialEq,Debug,Clone)]
pub enum Value<'d> {
    Boolean(bool),
    Number(f64),
    String(string::String),
    Nodeset(Nodeset<'d>),
}

fn str_to_num(s: &str) -> f64 {
    s.trim().parse().unwrap_or(::std::f64::NAN)
}

impl<'d> Value<'d> {
    pub fn boolean(&self) -> bool {
        use Value::*;
        match *self {
            Boolean(val) => val,
            Number(n) => n != 0.0 && ! n.is_nan(),
            String(ref s) => ! s.is_empty(),
            Nodeset(ref nodeset) => nodeset.size() > 0,
        }
    }

    pub fn number(&self) -> f64 {
        use Value::*;
        match *self {
            Boolean(val) => if val { 1.0 } else { 0.0 },
            Number(val) => val,
            String(ref s) => str_to_num(s),
            Nodeset(..) => str_to_num(&self.string()),
        }
    }

    pub fn string(&self) -> string::String {
        use Value::*;
        match *self {
            Boolean(v) => v.to_string(),
            Number(n) => {
                if n.is_infinite() {
                    if n.signum() < 0.0 {
                        "-Infinity".to_owned()
                    } else {
                        "Infinity".to_owned()
                    }
                } else {
                    n.to_string()
                }
            },
            String(ref val) => val.clone(),
            Nodeset(ref ns) => match ns.document_order_first() {
                Some(n) => n.string_value(),
                None => "".to_owned(),
            },
        }
    }

    #[allow(dead_code)]
    fn into_literal_value(self) -> LiteralValue {
        use Value::*;
        match self {
            Boolean(v) => LiteralValue::Boolean(v),
            Number(v)  => LiteralValue::Number(v),
            String(v)  => LiteralValue::String(v),
            Nodeset(_) => panic!("Cannot convert a nodeset to a literal"),
        }
    }
}

type BoxFunc = Box<Function + 'static>;
pub type Functions = HashMap<string::String, BoxFunc>;
pub type Variables<'d> = HashMap<string::String, Value<'d>>;
pub type Namespaces = HashMap<string::String, string::String>;

#[derive(Copy,Clone)]
pub struct EvaluationContext<'a, 'd : 'a> {
    node: Node<'d>,
    functions: &'a Functions,
    variables: &'a Variables<'d>,
    namespaces: &'a Namespaces,
    position: usize,
    size: usize,
}

impl<'a, 'd> EvaluationContext<'a, 'd> {
    pub fn new<N>(node: N,
                  functions: &'a Functions,
                  variables: &'a Variables<'d>,
                  namespaces: &'a Namespaces)
                  -> EvaluationContext<'a, 'd>
        where N: Into<Node<'d>>
    {
        EvaluationContext {
            node: node.into(),
            functions: functions,
            variables: variables,
            namespaces: namespaces,
            position: 1,
            size: 1,
        }
    }

    fn new_context_for<N>(&self, node: N) -> EvaluationContext<'a, 'd>
        where N: Into<Node<'d>>
    {
        EvaluationContext {
            node: node.into(),
            .. *self
        }
    }

    fn position(&self) -> usize {
        self.position
    }

    fn size(&self) -> usize {
        self.size
    }

    fn function_for_name(&self, name: &str) -> Option<&'a BoxFunc> {
        self.functions.get(&name.to_owned())
    }

    fn value_of(&self, name: &str) -> Option<&Value<'d>> {
        self.variables.get(&name.to_owned())
    }

    fn namespace_for(&self, prefix: &str) -> Option<&str> {
        self.namespaces.get(prefix).map(|ns| &ns[..])
    }

    fn predicate_iter<'p>(&'p self, nodes: Nodeset<'d>)
                          -> EvaluationContextPredicateIter<'a, 'd, 'p>
    {
        let sz = nodes.size();
        EvaluationContextPredicateIter {
            parent: self,
            nodes: nodes.into_iter().enumerate(),
            size: sz,
        }
    }
}

struct EvaluationContextPredicateIter<'a : 'p, 'd : 'a + 'p, 'p> {
    parent: &'p EvaluationContext<'a, 'd>,
    nodes: iter::Enumerate<nodeset::IntoIter<'d>>,
    size: usize,
}

impl<'a, 'd, 'p> Iterator for EvaluationContextPredicateIter<'a, 'd, 'p> {
    type Item = EvaluationContext<'a, 'd>;

    fn next(&mut self) -> Option<EvaluationContext<'a, 'd>> {
        self.nodes.next().map(|(idx, node)| {
            EvaluationContext {
                node: node,
                position: idx + 1,
                size: self.size,
                .. *self.parent
            }
        })
    }
}

pub struct Factory {
    parser: Parser,
}

impl Factory {
    pub fn new() -> Factory {
        Factory { parser: Parser::new() }
    }

    pub fn build(&self, xpath: &str) -> parser::ParseResult {
        let tokenizer = Tokenizer::new(xpath);
        let deabbreviator = TokenDeabbreviator::new(tokenizer);

        self.parser.parse(deabbreviator)
    }
}

#[cfg(test)]
mod test {
    use std::borrow::ToOwned;

    use sxd_document::Package;

    use super::Value;

    #[test]
    fn number_of_string_is_ieee_754_number() {
        let v = Value::String("1.5".to_owned());
        assert_eq!(1.5, v.number());
    }

    #[test]
    fn number_of_string_with_negative_is_negative_number() {
        let v = Value::String("-1.5".to_owned());
        assert_eq!(-1.5, v.number());
    }

    #[test]
    fn number_of_string_with_surrounding_whitespace_is_number_without_whitespace() {
        let v = Value::String("\r\n1.5 \t".to_owned());
        assert_eq!(1.5, v.number());
    }

    #[test]
    fn number_of_garbage_string_is_nan() {
        let v = Value::String("I am not an IEEE 754 number".to_owned());
        assert!(v.number().is_nan());
    }

    #[test]
    fn number_of_boolean_true_is_1() {
        let v = Value::Boolean(true);
        assert_eq!(1.0, v.number());
    }

    #[test]
    fn number_of_boolean_false_is_0() {
        let v = Value::Boolean(false);
        assert_eq!(0.0, v.number());
    }

    #[test]
    fn number_of_nodeset_is_number_value_of_first_node_in_document_order() {
        let package = Package::new();
        let doc = package.as_document();

        let c1 = doc.create_comment("42.42");
        let c2 = doc.create_comment("1234");
        doc.root().append_child(c1);
        doc.root().append_child(c2);

        let v = Value::Nodeset(nodeset![c2, c1]);
        assert_eq!(42.42, v.number());
    }

    #[test]
    fn string_of_true_is_true() {
        let v = Value::Boolean(true);
        assert_eq!("true", v.string());
    }

    #[test]
    fn string_of_false_is_false() {
        let v = Value::Boolean(false);
        assert_eq!("false", v.string());
    }

    #[test]
    fn string_of_nan_is_nan() {
        let v = Value::Number(::std::f64::NAN);
        assert_eq!("NaN", v.string());
    }

    #[test]
    fn string_of_positive_zero_is_zero() {
        let v = Value::Number(0.0);
        assert_eq!("0", v.string());
    }

    #[test]
    fn string_of_negative_zero_is_zero() {
        let v = Value::Number(-0.0);
        assert_eq!("0", v.string());
    }

    #[test]
    fn string_of_positive_infinity_is_infinity() {
        let v = Value::Number(::std::f64::INFINITY);
        assert_eq!("Infinity", v.string());
    }

    #[test]
    fn string_of_negative_infinity_is_minus_infinity() {
        let v = Value::Number(::std::f64::NEG_INFINITY);
        assert_eq!("-Infinity", v.string());
    }

    #[test]
    fn string_of_integer_has_no_decimal() {
        let v = Value::Number(-42.0);
        assert_eq!("-42", v.string());
    }

    #[test]
    fn string_of_decimal_has_fractional_part() {
        let v = Value::Number(1.2);
        assert_eq!("1.2", v.string());
    }

    #[test]
    fn string_of_nodeset_is_string_value_of_first_node_in_document_order() {
        let package = Package::new();
        let doc = package.as_document();

        let c1 = doc.create_comment("comment 1");
        let c2 = doc.create_comment("comment 2");
        doc.root().append_child(c1);
        doc.root().append_child(c2);

        let v = Value::Nodeset(nodeset![c2, c1]);
        assert_eq!("comment 1", v.string());
    }
}
<|endoftext|>


<commit_before>
mod error;
mod pair;
mod parser;
mod token;
mod scanner;

pub use error::*;
pub use pair::*;
pub use parser::*;
pub use token::*;
pub use scanner::*;

use std::path::Path;
use std::fs::File;
use std::io::Read;

pub fn parse_file(filename: &str) -> Result<Pair, Vec<NcclError>> {
    if let Ok(mut file) = File::open(Path::new(filename)) {
        let mut data = String::new();
        file.read_to_string(&mut data).unwrap();
        //let tokens = Scanner::new(data).scan_tokens()?;
        Parser::new(Scanner::new(data).scan_tokens()?).parse()
    } else {
        Err(vec![NcclError::new(ErrorKind::FileError, "Could not find file.", 0)])
    }
}

pub fn print_tokens(tokens: Vec<Token>) {
    for token in tokens {
        match token.kind {
            TokenKind::Value => print!("\"{}\"", token.lexeme),
            TokenKind::Colon => print!(" : "),
            TokenKind::Indent => print!(" >> "),
            TokenKind::Newline => println!(""),
            TokenKind::EOF => println!("end"),
        }
    }
}

<commit_msg>add more top-level functions<commit_after>
mod error;
mod pair;
mod parser;
mod token;
mod scanner;

pub use error::*;
pub use pair::*;
pub use parser::*;
pub use token::*;
pub use scanner::*;

use std::path::Path;
use std::fs::File;
use std::io::Read;

pub fn parse_file(filename: &str) -> Result<Pair, Vec<NcclError>> {
    if let Ok(mut file) = File::open(Path::new(filename)) {
        let mut data = String::new();
        file.read_to_string(&mut data).unwrap();
        Parser::new(Scanner::new(data).scan_tokens()?).parse()
    } else {
        Err(vec![NcclError::new(ErrorKind::FileError, "Could not find file.", 0)])
    }
}

pub fn parse_string(data: &str) -> Result<Pair, Vec<NcclError>> {
    Parser::new(Scanner::new(data.to_owned()).scan_tokens()?).parse()
}

pub fn parse_file_with(filename: &str, pair: Pair) -> Result<Pair, Vec<NcclError>> {
    if let Ok(mut file) = File::open(Path::new(filename)) {
        let mut data = String::new();
        file.read_to_string(&mut data).unwrap();
        Parser::new_with(Scanner::new(data).scan_tokens()?, pair).parse()
    } else {
        Err(vec![NcclError::new(ErrorKind::FileError, "Could not find file.", 0)])
    }
}

<|endoftext|>


<commit_before>extern crate shared_slice;

use std::collections::{BTreeMap, HashSet};
use std::hash::{Hash, Hasher};
use std::str::from_utf8;
use shared_slice::rc::RcStr;

/// A Document contains an id and content, both of which are reference-counted strings.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Eq)]
pub struct Document {
    pub id: RcStr,
    pub content: RcStr,
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document {
            id: into_rc_str(id),
            content: into_rc_str(content),
        }
    }
}

fn into_rc_str<S: Into<String>>(id: S) -> RcStr {
    let id = id.into();
    let id: Vec<u8> = id.into();
    RcStr::new(id.into_boxed_slice())
}

impl Hash for Document {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        state.write(&*self.id);
    }
}

impl PartialEq<Document> for Document {
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

impl std::fmt::Debug for Document {
    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(fmt, "Document {{ id: {:?}, content: {:?} }}",
               from_utf8(&*self.id).unwrap(), from_utf8(&*self.content).unwrap())
    }
}

/// An Index is anything that can store Documents and return matching Documents for a query.
trait Index {
    /// Given a Document, stores it in the index.
    fn index(&mut self, doc: Document);
    /// Given a query, return the set of Documents that match the query.
    /// What constitutes a "match" is implementation-specific; loosely, a Document should
    /// match a query if they are somehow related. A naive implementation might simply
    /// return the set of Documents for which `query` is a substring of their content.
    fn search(&self, query: &str) -> HashSet<Document>;
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to sets of Documents.
type InvertedIndex = BTreeMap<RcStr, HashSet<Document>>;

impl Index for InvertedIndex {
    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    fn index(&mut self, doc: Document) {
        for word in doc.content.clone().split_whitespace() {
            self.entry(word).or_insert_with(|| HashSet::new()).insert(doc.clone());
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    fn search(&self, query: &str) -> HashSet<Document> {
        query.split_whitespace()
            .flat_map(|word| self.get(&into_rc_str(word)))
            .flat_map(|docs| docs)
            .cloned()
            .collect()
    }
}

#[test]
fn test_index() {
    let mut index = BTreeMap::new();

    let doc1 = Document::new("1", "learn to program in rust today");
    index.index(doc1.clone());

    let doc2 = Document::new("2", "what did you do today");
    index.index(doc2.clone());

    let search_results = index.search("today");
    assert!(search_results.contains(&doc1));
    assert!(search_results.contains(&doc2));
    assert_eq!(2, index.search("rust today").len())
}
<commit_msg>Remove unsafe; make Document fields private; use getters.<commit_after>#![feature(collections)]
extern crate shared_slice;

use std::collections::{BTreeMap, HashSet};
use std::hash::{Hash, Hasher};
use std::str::from_utf8_unchecked;
use shared_slice::rc::RcStr;

/// A Document contains an id and content, both of which are reference-counted strings.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Eq)]
pub struct Document {
    id: RcStr,
    content: RcStr,
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document {
            id: into_rc_str(id),
            content: into_rc_str(content),
        }
    }

    pub fn id(&self) -> &str {
        unsafe { from_utf8_unchecked(&*self.id) }
    }

    pub fn content(&self) -> &str {
        unsafe { from_utf8_unchecked(&*self.content) }
    }
}

fn into_rc_str<S: Into<String>>(id: S) -> RcStr {
    let id = id.into();
    let id: Vec<u8> = id.into();
    RcStr::new(id.into_boxed_slice())
}

impl Hash for Document {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        state.write(&*self.id);
    }
}

impl PartialEq<Document> for Document {
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

impl std::fmt::Debug for Document {
    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(fmt, "Document {{ id: {:?}, content: {:?} }}", self.id(), self.content())
    }
}

/// An Index is anything that can store Documents and return matching Documents for a query.
trait Index {
    /// Given a Document, stores it in the index.
    fn index(&mut self, doc: Document);
    /// Given a query, return the set of Documents that match the query.
    /// What constitutes a "match" is implementation-specific; loosely, a Document should
    /// match a query if they are somehow related. A naive implementation might simply
    /// return the set of Documents for which `query` is a substring of their content.
    fn search(&self, query: &str) -> HashSet<Document>;
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to sets of Documents.
type InvertedIndex = BTreeMap<RcStr, HashSet<Document>>;

impl Index for InvertedIndex {
    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    fn index(&mut self, doc: Document) {
        let lowercased = doc.content().to_lowercase();
        let ngrams = into_rc_str(lowercased)
            .split_whitespace()
            .flat_map(|word: RcStr| (1..word.len() + 1).map(move |to| word.clone().slice_to(to)));

        for ngram in ngrams {
            self.entry(ngram).or_insert_with(|| HashSet::new()).insert(doc.clone());
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    fn search(&self, query: &str) -> HashSet<Document> {
        query.split_whitespace()
            .flat_map(|word| self.get(&into_rc_str(word)))
            .flat_map(|docs| docs)
            .cloned()
            .collect()
    }
}

#[test]
fn test_search1() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    let doc2 = Document::new("2", "what did you do today");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("today");
    assert!(search_results.contains(&doc1));
    assert!(search_results.contains(&doc2));
}

#[test]
fn test_ngrams() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    let doc2 = Document::new("2", "what did you do today");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    assert!(search_results.contains(&doc1));
    assert!(search_results.contains(&doc2));
}

#[test]
fn test_search2() {
    let mut index = InvertedIndex::new();
    index.index(Document::new("2", "what did you do today"));
    index.index(Document::new("3", "hey today"));
    let search_results = index.search("today");
    assert_eq!(search_results.len(), 2);
}
<|endoftext|>


<commit_before>#![crate_name="civet"]
#![crate_type="rlib"]
#![feature(unsafe_destructor)]

extern crate libc;
extern crate native;
extern crate collections;
extern crate semver;
extern crate conduit;

use std::io;
use std::io::net::ip::{IpAddr, Ipv4Addr};
use std::io::{IoResult, util, BufferedWriter};
use std::collections::HashMap;

use conduit::{Request, HeaderEntries, Handler, Extensions, TypeMap};

use raw::{RequestInfo,Header};
use raw::{get_header,get_headers,get_request_info};
use status::{ToStatusCode};

pub use raw::Config;

mod raw;
pub mod status;

pub struct Connection<'a> {
    request: CivetRequest<'a>,
    written: bool,
}

pub struct CivetRequest<'a> {
    conn: &'a raw::Connection,
    request_info: RequestInfo<'a>,
    headers: Headers<'a>,
    extensions: Extensions
}

fn ver(major: uint, minor: uint) -> semver::Version {
    semver::Version {
        major: major,
        minor: minor,
        patch: 0,
        pre: vec!(),
        build: vec!()
    }
}

impl<'a> conduit::Request for CivetRequest<'a> {
    fn http_version(&self) -> semver::Version {
        let version = self.request_info.http_version().unwrap();
        match version {
            "1.0" => ver(1, 0),
            "1.1" => ver(1, 1),
            _ => ver(1, 1)
        }
    }

    fn conduit_version(&self) -> semver::Version {
        ver(0, 1)
    }

    fn method(&self) -> conduit::Method {
        match self.request_info.method().unwrap() {
            "HEAD" => conduit::Head,
            "GET" => conduit::Get,
            "POST" => conduit::Post,
            "PUT" => conduit::Put,
            "DELETE" => conduit::Delete,
            "PATCH" => conduit::Patch,
            "PURGE" => conduit::Purge,
            "CONNECT" => conduit::Connect,
            "OPTIONS" => conduit::Options,
            "TRACE" => conduit::Trace,
            other @ _ => panic!("Civet does not support {} requests", other)
        }
    }

    fn scheme(&self) -> conduit::Scheme {
        if self.request_info.is_ssl() {
            conduit::Https
        } else {
            conduit::Http
        }
    }

    fn host<'a>(&'a self) -> conduit::Host<'a> {
        conduit::HostName(get_header(self.conn, "Host").unwrap())
    }

    fn virtual_root<'a>(&'a self) -> Option<&'a str> {
        None
    }

    fn path<'a>(&'a self) -> &'a str {
        self.request_info.url().unwrap()
    }

    fn query_string<'a>(&'a self) -> Option<&'a str> {
        self.request_info.query_string()
    }

    fn remote_ip(&self) -> IpAddr {
        let ip = self.request_info.remote_ip();
        Ipv4Addr((ip >> 24) as u8,
                 (ip >> 16) as u8,
                 (ip >>  8) as u8,
                 (ip >>  0) as u8)
    }

    fn content_length(&self) -> Option<uint> {
        get_header(self.conn, "Content-Length").and_then(from_str)
    }

    fn headers<'a>(&'a self) -> &'a conduit::Headers {
        &self.headers as &conduit::Headers
    }

    fn body<'a>(&'a mut self) -> &'a mut Reader {
        self as &mut Reader
    }

    fn extensions<'a>(&'a self) -> &'a Extensions {
        &self.extensions
    }

    fn mut_extensions<'a>(&'a mut self) -> &'a mut Extensions {
        &mut self.extensions
    }
}

pub fn response<S: ToStatusCode, R: Reader + Send>(status: S,
    headers: HashMap<String, Vec<String>>, body: R) -> conduit::Response
{
    conduit::Response {
        status: status.to_status().unwrap().to_code(),
        headers: headers,
        body: box body as Box<Reader + Send>
    }
}

impl<'a> Connection<'a> {
    fn new<'a>(conn: &'a raw::Connection) -> Result<Connection<'a>, String> {
        match request_info(conn) {
            Ok(info) => {
                let request = CivetRequest {
                    conn: conn,
                    request_info: info,
                    headers: Headers { conn: conn },
                    extensions: TypeMap::new()
                };

                Ok(Connection {
                    request: request,
                    written: false,
                })
            },
            Err(err) => Err(err)
        }
    }

}

impl<'a> Writer for Connection<'a> {
    fn write(&mut self, buf: &[u8]) -> IoResult<()> {
        self.written = true;
        write_bytes(self.request.conn, buf).map_err(|_| {
            io::standard_error(io::IoUnavailable)
        })
    }
}

impl<'a> Reader for CivetRequest<'a> {
    fn read(&mut self, buf: &mut[u8]) -> IoResult<uint> {
        let ret = raw::read(self.conn, buf);

        if ret == 0 {
            Err(io::standard_error(io::EndOfFile))
        } else {
            Ok(ret as uint)
        }
    }
}

#[unsafe_destructor]
impl<'a> Drop for Connection<'a> {
    fn drop(&mut self) {
        if !self.written {
            let _ = writeln!(self, "HTTP/1.1 500 Internal Server Error");
        }
    }
}

pub struct Headers<'a> {
    conn: &'a raw::Connection
}

impl<'a> conduit::Headers for Headers<'a> {
    fn find<'a>(&'a self, string: &str) -> Option<Vec<&'a str>> {
        get_header(self.conn, string).map(|s| vec!(s))
    }

    fn has(&self, string: &str) -> bool {
        get_header(self.conn, string).is_some()
    }

    fn iter<'a>(&'a self) -> conduit::HeaderEntries<'a> {
        box HeaderIterator::new(self.conn) as conduit::HeaderEntries<'a>
    }
}

pub struct HeaderIterator<'a> {
    headers: Vec<Header<'a>>,
    position: uint
}

impl<'a> HeaderIterator<'a> {
    fn new<'b>(conn: &'b raw::Connection) -> HeaderIterator<'b> {
        HeaderIterator { headers: get_headers(conn), position: 0 }
    }
}

impl<'a> Iterator<(&'a str, Vec<&'a str>)> for HeaderIterator<'a> {
    fn next(&mut self) -> Option<(&'a str, Vec<&'a str>)> {
        let pos = self.position;
        let headers = &self.headers;

        if self.headers.len() <= pos {
            None
        } else {
            let header = &headers[pos];
            self.position += 1;
            header.name().map(|name| (name, vec!(header.value().unwrap())))
        }
    }
}

pub struct Server(raw::Server<Box<Handler + 'static + Sync>>);

impl Server {
    pub fn start<H: Handler + 'static + Sync>(options: Config, handler: H)
        -> IoResult<Server>
    {
        fn internal_handler(conn: &mut raw::Connection,
                            handler: &Box<Handler + 'static + Sync>)
                            -> Result<(), ()> {
            let mut connection = Connection::new(conn).unwrap();
            let response = handler.call(&mut connection.request);
            let mut writer = BufferedWriter::new(connection);

            fn err<W: Writer>(writer: &mut W) {
                let _ = writeln!(writer, "HTTP/1.1 500 Internal Server Error");
            }

            let conduit::Response { status, headers, mut body } = match response {
                Ok(r) => r,
                Err(_) => return Err(err(&mut writer)),
            };
            let (code, string) = status;
            try!(write!(&mut writer, "HTTP/1.1 {:u} {:s}\r\n", code, string).map_err(|_| ()));

            for (key, value) in headers.iter() {
                for header in value.iter() {
                    try!(write!(&mut writer, "{:s}: {:s}\r\n", *key, *header).map_err(|_| ()));
                }
            }

            try!(write!(&mut writer, "\r\n").map_err(|_| ()));
            let mut body: &mut Reader = &mut *body;
            try!(util::copy(&mut body, &mut writer).map_err(|_| ()));

            Ok(())
        }

        let handler = box handler;
        let raw_callback = raw::ServerCallback::new(internal_handler, handler);
        Ok(Server(try!(raw::Server::start(options, raw_callback))))
    }
}

fn write_bytes(connection: &raw::Connection, bytes: &[u8]) -> Result<(), String> {
    let ret = raw::write(connection, bytes);

    if ret == -1 {
        return Err("Couldn't write bytes to the connection".to_string())
    }

    Ok(())
}

fn request_info<'a>(connection: &'a raw::Connection)
    -> Result<RequestInfo<'a>, String>
{
    match get_request_info(connection) {
        Some(info) => Ok(info),
        None => Err("Couldn't get request info for connection".to_string())
    }
}

#[cfg(test)]
mod test {
    use std::collections::HashMap;
    use std::io::net::ip::SocketAddr;
    use std::io::net::tcp::TcpStream;
    use std::io::test::next_test_ip4;
    use std::io::{IoResult, MemReader};
    use std::fmt::Show;
    use std::sync::Mutex;
    use super::{Server, Config, response};
    use conduit::{Request, Response, Handler};

    fn noop(_: &mut Request) -> IoResult<Response> { unreachable!() }

    fn request(addr: SocketAddr, req: &str) -> String {
        let mut s = TcpStream::connect(addr.ip.to_string().as_slice(),
                                       addr.port).unwrap();
        s.write_str(req.trim_left()).unwrap();
        s.read_to_string().unwrap()
    }

    #[test]
    fn smoke() {
        let addr = next_test_ip4();
        Server::start(Config { port: addr.port, threads: 1 }, noop).unwrap();
    }

    #[test]
    fn dupe_port() {
        let addr = next_test_ip4();
        let s1 = Server::start(Config { port: addr.port, threads: 1 }, noop);
        assert!(s1.is_ok());
        let s2 = Server::start(Config { port: addr.port, threads: 1 }, noop);
        assert!(s2.is_err());
    }

    #[test]
    fn drops_handler() {
        static mut DROPPED: bool = false;
        struct Foo;
        impl Handler for Foo {
            fn call(&self, _req: &mut Request) -> Result<Response, Box<Show + 'static>> {
                panic!()
            }
        }
        impl Drop for Foo {
            fn drop(&mut self) { unsafe { DROPPED = true; } }
        }

        let addr = next_test_ip4();
        drop(Server::start(Config { port: addr.port, threads: 1 }, Foo));
        unsafe { assert!(DROPPED); }
    }

    #[test]
    fn invokes() {
        struct Foo(Mutex<Sender<()>>);
        impl Handler for Foo {
            fn call(&self, _req: &mut Request) -> Result<Response, Box<Show + 'static>> {
                let Foo(ref tx) = *self;
                tx.lock().send(());
                Ok(response(200i, HashMap::new(), MemReader::new(vec![])))
            }
        }

        let addr = next_test_ip4();
        let (tx, rx) = channel();
        let handler = Foo(Mutex::new(tx));
        let _s = Server::start(Config { port: addr.port, threads: 1 }, handler);
        request(addr, r"
GET / HTTP/1.1

");
        rx.recv();
    }

    #[test]
    fn header_sent() {
        struct Foo(Mutex<Sender<String>>);
        impl Handler for Foo {
            fn call(&self, req: &mut Request) -> Result<Response, Box<Show + 'static>> {
                let Foo(ref tx) = *self;
                tx.lock().send(req.headers().find("Foo").unwrap().connect(""));
                Ok(response(200i, HashMap::new(), MemReader::new(vec![])))
            }
        }

        let addr = next_test_ip4();
        let (tx, rx) = channel();
        let handler = Foo(Mutex::new(tx));
        let _s = Server::start(Config { port: addr.port, threads: 1 }, handler);
        request(addr, r"
GET / HTTP/1.1
Foo: bar

");
        assert_eq!(rx.recv().as_slice(), "bar");
    }

    #[test]
    fn failing_handler() {
        struct Foo;
        impl Handler for Foo {
            fn call(&self, _req: &mut Request) -> Result<Response, Box<Show + 'static>> {
                panic!()
            }
        }

        let addr = next_test_ip4();
        let _s = Server::start(Config { port: addr.port, threads: 1 }, Foo);
        request(addr, r"
GET / HTTP/1.1
Foo: bar

");
    }

    #[test]
    fn failing_handler_is_500() {
        struct Foo;
        impl Handler for Foo {
            fn call(&self, _req: &mut Request) -> Result<Response, Box<Show + 'static>> {
                panic!()
            }
        }

        let addr = next_test_ip4();
        let _s = Server::start(Config { port: addr.port, threads: 1 }, Foo);
        let response = request(addr, r"
GET / HTTP/1.1
Foo: bar

");
        assert!(response.as_slice().contains("500 Internal"),
                "not a failing response: {}", response);
    }
}
<commit_msg>Update to conduit master<commit_after>#![crate_name="civet"]
#![crate_type="rlib"]
#![feature(unsafe_destructor)]

extern crate libc;
extern crate native;
extern crate collections;
extern crate semver;
extern crate conduit;

use std::io;
use std::io::net::ip::{IpAddr, Ipv4Addr};
use std::io::{IoResult, util, BufferedWriter};
use std::collections::HashMap;

use conduit::{Request, Handler, Extensions, TypeMap};

use raw::{RequestInfo,Header};
use raw::{get_header,get_headers,get_request_info};
use status::{ToStatusCode};

pub use raw::Config;

mod raw;
pub mod status;

pub struct Connection<'a> {
    request: CivetRequest<'a>,
    written: bool,
}

pub struct CivetRequest<'a> {
    conn: &'a raw::Connection,
    request_info: RequestInfo<'a>,
    headers: Headers<'a>,
    extensions: Extensions
}

fn ver(major: uint, minor: uint) -> semver::Version {
    semver::Version {
        major: major,
        minor: minor,
        patch: 0,
        pre: vec!(),
        build: vec!()
    }
}

impl<'a> conduit::Request for CivetRequest<'a> {
    fn http_version(&self) -> semver::Version {
        let version = self.request_info.http_version().unwrap();
        match version {
            "1.0" => ver(1, 0),
            "1.1" => ver(1, 1),
            _ => ver(1, 1)
        }
    }

    fn conduit_version(&self) -> semver::Version {
        ver(0, 1)
    }

    fn method(&self) -> conduit::Method {
        match self.request_info.method().unwrap() {
            "HEAD" => conduit::Head,
            "GET" => conduit::Get,
            "POST" => conduit::Post,
            "PUT" => conduit::Put,
            "DELETE" => conduit::Delete,
            "PATCH" => conduit::Patch,
            "PURGE" => conduit::Purge,
            "CONNECT" => conduit::Connect,
            "OPTIONS" => conduit::Options,
            "TRACE" => conduit::Trace,
            other @ _ => panic!("Civet does not support {} requests", other)
        }
    }

    fn scheme(&self) -> conduit::Scheme {
        if self.request_info.is_ssl() {
            conduit::Https
        } else {
            conduit::Http
        }
    }

    fn host<'a>(&'a self) -> conduit::Host<'a> {
        conduit::HostName(get_header(self.conn, "Host").unwrap())
    }

    fn virtual_root<'a>(&'a self) -> Option<&'a str> {
        None
    }

    fn path<'a>(&'a self) -> &'a str {
        self.request_info.url().unwrap()
    }

    fn query_string<'a>(&'a self) -> Option<&'a str> {
        self.request_info.query_string()
    }

    fn remote_ip(&self) -> IpAddr {
        let ip = self.request_info.remote_ip();
        Ipv4Addr((ip >> 24) as u8,
                 (ip >> 16) as u8,
                 (ip >>  8) as u8,
                 (ip >>  0) as u8)
    }

    fn content_length(&self) -> Option<uint> {
        get_header(self.conn, "Content-Length").and_then(from_str)
    }

    fn headers<'a>(&'a self) -> &'a conduit::Headers {
        &self.headers as &conduit::Headers
    }

    fn body<'a>(&'a mut self) -> &'a mut Reader {
        self as &mut Reader
    }

    fn extensions<'a>(&'a self) -> &'a Extensions {
        &self.extensions
    }

    fn mut_extensions<'a>(&'a mut self) -> &'a mut Extensions {
        &mut self.extensions
    }
}

pub fn response<S: ToStatusCode, R: Reader + Send>(status: S,
    headers: HashMap<String, Vec<String>>, body: R) -> conduit::Response
{
    conduit::Response {
        status: status.to_status().unwrap().to_code(),
        headers: headers,
        body: box body as Box<Reader + Send>
    }
}

impl<'a> Connection<'a> {
    fn new<'a>(conn: &'a raw::Connection) -> Result<Connection<'a>, String> {
        match request_info(conn) {
            Ok(info) => {
                let request = CivetRequest {
                    conn: conn,
                    request_info: info,
                    headers: Headers { conn: conn },
                    extensions: TypeMap::new()
                };

                Ok(Connection {
                    request: request,
                    written: false,
                })
            },
            Err(err) => Err(err)
        }
    }

}

impl<'a> Writer for Connection<'a> {
    fn write(&mut self, buf: &[u8]) -> IoResult<()> {
        self.written = true;
        write_bytes(self.request.conn, buf).map_err(|_| {
            io::standard_error(io::IoUnavailable)
        })
    }
}

impl<'a> Reader for CivetRequest<'a> {
    fn read(&mut self, buf: &mut[u8]) -> IoResult<uint> {
        let ret = raw::read(self.conn, buf);

        if ret == 0 {
            Err(io::standard_error(io::EndOfFile))
        } else {
            Ok(ret as uint)
        }
    }
}

#[unsafe_destructor]
impl<'a> Drop for Connection<'a> {
    fn drop(&mut self) {
        if !self.written {
            let _ = writeln!(self, "HTTP/1.1 500 Internal Server Error");
        }
    }
}

pub struct Headers<'a> {
    conn: &'a raw::Connection
}

impl<'a> conduit::Headers for Headers<'a> {
    fn find<'a>(&'a self, string: &str) -> Option<Vec<&'a str>> {
        get_header(self.conn, string).map(|s| vec!(s))
    }

    fn has(&self, string: &str) -> bool {
        get_header(self.conn, string).is_some()
    }

    fn all(&self) -> Vec<(&str, Vec<&str>)> {
        HeaderIterator::new(self.conn).collect()
    }
}

pub struct HeaderIterator<'a> {
    headers: Vec<Header<'a>>,
    position: uint
}

impl<'a> HeaderIterator<'a> {
    fn new<'b>(conn: &'b raw::Connection) -> HeaderIterator<'b> {
        HeaderIterator { headers: get_headers(conn), position: 0 }
    }
}

impl<'a> Iterator<(&'a str, Vec<&'a str>)> for HeaderIterator<'a> {
    fn next(&mut self) -> Option<(&'a str, Vec<&'a str>)> {
        let pos = self.position;
        let headers = &self.headers;

        if self.headers.len() <= pos {
            None
        } else {
            let header = &headers[pos];
            self.position += 1;
            header.name().map(|name| (name, vec!(header.value().unwrap())))
        }
    }
}

pub struct Server(raw::Server<Box<Handler + 'static + Sync>>);

impl Server {
    pub fn start<H: Handler + 'static + Sync>(options: Config, handler: H)
        -> IoResult<Server>
    {
        fn internal_handler(conn: &mut raw::Connection,
                            handler: &Box<Handler + 'static + Sync>)
                            -> Result<(), ()> {
            let mut connection = Connection::new(conn).unwrap();
            let response = handler.call(&mut connection.request);
            let mut writer = BufferedWriter::new(connection);

            fn err<W: Writer>(writer: &mut W) {
                let _ = writeln!(writer, "HTTP/1.1 500 Internal Server Error");
            }

            let conduit::Response { status, headers, mut body } = match response {
                Ok(r) => r,
                Err(_) => return Err(err(&mut writer)),
            };
            let (code, string) = status;
            try!(write!(&mut writer, "HTTP/1.1 {:u} {:s}\r\n", code, string).map_err(|_| ()));

            for (key, value) in headers.iter() {
                for header in value.iter() {
                    try!(write!(&mut writer, "{:s}: {:s}\r\n", *key, *header).map_err(|_| ()));
                }
            }

            try!(write!(&mut writer, "\r\n").map_err(|_| ()));
            let mut body: &mut Reader = &mut *body;
            try!(util::copy(&mut body, &mut writer).map_err(|_| ()));

            Ok(())
        }

        let handler = box handler;
        let raw_callback = raw::ServerCallback::new(internal_handler, handler);
        Ok(Server(try!(raw::Server::start(options, raw_callback))))
    }
}

fn write_bytes(connection: &raw::Connection, bytes: &[u8]) -> Result<(), String> {
    let ret = raw::write(connection, bytes);

    if ret == -1 {
        return Err("Couldn't write bytes to the connection".to_string())
    }

    Ok(())
}

fn request_info<'a>(connection: &'a raw::Connection)
    -> Result<RequestInfo<'a>, String>
{
    match get_request_info(connection) {
        Some(info) => Ok(info),
        None => Err("Couldn't get request info for connection".to_string())
    }
}

#[cfg(test)]
mod test {
    use std::collections::HashMap;
    use std::io::net::ip::SocketAddr;
    use std::io::net::tcp::TcpStream;
    use std::io::test::next_test_ip4;
    use std::io::{IoResult, MemReader};
    use std::fmt::Show;
    use std::sync::Mutex;
    use super::{Server, Config, response};
    use conduit::{Request, Response, Handler};

    fn noop(_: &mut Request) -> IoResult<Response> { unreachable!() }

    fn request(addr: SocketAddr, req: &str) -> String {
        let mut s = TcpStream::connect(addr.ip.to_string().as_slice(),
                                       addr.port).unwrap();
        s.write_str(req.trim_left()).unwrap();
        s.read_to_string().unwrap()
    }

    #[test]
    fn smoke() {
        let addr = next_test_ip4();
        Server::start(Config { port: addr.port, threads: 1 }, noop).unwrap();
    }

    #[test]
    fn dupe_port() {
        let addr = next_test_ip4();
        let s1 = Server::start(Config { port: addr.port, threads: 1 }, noop);
        assert!(s1.is_ok());
        let s2 = Server::start(Config { port: addr.port, threads: 1 }, noop);
        assert!(s2.is_err());
    }

    #[test]
    fn drops_handler() {
        static mut DROPPED: bool = false;
        struct Foo;
        impl Handler for Foo {
            fn call(&self, _req: &mut Request) -> Result<Response, Box<Show + 'static>> {
                panic!()
            }
        }
        impl Drop for Foo {
            fn drop(&mut self) { unsafe { DROPPED = true; } }
        }

        let addr = next_test_ip4();
        drop(Server::start(Config { port: addr.port, threads: 1 }, Foo));
        unsafe { assert!(DROPPED); }
    }

    #[test]
    fn invokes() {
        struct Foo(Mutex<Sender<()>>);
        impl Handler for Foo {
            fn call(&self, _req: &mut Request) -> Result<Response, Box<Show + 'static>> {
                let Foo(ref tx) = *self;
                tx.lock().send(());
                Ok(response(200i, HashMap::new(), MemReader::new(vec![])))
            }
        }

        let addr = next_test_ip4();
        let (tx, rx) = channel();
        let handler = Foo(Mutex::new(tx));
        let _s = Server::start(Config { port: addr.port, threads: 1 }, handler);
        request(addr, r"
GET / HTTP/1.1

");
        rx.recv();
    }

    #[test]
    fn header_sent() {
        struct Foo(Mutex<Sender<String>>);
        impl Handler for Foo {
            fn call(&self, req: &mut Request) -> Result<Response, Box<Show + 'static>> {
                let Foo(ref tx) = *self;
                tx.lock().send(req.headers().find("Foo").unwrap().connect(""));
                Ok(response(200i, HashMap::new(), MemReader::new(vec![])))
            }
        }

        let addr = next_test_ip4();
        let (tx, rx) = channel();
        let handler = Foo(Mutex::new(tx));
        let _s = Server::start(Config { port: addr.port, threads: 1 }, handler);
        request(addr, r"
GET / HTTP/1.1
Foo: bar

");
        assert_eq!(rx.recv().as_slice(), "bar");
    }

    #[test]
    fn failing_handler() {
        struct Foo;
        impl Handler for Foo {
            fn call(&self, _req: &mut Request) -> Result<Response, Box<Show + 'static>> {
                panic!()
            }
        }

        let addr = next_test_ip4();
        let _s = Server::start(Config { port: addr.port, threads: 1 }, Foo);
        request(addr, r"
GET / HTTP/1.1
Foo: bar

");
    }

    #[test]
    fn failing_handler_is_500() {
        struct Foo;
        impl Handler for Foo {
            fn call(&self, _req: &mut Request) -> Result<Response, Box<Show + 'static>> {
                panic!()
            }
        }

        let addr = next_test_ip4();
        let _s = Server::start(Config { port: addr.port, threads: 1 }, Foo);
        let response = request(addr, r"
GET / HTTP/1.1
Foo: bar

");
        assert!(response.as_slice().contains("500 Internal"),
                "not a failing response: {}", response);
    }
}
<|endoftext|>


<commit_before>/*!
Simple persistent generic HashMap/Key-value store.

Basic usage:

```
extern crate kv_cab;

use kv_cab::{ KV, Value };

fn main() {
    let mut test_store = KV::<String, Value>::new("./db.cab").unwrap();

    let _ = test_store.insert("key".to_string(), Value::String("value".to_string()));
    println!("{:?}", test_store.get("key".to_string()));
    let _ = test_store.remove("key".to_string());
}
```

Usage with user defined Key and Value types:

```
extern crate kv_cab;
extern crate rustc_serialize;

use kv_cab::KV;

#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
enum MyKey {
    String(String),
    Int(i32),
}

#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
enum MyValue {
    String(String),
    Int(i32),
}

fn main() {
    let mut test_store = KV::<MyKey, MyValue>::new("./db.cab").unwrap();

    let _ = test_store.insert(MyKey::Int(1i32), MyValue::String("value".to_string()));
    println!("{:?}", test_store.get(MyKey::Int(1i32)));
    let _ = test_store.remove(MyKey::Int(1i32));
}
```

*/

extern crate bincode;
extern crate rustc_serialize;
#[macro_use]
extern crate log;

use std::collections::HashMap;
use std::fs;
use std::fs::{ File, OpenOptions };
use std::io::prelude::*;
use std::hash::Hash;

use bincode::SizeLimit;
use bincode::rustc_serialize::{ encode, decode };

use rustc_serialize::{ Encodable, Decodable };

/// The maximum number of retries the cab will make
const MAX_RETRIES:i32 = 10;

/// Definition of a macro for retrying an operation
macro_rules! retry {
    ($i:ident, $b:block) => (
        for $i in 0..MAX_RETRIES {
            if $i != 0 {
                std::thread::park_timeout(std::time::Duration::new(0u64, 1000u32));
            }

            $b
        }
    )
}

/// A default value type to use with KV
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug)]
pub enum Value {
    String(String),
    Int(i32),
    Float(f32),
    Map(HashMap<String, Value>),
}

/// Type alias for results from KV
type KVResult = Result<bool, KVError>;

/// Errors that KV might have
#[derive(Debug, PartialEq)]
pub enum KVError {
    /// Cab did not contain anything on disk
    CabEmpty,
    /// Could not decode the cab from disk
    CouldntDecode,
    /// Cab was either not readable from disk or doesn't exist
    DoesntExistOrNotReadable,
    /// Couldn't encode the hashmap for writing to disk
    CouldntEncode,
    /// Could not write to the cab on disk
    CouldntWrite,
    /// Could not set the access permissions for the cab
    CouldntSetPermissions,
    /// Could get the permissions that the cab has on disk
    CoudlntGetPermissions,
    /// Failed to read the cab from disk
    FailedToRead,
    /// Could not sync the metadata with the disk
    CouldntSyncMetadata,
    /// Could not flush the data to disk
    CouldntFlush,
    /// Could not open the file from disk
    CouldntOpen,
}

/// The type that represents the key-value store
pub struct KV<K,V> {
    cab: HashMap<K, V>,
    file: File,
}

impl<K: Clone + Encodable + Decodable + Eq + Hash, V: Clone + Encodable + Decodable> KV<K,V> {
    /// Creates a new instance of the KV store
    pub fn new(p:&'static str) -> Result<KV<K,V>, KVError> {
        // create the KV instance
        let mut store = KV {
            cab: HashMap::new(),
            file: KV::<K, V>::retry_get_file(p).unwrap(),
        };

        // lock the cab for writes
        if let Err(e) = store.lock_cab(true) {
            return Err(e);
        }

        match store.load_from_persist() {
            Ok(f) => trace!("{}", f),
            Err(e) => {
                if e != KVError::CabEmpty {
                    return Err(e);
                }
                warn!("{:?}", e);
            },
        };

        Ok(store)
    }

    /// Sets file permission for a path to not be readonly
    fn set_path_permission(p: &'static str) {
        match fs::metadata(p) {
            Ok(f) => {
                let mut perms = f.permissions();
                perms.set_readonly(false);

                match fs::set_permissions(p, perms) {
                    Ok(_) => (),
                    Err(e) => {
                        error!("{}", e);
                    }
                }
            },
            Err(e) => {
                if e.kind() != std::io::ErrorKind::NotFound {
                    error!("{}", e);
                }
            },
        }
    }

    /// Retry to get a reference to the cab file at path p and create if doesn't exist
    fn retry_get_file(p:&'static str) -> Result<File, KVError> {
        retry!(i, {
            match OpenOptions::new().read(true).write(true).create(true).open(p) {
                Ok(f) => { return Ok(f); },
                Err(e) => {
                    error!("{}", e);
                    if i >= MAX_RETRIES - 1 {
                        return Err(KVError::CouldntOpen);
                    }

                    KV::<K, V>::set_path_permission(p);
                    continue;
                }
            };

        });

        Err(KVError::CouldntOpen)
    }

    /// Inserta a key, value pair into the key-value store
    pub fn insert(&mut self, key: K, value: V) -> KVResult {
        // make sure mem version up to date
        if let Err(e) = self.load_from_persist() {
            return Err(e);
        }
        // insert into the HashMap
        self.cab.insert(key, value);
        // persist
        self.write_to_persist()
    }

    /// Get the value from a key
    pub fn get(&mut self, key: K) -> Result<Option<V>, KVError> {
        // make sure mem version up to date
        if let Err(e) = self.load_from_persist() {
            return Err(e);
        }
        // get the value from the cab
        match self.cab.get(&key) {
            Some(v) => Ok(Some((*v).clone())),
            None => Ok(None),
        }
    }

    /// Removes a key and associated value from the key-value Store
    pub fn remove(&mut self, key: K) -> KVResult {
        // make sure mem version up to date
        if let Err(e) = self.load_from_persist() {
            return Err(e);
        }
        // remove from the HashMap
        self.cab.remove(&key);
        // persist
        self.write_to_persist()
    }

    /// get all the keys contained in the KV Store
    pub fn keys(&mut self) -> Result<Vec<K>, KVError> {
        // make sure mem version up to date
        if let Err(e) = self.load_from_persist() {
            return Err(e);
        }
        // create a vec from the cabs keys
        Ok(self.cab.keys().map(|k| k.clone()).collect())
    }

    /// Locks/unlocks cab for writing purposes
    fn lock_cab(&mut self, readonly:bool) -> KVResult {
        retry!(i, {
            // set not readonly while writing
            let mut perms = match self.file.metadata() {
                Ok(f) => f.permissions(),
                Err(e) => {
                    error!("{}", e);
                    if i >= MAX_RETRIES - 1 {
                        return Err(KVError::CoudlntGetPermissions);
                    }
                    continue;
                },
            };
            perms.set_readonly(readonly);

            match self.file.set_permissions(perms) {
                Ok(_) => {
                    if let Err(e) = self.file.sync_all() {
                        error!("{}", e);
                        return Err(KVError::CouldntSyncMetadata);
                    }
                    break;
                },
                Err(e) => {
                    error!("{}", e);
                    if i >= MAX_RETRIES - 1 {
                        return Err(KVError::CouldntSetPermissions);
                    }
                    continue;
                },
            }
        });

        Ok(true)
    }

    /// Waits for the cab to become free
    fn wait_for_free(&mut self, lock:bool) -> KVResult {
        loop {
            // check if the cab is being written to
            if let Err(e) = self.file.sync_all() {
                error!("{}", e);
                return Err(KVError::CouldntSyncMetadata);
            }
            let metadata = match self.file.metadata() {
                Ok(m) => m,
                Err(_) => return Err(KVError::DoesntExistOrNotReadable),
            };

            if metadata.permissions().readonly() {
                if lock {
                    if let Err(e) = self.lock_cab(false) {
                        return Err(e);
                    }
                }
                break;
            }
        }

        Ok(true)
    }

    /// Writes the key-value Store to file
    fn write_to_persist(&mut self) -> KVResult {
        // encode the cab as a u8 vec
        let byte_vec: Vec<u8> = match encode(&mut self.cab, SizeLimit::Infinite) {
            Ok(bv) => bv,
            Err(e) => {
                error!("encode: {}", e);
                return Err(KVError::CouldntEncode);
            },
        };

        if !self.wait_for_free(true).is_ok() {
            return Err(KVError::DoesntExistOrNotReadable);
        }
        // attempt to write to the cab
        retry!(i, {
            // write the bytes to it
            match self.file.write_all(byte_vec.as_slice()) {
                Ok(_) => (),
                Err(e) => {
                    error!("file.write_all/retry: {}", e);
                    if i >= MAX_RETRIES - 1 {
                        return Err(KVError::CouldntWrite);
                    }
                    continue;
                },
            }

            // flush to disk
            if let Err(e) = self.file.flush() {
                error!("{}", e);
                return Err(KVError::CouldntFlush);
            }
            if let Err(e) = self.lock_cab(true) {
                error!("{:?}", e);
                if i >= MAX_RETRIES - 1 {
                    return Err(e);
                }
            }
            return Ok(true);
        });

        Err(KVError::CouldntWrite)
    }

    /// Loads key-value store from file
    fn load_from_persist(&mut self) -> KVResult {
        // byte vec to read into
        let mut byte_vec = Vec::new();

        // wait/lock the cab and read the bytes
        if !self.wait_for_free(false).is_ok() {
            return Err(KVError::DoesntExistOrNotReadable);
        }

        // read the file into the buffer
        match self.file.read_to_end(&mut byte_vec) {
            Ok(count) => {
                if count == 0 {
                    // don't attempt to decode as empty
                    return Ok(true);
                }
            },
            Err(e) => {
                error!("{}", e);
                return Err(KVError::FailedToRead);
            },
        }

        // decode u8 vec back into HashMap
        let decoded: HashMap<K, V> = match decode(byte_vec.as_slice()) {
            Ok(f) => f,
            Err(e) => {
                error!("{}", e);
                return Err(KVError::CouldntDecode);
            },
        };
        // assign read HashMap back to self
        self.cab = decoded;

        Ok(true)
    }
}
<commit_msg>set_path_permissions now returns a KVResult for better error handling<commit_after>/*!
Simple persistent generic HashMap/Key-value store.

Basic usage:

```
extern crate kv_cab;

use kv_cab::{ KV, Value };

fn main() {
    let mut test_store = KV::<String, Value>::new("./db.cab").unwrap();

    let _ = test_store.insert("key".to_string(), Value::String("value".to_string()));
    println!("{:?}", test_store.get("key".to_string()));
    let _ = test_store.remove("key".to_string());
}
```

Usage with user defined Key and Value types:

```
extern crate kv_cab;
extern crate rustc_serialize;

use kv_cab::KV;

#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
enum MyKey {
    String(String),
    Int(i32),
}

#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
enum MyValue {
    String(String),
    Int(i32),
}

fn main() {
    let mut test_store = KV::<MyKey, MyValue>::new("./db.cab").unwrap();

    let _ = test_store.insert(MyKey::Int(1i32), MyValue::String("value".to_string()));
    println!("{:?}", test_store.get(MyKey::Int(1i32)));
    let _ = test_store.remove(MyKey::Int(1i32));
}
```

*/

extern crate bincode;
extern crate rustc_serialize;
#[macro_use]
extern crate log;

use std::collections::HashMap;
use std::fs;
use std::fs::{ File, OpenOptions };
use std::io::prelude::*;
use std::hash::Hash;

use bincode::SizeLimit;
use bincode::rustc_serialize::{ encode, decode };

use rustc_serialize::{ Encodable, Decodable };

/// The maximum number of retries the cab will make
const MAX_RETRIES:i32 = 10;

/// Definition of a macro for retrying an operation
macro_rules! retry {
    ($i:ident, $b:block) => (
        for $i in 0..MAX_RETRIES {
            if $i != 0 {
                std::thread::park_timeout(std::time::Duration::new(0u64, 1000u32));
            }

            $b
        }
    )
}

/// A default value type to use with KV
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Debug)]
pub enum Value {
    String(String),
    Int(i32),
    Float(f32),
    Map(HashMap<String, Value>),
}

/// Type alias for results from KV
type KVResult = Result<bool, KVError>;

/// Errors that KV might have
#[derive(Debug, PartialEq)]
pub enum KVError {
    /// Cab did not contain anything on disk
    CabEmpty,
    /// Could not decode the cab from disk
    CouldntDecode,
    /// Cab was either not readable from disk or doesn't exist
    DoesntExistOrNotReadable,
    /// Couldn't encode the hashmap for writing to disk
    CouldntEncode,
    /// Could not write to the cab on disk
    CouldntWrite,
    /// Could not set the access permissions for the cab
    CouldntSetPermissions,
    /// Could get the permissions that the cab has on disk
    CoudlntGetPermissions,
    /// Failed to read the cab from disk
    FailedToRead,
    /// Could not sync the metadata with the disk
    CouldntSyncMetadata,
    /// Could not flush the data to disk
    CouldntFlush,
    /// Could not open the file from disk
    CouldntOpen,
}

/// The type that represents the key-value store
pub struct KV<K,V> {
    cab: HashMap<K, V>,
    file: File,
}

impl<K: Clone + Encodable + Decodable + Eq + Hash, V: Clone + Encodable + Decodable> KV<K,V> {
    /// Creates a new instance of the KV store
    pub fn new(p:&'static str) -> Result<KV<K,V>, KVError> {
        // create the KV instance
        let mut store = KV {
            cab: HashMap::new(),
            file: KV::<K, V>::retry_get_file(p).unwrap(),
        };

        // lock the cab for writes
        if let Err(e) = store.lock_cab(true) {
            return Err(e);
        }

        match store.load_from_persist() {
            Ok(f) => trace!("{}", f),
            Err(e) => {
                if e != KVError::CabEmpty {
                    return Err(e);
                }
                warn!("{:?}", e);
            },
        };

        Ok(store)
    }

    /// Sets file permission for a path to not be readonly
    fn set_path_permission(p: &'static str) -> KVResult {
        match fs::metadata(p) {
            Ok(f) => {
                let mut perms = f.permissions();
                perms.set_readonly(false);

                match fs::set_permissions(p, perms) {
                    Ok(_) => return Ok(true),
                    Err(e) => {
                        error!("{}", e);
                        return Err(KVError::CouldntSetPermissions);
                    }
                }
            },
            Err(e) => {
                if e.kind() != std::io::ErrorKind::NotFound {
                    error!("{}", e);
                    return Err(KVError::CoudlntGetPermissions);
                }
                return Ok(true);
            },
        }
    }

    /// Retry to get a reference to the cab file at path p and create if doesn't exist
    fn retry_get_file(p:&'static str) -> Result<File, KVError> {
        retry!(i, {
            match OpenOptions::new().read(true).write(true).create(true).open(p) {
                Ok(f) => { return Ok(f); },
                Err(e) => {
                    error!("{}", e);
                    if i >= MAX_RETRIES - 1 {
                        return Err(KVError::CouldntOpen);
                    }

                    if let Err(e) = KV::<K, V>::set_path_permission(p) {
                        if i >= MAX_RETRIES - 1 {
                            return Err(e);
                        }
                    }
                    continue;
                }
            };

        });

        Err(KVError::CouldntOpen)
    }

    /// Inserta a key, value pair into the key-value store
    pub fn insert(&mut self, key: K, value: V) -> KVResult {
        // make sure mem version up to date
        if let Err(e) = self.load_from_persist() {
            return Err(e);
        }
        // insert into the HashMap
        self.cab.insert(key, value);
        // persist
        self.write_to_persist()
    }

    /// Get the value from a key
    pub fn get(&mut self, key: K) -> Result<Option<V>, KVError> {
        // make sure mem version up to date
        if let Err(e) = self.load_from_persist() {
            return Err(e);
        }
        // get the value from the cab
        match self.cab.get(&key) {
            Some(v) => Ok(Some((*v).clone())),
            None => Ok(None),
        }
    }

    /// Removes a key and associated value from the key-value Store
    pub fn remove(&mut self, key: K) -> KVResult {
        // make sure mem version up to date
        if let Err(e) = self.load_from_persist() {
            return Err(e);
        }
        // remove from the HashMap
        self.cab.remove(&key);
        // persist
        self.write_to_persist()
    }

    /// get all the keys contained in the KV Store
    pub fn keys(&mut self) -> Result<Vec<K>, KVError> {
        // make sure mem version up to date
        if let Err(e) = self.load_from_persist() {
            return Err(e);
        }
        // create a vec from the cabs keys
        Ok(self.cab.keys().map(|k| k.clone()).collect())
    }

    /// Locks/unlocks cab for writing purposes
    fn lock_cab(&mut self, readonly:bool) -> KVResult {
        retry!(i, {
            // set not readonly while writing
            let mut perms = match self.file.metadata() {
                Ok(f) => f.permissions(),
                Err(e) => {
                    error!("{}", e);
                    if i >= MAX_RETRIES - 1 {
                        return Err(KVError::CoudlntGetPermissions);
                    }
                    continue;
                },
            };
            perms.set_readonly(readonly);

            match self.file.set_permissions(perms) {
                Ok(_) => {
                    if let Err(e) = self.file.sync_all() {
                        error!("{}", e);
                        return Err(KVError::CouldntSyncMetadata);
                    }
                    break;
                },
                Err(e) => {
                    error!("{}", e);
                    if i >= MAX_RETRIES - 1 {
                        return Err(KVError::CouldntSetPermissions);
                    }
                    continue;
                },
            }
        });

        Ok(true)
    }

    /// Waits for the cab to become free
    fn wait_for_free(&mut self, lock:bool) -> KVResult {
        loop {
            // check if the cab is being written to
            if let Err(e) = self.file.sync_all() {
                error!("{}", e);
                return Err(KVError::CouldntSyncMetadata);
            }
            let metadata = match self.file.metadata() {
                Ok(m) => m,
                Err(_) => return Err(KVError::DoesntExistOrNotReadable),
            };

            if metadata.permissions().readonly() {
                if lock {
                    if let Err(e) = self.lock_cab(false) {
                        return Err(e);
                    }
                }
                break;
            }
        }

        Ok(true)
    }

    /// Writes the key-value Store to file
    fn write_to_persist(&mut self) -> KVResult {
        // encode the cab as a u8 vec
        let byte_vec: Vec<u8> = match encode(&mut self.cab, SizeLimit::Infinite) {
            Ok(bv) => bv,
            Err(e) => {
                error!("encode: {}", e);
                return Err(KVError::CouldntEncode);
            },
        };

        if !self.wait_for_free(true).is_ok() {
            return Err(KVError::DoesntExistOrNotReadable);
        }
        // attempt to write to the cab
        retry!(i, {
            // write the bytes to it
            match self.file.write_all(byte_vec.as_slice()) {
                Ok(_) => (),
                Err(e) => {
                    error!("file.write_all/retry: {}", e);
                    if i >= MAX_RETRIES - 1 {
                        return Err(KVError::CouldntWrite);
                    }
                    continue;
                },
            }

            // flush to disk
            if let Err(e) = self.file.flush() {
                error!("{}", e);
                return Err(KVError::CouldntFlush);
            }
            if let Err(e) = self.lock_cab(true) {
                error!("{:?}", e);
                if i >= MAX_RETRIES - 1 {
                    return Err(e);
                }
            }
            return Ok(true);
        });

        Err(KVError::CouldntWrite)
    }

    /// Loads key-value store from file
    fn load_from_persist(&mut self) -> KVResult {
        // byte vec to read into
        let mut byte_vec = Vec::new();

        // wait/lock the cab and read the bytes
        if !self.wait_for_free(false).is_ok() {
            return Err(KVError::DoesntExistOrNotReadable);
        }

        // read the file into the buffer
        match self.file.read_to_end(&mut byte_vec) {
            Ok(count) => {
                if count == 0 {
                    // don't attempt to decode as empty
                    return Ok(true);
                }
            },
            Err(e) => {
                error!("{}", e);
                return Err(KVError::FailedToRead);
            },
        }

        // decode u8 vec back into HashMap
        let decoded: HashMap<K, V> = match decode(byte_vec.as_slice()) {
            Ok(f) => f,
            Err(e) => {
                error!("{}", e);
                return Err(KVError::CouldntDecode);
            },
        };
        // assign read HashMap back to self
        self.cab = decoded;

        Ok(true)
    }
}
<|endoftext|>


<commit_before>extern crate libc;

use std::io;

mod ffi;

macro_rules! unsafe_try {
    ( $x:expr ) => {
        try!(int_result(unsafe { $x }))
    };
}

pub fn fork() -> io::Result<(libc::pid_t, libc::c_int)>
{
    let pty_master = unsafe_try!(ffi::posix_openpt(libc::O_RDWR));

    unsafe_try!(ffi::grantpt(pty_master));
    unsafe_try!(ffi::unlockpt(pty_master));

    let pid = unsafe_try!(libc::fork());

    if pid == 0 {
        let pts_name = unsafe { ffi::ptsname(pty_master) };

        if (pts_name as i32) == 0 {
            return Err(io::Error::last_os_error())
        }

        unsafe_try!(libc::close(pty_master));
        unsafe_try!(libc::setsid());

        let pty_slave = unsafe_try!(libc::open(pts_name, libc::O_RDWR, 0));

        unsafe_try!(libc::dup2(pty_slave, libc::STDIN_FILENO));
        unsafe_try!(libc::dup2(pty_slave, libc::STDOUT_FILENO));
        unsafe_try!(libc::dup2(pty_slave, libc::STDERR_FILENO));

        unsafe_try!(libc::close(pty_slave));

        return Ok((0, -1));
    }
    else {
        return Ok((pid, pty_master));
    }
}

#[inline]
fn int_result(value: libc::c_int) -> io::Result<libc::c_int> {
    if value < 0 {
        return Err(io::Error::last_os_error())
    }

    Ok(value)
}

#[test]
fn it_works() {
}
<commit_msg>Extract open_ptm and attach_pts<commit_after>extern crate libc;

use std::io;

mod ffi;

macro_rules! unsafe_try {
    ( $x:expr ) => {
        try!(int_result(unsafe { $x }))
    };
}

pub fn fork() -> io::Result<(libc::pid_t, libc::c_int)>
{
    let pty_master = try!(open_ptm());
    let pid        = unsafe_try!(libc::fork());

    if pid == 0 {
        try!(attach_pts(pty_master));

        return Ok((0, -1));
    }
    else {
        return Ok((pid, pty_master));
    }
}

fn open_ptm() -> io::Result<libc::c_int> {
    let pty_master = unsafe_try!(ffi::posix_openpt(libc::O_RDWR));

    unsafe_try!(ffi::grantpt(pty_master));
    unsafe_try!(ffi::unlockpt(pty_master));

    Ok(pty_master)
}

fn attach_pts(pty_master: libc::c_int) -> io::Result<()> {
    let pts_name = unsafe { ffi::ptsname(pty_master) };

    if (pts_name as i32) == 0 {
        return Err(io::Error::last_os_error())
    }

    unsafe_try!(libc::close(pty_master));
    unsafe_try!(libc::setsid());

    let pty_slave = unsafe_try!(libc::open(pts_name, libc::O_RDWR, 0));

    unsafe_try!(libc::dup2(pty_slave, libc::STDIN_FILENO));
    unsafe_try!(libc::dup2(pty_slave, libc::STDOUT_FILENO));
    unsafe_try!(libc::dup2(pty_slave, libc::STDERR_FILENO));

    unsafe_try!(libc::close(pty_slave));

    Ok(())
}

#[inline]
fn int_result(value: libc::c_int) -> io::Result<libc::c_int> {
    if value < 0 {
        return Err(io::Error::last_os_error())
    }

    Ok(value)
}

#[test]
fn it_works() {
}
<|endoftext|>


<commit_before>pub struct Lexer<'a> {
	src: &'a [u8],
	pos: usize
}

impl<'a> Lexer<'a> {
	pub fn new(src: &'a [u8]) -> Lexer {
		Lexer { src: src, pos: 0 }
	}

	fn is_ws(ch: u8) -> bool {
		match ch {
			b' '	=> true,
			b'\r'	=> true,
			b'\n'	=> true,
			b'\t'	=> true,
			_		=> false,
		}
	}
}

impl<'a> Lexer<'a> {
	fn eof(&self) -> bool {
		self.pos >= self.src.len()
	}

	fn ch(&self) -> Option<u8> {
		match self.eof() {
			true	=> None,
			false	=> Some(self.src[self.pos])
		}
	}

	fn string(&mut self) -> Option<String> {
		let mut tok = Vec::<u8>::new();

		loop {
			let ch = self.ch();
			match ch {
				Some(c) if Lexer::is_ws(c)	=> break,
				Some(b'"')					=> break,
				None						=> break,
				_							=> ()
			}

			tok.push(ch.unwrap());
			self.pos += 1;
		}

		match tok.len() {
			0	=> None,
			_	=> Some(String::from_utf8(tok).unwrap())
		}
	}

	fn quoted_string(&mut self) -> Option<String> {
		let mut tok = Vec::<u8>::new();
		self.pos += 1;

		loop {
			let ch = self.ch();
			match ch {
				Some(b'"')		=> { self.pos += 1; break; },
				None			=> break,
				_				=> ()
			}

			tok.push(ch.unwrap());
			self.pos += 1;
		}

		match tok.len() {
			0	=> None,
			_	=> Some(String::from_utf8(tok).unwrap())
		}
	}

	fn skip_ws(&mut self) -> usize {
		let prev_pos = self.pos;
		while self.valid_ws() {
			self.pos += 1;
		}
		self.pos - prev_pos
	}

	fn valid_ws(&self) -> bool {
		match self.ch() {
			Some(c)		=> Lexer::is_ws(c),
			None		=> false
		}
	}
}

impl<'a> Iterator for Lexer<'a> {
	type Item = String;

	fn next(&mut self) -> Option<String> {
		self.skip_ws();
		match self.ch() {
			Some(b'"')	=> self.quoted_string(),
			Some(_)		=> self.string(),
			None		=> None
		}
	}
}

#[test]
fn eof() {
	assert!(Lexer::new(b"").eof());
	assert!(!Lexer::new(b"a").eof());
}

#[test]
fn valid_ws() {
	let tests = vec![
		(b" ", 	true),
		(b"\t",	true),
		(b"\n",	true),
		(b"\r",	true),
		(b"a", 	false)
	];

	let mut lexer;
	for (src, expected) in tests {
		lexer = Lexer::new(src);
		assert_eq!(expected, lexer.valid_ws());
	}
}

#[test]
fn tokenize() {
	let lexer = Lexer::new(b"foo bar baz \"a b\" zoz");
	let tokens: Vec<_> = lexer.collect();
	assert_eq!("foo".to_string(), tokens[0]);
	assert_eq!("bar".to_string(), tokens[1]);
	assert_eq!("baz".to_string(), tokens[2]);
	assert_eq!("a b".to_string(), tokens[3]);
	assert_eq!("zoz".to_string(), tokens[4]);
}<commit_msg>Formatting<commit_after>pub struct Lexer<'a> {
	src: &'a [u8],
	pos: usize
}

impl<'a> Lexer<'a> {
	pub fn new(src: &'a [u8]) -> Lexer {
		Lexer { src: src, pos: 0 }
	}

	fn is_ws(ch: u8) -> bool {
		match ch {
			b' '  => true,
			b'\r' => true,
			b'\n' => true,
			b'\t' => true,
			_     => false,
		}
	}
}

impl<'a> Lexer<'a> {
	fn eof(&self) -> bool {
		self.pos >= self.src.len()
	}

	fn ch(&self) -> Option<u8> {
		match self.eof() {
			true  => None,
			false => Some(self.src[self.pos])
		}
	}

	fn string(&mut self) -> Option<String> {
		let mut tok = Vec::<u8>::new();

		loop {
			let ch = self.ch();
			match ch {
				Some(c) if Lexer::is_ws(c) => break,
				Some(b'"')                 => break,
				None                       => break,
				_                          => ()
			}

			tok.push(ch.unwrap());
			self.pos += 1;
		}

		match tok.len() {
			0	=> None,
			_	=> Some(String::from_utf8(tok).unwrap())
		}
	}

	fn quoted_string(&mut self) -> Option<String> {
		let mut tok = Vec::<u8>::new();
		self.pos += 1;

		loop {
			let ch = self.ch();
			match ch {
				Some(b'"') => { self.pos += 1; break; },
				None       => break,
				_          => ()
			}

			tok.push(ch.unwrap());
			self.pos += 1;
		}

		match tok.len() {
			0	=> None,
			_	=> Some(String::from_utf8(tok).unwrap())
		}
	}

	fn skip_ws(&mut self) -> usize {
		let prev_pos = self.pos;
		while self.valid_ws() {
			self.pos += 1;
		}
		self.pos - prev_pos
	}

	fn valid_ws(&self) -> bool {
		match self.ch() {
			Some(c)		=> Lexer::is_ws(c),
			None		=> false
		}
	}
}

impl<'a> Iterator for Lexer<'a> {
	type Item = String;

	fn next(&mut self) -> Option<String> {
		self.skip_ws();
		match self.ch() {
			Some(b'"') => self.quoted_string(),
			Some(_)    => self.string(),
			None       => None
		}
	}
}

#[test]
fn eof() {
	assert!(Lexer::new(b"").eof());
	assert!(!Lexer::new(b"a").eof());
}

#[test]
fn valid_ws() {
	let tests = vec![
		(b" ", 	true),
		(b"\t",	true),
		(b"\n",	true),
		(b"\r",	true),
		(b"a", 	false)
	];

	let mut lexer;
	for (src, expected) in tests {
		lexer = Lexer::new(src);
		assert_eq!(expected, lexer.valid_ws());
	}
}

#[test]
fn tokenize() {
	let lexer = Lexer::new(b"foo bar baz \"a b\" zoz");
	let tokens: Vec<_> = lexer.collect();
	assert_eq!("foo".to_string(), tokens[0]);
	assert_eq!("bar".to_string(), tokens[1]);
	assert_eq!("baz".to_string(), tokens[2]);
	assert_eq!("a b".to_string(), tokens[3]);
	assert_eq!("zoz".to_string(), tokens[4]);
}<|endoftext|>