<commit_before>//! A simple library for parsing an XML file into an in-memory tree structure
//!
//! Not recommended for large XML files, as it will load the entire file into memory.
//!
//! # Example
//!
//! ```no_run
//! use xmltree::Element;
//! use std::fs::File;
//!
//! let data: &'static str = r##"
//! <?xml version="1.0" encoding="utf-8" standalone="yes"?>
//! <names>
//!     <name first="bob" last="jones" />
//!     <name first="elizabeth" last="smith" />
//! </names>
//! "##;
//!
//! let mut names_element = Element::parse(data.as_bytes()).unwrap();
//!
//! println!("{:#?}", names_element);
//! {
//!     // get first `name` element
//!     let name = names_element.get_mut_child("name").expect("Can't find name element");
//!     name.attributes.insert("suffix".to_owned(), "mr".to_owned());
//! }
//! names_element.write(File::create("result.xml").unwrap());
//!
//!
//! ```
extern crate xml;

use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt;
use std::io::{Read, Write};

pub use xml::namespace::Namespace;
use xml::reader::{EventReader, XmlEvent};
pub use xml::writer::{EmitterConfig, Error};

/// Represents an XML element.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Element {
    /// This elements prefix, if any
    pub prefix: Option<String>,

    /// This elements namespace, if any
    pub namespace: Option<String>,

    /// The full list of namespaces, if any
    ///
    /// The `Namespace` type is exported from the `xml-rs` crate.
    pub namespaces: Option<Namespace>,

    /// The name of the Element.  Does not include any namespace info
    pub name: String,

    /// The Element attributes
    pub attributes: HashMap<String, String>,

    /// Children
    pub children: Vec<Element>,

    /// The text data for this element
    pub text: Option<String>,
}

/// Errors that can occur parsing XML
#[derive(Debug)]
pub enum ParseError {
    /// The XML is invalid
    MalformedXml(xml::reader::Error),
    /// This library is unable to process this XML. This can occur if, for
    /// example, the XML contains processing instructions.
    CannotParse,
}

impl fmt::Display for ParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            ParseError::MalformedXml(ref e) => write!(f, "Malformed XML. {}", e),
            ParseError::CannotParse => write!(f, "Cannot parse"),
        }
    }
}

impl std::error::Error for ParseError {
    fn description(&self) -> &str {
        match *self {
            ParseError::MalformedXml(..) => "Malformed XML",
            ParseError::CannotParse => "Cannot parse",
        }
    }

    fn cause(&self) -> Option<&dyn std::error::Error> {
        match *self {
            ParseError::MalformedXml(ref e) => Some(e),
            ParseError::CannotParse => None,
        }
    }
}

fn build<B: Read>(reader: &mut EventReader<B>, mut elem: Element) -> Result<Element, ParseError> {
    loop {
        match reader.next() {
            Ok(XmlEvent::EndElement { ref name }) => {
                if name.local_name == elem.name {
                    return Ok(elem);
                } else {
                    return Err(ParseError::CannotParse);
                }
            }
            Ok(XmlEvent::StartElement {
                name,
                attributes,
                namespace,
            }) => {
                let mut attr_map = HashMap::new();
                for attr in attributes {
                    attr_map.insert(attr.name.local_name, attr.value);
                }

                let new_elem = Element {
                    prefix: name.prefix,
                    namespace: name.namespace,
                    namespaces: if namespace.is_essentially_empty() {
                        None
                    } else {
                        Some(namespace)
                    },
                    name: name.local_name,
                    attributes: attr_map,
                    children: Vec::new(),
                    text: None,
                };
                elem.children.push(build(reader, new_elem)?);
            }
            Ok(XmlEvent::Characters(s)) => {
                elem.text = Some(s);
            }
            Ok(XmlEvent::Whitespace(..)) | Ok(XmlEvent::Comment(..)) => (),
            Ok(XmlEvent::CData(s)) => elem.text = Some(s),
            Ok(XmlEvent::StartDocument { .. })
            | Ok(XmlEvent::EndDocument)
            | Ok(XmlEvent::ProcessingInstruction { .. }) => return Err(ParseError::CannotParse),
            Err(e) => return Err(ParseError::MalformedXml(e)),
        }
    }
}

impl Element {
    /// Create a new empty element with given name
    ///
    /// All other fields are empty
    pub fn new(name: &str) -> Element {
        Element {
            name: String::from(name),
            prefix: None,
            namespace: None,
            namespaces: None,
            attributes: HashMap::new(),
            children: Vec::new(),
            text: None,
        }
    }

    /// Parses some data into an Element
    pub fn parse<R: Read>(r: R) -> Result<Element, ParseError> {
        let mut reader = EventReader::new(r);
        loop {
            match reader.next() {
                Ok(XmlEvent::StartElement {
                    name,
                    attributes,
                    namespace,
                }) => {
                    let mut attr_map = HashMap::new();
                    for attr in attributes {
                        attr_map.insert(attr.name.local_name, attr.value);
                    }

                    let root = Element {
                        prefix: name.prefix,
                        namespace: name.namespace,
                        namespaces: if namespace.is_essentially_empty() {
                            None
                        } else {
                            Some(namespace)
                        },
                        name: name.local_name,
                        attributes: attr_map,
                        children: Vec::new(),
                        text: None,
                    };
                    return build(&mut reader, root);
                }
                Ok(XmlEvent::Comment(..))
                | Ok(XmlEvent::Whitespace(..))
                | Ok(XmlEvent::StartDocument { .. }) => continue,
                Ok(XmlEvent::EndDocument)
                | Ok(XmlEvent::EndElement { .. })
                | Ok(XmlEvent::Characters(..))
                | Ok(XmlEvent::CData(..))
                | Ok(XmlEvent::ProcessingInstruction { .. }) => {
                    return Err(ParseError::CannotParse)
                }
                Err(e) => return Err(ParseError::MalformedXml(e)),
            }
        }
    }

    fn _write<B: Write>(&self, emitter: &mut xml::writer::EventWriter<B>) -> Result<(), Error> {
        use xml::attribute::Attribute;
        use xml::name::Name;
        use xml::writer::events::XmlEvent;

        let mut name = Name::local(&self.name);
        if let Some(ref ns) = self.namespace {
            name.namespace = Some(ns);
        }
        if let Some(ref p) = self.prefix {
            name.prefix = Some(p);
        }

        let mut attributes = Vec::with_capacity(self.attributes.len());
        for (k, v) in &self.attributes {
            attributes.push(Attribute {
                name: Name::local(k),
                value: v,
            });
        }

        let empty_ns = Namespace::empty();
        let namespace = if let Some(ref ns) = self.namespaces {
            Cow::Borrowed(ns)
        } else {
            Cow::Borrowed(&empty_ns)
        };

        emitter.write(XmlEvent::StartElement {
            name,
            attributes: Cow::Owned(attributes),
            namespace,
        })?;
        if let Some(ref t) = self.text {
            emitter.write(XmlEvent::Characters(t))?;
        }
        for elem in &self.children {
            elem._write(emitter)?;
        }
        emitter.write(XmlEvent::EndElement { name: Some(name) })?;

        Ok(())
    }

    /// Writes out this element as the root element in an new XML document
    pub fn write<W: Write>(&self, w: W) -> Result<(), Error> {
        self.write_with_config(w, EmitterConfig::new())
    }

    /// Writes out this element as the root element in a new XML document using the provided configuration
    pub fn write_with_config<W: Write>(&self, w: W, config: EmitterConfig) -> Result<(), Error> {
        use xml::common::XmlVersion;
        use xml::writer::events::XmlEvent;
        use xml::writer::EventWriter;

        let mut emitter = EventWriter::new_with_config(w, config);
        emitter.write(XmlEvent::StartDocument {
            version: XmlVersion::Version10,
            encoding: None,
            standalone: None,
        })?;
        self._write(&mut emitter)
    }

    /// Find a child element with the given name and return a reference to it.
    pub fn get_child<P: ElementPredicate>(&self, k: P) -> Option<&Element> {
        self.children.iter().find(|e| k.match_element(e))
    }

    /// Find a child element with the given name and return a mutable reference to it.
    pub fn get_mut_child<P: ElementPredicate>(&mut self, k: P) -> Option<&mut Element> {
        self.children.iter_mut().find(|e| k.match_element(e))
    }

    /// Find a child element with the given name, remove and return it.
    pub fn take_child<P: ElementPredicate>(&mut self, k: P) -> Option<Element> {
        self.children
            .iter()
            .position(|e| k.match_element(e))
            .map(|i| self.children.remove(i))
    }
}

/// A predicate for matching elements.
///
/// The default implementations allow you to match by tag name or a tuple of
/// tag name and namespace.
pub trait ElementPredicate {
    fn match_element(&self, e: &Element) -> bool;
}

// Unfortunately,
// `impl<TN> ElementPredicate for TN where String: PartialEq<TN>` and
// `impl<TN, NS> ElementPredicate for (TN, NS) where String: PartialEq<TN>, String: PartialEq<NS>`
// are conflicting implementations, even though we know that there is no
// implementation for tuples. We just manually implement `ElementPredicate` for
// all `PartialEq` impls of `String` and forward them to the 1-tuple version.
//
// This can probably be fixed once specialization is stable.
impl<TN> ElementPredicate for (TN,)
where
    String: PartialEq<TN>,
{
    fn match_element(&self, e: &Element) -> bool {
        e.name == self.0
    }
}

impl<'a> ElementPredicate for &'a str {
    /// Search by tag name
    fn match_element(&self, e: &Element) -> bool {
        (*self,).match_element(e)
    }
}

impl<'a> ElementPredicate for Cow<'a, str> {
    /// Search by tag name
    fn match_element(&self, e: &Element) -> bool {
        (&**self,).match_element(e)
    }
}

impl ElementPredicate for String {
    /// Search by tag name
    fn match_element(&self, e: &Element) -> bool {
        (&**self,).match_element(e)
    }
}

impl<TN, NS> ElementPredicate for (TN, NS)
where
    String: PartialEq<TN>,
    String: PartialEq<NS>,
{
    /// Search by a tuple of (tagname, namespace)
    fn match_element(&self, e: &Element) -> bool {
        e.name == self.0
            && e.namespace
                .as_ref()
                .map(|ns| ns == &self.1)
                .unwrap_or(false)
    }
}
<commit_msg>[FEAT] Slight restructuring to allow for comments, better text handling, and allowing more information to be passed through<commit_after>//! A simple library for parsing an XML file into an in-memory tree structure
//!
//! Not recommended for large XML files, as it will load the entire file into memory.
//!
//! # Example
//!
//! ```no_run
//! use xmltree::Element;
//! use std::fs::File;
//!
//! let data: &'static str = r##"
//! <?xml version="1.0" encoding="utf-8" standalone="yes"?>
//! <names>
//!     <name first="bob" last="jones" />
//!     <name first="elizabeth" last="smith" />
//! </names>
//! "##;
//!
//! let mut names_element = Element::parse(data.as_bytes()).unwrap();
//!
//! println!("{:#?}", names_element);
//! {
//!     // get first `name` element
//!     let name = names_element.get_mut_child("name").expect("Can't find name element");
//!     name.attributes.insert("suffix".to_owned(), "mr".to_owned());
//! }
//! names_element.write(File::create("result.xml").unwrap());
//!
//!
//! ```
extern crate xml;

use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt;
use std::io::{Read, Write};

pub use xml::namespace::Namespace;
use xml::reader::{EventReader, XmlEvent};
pub use xml::writer::{EmitterConfig, Error};

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum XMLNode {
    Element(Element),
    Comment(String),
    CData(String),
    Text(String),
    ProcessingInstruction(String, Option<String>),
}

/// Represents an XML element.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Element {
    /// This elements prefix, if any
    pub prefix: Option<String>,

    /// This elements namespace, if any
    pub namespace: Option<String>,

    /// The full list of namespaces, if any
    ///
    /// The `Namespace` type is exported from the `xml-rs` crate.
    pub namespaces: Option<Namespace>,

    /// The name of the Element.  Does not include any namespace info
    pub name: String,

    /// The Element attributes
    pub attributes: HashMap<String, String>,

    /// Children
    pub children: Vec<XMLNode>,
}

/// Errors that can occur parsing XML
#[derive(Debug)]
pub enum ParseError {
    /// The XML is invalid
    MalformedXml(xml::reader::Error),
    /// This library is unable to process this XML. This can occur if, for
    /// example, the XML contains processing instructions.
    CannotParse,
}

impl fmt::Display for ParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            ParseError::MalformedXml(ref e) => write!(f, "Malformed XML. {}", e),
            ParseError::CannotParse => write!(f, "Cannot parse"),
        }
    }
}

impl std::error::Error for ParseError {
    fn description(&self) -> &str {
        match *self {
            ParseError::MalformedXml(..) => "Malformed XML",
            ParseError::CannotParse => "Cannot parse",
        }
    }

    fn cause(&self) -> Option<&dyn std::error::Error> {
        match *self {
            ParseError::MalformedXml(ref e) => Some(e),
            ParseError::CannotParse => None,
        }
    }
}

fn build<B: Read>(reader: &mut EventReader<B>, mut elem: Element) -> Result<Element, ParseError> {
    loop {
        match reader.next() {
            Ok(XmlEvent::EndElement { ref name }) => {
                if name.local_name == elem.name {
                    return Ok(elem);
                } else {
                    return Err(ParseError::CannotParse);
                }
            }
            Ok(XmlEvent::StartElement {
                name,
                attributes,
                namespace,
            }) => {
                let mut attr_map = HashMap::new();
                for attr in attributes {
                    attr_map.insert(attr.name.local_name, attr.value);
                }

                let new_elem = Element {
                    prefix: name.prefix,
                    namespace: name.namespace,
                    namespaces: if namespace.is_essentially_empty() {
                        None
                    } else {
                        Some(namespace)
                    },
                    name: name.local_name,
                    attributes: attr_map,
                    children: Vec::new(),
                };
                elem.children
                    .push(XMLNode::Element(build(reader, new_elem)?));
            }
            Ok(XmlEvent::Characters(s)) => elem.children.push(XMLNode::Text(s)),
            Ok(XmlEvent::Whitespace(..)) => (),
            Ok(XmlEvent::Comment(s)) => elem.children.push(XMLNode::Comment(s)),
            Ok(XmlEvent::CData(s)) => elem.children.push(XMLNode::Text(s)),
            Ok(XmlEvent::ProcessingInstruction { name, data }) => elem
                .children
                .push(XMLNode::ProcessingInstruction(name, data)),
            Ok(XmlEvent::StartDocument { .. }) | Ok(XmlEvent::EndDocument) => {
                return Err(ParseError::CannotParse)
            }
            Err(e) => return Err(ParseError::MalformedXml(e)),
        }
    }
}

impl Element {
    /// Create a new empty element with given name
    ///
    /// All other fields are empty
    pub fn new(name: &str) -> Element {
        Element {
            name: String::from(name),
            prefix: None,
            namespace: None,
            namespaces: None,
            attributes: HashMap::new(),
            children: Vec::new(),
        }
    }

    /// Parses some data into an Element
    pub fn parse<R: Read>(r: R) -> Result<Vec<XMLNode>, ParseError> {
        let mut reader = EventReader::new(r);
        let mut root_nodes = Vec::new();
        loop {
            match reader.next() {
                Ok(XmlEvent::StartElement {
                    name,
                    attributes,
                    namespace,
                }) => {
                    let mut attr_map = HashMap::with_capacity(attributes.len());
                    for attr in attributes {
                        attr_map.insert(attr.name.local_name, attr.value);
                    }

                    let root = Element {
                        prefix: name.prefix,
                        namespace: name.namespace,
                        namespaces: if namespace.is_essentially_empty() {
                            None
                        } else {
                            Some(namespace)
                        },
                        name: name.local_name,
                        attributes: attr_map,
                        children: Vec::new(),
                    };
                    root_nodes.push(XMLNode::Element(build(&mut reader, root)?));
                }
                Ok(XmlEvent::Comment(comment_string)) => {
                    root_nodes.push(XMLNode::Comment(comment_string))
                }
                Ok(XmlEvent::Characters(text_string)) => {
                    root_nodes.push(XMLNode::Text(text_string))
                }
                Ok(XmlEvent::CData(cdata_string)) => root_nodes.push(XMLNode::CData(cdata_string)),
                Ok(XmlEvent::Whitespace(..)) | Ok(XmlEvent::StartDocument { .. }) => continue,
                Ok(XmlEvent::ProcessingInstruction { name, data }) => {
                    root_nodes.push(XMLNode::ProcessingInstruction(name, data))
                }
                Ok(XmlEvent::EndDocument) | Ok(XmlEvent::EndElement { .. }) => {
                    return Err(ParseError::CannotParse)
                }
                Err(e) => return Err(ParseError::MalformedXml(e)),
            }
        }
    }

    fn _write<B: Write>(&self, emitter: &mut xml::writer::EventWriter<B>) -> Result<(), Error> {
        use xml::attribute::Attribute;
        use xml::name::Name;
        use xml::writer::events::XmlEvent;

        let mut name = Name::local(&self.name);
        if let Some(ref ns) = self.namespace {
            name.namespace = Some(ns);
        }
        if let Some(ref p) = self.prefix {
            name.prefix = Some(p);
        }

        let mut attributes = Vec::with_capacity(self.attributes.len());
        for (k, v) in &self.attributes {
            attributes.push(Attribute {
                name: Name::local(k),
                value: v,
            });
        }

        let empty_ns = Namespace::empty();
        let namespace = if let Some(ref ns) = self.namespaces {
            Cow::Borrowed(ns)
        } else {
            Cow::Borrowed(&empty_ns)
        };

        emitter.write(XmlEvent::StartElement {
            name,
            attributes: Cow::Owned(attributes),
            namespace,
        })?;
        for node in &self.children {
            match node {
                XMLNode::Element(elem) => elem._write(emitter)?,
                XMLNode::Text(text) => emitter.write(XmlEvent::Characters(text))?,
                XMLNode::Comment(comment) => emitter.write(XmlEvent::Comment(comment))?,
                XMLNode::CData(comment) => emitter.write(XmlEvent::CData(comment))?,
                XMLNode::ProcessingInstruction(name, data) => match data.to_owned() {
                    Some(string) => emitter.write(XmlEvent::ProcessingInstruction {
                        name,
                        data: Some(&string),
                    })?,
                    None => emitter.write(XmlEvent::ProcessingInstruction { name, data: None })?,
                },
            }
            // elem._write(emitter)?;
        }
        emitter.write(XmlEvent::EndElement { name: Some(name) })?;

        Ok(())
    }

    /// Writes out this element as the root element in an new XML document
    pub fn write<W: Write>(&self, w: W) -> Result<(), Error> {
        self.write_with_config(w, EmitterConfig::new())
    }

    /// Writes out this element as the root element in a new XML document using the provided configuration
    pub fn write_with_config<W: Write>(&self, w: W, config: EmitterConfig) -> Result<(), Error> {
        use xml::common::XmlVersion;
        use xml::writer::events::XmlEvent;
        use xml::writer::EventWriter;

        let mut emitter = EventWriter::new_with_config(w, config);
        emitter.write(XmlEvent::StartDocument {
            version: XmlVersion::Version10,
            encoding: None,
            standalone: None,
        })?;
        self._write(&mut emitter)
    }

    /// Find a child element with the given name and return a reference to it.
    pub fn get_child<P: ElementPredicate>(&self, k: P) -> Option<&Element> {
        self.children
            .iter()
            .filter_map(|e| match e {
                XMLNode::Element(elem) => Some(elem),
                _ => None,
            })
            .find(|e| k.match_element(e))
    }

    /// Find a child element with the given name and return a mutable reference to it.
    pub fn get_mut_child<P: ElementPredicate>(&mut self, k: P) -> Option<&mut Element> {
        self.children
            .iter_mut()
            .filter_map(|e| match e {
                XMLNode::Element(elem) => Some(elem),
                _ => None,
            })
            .find(|e| k.match_element(e))
    }

    /// Find a child element with the given name, remove and return it.
    pub fn take_child<P: ElementPredicate>(&mut self, k: P) -> Option<Element> {
        let index = self.children.iter().position(|e| match e {
            XMLNode::Element(elem) => k.match_element(elem),
            _ => false,
        });
        match index {
            Some(index) => match self.children.remove(index) {
                XMLNode::Element(elem) => Some(elem),
                _ => None,
            },
            None => None,
        }
    }
}

/// A predicate for matching elements.
///
/// The default implementations allow you to match by tag name or a tuple of
/// tag name and namespace.
pub trait ElementPredicate {
    fn match_element(&self, e: &Element) -> bool;
}

// Unfortunately,
// `impl<TN> ElementPredicate for TN where String: PartialEq<TN>` and
// `impl<TN, NS> ElementPredicate for (TN, NS) where String: PartialEq<TN>, String: PartialEq<NS>`
// are conflicting implementations, even though we know that there is no
// implementation for tuples. We just manually implement `ElementPredicate` for
// all `PartialEq` impls of `String` and forward them to the 1-tuple version.
//
// This can probably be fixed once specialization is stable.
impl<TN> ElementPredicate for (TN,)
where
    String: PartialEq<TN>,
{
    fn match_element(&self, e: &Element) -> bool {
        e.name == self.0
    }
}

impl<'a> ElementPredicate for &'a str {
    /// Search by tag name
    fn match_element(&self, e: &Element) -> bool {
        (*self,).match_element(e)
    }
}

impl<'a> ElementPredicate for Cow<'a, str> {
    /// Search by tag name
    fn match_element(&self, e: &Element) -> bool {
        (&**self,).match_element(e)
    }
}

impl ElementPredicate for String {
    /// Search by tag name
    fn match_element(&self, e: &Element) -> bool {
        (&**self,).match_element(e)
    }
}

impl<TN, NS> ElementPredicate for (TN, NS)
where
    String: PartialEq<TN>,
    String: PartialEq<NS>,
{
    /// Search by a tuple of (tagname, namespace)
    fn match_element(&self, e: &Element) -> bool {
        e.name == self.0
            && e.namespace
                .as_ref()
                .map(|ns| ns == &self.1)
                .unwrap_or(false)
    }
}
<|endoftext|>


<commit_before>extern crate libc;

pub mod platform;
pub mod os;
<commit_msg>Forgotten file (runtime error custom implementation)<commit_after>#![feature(cstr_to_str)]

extern crate libc;

pub mod error;
pub mod platform;
pub mod os;
<|endoftext|>


<commit_before>extern crate json_schema;
extern crate serde_json;

#[macro_use]
extern crate quote;

use std::error::Error;

use json_schema::{Schema, Type};

use quote::{Tokens, ToTokens};

struct Ident<S>(S);

impl<S: AsRef<str>> ToTokens for Ident<S> {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(self.0.as_ref())
    }
}

fn field(s: &str) -> Tokens {
    if ["type", "struct", "enum"].iter().any(|&keyword| keyword == s) {
        let n = Ident(format!("{}_", s));
        quote!( #n )
    } else {
        let mut snake = String::new();
        let mut chars = s.chars();
        let mut prev_was_upper = false;
        while let Some(c) = chars.next() {
            if c.is_uppercase() {
                if !prev_was_upper {
                    snake.push('_');
                }
                snake.extend(c.to_lowercase());
                prev_was_upper = true;
            } else {
                snake.push(c);
                prev_was_upper = false;
            }
        }
        if snake != s || snake.contains(|c: char| c == '$' || c == '#') {
            let field = Ident(snake.replace('$', "").replace('#', ""));
            quote!{
                #[serde(rename = #s)]
                pub #field
            }
        } else {
            let field = Ident(s);
            quote!( pub #field )
        }
    }
}

fn merge(result: &mut Schema, r: &Schema) {
    use std::collections::hash_map::Entry;

    for (k, v) in &r.properties {
        match result.properties.entry(k.clone()) {
            Entry::Vacant(entry) => {
                entry.insert(v.clone());
            }
            Entry::Occupied(mut entry) => merge(entry.get_mut(), v),
        }
    }
}

struct Expander<'r> {
    root_name: Option<&'r str>,
    root: &'r Schema,
}

impl<'r> Expander<'r> {
    fn type_ref(&self, s: &str) -> String {
        if s == "#" {
            self.root_name.expect("Root name").into()
        } else {
            s.split('/').last().expect("Component").into()
        }
    }

    fn schema(&self, s: &'r Schema) -> &'r Schema {
        if let Some(ref ref_) = s.ref_ {
            self.schema_ref(ref_)
        } else {
            s
        }
    }

    fn schema_ref(&self, s: &str) -> &'r Schema {
        s.split('/').fold(self.root, |schema, comp| {
            if comp == "#" {
                self.root
            } else if comp == "definitions" {
                schema
            } else {
                schema.definitions
                    .get(comp)
                    .unwrap_or_else(|| panic!("Expected definition: `{}` {}", s, comp))
            }
        })
    }

    fn expand_type(&mut self, type_name: &str, typ: &Schema) -> String {
        let result = self.expand_type_(typ);
        if type_name == result {
            format!("Box<{}>", result)
        } else {
            result
        }
    }
    fn expand_type_(&mut self, typ: &Schema) -> String {
        if let Some(ref ref_) = typ.ref_ {
            self.type_ref(ref_)
        } else if typ.type_.len() == 1 {
            match typ.type_[0] {
                Type::String => {
                    if !typ.enum_.is_empty() {
                        "serde_json::Value".into()
                    } else {
                        "String".into()
                    }
                }
                Type::Integer => "i64".into(),
                Type::Boolean => "bool".into(),
                Type::Number => "f64".into(),
                Type::Object => "serde_json::Value".into(),
                Type::Array => {
                    let item_type =
                        typ.items.as_ref().map_or("serde_json::Value".into(),
                                                  |item_schema| self.expand_type_(item_schema));
                    format!("Vec<{}>", item_type)
                }
                _ => panic!("Type"),
            }
        } else {
            "serde_json::Value".into()
        }
    }

    fn expand_fields(&mut self, type_name: &str, schema: &Schema) -> Vec<Tokens> {
        if let Some(ref ref_) = schema.ref_ {
            let schema = self.schema_ref(ref_);
            self.expand_fields(type_name, schema)
        } else if !schema.allOf.is_empty() {
            let first = schema.allOf.first().unwrap().clone();
            let result = schema.allOf
                .iter()
                .skip(1)
                .fold(first, |mut result, def| {
                    merge(&mut result, self.schema(def));
                    result
                });
            self.expand_fields(type_name, &result)
        } else {
            schema.properties
                .iter()
                .map(|(key, value)| {
                    let key = field(key);
                    let typ = Ident(self.expand_type(type_name, value));
                    quote!( #key : #typ )
                })
                .collect()
        }
    }

    pub fn expand_definitions(&mut self, schema: &Schema) -> Vec<Tokens> {
        let mut types = Vec::new();
        for (name, def) in &schema.definitions {
            types.push(self.expand_schema(name, def));
        }
        types
    }

    pub fn expand_schema(&mut self, name: &str, schema: &Schema) -> Tokens {
        let fields = self.expand_fields(name, schema);
        let name = Ident(name);
        quote! {
            #[derive(Deserialize, Serialize)]
            pub struct #name {
                #(#fields),*
            }
        }
    }

    pub fn expand(&mut self, schema: &Schema) -> Tokens {
        let mut types = self.expand_definitions(schema);
        if let Some(name) = self.root_name {
            types.push(self.expand_schema(name, schema));
        }

        quote! { #(
            #types
            )*
        }
    }
}

pub fn generate(root_name: Option<&str>, s: &str) -> Result<String, Box<Error>> {
    use std::process::{Command, Stdio};
    use std::io::Write;

    let schema = serde_json::from_str(s).unwrap();
    let mut expander = Expander {
        root_name: root_name,
        root: &schema,
    };
    let output = expander.expand(&schema).to_string();
    let mut child =
        try!(Command::new("rustfmt").stdin(Stdio::piped()).stdout(Stdio::piped()).spawn());
    try!(child.stdin.as_mut().expect("stdin").write_all(output.as_bytes()));
    let output = try!(child.wait_with_output());
    Ok(try!(String::from_utf8(output.stdout)))
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::fs::File;
    use std::io::Write;
    use std::path::PathBuf;
    use std::process::{Command, Stdio};

    #[test]
    fn generate_schema() {
        let s = include_str!("../../json-schema/tests/schema.json");

        let s = generate(Some("Schema"), s).unwrap().to_string();

        assert!(s.contains("pub struct Schema"), "{}", s);

        verify_compile("schema.rs", &s);
    }

    fn verify_compile(name: &str, s: &str) {

        let mut filename = PathBuf::from("target/debug");
        filename.push(name);
        {
            let mut file = File::create(&filename).unwrap();
            let header = r#"
            #![feature(proc_macro)]

            #[macro_use]
            extern crate serde_derive;
            extern crate serde_json;
            "#;
            file.write_all(header.as_bytes()).unwrap();
            file.write_all(s.as_bytes()).unwrap();
        }

        let child = Command::new("rustc")
            .args(&["-L", "target/debug/deps/", "--crate-type=rlib", filename.to_str().unwrap()])
            .stderr(Stdio::piped())
            .spawn()
            .unwrap();

        let output = child.wait_with_output().unwrap();
        let error = String::from_utf8(output.stderr).unwrap();
        assert!(output.status.success(), "{}", error);
    }

    #[test]
    fn builds_with_rustc() {
        let s = include_str!("../../json-schema/tests/debugserver-schema.json");

        let s = generate(None, s).unwrap().to_string();

        verify_compile("debug-server.rs", &s)
    }
}
<commit_msg>fix: Emit simple types (just constraints) as type aliases<commit_after>extern crate json_schema;
extern crate serde_json;

#[macro_use]
extern crate quote;

use std::borrow::Cow;
use std::error::Error;

use json_schema::{Schema, Type};

use quote::{Tokens, ToTokens};

struct Ident<S>(S);

impl<S: AsRef<str>> ToTokens for Ident<S> {
    fn to_tokens(&self, tokens: &mut Tokens) {
        tokens.append(self.0.as_ref())
    }
}

fn field(s: &str) -> Tokens {
    if ["type", "struct", "enum"].iter().any(|&keyword| keyword == s) {
        let n = Ident(format!("{}_", s));
        quote!( #n )
    } else {
        let mut snake = String::new();
        let mut chars = s.chars();
        let mut prev_was_upper = false;
        while let Some(c) = chars.next() {
            if c.is_uppercase() {
                if !prev_was_upper {
                    snake.push('_');
                }
                snake.extend(c.to_lowercase());
                prev_was_upper = true;
            } else {
                snake.push(c);
                prev_was_upper = false;
            }
        }
        if snake != s || snake.contains(|c: char| c == '$' || c == '#') {
            let field = Ident(snake.replace('$', "").replace('#', ""));
            quote!{
                #[serde(rename = #s)]
                pub #field
            }
        } else {
            let field = Ident(s);
            quote!( pub #field )
        }
    }
}

fn merge(result: &mut Schema, r: &Schema) {
    use std::collections::hash_map::Entry;

    for (k, v) in &r.properties {
        match result.properties.entry(k.clone()) {
            Entry::Vacant(entry) => {
                entry.insert(v.clone());
            }
            Entry::Occupied(mut entry) => merge(entry.get_mut(), v),
        }
    }
}

struct Expander<'r> {
    root_name: Option<&'r str>,
    root: &'r Schema,
}

impl<'r> Expander<'r> {
    fn type_ref(&self, s: &str) -> String {
        if s == "#" {
            self.root_name.expect("Root name").into()
        } else {
            s.split('/').last().expect("Component").into()
        }
    }

    fn schema(&self, schema: &'r Schema) -> Cow<'r, Schema> {
        let result = match schema.allOf.first() {
            Some(result) => {
                schema.allOf
                    .iter()
                    .skip(1)
                    .fold(Cow::Borrowed(result), |mut result, def| {
                        merge(result.to_mut(), &self.schema(def));
                        result
                    })
            }
            None => Cow::Borrowed(schema),
        };
        if let Some(ref ref_) = result.ref_ {
            return Cow::Borrowed(self.schema_ref(ref_));
        }
        result
    }

    fn schema_ref(&self, s: &str) -> &'r Schema {
        s.split('/').fold(self.root, |schema, comp| {
            if comp == "#" {
                self.root
            } else if comp == "definitions" {
                schema
            } else {
                schema.definitions
                    .get(comp)
                    .unwrap_or_else(|| panic!("Expected definition: `{}` {}", s, comp))
            }
        })
    }

    fn expand_type(&mut self, type_name: &str, typ: &Schema) -> String {
        let result = self.expand_type_(typ);
        if type_name == result {
            format!("Box<{}>", result)
        } else {
            result
        }
    }
    fn expand_type_(&mut self, typ: &Schema) -> String {
        if let Some(ref ref_) = typ.ref_ {
            self.type_ref(ref_)
        } else if typ.type_.len() == 1 {
            match typ.type_[0] {
                Type::String => {
                    if !typ.enum_.is_empty() {
                        "serde_json::Value".into()
                    } else {
                        "String".into()
                    }
                }
                Type::Integer => "i64".into(),
                Type::Boolean => "bool".into(),
                Type::Number => "f64".into(),
                Type::Object => "serde_json::Value".into(),
                Type::Array => {
                    let item_type =
                        typ.items.as_ref().map_or("serde_json::Value".into(),
                                                  |item_schema| self.expand_type_(item_schema));
                    format!("Vec<{}>", item_type)
                }
                _ => panic!("Type"),
            }
        } else {
            "serde_json::Value".into()
        }
    }

    fn expand_fields(&mut self, type_name: &str, schema: &Schema) -> Vec<Tokens> {
        let schema = self.schema(schema);
        schema.properties
            .iter()
            .map(|(key, value)| {
                let key = field(key);
                let typ = Ident(self.expand_type(type_name, value));
                quote!( #key : #typ )
            })
            .collect()
    }

    pub fn expand_definitions(&mut self, schema: &Schema) -> Vec<Tokens> {
        let mut types = Vec::new();
        for (name, def) in &schema.definitions {
            types.push(self.expand_schema(name, def));
        }
        types
    }

    pub fn expand_schema(&mut self, name: &str, schema: &Schema) -> Tokens {
        let fields = self.expand_fields(name, schema);
        let name = Ident(name);
        if fields.is_empty() {
            let typ = Ident(self.expand_type("", schema));
            quote! {
                pub type #name = #typ;
            }
        } else {
            quote! {
                #[derive(Deserialize, Serialize)]
                pub struct #name {
                    #(#fields),*
                }
            }
        }
    }

    pub fn expand(&mut self, schema: &Schema) -> Tokens {
        let mut types = self.expand_definitions(schema);
        if let Some(name) = self.root_name {
            types.push(self.expand_schema(name, schema));
        }

        quote! { #(
            #types
            )*
        }
    }
}

pub fn generate(root_name: Option<&str>, s: &str) -> Result<String, Box<Error>> {
    use std::process::{Command, Stdio};
    use std::io::Write;

    let schema = serde_json::from_str(s).unwrap();
    let mut expander = Expander {
        root_name: root_name,
        root: &schema,
    };
    let output = expander.expand(&schema).to_string();
    let mut child =
        try!(Command::new("rustfmt").stdin(Stdio::piped()).stdout(Stdio::piped()).spawn());
    try!(child.stdin.as_mut().expect("stdin").write_all(output.as_bytes()));
    let output = try!(child.wait_with_output());
    Ok(try!(String::from_utf8(output.stdout)))
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::fs::File;
    use std::io::Write;
    use std::path::PathBuf;
    use std::process::{Command, Stdio};

    #[test]
    fn generate_schema() {
        let s = include_str!("../../json-schema/tests/schema.json");

        let s = generate(Some("Schema"), s).unwrap().to_string();

        assert!(s.contains("pub struct Schema"), "{}", s);
        assert!(s.contains("pub type positiveInteger = i64"));

        verify_compile("schema.rs", &s);
    }

    fn verify_compile(name: &str, s: &str) {

        let mut filename = PathBuf::from("target/debug");
        filename.push(name);
        {
            let mut file = File::create(&filename).unwrap();
            let header = r#"
            #![feature(proc_macro)]

            #[macro_use]
            extern crate serde_derive;
            extern crate serde_json;
            "#;
            file.write_all(header.as_bytes()).unwrap();
            file.write_all(s.as_bytes()).unwrap();
        }

        let child = Command::new("rustc")
            .args(&["-L", "target/debug/deps/", "--crate-type=rlib", filename.to_str().unwrap()])
            .stderr(Stdio::piped())
            .spawn()
            .unwrap();

        let output = child.wait_with_output().unwrap();
        let error = String::from_utf8(output.stderr).unwrap();
        assert!(output.status.success(), "{}", error);
    }

    #[test]
    fn builds_with_rustc() {
        let s = include_str!("../../json-schema/tests/debugserver-schema.json");

        let s = generate(None, s).unwrap().to_string();

        verify_compile("debug-server.rs", &s)
    }
}
<|endoftext|>


<commit_before>// Copyright 2015 Jakob Lautrup Nysom

//! A simple string tokenizer, and a tokenizer of sentences based on it.
//! The tokenizer ignores all the given separator chars, and returns the
//! characters between as string slice
//!
//! # Examples
//! **General use (as an iterator)**
//!
//! This is how you will probably use it
//!
//! ```
//! let separators = vec![' ', '\n', '\t', '\r'];
//! let source: &str = "    Hello world \n  How do you do\t-Finely I hope";
//!
//! let mut tokenizer = token::Tokenizer::new(source.as_bytes(), separators);
//! println!("Tokenizing...");
//! for token in tokenizer {
//!     println!("- Got token: {}", token.unwrap());
//! }
//! println!("Done!");
//! ```
//!
//! **Behavior**
//!
//! This is what to expect when parsing a string (or input from a reader)
//!
//! ```
//! let separators = vec![' ', '\n', '\t', '\r'];
//! let source: &str = "    Hello world \n  How do you do\t-Finely I hope";
//!
//! let mut tokenizer = token::Tokenizer::new(source.as_bytes(), separators);
//! assert_eq!("Hello",     tokenizer.next().expect("1").unwrap());
//! assert_eq!("world",     tokenizer.next().expect("2").unwrap());
//! assert_eq!("How",       tokenizer.next().expect("3").unwrap());
//! assert_eq!("do",        tokenizer.next().expect("4").unwrap());
//! assert_eq!("you",       tokenizer.next().expect("5").unwrap());
//! assert_eq!("do",        tokenizer.next().expect("6").unwrap());
//! assert_eq!("-Finely",   tokenizer.next().expect("7").unwrap());
//! assert_eq!("I",         tokenizer.next().expect("8").unwrap());
//! assert_eq!("hope",      tokenizer.next().expect("9").unwrap());
//! assert_eq!(None,        tokenizer.next());
//! ```

#![feature(core)]
#![feature(io)]
use std::vec::Vec;
use std::iter::Iterator;
use std::io;
use std::io::{Read, ReadExt};

/// A tokenizer returning string slices from a reader
///
/// **Note:** The returned tokens are only valid until the next iteration
pub struct Tokenizer<R: Read> {
    separators: Vec<char>,
    chars: io::Chars<R>,
    current: String,
}

impl <R> Tokenizer<R> where R: Read {
    /// Creates a new tokenizer from a reader and a set of separating characters
    ///
    /// ```
    /// let separators = vec![' ', '\n', '\t'];
    /// let source: &str = "   Hello world\nHow do you do\t-Finely I hope";
    ///
    /// let mut tokenizer = token::Tokenizer::new(source.as_bytes(), separators);
    ///
    ///
    /// ```
    ///
    pub fn new(reader: R, separators: Vec<char>) -> Tokenizer<R> {
        Tokenizer {
            chars: reader.chars(),
            separators: separators,
            current: String::new(),
        }
    }
}

impl<'a, R: Read> Iterator for Tokenizer<R> {
    type Item = Result<&'a str, io::CharsError>;

    /// Returns a string slice of the next non-empty sequence that terminates
    /// in one of the specified separator strings
    fn next(&mut self) -> Option<Result<&str, io::CharsError>> {
        self.current.clear();
        'main: loop {
            match self.chars.next() {
                None => break,
                Some(res) => {
                    match res {
                        Ok(c) => {
                            // Check for token terminators
                            for &t in self.separators.iter() {
                                if c == t {
                                    if !self.current.is_empty() {
                                        return Some(Ok(self.current.as_slice()));
                                    }
                                    continue 'main; // No token: ignore
                                }
                            }

                            // Just add the char
                            self.current.push(c);
                        }

                        Err(e) => return Some(Err(e)),
                    }
                }
            }
        }
        // Handle leftover chars
        if !self.current.is_empty() {
            Some(Ok(self.current.as_slice()))
        } else {
            None // No more chars left
        }
    }

    /// Uses the underlying reader's size hint
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.chars.size_hint()
    }
}

/// A structure for iteratively splitting stringy things into sentences
pub struct SentenceSplitter<'a, R: Read> {
    tokenizer: Tokenizer<R>,
    terminators: Vec<&'a str>,
    current: String,
    quote_types: Vec<&'a str>,
}

impl <'a, R: Read> SentenceSplitter<'a, R> {

    /// Creates a new sentence-splitting iterator
    ///
    /// ```
    /// let separators  = vec![' ', '\n', '\t', '\r'];
    /// let terminators = vec![".", "!", "?"];
    /// let quote_types = vec![]; // For when whole lines are quoted
    ///
    /// let text = "I walked down the road.\n\"What did he say\", she asked me.\n\"Nothing\", I replied, and continued walking...\nit wasn't any of my business.\nOr was it?";
    ///
    /// let mut tokenizer = token::Tokenizer::new(text.as_bytes(), separators);
    /// let mut splitter = token::SentenceSplitter::new(
    ///     tokenizer, terminators, quote_types
    /// );
    ///
    /// assert_eq!("I walked down the road.",
    ///     splitter.next().expect("1").unwrap());
    /// assert_eq!("\"What did he say\", she asked me.",
    ///     splitter.next().expect("2").unwrap());
    /// assert_eq!("\"Nothing\", I replied, and continued walking... \
    ///     it wasn't any of my business.",
    ///     splitter.next().expect("3").unwrap());
    /// assert_eq!("Or was it?", splitter.next().expect("3").unwrap());
    /// assert_eq!(None, splitter.next());
    /// ```
    pub fn new(source: Tokenizer<R>, terminators: Vec<&'a str>,
        quote_types: Vec<&'a str>) -> SentenceSplitter<'a, R>
    {
        SentenceSplitter{
            tokenizer: source,
            current: String::new(),
            terminators: terminators,
            quote_types: quote_types
        }
    }
}

impl <'a, 'b, R: Read> Iterator for SentenceSplitter<'a, R> {
    type Item = Result<&'b str, io::CharsError>;

    /// Returns the next sentence
    fn next(&mut self) -> Option<Result<&str, io::CharsError>> {
        self.current.clear();
        let mut quote = "";
        'main: loop {
            match self.tokenizer.next() {
                Some(res) => {
                    match res {
                        Ok(s) => {
                            self.current.push_str(s);

                            // Inside a quote
                            if !quote.is_empty() {
                                if s.ends_with(quote) {
                                    return Some(Ok(self.current.as_slice()))
                                }
                            }

                            // Not inside a quote
                            else {
                                // Check whether a quote is starting
                                for &qt in self.quote_types.iter() {
                                    if s.starts_with(qt) {
                                        if s.ends_with(qt) { // It can end again
                                            return Some(Ok(self.current.as_slice()));
                                        }
                                        quote = qt;
                                        self.current.push_str(" ");
                                        continue 'main;
                                    }
                                }

                                // Check whether the token is ending normally
                                // It ends in a terminating character
                                if s.ends_with("..") { // Continue thought trails
                                    self.current.push_str(" ");
                                    continue;
                                }
                                for &t in self.terminators.iter() {
                                    if s.ends_with(t) {
                                        return Some(Ok(self.current.as_slice()));
                                    }
                                }
                            }
                            // SPAAAAAAAAACE
                            self.current.push_str(" ");
                        }

                        Err(e) => return Some(Err(e)),
                    }
                },
                None => {
                    if self.current.len() != 0 {
                        return Some(Ok(self.current.as_slice()));
                    } else {
                        return None;
                    }
                }
            }
        }
    }
    /// Uses the underlying tokenizer's size hint
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.tokenizer.size_hint()
    }
}<commit_msg>Removed obsolete import and flags<commit_after>// Copyright 2015 Jakob Lautrup Nysom

//! A simple string tokenizer, and a tokenizer of sentences based on it.
//! The tokenizer ignores all the given separator chars, and returns the
//! characters between as string slice
//!
//! # Examples
//! **General use (as an iterator)**
//!
//! This is how you will probably use it
//!
//! ```
//! let separators = vec![' ', '\n', '\t', '\r'];
//! let source: &str = "    Hello world \n  How do you do\t-Finely I hope";
//!
//! let mut tokenizer = token::Tokenizer::new(source.as_bytes(), separators);
//! println!("Tokenizing...");
//! for token in tokenizer {
//!     println!("- Got token: {}", token.unwrap());
//! }
//! println!("Done!");
//! ```
//!
//! **Behavior**
//!
//! This is what to expect when parsing a string (or input from a reader)
//!
//! ```
//! let separators = vec![' ', '\n', '\t', '\r'];
//! let source: &str = "    Hello world \n  How do you do\t-Finely I hope";
//!
//! let mut tokenizer = token::Tokenizer::new(source.as_bytes(), separators);
//! assert_eq!("Hello",     tokenizer.next().expect("1").unwrap());
//! assert_eq!("world",     tokenizer.next().expect("2").unwrap());
//! assert_eq!("How",       tokenizer.next().expect("3").unwrap());
//! assert_eq!("do",        tokenizer.next().expect("4").unwrap());
//! assert_eq!("you",       tokenizer.next().expect("5").unwrap());
//! assert_eq!("do",        tokenizer.next().expect("6").unwrap());
//! assert_eq!("-Finely",   tokenizer.next().expect("7").unwrap());
//! assert_eq!("I",         tokenizer.next().expect("8").unwrap());
//! assert_eq!("hope",      tokenizer.next().expect("9").unwrap());
//! assert_eq!(None,        tokenizer.next());
//! ```

#![feature(core)]
#![feature(io)]
use std::vec::Vec;
use std::iter::Iterator;
use std::io;
use std::io::Read;

/// A tokenizer returning string slices from a reader
///
/// **Note:** The returned tokens are only valid until the next iteration
pub struct Tokenizer<R: Read> {
    separators: Vec<char>,
    chars: io::Chars<R>,
    current: String,
}

impl <R> Tokenizer<R> where R: Read {
    /// Creates a new tokenizer from a reader and a set of separating characters
    ///
    /// ```
    /// let separators = vec![' ', '\n', '\t'];
    /// let source: &str = "   Hello world\nHow do you do\t-Finely I hope";
    ///
    /// let mut tokenizer = token::Tokenizer::new(source.as_bytes(), separators);
    ///
    ///
    /// ```
    ///
    pub fn new(reader: R, separators: Vec<char>) -> Tokenizer<R> {
        Tokenizer {
            chars: reader.chars(),
            separators: separators,
            current: String::new(),
        }
    }
}

impl<'a, R: Read> Iterator for Tokenizer<R> {
    type Item = Result<&'a str, io::CharsError>;

    /// Returns a string slice of the next non-empty sequence that terminates
    /// in one of the specified separator strings
    fn next(&mut self) -> Option<Result<&str, io::CharsError>> {
        self.current.clear();
        'main: loop {
            match self.chars.next() {
                None => break,
                Some(res) => {
                    match res {
                        Ok(c) => {
                            // Check for token terminators
                            for &t in self.separators.iter() {
                                if c == t {
                                    if !self.current.is_empty() {
                                        return Some(Ok(self.current.as_slice()));
                                    }
                                    continue 'main; // No token: ignore
                                }
                            }

                            // Just add the char
                            self.current.push(c);
                        }

                        Err(e) => return Some(Err(e)),
                    }
                }
            }
        }
        // Handle leftover chars
        if !self.current.is_empty() {
            Some(Ok(self.current.as_slice()))
        } else {
            None // No more chars left
        }
    }

    /// Uses the underlying reader's size hint
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.chars.size_hint()
    }
}

/// A structure for iteratively splitting stringy things into sentences
pub struct SentenceSplitter<'a, R: Read> {
    tokenizer: Tokenizer<R>,
    terminators: Vec<&'a str>,
    current: String,
    quote_types: Vec<&'a str>,
}

impl <'a, R: Read> SentenceSplitter<'a, R> {

    /// Creates a new sentence-splitting iterator
    ///
    /// ```
    /// let separators  = vec![' ', '\n', '\t', '\r'];
    /// let terminators = vec![".", "!", "?"];
    /// let quote_types = vec![]; // For when whole lines are quoted
    ///
    /// let text = "I walked down the road.\n\"What did he say\", she asked me.\n\"Nothing\", I replied, and continued walking...\nit wasn't any of my business.\nOr was it?";
    ///
    /// let mut tokenizer = token::Tokenizer::new(text.as_bytes(), separators);
    /// let mut splitter = token::SentenceSplitter::new(
    ///     tokenizer, terminators, quote_types
    /// );
    ///
    /// assert_eq!("I walked down the road.",
    ///     splitter.next().expect("1").unwrap());
    /// assert_eq!("\"What did he say\", she asked me.",
    ///     splitter.next().expect("2").unwrap());
    /// assert_eq!("\"Nothing\", I replied, and continued walking... \
    ///     it wasn't any of my business.",
    ///     splitter.next().expect("3").unwrap());
    /// assert_eq!("Or was it?", splitter.next().expect("3").unwrap());
    /// assert_eq!(None, splitter.next());
    /// ```
    pub fn new(source: Tokenizer<R>, terminators: Vec<&'a str>,
        quote_types: Vec<&'a str>) -> SentenceSplitter<'a, R>
    {
        SentenceSplitter{
            tokenizer: source,
            current: String::new(),
            terminators: terminators,
            quote_types: quote_types
        }
    }
}

impl <'a, 'b, R: Read> Iterator for SentenceSplitter<'a, R> {
    type Item = Result<&'b str, io::CharsError>;

    /// Returns the next sentence
    fn next(&mut self) -> Option<Result<&str, io::CharsError>> {
        self.current.clear();
        let mut quote = "";
        'main: loop {
            match self.tokenizer.next() {
                Some(res) => {
                    match res {
                        Ok(s) => {
                            self.current.push_str(s);

                            // Inside a quote
                            if !quote.is_empty() {
                                if s.ends_with(quote) {
                                    return Some(Ok(self.current.as_slice()))
                                }
                            }

                            // Not inside a quote
                            else {
                                // Check whether a quote is starting
                                for &qt in self.quote_types.iter() {
                                    if s.starts_with(qt) {
                                        if s.ends_with(qt) { // It can end again
                                            return Some(Ok(self.current.as_slice()));
                                        }
                                        quote = qt;
                                        self.current.push_str(" ");
                                        continue 'main;
                                    }
                                }

                                // Check whether the token is ending normally
                                // It ends in a terminating character
                                if s.ends_with("..") { // Continue thought trails
                                    self.current.push_str(" ");
                                    continue;
                                }
                                for &t in self.terminators.iter() {
                                    if s.ends_with(t) {
                                        return Some(Ok(self.current.as_slice()));
                                    }
                                }
                            }
                            // SPAAAAAAAAACE
                            self.current.push_str(" ");
                        }

                        Err(e) => return Some(Err(e)),
                    }
                },
                None => {
                    if self.current.len() != 0 {
                        return Some(Ok(self.current.as_slice()));
                    } else {
                        return None;
                    }
                }
            }
        }
    }
    /// Uses the underlying tokenizer's size hint
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.tokenizer.size_hint()
    }
}<|endoftext|>


<commit_before>use std::collections::BTreeMap;

#[derive(Copy, Clone, PartialEq, Eq)]
pub enum Token {
    Output,
    Input,
    Loop,
    EndLoop,
    Move(i32),
    Add(i32, i32),
    Set(i32, i32),
    MulCopy(i32, i32, i32),
    Scan(i32),
    LoadOut(i32, i32),
    LoadOutSet(i32),
    If(i32),
    EndIf,
}

pub fn parse(code: &str) -> Vec<Token> {
    let mut tokens = Vec::new();
    for i in code.chars() {
        match i {
            '+' => tokens.push(Token::Add(0, 1)),
            '-' => tokens.push(Token::Add(0, -1)),
            '>' => tokens.push(Token::Move(1)),
            '<' => tokens.push(Token::Move(-1)),
            '[' => tokens.push(Token::Loop),
            ']' => tokens.push(Token::EndLoop),
            ',' => tokens.push(Token::Input),
            '.' => {
                tokens.push(Token::LoadOut(0, 0));
                tokens.push(Token::Output);
            },
            _ => ()
        };
    }

    tokens
    }

pub fn optimize(tokens: Vec<Token>) -> Vec<Token> {
    let mut newtokens: Vec<Token> = Vec::new();
    let mut shift = 0;
    let mut do_output = false;
    // With HashMap, the order sometimes switches
    // in recursion, and the optimizer never exits.
    let mut adds: BTreeMap<i32, i32> = BTreeMap::new();
    let mut sets: BTreeMap<i32, i32> = BTreeMap::new();
    let mut pre_loop_sets: BTreeMap<i32, i32> = BTreeMap::new();

    for token in tokens.iter() {
        let mut prev_token = match newtokens.last() {
            Some(tok) => Some(*tok),
            None => None
        };
        if *token == Token::EndLoop && prev_token == Some(Token::Loop) && shift == 0 && adds.contains_key(&0) {
            if adds.len() == 1 {
                newtokens.pop(); // Remove Loop
                if !sets.is_empty() {
                    newtokens.push(Token::If(0));
                    for (offset, value) in sets.iter() {
                        newtokens.push(Token::Set(*offset, *value));
                    }
                    sets.clear();
                    newtokens.push(Token::Set(0, 0));
                    newtokens.push(Token::EndIf);
                } else {
                    sets.insert(0, 0);
                }
                pre_loop_sets.clear();
                adds.clear();
                continue
            } else if adds.get(&0) == Some(&-1) {
                newtokens.pop(); // Remove Loop
                if !sets.is_empty() {
                    newtokens.push(Token::If(0));
                    for (offset, value) in sets.iter() {
                        newtokens.push(Token::Set(*offset, *value));
                    }
                }
                for (offset, value) in adds.iter() {
                    if *offset != 0 {
                        let src = 0;
                        let dest = *offset;
                        let mul = *value;
                        if pre_loop_sets.contains_key(&src) {
                            let val = pre_loop_sets.get(&src).unwrap() * mul;
                            newtokens.push(Token::Add(dest, val));
                        } else {
                            newtokens.push(Token::MulCopy(src, dest, mul));
                        }
                    }
                }
                if !sets.is_empty() {
                    newtokens.push(Token::EndIf);
                }
                pre_loop_sets.clear();
                adds.clear();
                sets.clear();
                sets.insert(0, 0);
                continue
            }
        }

        match *token {
            Token::Loop =>
                for (offset, value) in sets.iter() {
                    pre_loop_sets.insert(*offset+shift, *value);
                },
            Token::Set(_, _) | Token::Add(_, _) | Token::Move(_) => {},
            _ => pre_loop_sets.clear()
        }

        match *token {
            Token::Set(_, _) | Token::Add(_, _) | Token::Move(_) | Token::LoadOut(_, _) | Token::LoadOutSet(_) | Token::Output => {},
            _ => {
               if do_output {
                   newtokens.push(Token::Output);
                   do_output = false;
               }

               for (offset, value) in sets.iter() {
                  newtokens.push(Token::Set(*offset, *value));
               }
               for (offset, value) in adds.iter() {
                  newtokens.push(Token::Add(*offset, *value));
               }
               sets.clear();
               adds.clear();
            }
        }

        if shift != 0 {
           match *token {
               Token::Loop | Token::Input | Token::Scan(_) => {
                   newtokens.push(Token::Move(shift));
                   shift = 0;
               },
               _ => {}
           }
        }

        match *token {
            Token::Set(mut offset, val) => {
                offset += shift;
                // Add before Set does nothing; remove it
                if adds.contains_key(&offset) {
                    adds.remove(&offset);
                }
                sets.insert(offset, val);
            },
            Token::Add(mut offset, mut val) => {
                offset += shift;
                if sets.contains_key(&offset) {
                    val = sets.get(&offset).unwrap() + val;
                    sets.insert(offset, val);
                } else {
                    val = adds.get(&offset).unwrap_or(&0) + val;
                    adds.insert(offset, val);
                }
            },
            Token::MulCopy(src, dest, mul) =>
                newtokens.push(Token::MulCopy(src+shift, dest+shift, mul)),
            // XXX Deal with shift in if, if those are ever generated
            Token::If(offset) =>
                newtokens.push(Token::If(offset+shift)),
            Token::Move(offset) =>
                shift += offset,
            Token::Output =>
                do_output = true,
            Token::LoadOut(mut offset, add) => {
                offset += shift;
                if sets.contains_key(&offset) {
                    newtokens.push(Token::LoadOutSet(sets.get(&offset).unwrap() + add));
                } else {
                    newtokens.push(Token::LoadOut(offset, adds.get(&offset).unwrap_or(&0) + add));
                }
            },
            Token::EndLoop => {
                prev_token = match newtokens.last() {
                            Some(tok) => Some(*tok),
                            None => None
                        };
                if prev_token == Some(Token::Loop) && shift != 0 && sets.is_empty() && adds.is_empty() {
                    newtokens.pop(); // Remove StartLoop
                    newtokens.push(Token::Scan(shift));
                } else {
                    if shift != 0 {
                        newtokens.push(Token::Move(shift));
                    }
                    shift = 0;
                    newtokens.push(Token::EndLoop);
                }
            },
            Token::EndIf | Token::LoadOutSet(_) | Token::Loop | Token::Input | Token::Scan(_) =>
                newtokens.push(*token),
        }
    }

    // Any remaining add/set/shift is ignored, as it would have no effect
    if do_output {
        newtokens.push(Token::Output);
    }

    // Optimize recursively
    if newtokens != tokens {
        optimize(newtokens)
    } else {
        newtokens
    }
}
<commit_msg>Ok, actually fix the rust version<commit_after>use std::collections::BTreeMap;

#[derive(Copy, Clone, PartialEq, Eq)]
pub enum Token {
    Output,
    Input,
    Loop,
    EndLoop,
    Move(i32),
    Add(i32, i32),
    Set(i32, i32),
    MulCopy(i32, i32, i32),
    Scan(i32),
    LoadOut(i32, i32),
    LoadOutSet(i32),
    If(i32),
    EndIf,
}

pub fn parse(code: &str) -> Vec<Token> {
    let mut tokens = Vec::new();
    for i in code.chars() {
        match i {
            '+' => tokens.push(Token::Add(0, 1)),
            '-' => tokens.push(Token::Add(0, -1)),
            '>' => tokens.push(Token::Move(1)),
            '<' => tokens.push(Token::Move(-1)),
            '[' => tokens.push(Token::Loop),
            ']' => tokens.push(Token::EndLoop),
            ',' => tokens.push(Token::Input),
            '.' => {
                tokens.push(Token::LoadOut(0, 0));
                tokens.push(Token::Output);
            },
            _ => ()
        };
    }

    tokens
    }

pub fn optimize(tokens: Vec<Token>) -> Vec<Token> {
    let mut newtokens: Vec<Token> = Vec::new();
    let mut shift = 0;
    let mut do_output = false;
    // With HashMap, the order sometimes switches
    // in recursion, and the optimizer never exits.
    let mut adds: BTreeMap<i32, i32> = BTreeMap::new();
    let mut sets: BTreeMap<i32, i32> = BTreeMap::new();
    let mut pre_loop_sets: BTreeMap<i32, i32> = BTreeMap::new();

    for token in tokens.iter() {
        let mut prev_token = match newtokens.last() {
            Some(tok) => Some(*tok),
            None => None
        };
        if *token == Token::EndLoop && prev_token == Some(Token::Loop) && shift == 0 && adds.contains_key(&0) {
            if adds.len() == 1 {
                newtokens.pop(); // Remove Loop
                if !sets.is_empty() {
                    newtokens.push(Token::If(0));
                    for (offset, value) in sets.iter() {
                        newtokens.push(Token::Set(*offset, *value));
                    }
                    sets.clear();
                    newtokens.push(Token::Set(0, 0));
                    newtokens.push(Token::EndIf);
                } else {
                    sets.insert(0, 0);
                }
                pre_loop_sets.clear();
                adds.clear();
                continue
            } else if adds.get(&0) == Some(&-1) {
                newtokens.pop(); // Remove Loop
                if !sets.is_empty() {
                    newtokens.push(Token::If(0));
                    for (offset, value) in sets.iter() {
                        newtokens.push(Token::Set(*offset, *value));
                    }
                }
                for (offset, value) in adds.iter() {
                    if *offset != 0 {
                        let src = 0;
                        let dest = *offset;
                        let mul = *value;
                        if pre_loop_sets.contains_key(&src) {
                            let val = pre_loop_sets.get(&src).unwrap() * mul;
                            newtokens.push(Token::Add(dest, val));
                        } else {
                            newtokens.push(Token::MulCopy(src, dest, mul));
                        }
                    }
                }
                if !sets.is_empty() {
                    newtokens.push(Token::EndIf);
                }
                pre_loop_sets.clear();
                adds.clear();
                sets.clear();
                sets.insert(0, 0);
                continue
            }
        }

        match *token {
            Token::Loop =>
                for (offset, value) in sets.iter() {
                    pre_loop_sets.insert(*offset+shift, *value);
                },
            Token::Set(_, _) | Token::Add(_, _) | Token::Move(_) => {},
            _ => pre_loop_sets.clear()
        }

        match *token {
            Token::Set(_, _) | Token::Add(_, _) | Token::Move(_) | Token::LoadOut(_, _) | Token::LoadOutSet(_) | Token::Output => {},
            _ => {
               if do_output {
                   newtokens.push(Token::Output);
                   do_output = false;
               }

               for (offset, value) in sets.iter() {
                  newtokens.push(Token::Set(*offset, *value));
               }
               for (offset, value) in adds.iter() {
                  newtokens.push(Token::Add(*offset, *value));
               }
               sets.clear();
               adds.clear();
            }
        }

        if shift != 0 {
           match *token {
               Token::Loop | Token::Input | Token::Scan(_) => {
                   newtokens.push(Token::Move(shift));
                   shift = 0;
               },
               _ => {}
           }
        }

        match *token {
            Token::Set(mut offset, val) => {
                offset += shift;
                // Add before Set does nothing; remove it
                if adds.contains_key(&offset) {
                    adds.remove(&offset);
                }
                sets.insert(offset, val);
            },
            Token::Add(mut offset, mut val) => {
                offset += shift;
                if sets.contains_key(&offset) {
                    val = sets.get(&offset).unwrap() + val;
                    sets.insert(offset, val);
                } else {
                    val = adds.get(&offset).unwrap_or(&0) + val;
                    adds.insert(offset, val);
                }
            },
            Token::MulCopy(src, dest, mul) =>
                newtokens.push(Token::MulCopy(src+shift, dest+shift, mul)),
            // XXX Deal with shift in if, if those are ever generated
            Token::If(offset) =>
                newtokens.push(Token::If(offset+shift)),
            Token::Move(offset) =>
                shift += offset,
            Token::Output =>
                do_output = true,
            Token::LoadOut(mut offset, add) => {
                offset += shift;
                if sets.contains_key(&offset) {
                    newtokens.push(Token::LoadOutSet(sets.get(&offset).unwrap() + add));
                } else {
                    newtokens.push(Token::LoadOut(offset, adds.get(&offset).unwrap_or(&0) + add));
                }
            },
            Token::EndLoop => {
                prev_token = match newtokens.last() {
                            Some(tok) => Some(*tok),
                            None => None
                        };
                if prev_token == Some(Token::Loop) && shift != 0 && sets.is_empty() && adds.is_empty() {
                    newtokens.pop(); // Remove StartLoop
                    newtokens.push(Token::Scan(shift));
                } else {
                    if shift != 0 {
                        newtokens.push(Token::Move(shift));
                    }
                    newtokens.push(Token::EndLoop);
                }
                shift = 0;
            },
            Token::EndIf | Token::LoadOutSet(_) | Token::Loop | Token::Input | Token::Scan(_) =>
                newtokens.push(*token),
        }
    }

    // Any remaining add/set/shift is ignored, as it would have no effect
    if do_output {
        newtokens.push(Token::Output);
    }

    // Optimize recursively
    if newtokens != tokens {
        optimize(newtokens)
    } else {
        newtokens
    }
}
<|endoftext|>


<commit_before>use std::io::prelude::*;
use std::fs::File;
use std::mem;
use std::ptr;

extern crate libc;

pub mod minidump_format;
use minidump_format as fmt;

#[allow(dead_code)]
pub struct Minidump {
    file : File,
    header : fmt::MDRawHeader,
    swap : bool,
}

#[derive(Debug)]
pub enum Error {
    MissingHeader,
    HeaderMismatch,
}

fn read<T>(mut f : &File) -> std::io::Result<T> {
    let size = mem::size_of::<T>();
    let mut buf = vec!(0; size);
    let bytes = &mut buf[..];
    try!(f.read(bytes));
    Ok(unsafe {
        let mut val : T = mem::uninitialized();
        ptr::copy(bytes.as_mut_ptr(), &mut val as *mut T as *mut u8, size);
        val
    })
}

impl Minidump {
    pub fn read(f : File) -> Result<Minidump, Error> {
        let header = try!(read::<fmt::MDRawHeader>(&f).or(Err(Error::MissingHeader)));
        let mut swap = false;
        if header.signature != fmt::MD_HEADER_SIGNATURE {
            if header.signature.swap_bytes() != fmt::MD_HEADER_SIGNATURE {
                return Err(Error::HeaderMismatch);
            }
            swap = true;
        }
        Ok(Minidump { file: f, header: header, swap: swap })
    }
}

#[cfg(test)]
mod tests {
    use std::path::PathBuf;
    use std::fs::File;
    use super::*;

    #[test]
    fn test_read_minidump() {
        let mut path = PathBuf::from(file!());
        path.pop();
        path.push("testdata/test.dmp");
        let f = File::open(&path).ok().expect(&format!("failed to open file: {:?}", path));
        Minidump::read(f).unwrap();
    }
}
<commit_msg>Read stream directory<commit_after>use std::io::prelude::*;
use std::fs::File;
use std::mem;
use std::ptr;
use std::collections::HashMap;

extern crate libc;

pub mod minidump_format;
use minidump_format as fmt;

#[allow(dead_code)]
pub struct Minidump {
    file : File,
    header : fmt::MDRawHeader,
    streams : HashMap<u32, fmt::MDRawDirectory>,
    swap : bool,
}

#[derive(Debug)]
pub enum Error {
    MissingHeader,
    HeaderMismatch,
    SwapNotImplemented,
    VersionMismatch,
    MissingDirectory,
    StreamReadFailure,
}

fn read<T>(mut f : &File) -> std::io::Result<T> {
    let size = mem::size_of::<T>();
    let mut buf = vec!(0; size);
    let bytes = &mut buf[..];
    try!(f.read(bytes));
    Ok(unsafe {
        let mut val : T = mem::uninitialized();
        ptr::copy(bytes.as_mut_ptr(), &mut val as *mut T as *mut u8, size);
        val
    })
}

impl Minidump {
    pub fn read(f : File) -> Result<Minidump, Error> {
        let header = try!(read::<fmt::MDRawHeader>(&f).or(Err(Error::MissingHeader)));
        let swap = false;
        if header.signature != fmt::MD_HEADER_SIGNATURE {
            if header.signature.swap_bytes() != fmt::MD_HEADER_SIGNATURE {
                return Err(Error::HeaderMismatch);
            }
            return Err(Error::SwapNotImplemented);
            // TODO: implement swapping
            //swap = true;
        }
        if (header.version & 0x0000ffff) != fmt::MD_HEADER_VERSION {
            return Err(Error::VersionMismatch);
        }
        let mut streams = HashMap::with_capacity(header.stream_count as usize);
        if header.stream_count != 0 {
            for _ in 0..header.stream_count {
                let dir = try!(read::<fmt::MDRawDirectory>(&f).or(Err(Error::MissingDirectory)));
                if dir.stream_type != fmt::MD_UNUSED_STREAM {
                    streams.insert(dir.stream_type, dir);
                }
            }
        }
        Ok(Minidump {
            file: f,
            header: header,
            streams: streams,
            swap: swap
        })
    }
}

#[cfg(test)]
mod tests {
    use std::path::PathBuf;
    use std::fs::File;
    use super::*;

    #[test]
    fn test_read_minidump() {
        let mut path = PathBuf::from(file!());
        path.pop();
        path.push("testdata/test.dmp");
        let f = File::open(&path).ok().expect(&format!("failed to open file: {:?}", path));
        let dump = Minidump::read(f).unwrap();
        assert_eq!(dump.streams.len(), 7);
    }
}
<|endoftext|>