<commit_before>use std::io;
use std::num::{Float, FpCategory};
use std::string::FromUtf8Error;

use ser;

/// A structure for implementing serialization to JSON.
pub struct Serializer<W, F=CompactFormatter> {
    writer: W,
    formatter: F,

    /// `first` is used to signify if we should print a comma when we are walking through a
    /// sequence.
    first: bool,
}

impl<W> Serializer<W>
    where W: io::Write,
{
    /// Creates a new JSON visitr whose output will be written to the writer
    /// specified.
    #[inline]
    pub fn new(writer: W) -> Serializer<W> {
        Serializer::new_with_formatter(writer, CompactFormatter)
    }
}

impl<W, F> Serializer<W, F>
    where W: io::Write,
          F: Formatter,
{
    /// Creates a new JSON visitr whose output will be written to the writer
    /// specified.
    #[inline]
    pub fn new_with_formatter(writer: W, formatter: F) -> Serializer<W, F> {
        Serializer {
            writer: writer,
            formatter: formatter,
            first: false,
        }
    }

    /// Unwrap the `Writer` from the `Serializer`.
    #[inline]
    pub fn into_inner(self) -> W {
        self.writer
    }
}

impl<W, F> ser::Serializer for Serializer<W, F>
    where W: io::Write,
          F: Formatter,
{
    type Error = io::Error;

    #[inline]
    fn visit_bool(&mut self, value: bool) -> io::Result<()> {
        if value {
            self.writer.write_all(b"true")
        } else {
            self.writer.write_all(b"false")
        }
    }

    #[inline]
    fn visit_isize(&mut self, value: isize) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_i8(&mut self, value: i8) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_i16(&mut self, value: i16) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_i32(&mut self, value: i32) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_i64(&mut self, value: i64) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_usize(&mut self, value: usize) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_u8(&mut self, value: u8) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_u16(&mut self, value: u16) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_u32(&mut self, value: u32) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_u64(&mut self, value: u64) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_f32(&mut self, value: f32) -> io::Result<()> {
        fmt_f32_or_null(&mut self.writer, value)
    }

    #[inline]
    fn visit_f64(&mut self, value: f64) -> io::Result<()> {
        fmt_f64_or_null(&mut self.writer, value)
    }

    #[inline]
    fn visit_char(&mut self, value: char) -> io::Result<()> {
        escape_char(&mut self.writer, value)
    }

    #[inline]
    fn visit_str(&mut self, value: &str) -> io::Result<()> {
        escape_str(&mut self.writer, value)
    }

    #[inline]
    fn visit_none(&mut self) -> io::Result<()> {
        self.visit_unit()
    }

    #[inline]
    fn visit_some<V>(&mut self, value: V) -> io::Result<()>
        where V: ser::Serialize
    {
        value.serialize(self)
    }

    #[inline]
    fn visit_unit(&mut self) -> io::Result<()> {
        self.writer.write_all(b"null")
    }

    #[inline]
    fn visit_enum_unit(&mut self, _name: &str, variant: &str) -> io::Result<()> {
        try!(self.formatter.open(&mut self.writer, b'{'));
        try!(self.formatter.comma(&mut self.writer, true));
        try!(self.visit_str(variant));
        try!(self.formatter.colon(&mut self.writer));
        try!(self.writer.write_all(b"[]"));
        self.formatter.close(&mut self.writer, b'}')
    }

    #[inline]
    fn visit_seq<V>(&mut self, mut visitor: V) -> io::Result<()>
        where V: ser::SeqVisitor,
    {
        match visitor.len() {
            Some(len) if len == 0 => {
                self.writer.write_all(b"[]")
            }
            _ => {
                try!(self.formatter.open(&mut self.writer, b'['));

                self.first = true;

                while let Some(()) = try!(visitor.visit(self)) { }

                self.formatter.close(&mut self.writer, b']')
            }
        }

    }

    #[inline]
    fn visit_enum_seq<V>(&mut self, _name: &str, variant: &str, visitor: V) -> io::Result<()>
        where V: ser::SeqVisitor,
    {
        try!(self.formatter.open(&mut self.writer, b'{'));
        try!(self.formatter.comma(&mut self.writer, true));
        try!(self.visit_str(variant));
        try!(self.formatter.colon(&mut self.writer));
        try!(self.visit_seq(visitor));
        self.formatter.close(&mut self.writer, b'}')
    }

    #[inline]
    fn visit_seq_elt<T>(&mut self, value: T) -> io::Result<()>
        where T: ser::Serialize,
    {
        try!(self.formatter.comma(&mut self.writer, self.first));
        self.first = false;

        value.serialize(self)
    }

    #[inline]
    fn visit_map<V>(&mut self, mut visitor: V) -> io::Result<()>
        where V: ser::MapVisitor,
    {
        match visitor.len() {
            Some(len) if len == 0 => {
                self.writer.write_all(b"{}")
            }
            _ => {
                try!(self.formatter.open(&mut self.writer, b'{'));

                self.first = true;

                while let Some(()) = try!(visitor.visit(self)) { }

                self.formatter.close(&mut self.writer, b'}')
            }
        }
    }

    #[inline]
    fn visit_enum_map<V>(&mut self, _name: &str, variant: &str, visitor: V) -> io::Result<()>
        where V: ser::MapVisitor,
    {
        try!(self.formatter.open(&mut self.writer, b'{'));
        try!(self.formatter.comma(&mut self.writer, true));
        try!(self.visit_str(variant));
        try!(self.formatter.colon(&mut self.writer));
        try!(self.visit_map(visitor));

        self.formatter.close(&mut self.writer, b'}')
    }

    #[inline]
    fn visit_map_elt<K, V>(&mut self, key: K, value: V) -> io::Result<()>
        where K: ser::Serialize,
              V: ser::Serialize,
    {
        try!(self.formatter.comma(&mut self.writer, self.first));
        self.first = false;

        try!(key.serialize(self));
        try!(self.formatter.colon(&mut self.writer));
        value.serialize(self)
    }
}

pub trait Formatter {
    fn open<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write;

    fn comma<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
        where W: io::Write;

    fn colon<W>(&mut self, writer: &mut W) -> io::Result<()>
        where W: io::Write;

    fn close<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write;
}

pub struct CompactFormatter;

impl Formatter for CompactFormatter {
    fn open<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write,
    {
        writer.write_all(&[ch])
    }

    fn comma<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
        where W: io::Write,
    {
        if first {
            Ok(())
        } else {
            writer.write_all(b",")
        }
    }

    fn colon<W>(&mut self, writer: &mut W) -> io::Result<()>
        where W: io::Write,
    {
        writer.write_all(b":")
    }

    fn close<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write,
    {
        writer.write_all(&[ch])
    }
}

pub struct PrettyFormatter {
    current_indent: usize,
    indent: usize,
}

impl Formatter for PrettyFormatter {
    fn open<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write,
    {
        self.current_indent += self.indent;
        writer.write_all(&[ch])
    }

    fn comma<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
        where W: io::Write,
    {
        if first {
            try!(writer.write_all(b"\n"));
        } else {
            try!(writer.write_all(b",\n"));
        }

        spaces(writer, self.current_indent)
    }

    fn colon<W>(&mut self, writer: &mut W) -> io::Result<()>
        where W: io::Write,
    {
        writer.write_all(b": ")
    }

    fn close<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write,
    {
        self.current_indent -= self.indent;
        try!(writer.write(b"\n"));
        try!(spaces(writer, self.current_indent));

        writer.write_all(&[ch])
    }
}

#[inline]
pub fn escape_bytes<W>(wr: &mut W, bytes: &[u8]) -> io::Result<()>
    where W: io::Write
{
    try!(wr.write_all(b"\""));

    let mut start = 0;

    for (i, byte) in bytes.iter().enumerate() {
        let escaped = match *byte {
            b'"' => b"\\\"",
            b'\\' => b"\\\\",
            b'\x08' => b"\\b",
            b'\x0c' => b"\\f",
            b'\n' => b"\\n",
            b'\r' => b"\\r",
            b'\t' => b"\\t",
            _ => { continue; }
        };

        if start < i {
            try!(wr.write_all(&bytes[start..i]));
        }

        try!(wr.write_all(escaped));

        start = i + 1;
    }

    if start != bytes.len() {
        try!(wr.write_all(&bytes[start..]));
    }

    try!(wr.write_all(b"\""));
    Ok(())
}

#[inline]
pub fn escape_str<W>(wr: &mut W, value: &str) -> io::Result<()>
    where W: io::Write
{
    escape_bytes(wr, value.as_bytes())
}

#[inline]
fn escape_char<W>(wr: &mut W, value: char) -> io::Result<()>
    where W: io::Write
{
    // FIXME: this allocation is required in order to be compatible with stable
    // rust, which doesn't support encoding a `char` into a stack buffer.
    escape_bytes(wr, value.to_string().as_bytes())
}

fn fmt_f32_or_null<W>(wr: &mut W, value: f32) -> io::Result<()>
    where W: io::Write
{
    match value.classify() {
        FpCategory::Nan | FpCategory::Infinite => wr.write_all(b"null"),
        _ => {
            let s = value.to_string();
            try!(wr.write_all(s.as_bytes()));
            if !s.contains('.') {
                try!(wr.write_all(b".0"))
            }
            Ok(())
        }
    }
}

fn fmt_f64_or_null<W>(wr: &mut W, value: f64) -> io::Result<()>
    where W: io::Write
{
    match value.classify() {
        FpCategory::Nan | FpCategory::Infinite => wr.write_all(b"null"),
        _ => {
            let s = value.to_string();
            try!(wr.write_all(s.as_bytes()));
            if !s.contains('.') {
                try!(wr.write_all(b".0"))
            }
            Ok(())
        }
    }
}

/// Encode the specified struct into a json `[u8]` writer.
#[inline]
pub fn to_writer<W, T>(writer: &mut W, value: &T) -> io::Result<()>
    where W: io::Write,
          T: ser::Serialize,
{
    let mut ser = Serializer::new(writer);
    try!(value.serialize(&mut ser));
    Ok(())
}

/// Encode the specified struct into a json `[u8]` writer.
#[inline]
pub fn to_writer_pretty<W, T>(writer: &mut W, value: &T) -> io::Result<()>
    where W: io::Write,
          T: ser::Serialize,
{
    let mut ser = Serializer::new_with_formatter(writer, PrettyFormatter {
        current_indent: 0,
        indent: 2,
    });
    try!(value.serialize(&mut ser));
    Ok(())
}

/// Encode the specified struct into a json `[u8]` buffer.
#[inline]
pub fn to_vec<T>(value: &T) -> Vec<u8>
    where T: ser::Serialize,
{
    // We are writing to a Vec, which doesn't fail. So we can ignore
    // the error.
    let mut writer = Vec::with_capacity(128);
    to_writer(&mut writer, value).unwrap();
    writer
}

/// Encode the specified struct into a json `[u8]` buffer.
#[inline]
pub fn to_vec_pretty<T>(value: &T) -> Vec<u8>
    where T: ser::Serialize,
{
    // We are writing to a Vec, which doesn't fail. So we can ignore
    // the error.
    let mut writer = Vec::with_capacity(128);
    to_writer_pretty(&mut writer, value).unwrap();
    writer
}

/// Encode the specified struct into a json `String` buffer.
#[inline]
pub fn to_string<T>(value: &T) -> Result<String, FromUtf8Error>
    where T: ser::Serialize
{
    let vec = to_vec(value);
    String::from_utf8(vec)
}

/// Encode the specified struct into a json `String` buffer.
#[inline]
pub fn to_string_pretty<T>(value: &T) -> Result<String, FromUtf8Error>
    where T: ser::Serialize
{
    let vec = to_vec_pretty(value);
    String::from_utf8(vec)
}

fn spaces<W>(wr: &mut W, mut n: usize) -> io::Result<()>
    where W: io::Write,
{
    const LEN: usize = 16;
    const BUF: &'static [u8; LEN] = &[b' '; 16];

    while n >= LEN {
        try!(wr.write_all(BUF));
        n -= LEN;
    }

    if n > 0 {
        wr.write_all(&BUF[..n])
    } else {
        Ok(())
    }
}
<commit_msg>Fix compilation for latest nightly<commit_after>use std::io;
use std::num::FpCategory;
use std::string::FromUtf8Error;

use ser;

/// A structure for implementing serialization to JSON.
pub struct Serializer<W, F=CompactFormatter> {
    writer: W,
    formatter: F,

    /// `first` is used to signify if we should print a comma when we are walking through a
    /// sequence.
    first: bool,
}

impl<W> Serializer<W>
    where W: io::Write,
{
    /// Creates a new JSON visitr whose output will be written to the writer
    /// specified.
    #[inline]
    pub fn new(writer: W) -> Serializer<W> {
        Serializer::new_with_formatter(writer, CompactFormatter)
    }
}

impl<W, F> Serializer<W, F>
    where W: io::Write,
          F: Formatter,
{
    /// Creates a new JSON visitr whose output will be written to the writer
    /// specified.
    #[inline]
    pub fn new_with_formatter(writer: W, formatter: F) -> Serializer<W, F> {
        Serializer {
            writer: writer,
            formatter: formatter,
            first: false,
        }
    }

    /// Unwrap the `Writer` from the `Serializer`.
    #[inline]
    pub fn into_inner(self) -> W {
        self.writer
    }
}

impl<W, F> ser::Serializer for Serializer<W, F>
    where W: io::Write,
          F: Formatter,
{
    type Error = io::Error;

    #[inline]
    fn visit_bool(&mut self, value: bool) -> io::Result<()> {
        if value {
            self.writer.write_all(b"true")
        } else {
            self.writer.write_all(b"false")
        }
    }

    #[inline]
    fn visit_isize(&mut self, value: isize) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_i8(&mut self, value: i8) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_i16(&mut self, value: i16) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_i32(&mut self, value: i32) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_i64(&mut self, value: i64) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_usize(&mut self, value: usize) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_u8(&mut self, value: u8) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_u16(&mut self, value: u16) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_u32(&mut self, value: u32) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_u64(&mut self, value: u64) -> io::Result<()> {
        write!(&mut self.writer, "{}", value)
    }

    #[inline]
    fn visit_f32(&mut self, value: f32) -> io::Result<()> {
        fmt_f32_or_null(&mut self.writer, value)
    }

    #[inline]
    fn visit_f64(&mut self, value: f64) -> io::Result<()> {
        fmt_f64_or_null(&mut self.writer, value)
    }

    #[inline]
    fn visit_char(&mut self, value: char) -> io::Result<()> {
        escape_char(&mut self.writer, value)
    }

    #[inline]
    fn visit_str(&mut self, value: &str) -> io::Result<()> {
        escape_str(&mut self.writer, value)
    }

    #[inline]
    fn visit_none(&mut self) -> io::Result<()> {
        self.visit_unit()
    }

    #[inline]
    fn visit_some<V>(&mut self, value: V) -> io::Result<()>
        where V: ser::Serialize
    {
        value.serialize(self)
    }

    #[inline]
    fn visit_unit(&mut self) -> io::Result<()> {
        self.writer.write_all(b"null")
    }

    #[inline]
    fn visit_enum_unit(&mut self, _name: &str, variant: &str) -> io::Result<()> {
        try!(self.formatter.open(&mut self.writer, b'{'));
        try!(self.formatter.comma(&mut self.writer, true));
        try!(self.visit_str(variant));
        try!(self.formatter.colon(&mut self.writer));
        try!(self.writer.write_all(b"[]"));
        self.formatter.close(&mut self.writer, b'}')
    }

    #[inline]
    fn visit_seq<V>(&mut self, mut visitor: V) -> io::Result<()>
        where V: ser::SeqVisitor,
    {
        match visitor.len() {
            Some(len) if len == 0 => {
                self.writer.write_all(b"[]")
            }
            _ => {
                try!(self.formatter.open(&mut self.writer, b'['));

                self.first = true;

                while let Some(()) = try!(visitor.visit(self)) { }

                self.formatter.close(&mut self.writer, b']')
            }
        }

    }

    #[inline]
    fn visit_enum_seq<V>(&mut self, _name: &str, variant: &str, visitor: V) -> io::Result<()>
        where V: ser::SeqVisitor,
    {
        try!(self.formatter.open(&mut self.writer, b'{'));
        try!(self.formatter.comma(&mut self.writer, true));
        try!(self.visit_str(variant));
        try!(self.formatter.colon(&mut self.writer));
        try!(self.visit_seq(visitor));
        self.formatter.close(&mut self.writer, b'}')
    }

    #[inline]
    fn visit_seq_elt<T>(&mut self, value: T) -> io::Result<()>
        where T: ser::Serialize,
    {
        try!(self.formatter.comma(&mut self.writer, self.first));
        self.first = false;

        value.serialize(self)
    }

    #[inline]
    fn visit_map<V>(&mut self, mut visitor: V) -> io::Result<()>
        where V: ser::MapVisitor,
    {
        match visitor.len() {
            Some(len) if len == 0 => {
                self.writer.write_all(b"{}")
            }
            _ => {
                try!(self.formatter.open(&mut self.writer, b'{'));

                self.first = true;

                while let Some(()) = try!(visitor.visit(self)) { }

                self.formatter.close(&mut self.writer, b'}')
            }
        }
    }

    #[inline]
    fn visit_enum_map<V>(&mut self, _name: &str, variant: &str, visitor: V) -> io::Result<()>
        where V: ser::MapVisitor,
    {
        try!(self.formatter.open(&mut self.writer, b'{'));
        try!(self.formatter.comma(&mut self.writer, true));
        try!(self.visit_str(variant));
        try!(self.formatter.colon(&mut self.writer));
        try!(self.visit_map(visitor));

        self.formatter.close(&mut self.writer, b'}')
    }

    #[inline]
    fn visit_map_elt<K, V>(&mut self, key: K, value: V) -> io::Result<()>
        where K: ser::Serialize,
              V: ser::Serialize,
    {
        try!(self.formatter.comma(&mut self.writer, self.first));
        self.first = false;

        try!(key.serialize(self));
        try!(self.formatter.colon(&mut self.writer));
        value.serialize(self)
    }
}

pub trait Formatter {
    fn open<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write;

    fn comma<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
        where W: io::Write;

    fn colon<W>(&mut self, writer: &mut W) -> io::Result<()>
        where W: io::Write;

    fn close<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write;
}

pub struct CompactFormatter;

impl Formatter for CompactFormatter {
    fn open<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write,
    {
        writer.write_all(&[ch])
    }

    fn comma<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
        where W: io::Write,
    {
        if first {
            Ok(())
        } else {
            writer.write_all(b",")
        }
    }

    fn colon<W>(&mut self, writer: &mut W) -> io::Result<()>
        where W: io::Write,
    {
        writer.write_all(b":")
    }

    fn close<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write,
    {
        writer.write_all(&[ch])
    }
}

pub struct PrettyFormatter {
    current_indent: usize,
    indent: usize,
}

impl Formatter for PrettyFormatter {
    fn open<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write,
    {
        self.current_indent += self.indent;
        writer.write_all(&[ch])
    }

    fn comma<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
        where W: io::Write,
    {
        if first {
            try!(writer.write_all(b"\n"));
        } else {
            try!(writer.write_all(b",\n"));
        }

        spaces(writer, self.current_indent)
    }

    fn colon<W>(&mut self, writer: &mut W) -> io::Result<()>
        where W: io::Write,
    {
        writer.write_all(b": ")
    }

    fn close<W>(&mut self, writer: &mut W, ch: u8) -> io::Result<()>
        where W: io::Write,
    {
        self.current_indent -= self.indent;
        try!(writer.write(b"\n"));
        try!(spaces(writer, self.current_indent));

        writer.write_all(&[ch])
    }
}

#[inline]
pub fn escape_bytes<W>(wr: &mut W, bytes: &[u8]) -> io::Result<()>
    where W: io::Write
{
    try!(wr.write_all(b"\""));

    let mut start = 0;

    for (i, byte) in bytes.iter().enumerate() {
        let escaped = match *byte {
            b'"' => b"\\\"",
            b'\\' => b"\\\\",
            b'\x08' => b"\\b",
            b'\x0c' => b"\\f",
            b'\n' => b"\\n",
            b'\r' => b"\\r",
            b'\t' => b"\\t",
            _ => { continue; }
        };

        if start < i {
            try!(wr.write_all(&bytes[start..i]));
        }

        try!(wr.write_all(escaped));

        start = i + 1;
    }

    if start != bytes.len() {
        try!(wr.write_all(&bytes[start..]));
    }

    try!(wr.write_all(b"\""));
    Ok(())
}

#[inline]
pub fn escape_str<W>(wr: &mut W, value: &str) -> io::Result<()>
    where W: io::Write
{
    escape_bytes(wr, value.as_bytes())
}

#[inline]
fn escape_char<W>(wr: &mut W, value: char) -> io::Result<()>
    where W: io::Write
{
    // FIXME: this allocation is required in order to be compatible with stable
    // rust, which doesn't support encoding a `char` into a stack buffer.
    escape_bytes(wr, value.to_string().as_bytes())
}

fn fmt_f32_or_null<W>(wr: &mut W, value: f32) -> io::Result<()>
    where W: io::Write
{
    match value.classify() {
        FpCategory::Nan | FpCategory::Infinite => wr.write_all(b"null"),
        _ => {
            let s = value.to_string();
            try!(wr.write_all(s.as_bytes()));
            if !s.contains('.') {
                try!(wr.write_all(b".0"))
            }
            Ok(())
        }
    }
}

fn fmt_f64_or_null<W>(wr: &mut W, value: f64) -> io::Result<()>
    where W: io::Write
{
    match value.classify() {
        FpCategory::Nan | FpCategory::Infinite => wr.write_all(b"null"),
        _ => {
            let s = value.to_string();
            try!(wr.write_all(s.as_bytes()));
            if !s.contains('.') {
                try!(wr.write_all(b".0"))
            }
            Ok(())
        }
    }
}

/// Encode the specified struct into a json `[u8]` writer.
#[inline]
pub fn to_writer<W, T>(writer: &mut W, value: &T) -> io::Result<()>
    where W: io::Write,
          T: ser::Serialize,
{
    let mut ser = Serializer::new(writer);
    try!(value.serialize(&mut ser));
    Ok(())
}

/// Encode the specified struct into a json `[u8]` writer.
#[inline]
pub fn to_writer_pretty<W, T>(writer: &mut W, value: &T) -> io::Result<()>
    where W: io::Write,
          T: ser::Serialize,
{
    let mut ser = Serializer::new_with_formatter(writer, PrettyFormatter {
        current_indent: 0,
        indent: 2,
    });
    try!(value.serialize(&mut ser));
    Ok(())
}

/// Encode the specified struct into a json `[u8]` buffer.
#[inline]
pub fn to_vec<T>(value: &T) -> Vec<u8>
    where T: ser::Serialize,
{
    // We are writing to a Vec, which doesn't fail. So we can ignore
    // the error.
    let mut writer = Vec::with_capacity(128);
    to_writer(&mut writer, value).unwrap();
    writer
}

/// Encode the specified struct into a json `[u8]` buffer.
#[inline]
pub fn to_vec_pretty<T>(value: &T) -> Vec<u8>
    where T: ser::Serialize,
{
    // We are writing to a Vec, which doesn't fail. So we can ignore
    // the error.
    let mut writer = Vec::with_capacity(128);
    to_writer_pretty(&mut writer, value).unwrap();
    writer
}

/// Encode the specified struct into a json `String` buffer.
#[inline]
pub fn to_string<T>(value: &T) -> Result<String, FromUtf8Error>
    where T: ser::Serialize
{
    let vec = to_vec(value);
    String::from_utf8(vec)
}

/// Encode the specified struct into a json `String` buffer.
#[inline]
pub fn to_string_pretty<T>(value: &T) -> Result<String, FromUtf8Error>
    where T: ser::Serialize
{
    let vec = to_vec_pretty(value);
    String::from_utf8(vec)
}

fn spaces<W>(wr: &mut W, mut n: usize) -> io::Result<()>
    where W: io::Write,
{
    const LEN: usize = 16;
    const BUF: &'static [u8; LEN] = &[b' '; 16];

    while n >= LEN {
        try!(wr.write_all(BUF));
        n -= LEN;
    }

    if n > 0 {
        wr.write_all(&BUF[..n])
    } else {
        Ok(())
    }
}
<|endoftext|>


<commit_before>//! Protocol messages
//!
//! All protocol messages are serialized by serde according to the bincode data format. For
//! transmission on the wire, `tokio_io::codec::length_delimited` is used to prepend protocol
//! messages by an additional length header, thereby creating frames. It is crucial to
//!
//! The main type `Message` and the types in its fields are dump containers, which are not
//! responsible for any protocol logic (except for syntactic validation including validation of
//! `PublicKey` and `SecretKey` fields). Consequently, all fields of `Message` and all fields of
//! its contained types such as `Header` and `Payload` are public.

use secp256k1::key::{PublicKey, SecretKey};
use secp256k1::Signature;
use ::{SessionId, PeerIndex, SymmetricKey, SequenceNum};
use field::Fp;

/// A protocol message
///
/// Protocol messages consist of a header and a payload.
#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct Message {
    pub header: Header,
    pub payload: Payload,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct Header {
    pub session_id: SessionId,
    pub peer_index: PeerIndex,
    pub sequence_num: SequenceNum,
    pub signature: Signature,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) enum Payload {
    KeyExchange(KeyExchange),
    DcExponential(DcExponential),
    DcMain(DcMain),
    Blame(Blame),
    Confirm(Confirm),
    Reveal(Reveal),
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct KeyExchange {
    pub ke_pk: PublicKey,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct DcExponential {
    pub commitment: [u8; 32],
    pub dc_exp: Vec<Fp>,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct DcMain {
    pub ok: bool,
    pub dc_xor: Vec<Vec<u8>>,
    pub ke_pk: PublicKey,
    pub extension: Extension,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) enum Extension {
    None,
    DcAddSecp256k1Scalar(/* TODO */),
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct Blame {
    pub ke_sk: SecretKey,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub struct Confirm {
    pub data: Vec<u8>,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct Reveal {
    pub keys: Vec<(PeerIndex, SymmetricKey)>,
}

#[cfg(test)]
mod tests {
    use std::io::Cursor;
    use tokio_io::codec::length_delimited;
    use tokio_serde_bincode::{ReadBincode, WriteBincode};
    use futures::sink::Sink;
    use futures::{Future, Stream};

    use super::*;
    use secp256k1::key::SecretKey;

    #[test]
    fn roundtrip_serde_bincode() {
        // Create secret key, public key, message digest and signature
        let slice: [u8; 32] = [0xab; 32];
        let sk = SecretKey::from_slice(&::SECP256K1, &slice).unwrap();
        let ke_pk = PublicKey::from_secret_key(&::SECP256K1, &sk).unwrap();
        assert!(ke_pk.is_valid());
        let slice: [u8; 32] = [0x01; 32];
        let msg = ::secp256k1::Message::from_slice(&slice).unwrap();
        let sig = ::SECP256K1.sign(&msg, &sk).unwrap();

        // Create message
        let msg1 = Message {
            header: Header {
                peer_index: 17,
                session_id: SessionId([56; 32]),
                sequence_num: 14,
                signature: sig,
            },
            payload: Payload::KeyExchange(KeyExchange {
                ke_pk: ke_pk,
            }),
        };

        // Write message
        let pipe = Cursor::new(Vec::new());
        let write = WriteBincode::new(length_delimited::FramedWrite::new(pipe));
        let write = write.send(msg1.clone()).wait().unwrap();

        // Read message again
        let pipe = Cursor::new(write.into_inner().into_inner().into_inner());
        let read = ReadBincode::<_ ,Message>::new(length_delimited::FramedRead::new(pipe));
        let msg2 = read.wait().next().unwrap().unwrap();

        assert_eq!(msg1, msg2);
    }
}
<commit_msg>Add serde roundtrip test for DcExponential<commit_after>//! Protocol messages
//!
//! All protocol messages are serialized by serde according to the bincode data format. For
//! transmission on the wire, `tokio_io::codec::length_delimited` is used to prepend protocol
//! messages by an additional length header, thereby creating frames. It is crucial to
//!
//! The main type `Message` and the types in its fields are dump containers, which are not
//! responsible for any protocol logic (except for syntactic validation including validation of
//! `PublicKey` and `SecretKey` fields). Consequently, all fields of `Message` and all fields of
//! its contained types such as `Header` and `Payload` are public.

use secp256k1::key::{PublicKey, SecretKey};
use secp256k1::Signature;
use ::{SessionId, PeerIndex, SymmetricKey, SequenceNum};
use field::Fp;

/// A protocol message
///
/// Protocol messages consist of a header and a payload.
#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct Message {
    pub header: Header,
    pub payload: Payload,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct Header {
    pub session_id: SessionId,
    pub peer_index: PeerIndex,
    pub sequence_num: SequenceNum,
    pub signature: Signature,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) enum Payload {
    KeyExchange(KeyExchange),
    DcExponential(DcExponential),
    DcMain(DcMain),
    Blame(Blame),
    Confirm(Confirm),
    Reveal(Reveal),
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct KeyExchange {
    pub ke_pk: PublicKey,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct DcExponential {
    pub commitment: [u8; 32],
    pub dc_exp: Vec<Fp>,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct DcMain {
    pub ok: bool,
    pub dc_xor: Vec<Vec<u8>>,
    pub ke_pk: PublicKey,
    pub extension: Extension,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) enum Extension {
    None,
    DcAddSecp256k1Scalar(/* TODO */),
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct Blame {
    pub ke_sk: SecretKey,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub struct Confirm {
    pub data: Vec<u8>,
}

#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
pub(crate) struct Reveal {
    pub keys: Vec<(PeerIndex, SymmetricKey)>,
}

#[cfg(test)]
mod tests {
    use std::io::Cursor;
    use tokio_io::codec::length_delimited;
    use tokio_serde_bincode::{ReadBincode, WriteBincode};
    use futures::sink::Sink;
    use futures::{Future, Stream};

    use super::*;
    use secp256k1::key::SecretKey;

    #[cfg(test)]
    fn roundtrip_serde_bincode(payload: Payload) {
        let msg1 = Message {
            header: dummy_header(),
            payload: payload,
        };

        // Write message
        let pipe = Cursor::new(Vec::new());
        let write = WriteBincode::new(length_delimited::FramedWrite::new(pipe));
        let write = write.send(msg1.clone()).wait().unwrap();

        // Read message again
        let pipe = Cursor::new(write.into_inner().into_inner().into_inner());
        let read = ReadBincode::<_ ,Message>::new(length_delimited::FramedRead::new(pipe));
        let msg2 = read.wait().next().unwrap().unwrap();

        assert_eq!(msg1, msg2);
    }

    #[cfg(test)]
    fn dummy_header() -> Header {
        // Create secret key, public key, message digest and signature
        let slice: [u8; 32] = [0xab; 32];
        let sk = SecretKey::from_slice(&::SECP256K1, &slice).unwrap();
        let slice: [u8; 32] = [0x01; 32];
        let sign_msg = ::secp256k1::Message::from_slice(&slice).unwrap();
        let sig = ::SECP256K1.sign(&sign_msg, &sk).unwrap();

        Header {
            peer_index: 2,
            session_id: SessionId([56; 32]),
            sequence_num: 14,
            signature: sig,
        }
    }

    #[test]
    fn roundtrip_keyexchange() {
        let slice: [u8; 32] = [0x4f; 32];
        let sk = SecretKey::from_slice(&::SECP256K1, &slice).unwrap();
        let ke_pk = PublicKey::from_secret_key(&::SECP256K1, &sk).unwrap();
        assert!(ke_pk.is_valid());

        let payload = Payload::KeyExchange(KeyExchange {
            ke_pk: ke_pk,
        });

        roundtrip_serde_bincode(payload);
    }

    #[test]
    fn roundtrip_dcexponential() {
        let payload = Payload::DcExponential(DcExponential {
            commitment: [9; 32],
            dc_exp: vec![Fp::from_u127(Fp::prime()), Fp::from_u127(0), Fp::from_u127(656)],
        });

        roundtrip_serde_bincode(payload);
    }
}
<|endoftext|>


<commit_before>pub mod get;
pub mod put;
pub mod list;

use std::fmt;
use ::rustc_serialize::base64::FromBase64;

// struct that mirrors the json values from consul
// using snake case as #[serde(alias="")] wouldn't work for me
#[allow(non_snake_case)]
#[derive(RustcDecodable, Serialize, Deserialize, Debug, Clone)]
pub struct ValueData {
  pub CreateIndex: i32,
  pub ModifyIndex: i32,
  pub LockIndex: i32,
  pub Key: String,
  pub Flags: i32,
  pub Value: String,
}

impl fmt::Display for ValueData {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let value = String::from_utf8(self.Value.to_owned().from_base64().unwrap()).unwrap();
        write!(f, "{} {} {} {} {} {}",
            self.CreateIndex,
            self.ModifyIndex,
            self.LockIndex,
            self.Key,
            self.Flags,
            value,
        )
    }
}

pub fn decode_json (json: Vec<ValueData>) -> Result<String, ::serde_json::error::Error> {
    // let value = String::from_utf8(value.to_owned().from_base64().unwrap()).unwrap();
    // let mut json_new = String::new();
    // json_new = "poop".to_owned()
    ::serde_json::to_string_pretty(&json)
}
<commit_msg>revert back.<commit_after>pub mod get;
pub mod put;
pub mod list;

use std::fmt;
use ::rustc_serialize::base64::FromBase64;

// struct that mirrors the json values from consul
// using snake case as #[serde(alias="")] wouldn't work for me
#[allow(non_snake_case)]
#[derive(RustcDecodable, Serialize, Deserialize, Debug, Clone)]
pub struct ValueData {
  pub CreateIndex: i32,
  pub ModifyIndex: i32,
  pub LockIndex: i32,
  pub Key: String,
  pub Flags: i32,
  pub Value: String,
}

impl fmt::Display for ValueData {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{} {} {} {} {} {}",
            self.CreateIndex,
            self.ModifyIndex,
            self.LockIndex,
            self.Key,
            self.Flags,
            self.value,
        )
    }
}

pub fn decode_json (json: Vec<ValueData>) -> Result<String, ::serde_json::error::Error> {
    // let value = String::from_utf8(value.to_owned().from_base64().unwrap()).unwrap();
    // let mut json_new = String::new();
    // json_new = "poop".to_owned()
    ::serde_json::to_string_pretty(&json)
}
<|endoftext|>


<commit_before>use std::mem::swap;
use std::ops::Deref;

use unicode_width::UnicodeWidthStr;

use strcursor::StrCursor;

use builder::Builder;

pub struct Buffer {
    front_buf: String,
    back_buf: String,
    pos: usize
}

impl Buffer {

    pub fn new() -> Buffer {
        Buffer {
            front_buf: String::new(),
            back_buf: String::new(),
            pos: 0
        }
    }

    pub fn swap(&mut self) {
        swap(&mut self.front_buf, &mut self.back_buf);
        self.pos = self.front_buf.len();
    }

    pub fn replace(&mut self, s: &str) {
        self.front_buf.clear();
        self.front_buf.extend(s.chars());
        self.pos = s.len();
    }

    pub fn insert_char_at_cursor(&mut self, c: char) {
        self.front_buf.insert(self.pos, c);
        self.pos += c.len_utf8();
    }

    pub fn insert_chars_at_cursor(&mut self, s: String) {
        for c in s.chars() {
            self.insert_char_at_cursor(c);
        }
    }

    pub fn replace_chars_at_cursor(&mut self, s: String) {
        self.delete_char_right_of_cursor();
        let insert_len = s.chars().count();
        self.insert_chars_at_cursor(s);
        self.pos -= insert_len;
    }

    pub fn delete_char_left_of_cursor(&mut self) {
        if self.move_left() {
            self.front_buf.remove(self.pos);
        }
    }

    pub fn delete_char_right_of_cursor(&mut self) -> bool {
        if self.pos < self.front_buf.len() {
            self.front_buf.remove(self.pos);
            return true;
        } else {
            return false;
        }
    }

    fn cursor(&self) -> StrCursor {
        StrCursor::new_at_left_of_byte_pos(self.front_buf.deref(), self.pos)
    }

    fn prev_pos(&self) -> Option<usize> {
        self.cursor().at_prev().map(|c| c.byte_pos())
    }

    pub fn move_left(&mut self) -> bool {
        match self.prev_pos() {
            Some(pos) => {
                self.pos = pos;
                true
            },
            None => false
        }
    }

    fn next_pos(&self) -> Option<usize> {
        self.cursor().at_next().map(|c| c.byte_pos())
    }

    pub fn move_right(&mut self) -> bool {
        match self.next_pos() {
            Some(pos) => {
                self.pos = pos;
                true
            },
            None => false
        }
    }

    /// If the cursor is one past the end of the line, move one character to the left.
    pub fn exclude_eol(&mut self) {
        self.move_right();
        self.move_left();
    }

    pub fn move_start(&mut self) {
        self.pos = 0;
    }

    pub fn move_end(&mut self) {
        self.pos = self.front_buf.len();
    }

    pub fn move_to_end_of_word(&mut self) {
        self.vi_move_word_end();
    }

    fn vi_move_word_end(&mut self, move_mode: ViMoveMode) {
        enum State {
            Whitespace,
            EndOnWord,
            EndOnOther,
        };

        let mut state = State::Whitespace;

        while self.move_right() {

            // XXX maybe use for self.cursor().slice_after().char_indicies()
            // XXX should we use self.cursor().after()?
            let c = match self.cursor().cp_after() {
                Some(c) => c,
                _ => return,
            };

            match state {
                State::Whitespace => match c {
                    // skip initial whitespace
                    c if c.is_whitespace() => {},
                    // if we are in keyword mode and found a keyword, stop on word
                    c if is_vi_keyword(c) => {
                        state = State::EndOnWord;
                    },
                    // in keyword mode, found non-whitespace non-keyword, stop on anything
                    _ => {
                        state = State::EndOnOther;
                    }
                },
                State::EndOnWord if !is_vi_keyword(c) => {
                    self.move_left();
                    return;
                },
                State::EndOnOther if c.is_whitespace() || is_vi_keyword(c) => {
                    self.move_left();
                    return;
                },
                _ => {},
            }
        }
    }

    fn char_pos(&self) -> usize {
         UnicodeWidthStr::width(self.cursor().slice_before())
    }

    fn move_to_pos(&mut self, pos: usize) {
        if pos > self.front_buf.len() {
            self.move_end();
        }
        else {
            self.pos = pos;
        }
    }

    pub fn get_line(&self, prompt: &str, clear: bool) -> Vec<u8> {
        let mut line = Builder::new();
        if clear {
            line.clear_screen();
        }
        line.carriage_return();
        line.append(prompt);
        line.append(&self.front_buf);
        line.erase_to_right();
        line.set_cursor_pos(prompt.len() + self.char_pos());
        line.build()
    }

    pub fn to_string(self) -> String {
        self.front_buf
    }

    pub fn drain(&mut self) -> String {
        let mut s = String::new();
        swap(&mut s, &mut self.front_buf);
        self.pos = 0;
        s
    }
}

/// All alphanumeric characters and _ are considered valid for keywords in vi by default.
fn is_vi_keyword(c: char) -> bool {
    match c {
        '_' => true,
        c if c.is_alphanumeric() => true,
        _ => false,
    }
}

#[test]
fn move_and_insert_ascii() {
    let mut buf = Buffer::new();
    buf.insert_char_at_cursor('a');
    buf.move_left();
    buf.insert_char_at_cursor('x');
    buf.move_left();
    buf.move_right();
    buf.move_right();
    buf.insert_char_at_cursor('b');
    buf.move_start();
    buf.insert_char_at_cursor('w');
    buf.move_end();
    buf.insert_char_at_cursor('c');
    buf.move_left();
    buf.move_left();
    buf.delete_char_left_of_cursor();
    assert_eq!(buf.to_string(), "wxbc".to_string());
}

#[test]
fn move_and_insert_cyrillic() {
    let mut buf = Buffer::new();
    buf.insert_char_at_cursor('Й');
    buf.move_left();
    buf.insert_char_at_cursor('ч');
    buf.move_left();
    buf.move_right();
    buf.move_right();
    buf.insert_char_at_cursor('Њ');
    buf.move_start();
    buf.insert_char_at_cursor('Ѿ');
    buf.move_end();
    buf.insert_char_at_cursor('Җ');
    buf.move_left();
    buf.move_left();
    buf.delete_char_left_of_cursor();
    assert_eq!(buf.to_string(), "ѾчЊҖ".to_string());
}

#[test]
fn move_and_insert_cjk() {
    let mut buf = Buffer::new();
    buf.insert_char_at_cursor('䩖');
    buf.move_left();
    buf.insert_char_at_cursor('䨻');
    buf.move_left();
    buf.move_right();
    buf.move_right();
    buf.insert_char_at_cursor('䦴');
    buf.move_start();
    buf.insert_char_at_cursor('乫');
    buf.move_end();
    buf.insert_char_at_cursor('憛');
    buf.move_left();
    buf.move_left();
    buf.delete_char_left_of_cursor();
    assert_eq!(buf.to_string(), "乫䨻䦴憛".to_string());
}

#[test]
fn move_to_pos() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("pos".to_string());
    let pos = buf.char_pos();
    buf.insert_chars_at_cursor("pos".to_string());

    assert!(buf.char_pos() != pos);
    buf.move_to_pos(pos);
    assert_eq!(buf.char_pos(), pos);
}

#[test]
fn move_to_pos_past_end() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("pos".to_string());
    let end_pos = buf.char_pos();

    assert_eq!(buf.char_pos(), end_pos);
    buf.move_to_pos(10_000);
    assert_eq!(buf.char_pos(), end_pos);
}

#[test]
fn move_to_end_of_word_simple() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("here are".to_string());
    let start_pos = buf.char_pos();
    buf.insert_chars_at_cursor(" som".to_string());
    let end_pos = buf.char_pos();
    buf.insert_chars_at_cursor("e words".to_string());
    buf.move_to_pos(start_pos);

    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos);
}

#[test]
fn move_to_end_of_word_comma() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("here ar".to_string());
    let start_pos = buf.char_pos();
    buf.insert_char_at_cursor('e');
    let end_pos1 = buf.char_pos();
    buf.insert_chars_at_cursor(", som".to_string());
    let end_pos2 = buf.char_pos();
    buf.insert_chars_at_cursor("e words".to_string());
    buf.move_to_pos(start_pos);

    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos1);
    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos2);
}

#[test]
fn move_to_end_of_word_nonkeywords() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("here ar".to_string());
    let start_pos = buf.char_pos();
    buf.insert_chars_at_cursor("e,,,".to_string());
    let end_pos1 = buf.char_pos();
    buf.insert_chars_at_cursor(",som".to_string());
    let end_pos2 = buf.char_pos();
    buf.insert_chars_at_cursor("e words".to_string());
    buf.move_to_pos(start_pos);

    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos1);
    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos2);
}

#[test]
fn move_to_end_of_word_whitespace() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("here are".to_string());
    let start_pos = buf.char_pos();
    buf.insert_chars_at_cursor("      som".to_string());
    let end_pos = buf.char_pos();
    buf.insert_chars_at_cursor("e words".to_string());
    buf.move_to_pos(start_pos);

    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos);
}

#[test]
fn move_to_end_of_word_whitespace_nonkeywords() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("here ar".to_string());
    let start_pos = buf.char_pos();
    buf.insert_chars_at_cursor("e   ,,,".to_string());
    let end_pos1 = buf.char_pos();
    buf.insert_chars_at_cursor(", som".to_string());
    let end_pos2 = buf.char_pos();
    buf.insert_chars_at_cursor("e words".to_string());
    buf.move_to_pos(start_pos);

    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos1);
    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos2);
}

#[test]
fn replace_chars_at_cursor() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("text".to_string());
    let pos = buf.char_pos();
    buf.insert_chars_at_cursor(" string".to_string());
    for _ in 0..buf.char_pos() - pos {
        buf.move_left();
    }

    // replace should not move the cursor
    assert_eq!(buf.char_pos(), pos);
    buf.replace_chars_at_cursor("_".to_string());
    assert_eq!(buf.to_string(), "text_string".to_string());
}

#[test]
fn exclude_eol() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("text".to_string());
    let end = buf.char_pos();
    buf.move_left();

    let target = buf.char_pos();
    buf.move_right();
    buf.move_right();

    // should be at the end of the string
    assert_eq!(buf.char_pos(), end);

    // a call to exclude_eol() should move the cursor
    buf.exclude_eol();
    assert_eq!(buf.char_pos(), target);

    // further calls should not move the cursor
    buf.exclude_eol();
    assert_eq!(buf.char_pos(), target);
}
<commit_msg>add support for end of WORD vi moves<commit_after>use std::mem::swap;
use std::ops::Deref;

use unicode_width::UnicodeWidthStr;

use strcursor::StrCursor;

use builder::Builder;

pub struct Buffer {
    front_buf: String,
    back_buf: String,
    pos: usize
}

impl Buffer {

    pub fn new() -> Buffer {
        Buffer {
            front_buf: String::new(),
            back_buf: String::new(),
            pos: 0
        }
    }

    pub fn swap(&mut self) {
        swap(&mut self.front_buf, &mut self.back_buf);
        self.pos = self.front_buf.len();
    }

    pub fn replace(&mut self, s: &str) {
        self.front_buf.clear();
        self.front_buf.extend(s.chars());
        self.pos = s.len();
    }

    pub fn insert_char_at_cursor(&mut self, c: char) {
        self.front_buf.insert(self.pos, c);
        self.pos += c.len_utf8();
    }

    pub fn insert_chars_at_cursor(&mut self, s: String) {
        for c in s.chars() {
            self.insert_char_at_cursor(c);
        }
    }

    pub fn replace_chars_at_cursor(&mut self, s: String) {
        self.delete_char_right_of_cursor();
        let insert_len = s.chars().count();
        self.insert_chars_at_cursor(s);
        self.pos -= insert_len;
    }

    pub fn delete_char_left_of_cursor(&mut self) {
        if self.move_left() {
            self.front_buf.remove(self.pos);
        }
    }

    pub fn delete_char_right_of_cursor(&mut self) -> bool {
        if self.pos < self.front_buf.len() {
            self.front_buf.remove(self.pos);
            return true;
        } else {
            return false;
        }
    }

    fn cursor(&self) -> StrCursor {
        StrCursor::new_at_left_of_byte_pos(self.front_buf.deref(), self.pos)
    }

    fn prev_pos(&self) -> Option<usize> {
        self.cursor().at_prev().map(|c| c.byte_pos())
    }

    pub fn move_left(&mut self) -> bool {
        match self.prev_pos() {
            Some(pos) => {
                self.pos = pos;
                true
            },
            None => false
        }
    }

    fn next_pos(&self) -> Option<usize> {
        self.cursor().at_next().map(|c| c.byte_pos())
    }

    pub fn move_right(&mut self) -> bool {
        match self.next_pos() {
            Some(pos) => {
                self.pos = pos;
                true
            },
            None => false
        }
    }

    /// If the cursor is one past the end of the line, move one character to the left.
    pub fn exclude_eol(&mut self) {
        self.move_right();
        self.move_left();
    }

    pub fn move_start(&mut self) {
        self.pos = 0;
    }

    pub fn move_end(&mut self) {
        self.pos = self.front_buf.len();
    }

    pub fn move_to_end_of_word(&mut self) {
        self.vi_move_word_end(ViMoveMode::Keyword);
    }

    pub fn move_to_end_of_word_ws(&mut self) {
        self.vi_move_word_end(ViMoveMode::Whitespace);
    }

    fn vi_move_word_end(&mut self, move_mode: ViMoveMode) {
        enum State {
            Whitespace,
            EndOnWord,
            EndOnOther,
            EndOnWhitespace,
        };

        let mut state = State::Whitespace;

        while self.move_right() {

            // XXX maybe use for self.cursor().slice_after().char_indicies()
            // XXX should we use self.cursor().after()?
            let c = match self.cursor().cp_after() {
                Some(c) => c,
                _ => return,
            };

            match state {
                State::Whitespace => match c {
                    // skip initial whitespace
                    c if c.is_whitespace() => {},
                    // if we are in keyword mode and found a keyword, stop on word
                    c if move_mode == ViMoveMode::Keyword
                        && is_vi_keyword(c) =>
                    {
                        state = State::EndOnWord;
                    },
                    // not in keyword mode, stop on whitespace
                    _ if move_mode == ViMoveMode::Whitespace => {
                        state = State::EndOnWhitespace;
                    }
                    // in keyword mode, found non-whitespace non-keyword, stop on anything
                    _ => {
                        state = State::EndOnOther;
                    }
                },
                State::EndOnWord if !is_vi_keyword(c) => {
                    self.move_left();
                    return;
                },
                State::EndOnWhitespace if c.is_whitespace() => {
                    self.move_left();
                    return;
                },
                State::EndOnOther if c.is_whitespace() || is_vi_keyword(c) => {
                    self.move_left();
                    return;
                },
                _ => {},
            }
        }
    }

    fn char_pos(&self) -> usize {
         UnicodeWidthStr::width(self.cursor().slice_before())
    }

    fn move_to_pos(&mut self, pos: usize) {
        if pos > self.front_buf.len() {
            self.move_end();
        }
        else {
            self.pos = pos;
        }
    }

    pub fn get_line(&self, prompt: &str, clear: bool) -> Vec<u8> {
        let mut line = Builder::new();
        if clear {
            line.clear_screen();
        }
        line.carriage_return();
        line.append(prompt);
        line.append(&self.front_buf);
        line.erase_to_right();
        line.set_cursor_pos(prompt.len() + self.char_pos());
        line.build()
    }

    pub fn to_string(self) -> String {
        self.front_buf
    }

    pub fn drain(&mut self) -> String {
        let mut s = String::new();
        swap(&mut s, &mut self.front_buf);
        self.pos = 0;
        s
    }
}

#[derive(PartialEq)]
enum ViMoveMode {
    Keyword,
    Whitespace,
}

/// All alphanumeric characters and _ are considered valid for keywords in vi by default.
fn is_vi_keyword(c: char) -> bool {
    match c {
        '_' => true,
        c if c.is_alphanumeric() => true,
        _ => false,
    }
}

#[test]
fn move_and_insert_ascii() {
    let mut buf = Buffer::new();
    buf.insert_char_at_cursor('a');
    buf.move_left();
    buf.insert_char_at_cursor('x');
    buf.move_left();
    buf.move_right();
    buf.move_right();
    buf.insert_char_at_cursor('b');
    buf.move_start();
    buf.insert_char_at_cursor('w');
    buf.move_end();
    buf.insert_char_at_cursor('c');
    buf.move_left();
    buf.move_left();
    buf.delete_char_left_of_cursor();
    assert_eq!(buf.to_string(), "wxbc".to_string());
}

#[test]
fn move_and_insert_cyrillic() {
    let mut buf = Buffer::new();
    buf.insert_char_at_cursor('Й');
    buf.move_left();
    buf.insert_char_at_cursor('ч');
    buf.move_left();
    buf.move_right();
    buf.move_right();
    buf.insert_char_at_cursor('Њ');
    buf.move_start();
    buf.insert_char_at_cursor('Ѿ');
    buf.move_end();
    buf.insert_char_at_cursor('Җ');
    buf.move_left();
    buf.move_left();
    buf.delete_char_left_of_cursor();
    assert_eq!(buf.to_string(), "ѾчЊҖ".to_string());
}

#[test]
fn move_and_insert_cjk() {
    let mut buf = Buffer::new();
    buf.insert_char_at_cursor('䩖');
    buf.move_left();
    buf.insert_char_at_cursor('䨻');
    buf.move_left();
    buf.move_right();
    buf.move_right();
    buf.insert_char_at_cursor('䦴');
    buf.move_start();
    buf.insert_char_at_cursor('乫');
    buf.move_end();
    buf.insert_char_at_cursor('憛');
    buf.move_left();
    buf.move_left();
    buf.delete_char_left_of_cursor();
    assert_eq!(buf.to_string(), "乫䨻䦴憛".to_string());
}

#[test]
fn move_to_pos() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("pos".to_string());
    let pos = buf.char_pos();
    buf.insert_chars_at_cursor("pos".to_string());

    assert!(buf.char_pos() != pos);
    buf.move_to_pos(pos);
    assert_eq!(buf.char_pos(), pos);
}

#[test]
fn move_to_pos_past_end() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("pos".to_string());
    let end_pos = buf.char_pos();

    assert_eq!(buf.char_pos(), end_pos);
    buf.move_to_pos(10_000);
    assert_eq!(buf.char_pos(), end_pos);
}

#[test]
fn move_to_end_of_word_simple() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("here are".to_string());
    let start_pos = buf.char_pos();
    buf.insert_chars_at_cursor(" som".to_string());
    let end_pos = buf.char_pos();
    buf.insert_chars_at_cursor("e words".to_string());
    buf.move_to_pos(start_pos);

    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos);
}

#[test]
fn move_to_end_of_word_comma() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("here ar".to_string());
    let start_pos = buf.char_pos();
    buf.insert_char_at_cursor('e');
    let end_pos1 = buf.char_pos();
    buf.insert_chars_at_cursor(", som".to_string());
    let end_pos2 = buf.char_pos();
    buf.insert_chars_at_cursor("e words".to_string());
    buf.move_to_pos(start_pos);

    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos1);
    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos2);
}

#[test]
fn move_to_end_of_word_nonkeywords() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("here ar".to_string());
    let start_pos = buf.char_pos();
    buf.insert_chars_at_cursor("e,,,".to_string());
    let end_pos1 = buf.char_pos();
    buf.insert_chars_at_cursor(",som".to_string());
    let end_pos2 = buf.char_pos();
    buf.insert_chars_at_cursor("e words".to_string());
    buf.move_to_pos(start_pos);

    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos1);
    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos2);
}

#[test]
fn move_to_end_of_word_whitespace() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("here are".to_string());
    let start_pos = buf.char_pos();
    buf.insert_chars_at_cursor("      som".to_string());
    let end_pos = buf.char_pos();
    buf.insert_chars_at_cursor("e words".to_string());
    buf.move_to_pos(start_pos);

    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos);
}

#[test]
fn move_to_end_of_word_whitespace_nonkeywords() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("here ar".to_string());
    let start_pos = buf.char_pos();
    buf.insert_chars_at_cursor("e   ,,,".to_string());
    let end_pos1 = buf.char_pos();
    buf.insert_chars_at_cursor(", som".to_string());
    let end_pos2 = buf.char_pos();
    buf.insert_chars_at_cursor("e words".to_string());
    buf.move_to_pos(start_pos);

    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos1);
    buf.move_to_end_of_word();
    assert_eq!(buf.char_pos(), end_pos2);
}

#[test]
fn replace_chars_at_cursor() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("text".to_string());
    let pos = buf.char_pos();
    buf.insert_chars_at_cursor(" string".to_string());
    for _ in 0..buf.char_pos() - pos {
        buf.move_left();
    }

    // replace should not move the cursor
    assert_eq!(buf.char_pos(), pos);
    buf.replace_chars_at_cursor("_".to_string());
    assert_eq!(buf.to_string(), "text_string".to_string());
}

#[test]
fn exclude_eol() {
    let mut buf = Buffer::new();
    buf.insert_chars_at_cursor("text".to_string());
    let end = buf.char_pos();
    buf.move_left();

    let target = buf.char_pos();
    buf.move_right();
    buf.move_right();

    // should be at the end of the string
    assert_eq!(buf.char_pos(), end);

    // a call to exclude_eol() should move the cursor
    buf.exclude_eol();
    assert_eq!(buf.char_pos(), target);

    // further calls should not move the cursor
    buf.exclude_eol();
    assert_eq!(buf.char_pos(), target);
}
<|endoftext|>


<commit_before>use std::error;
use std::fmt;
use std::io;

/// Errors returned during parsing/reading buffers
#[derive(Clone, Debug, PartialEq)]
pub enum ParseError {
    NeedMore,
    EOF,
    PrematureEOF,
    Invalid(String),
}

impl fmt::Display for ParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Parse Error: {}", *self)
    }
}

impl error::Error for ParseError {
    fn description(&self) -> &str {
        "ParseError"
    }

    fn cause(&self) -> Option<&error::Error> {
        None
    }
}

impl From<io::Error> for ParseError {
    fn from(err: io::Error) -> ParseError {
        ParseError::Invalid(err.to_string())
    }
}

pub struct RecBuffer<'a> {
    file: &'a mut io::Read,
    buf: Vec<u8>,
    offset: usize,
    record_start: usize,
    marks: Vec<usize>,
}

/// A buffer that wraps an object with the `Read` trait and allows extracting
/// a set of slices to data. Acts as a lower-level primitive for our FASTX
/// readers.
impl<'a> RecBuffer<'a> {
    /// Instantiate a new buffer.
    ///
    /// # Panics
    ///
    /// Under some very rare circumstances (setting a `buf_size` larger than 2 Gb
    /// on Mac OS X) a panic can occur. Please use a smaller buffer in this case.
    pub fn new(file: &'a mut io::Read, buf_size: usize) -> RecBuffer {
        let mut buf = Vec::with_capacity(buf_size);
        unsafe {
            buf.set_len(buf_size);
        }
        let amt_read = file.read(&mut buf).unwrap();
        unsafe {
            buf.set_len(amt_read);
        }

        RecBuffer {
            file: file,
            buf: buf,
            offset: 0,
            record_start: 0,
            marks: Vec::new(),
        }
    }

    /// Internal method to refill the internal buffer (and optionally increase
    /// it's capacity if it's not big enough)
    fn refresh(&mut self) -> Result<bool, ParseError> {
        let cur_length = self.buf.len() - self.record_start;
        let new_length = cur_length + self.buf.capacity();

        let mut new_buf = Vec::with_capacity(new_length);
        unsafe {
            new_buf.set_len(new_length);
        }
        new_buf[..cur_length].copy_from_slice(&self.buf[self.record_start..]);
        let amt_read = self.file.read(&mut new_buf[cur_length..])?;
        unsafe {
            new_buf.set_len(cur_length + amt_read);
        }
        self.buf = new_buf;
        self.offset -= self.record_start;
        self.record_start = 0;

        // return eof if we didn't get anything
        Ok(amt_read == 0)
    }

    pub fn mark_field<F>(&mut self, mut rec_fn: F) -> Result<usize, ParseError>
        where F: FnMut(&[u8], bool) -> Result<usize, ParseError>
    {
        let mut eof = false;
        loop {
            if self.offset == self.buf.len() {
                eof = self.refresh()?;
                if eof {
                    return Err(ParseError::EOF);
                }
            }
            let result = rec_fn(&self.buf[self.offset..], eof);
            match (result, eof) {
                (Ok(v), _) => {
                    self.marks.push(v);
                    self.offset += v;
                    return Ok(v);
                },
                (Err(ParseError::NeedMore), false) => {
                    eof = self.refresh()?;
                },
                (Err(ParseError::NeedMore), true) => return Err(ParseError::PrematureEOF),
                (Err(e), false) => return Err(e),
                _ => panic!("Unknown contingency; remove at some point and set Err(e), true above instead"),
            }
        }
    }

    pub fn fields<'b>(&'b mut self, fields: &mut [&'b [u8]]) {
        // TODO: it would be nice if this could just return the array so we don't
        // have to have an unsafe mem::uninitialized to set up for this
        let mut cum_pos = self.record_start;
        let mut i = 0;
        for mark in &self.marks {
            fields[i] = &self.buf[cum_pos..cum_pos + *mark];
            cum_pos += *mark;
            i += 1;
        }

        self.marks.clear();
        self.record_start = self.offset;
    }
}

#[test]
fn test_buffer() {
    use std::mem;

    let test: [u8; 4] = [1, 2, 3, 4];
    let mut slice = &test[..];
    let mut b = RecBuffer::new(&mut slice, 2usize);

    let field_len = b.mark_field(|buf, eof| {
        assert_eq!(buf[0], 1);
        assert_eq!(buf.len(), 2);
        assert!(!eof);
        Ok(1)
    });
    assert_eq!(field_len.unwrap(), 1);

    let field_len = b.mark_field(|buf, eof| {
        if buf.len() == 1 {
            assert_eq!(buf[0], 2);
            assert!(!eof);
            return Err(ParseError::NeedMore);
        }
        if !eof {
            assert_eq!(buf[0], 2);
            assert_eq!(buf.len(), 3);
            return Err(ParseError::NeedMore);
        }

        assert_eq!(buf[0], 2);
        assert!(eof);
        assert_eq!(buf.len(), 3);
        Ok(3)
    });
    assert_eq!(field_len.unwrap(), 3);

    let mut fields: [&[u8]; 2];
    unsafe {
        fields = mem::uninitialized();
    }
    b.fields(&mut fields);
    assert_eq!(fields[0], &[1]);
    assert_eq!(fields[1], &[2, 3, 4]);
}
<commit_msg>Simplify error case in buffer field marking<commit_after>use std::error;
use std::fmt;
use std::io;

/// Errors returned during parsing/reading buffers
#[derive(Clone, Debug, PartialEq)]
pub enum ParseError {
    NeedMore,
    EOF,
    PrematureEOF,
    Invalid(String),
}

impl fmt::Display for ParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Parse Error: {}", *self)
    }
}

impl error::Error for ParseError {
    fn description(&self) -> &str {
        "ParseError"
    }

    fn cause(&self) -> Option<&error::Error> {
        None
    }
}

impl From<io::Error> for ParseError {
    fn from(err: io::Error) -> ParseError {
        ParseError::Invalid(err.to_string())
    }
}

pub struct RecBuffer<'a> {
    file: &'a mut io::Read,
    buf: Vec<u8>,
    offset: usize,
    record_start: usize,
    marks: Vec<usize>,
}

/// A buffer that wraps an object with the `Read` trait and allows extracting
/// a set of slices to data. Acts as a lower-level primitive for our FASTX
/// readers.
impl<'a> RecBuffer<'a> {
    /// Instantiate a new buffer.
    ///
    /// # Panics
    ///
    /// Under some very rare circumstances (setting a `buf_size` larger than 2 Gb
    /// on Mac OS X) a panic can occur. Please use a smaller buffer in this case.
    pub fn new(file: &'a mut io::Read, buf_size: usize) -> RecBuffer {
        let mut buf = Vec::with_capacity(buf_size);
        unsafe {
            buf.set_len(buf_size);
        }
        let amt_read = file.read(&mut buf).unwrap();
        unsafe {
            buf.set_len(amt_read);
        }

        RecBuffer {
            file: file,
            buf: buf,
            offset: 0,
            record_start: 0,
            marks: Vec::new(),
        }
    }

    /// Internal method to refill the internal buffer (and optionally increase
    /// it's capacity if it's not big enough)
    fn refresh(&mut self) -> Result<bool, ParseError> {
        let cur_length = self.buf.len() - self.record_start;
        let new_length = cur_length + self.buf.capacity();

        let mut new_buf = Vec::with_capacity(new_length);
        unsafe {
            new_buf.set_len(new_length);
        }
        new_buf[..cur_length].copy_from_slice(&self.buf[self.record_start..]);
        let amt_read = self.file.read(&mut new_buf[cur_length..])?;
        unsafe {
            new_buf.set_len(cur_length + amt_read);
        }
        self.buf = new_buf;
        self.offset -= self.record_start;
        self.record_start = 0;

        // return eof if we didn't get anything
        Ok(amt_read == 0)
    }

    pub fn mark_field<F>(&mut self, mut rec_fn: F) -> Result<usize, ParseError>
        where F: FnMut(&[u8], bool) -> Result<usize, ParseError>
    {
        let mut eof = false;
        loop {
            if self.offset == self.buf.len() {
                eof = self.refresh()?;
                if eof {
                    return Err(ParseError::EOF);
                }
            }
            let result = rec_fn(&self.buf[self.offset..], eof);
            match (result, eof) {
                (Ok(v), _) => {
                    self.marks.push(v);
                    self.offset += v;
                    return Ok(v);
                },
                (Err(ParseError::NeedMore), false) => {
                    eof = self.refresh()?;
                },
                (Err(ParseError::NeedMore), true) => return Err(ParseError::PrematureEOF),
                (Err(e), _) => return Err(e),
            }
        }
    }

    pub fn fields<'b>(&'b mut self, fields: &mut [&'b [u8]]) {
        // TODO: it would be nice if this could just return the array so we don't
        // have to have an unsafe mem::uninitialized to set up for this
        let mut cum_pos = self.record_start;
        let mut i = 0;
        for mark in &self.marks {
            fields[i] = &self.buf[cum_pos..cum_pos + *mark];
            cum_pos += *mark;
            i += 1;
        }

        self.marks.clear();
        self.record_start = self.offset;
    }
}

#[test]
fn test_buffer() {
    use std::mem;

    let test: [u8; 4] = [1, 2, 3, 4];
    let mut slice = &test[..];
    let mut b = RecBuffer::new(&mut slice, 2usize);

    let field_len = b.mark_field(|buf, eof| {
        assert_eq!(buf[0], 1);
        assert_eq!(buf.len(), 2);
        assert!(!eof);
        Ok(1)
    });
    assert_eq!(field_len.unwrap(), 1);

    let field_len = b.mark_field(|buf, eof| {
        if buf.len() == 1 {
            assert_eq!(buf[0], 2);
            assert!(!eof);
            return Err(ParseError::NeedMore);
        }
        if !eof {
            assert_eq!(buf[0], 2);
            assert_eq!(buf.len(), 3);
            return Err(ParseError::NeedMore);
        }

        assert_eq!(buf[0], 2);
        assert!(eof);
        assert_eq!(buf.len(), 3);
        Ok(3)
    });
    assert_eq!(field_len.unwrap(), 3);

    let mut fields: [&[u8]; 2];
    unsafe {
        fields = mem::uninitialized();
    }
    b.fields(&mut fields);
    assert_eq!(fields[0], &[1]);
    assert_eq!(fields[1], &[2, 3, 4]);
}
<|endoftext|>


<commit_before>//! Experiments with parallel processing
//!
//! The provided functions focus on the possibility of
//! returning results while the parser proceeds. Sequences are processesd in
//! batches (`RecordSet`) because sending across channels has a performance
//! impact. FASTA/FASTQ records can be accessed in both the 'worker' function and
//! (after processing) a function running in the main thread.
//!
//! # Search first occurrence of a sequence pattern
//!
//! ```no_run
//! use seq_io::fastq::{Reader,Record};
//! use seq_io::parallel::read_parallel;
//!
//! let reader = Reader::from_path("seqs.fastq").unwrap();
//!
//! read_parallel(reader, 4, 2, |record_set| {
//!     // this function does the heavy work
//!     for (i, record) in record_set.into_iter().enumerate() {
//!         // this is not very efficient code, just for demonstration
//!         if let Some(pos) = record.seq().windows(3).position(|s| s == b"AAA") {
//!             return Some((i, pos));
//!         }
//!     }
//!     None
//! }, |record_sets| {
//!     // This function runs in the main thread. It provides a streaming iterator over
//!     // record sets and the corresponding return values from the worker function
//!     // (not necessarily in the same order as in the file)
//!     while let Some(result) = record_sets.next() {
//!         let (record_set, found) = result.unwrap();
//!         if let Some((i, pos)) = found {
//!             let record = record_set.into_iter().nth(i).unwrap();
//!             println!("Found AAA in record {} at position {}", record.id().unwrap(), pos);
//!              // this will also stop the worker threads, although with some delay
//!             return;
//!         }
//!     }
//! });
//! ```
//!
//! # Per-record processsing
//! The `parallel_fasta` / `parallel_fastq` functions are designed to efficiently pass
//! results for **each record** to the main thread without having to care about record sets.
//! This example filters sequences by the occurrence of a pattern:
//!
//! ```no_run
//! use seq_io::fastq::{Reader,Record};
//! use std::fs::File;
//! use std::io::BufWriter;
//!
//! let reader = Reader::from_path("seqs.fastq").unwrap();
//! let mut writer = BufWriter::new(File::create("filtered.fastq").unwrap());
//!
//! seq_io::parallel::parallel_fastq(reader, 4, 2,
//!     |record, found| { // runs in worker
//!         *found = record.seq().windows(3).position(|s| s == b"AAA").is_some();
//!     },
//!     |record, found| { // runs in main thread
//!         if *found {
//!             record.write(&mut writer).unwrap();
//!         }
//!         // `false` will stop the reader
//!         true
//! }).unwrap();
//! ```

use std::io;

extern crate crossbeam;
extern crate scoped_threadpool;

use std::sync::mpsc;
use std::marker::PhantomData;

pub trait Reader: Send {
    type DataSet: Default + Send;
    type Err: Send;
    fn fill_data(&mut self, record: &mut Self::DataSet) -> Option<Result<(), Self::Err>>;
}

pub fn read_parallel<P, O, W, F, Out>(mut reader: P, n_threads: u32, queue_len: usize, work: W, func: F) -> Out
    where P: Reader,
          O: Default + Send,
          W: Send + Sync,
          W: Fn(&mut P::DataSet) -> O,
          F: FnOnce(&mut ParallelRecordsets<P::DataSet, P::Err, O>) -> Out,
{

    let (done_send, done_recv) = mpsc::sync_channel(queue_len);
    let (empty_send, empty_recv): (mpsc::SyncSender<Option<P::DataSet>>, _) = mpsc::sync_channel(queue_len);

    crossbeam::scope(|scope| {

        scope.spawn(move || {

            let mut pool = scoped_threadpool::Pool::new(n_threads);

            pool.scoped(|pool_scope| {

                let work = &work;

                loop {
                    // recycle an old DataSet sent back after use by the streaming iterator
                    let mut data =
                        if let Ok(Some(r)) = empty_recv.recv() {
                            r
                        } else {
                            // 'ParallelRecordsets::stop()' called
                            return;
                        };

                    // each time, we need a new reference
                    let done_send = done_send.clone();

                    if let Some(res) = reader.fill_data(&mut data) {
                        match res {
                            Ok(_) => {
                                // expensive work carried out by func()
                                pool_scope.execute(move || {

                                    let out = work(&mut data);

                                    done_send.send(Some(Ok((data, out)))).ok();
                                });
                            }
                            Err(e) => {
                                done_send.send(Some(Err(e))).ok();
                                break;
                            }
                        }
                    } else {
                        break;
                    }
                }

                // make sure that the 'done' signal is only sent after everything else is done
                pool_scope.join_all();

                done_send.send(None).ok();
            });
        });

        for _ in 0..queue_len  {
            empty_send.send(Some(P::DataSet::default())).unwrap();
        }

        let mut records = ParallelRecordsets {
            empty_send: empty_send,
            done_recv: done_recv,
            current_recordset: P::DataSet::default(),
        };

        let out = func(&mut records);

        records.stop();

        out
    })
}

pub struct ParallelRecordsets<R, E, O>
    where R: Default + Send,
          E: Send,
          O: Send,
{
    empty_send: mpsc::SyncSender<Option<R>>,
    done_recv: mpsc::Receiver<Option<Result<(R, O), E>>>,
    current_recordset: R,
}

impl<R, E, O> ParallelRecordsets<R, E, O>
    where R: Default + Send,
          E: Send,
          O: Send,
 {

    pub fn next<'b>(&'b mut self) -> Option<Result<(&'b R, O), E>> {
        if let Some(result) = self.done_recv.recv().unwrap() {
            match result {
                Ok((r, o)) => {
                     let prev_rset = ::std::mem::replace(&mut self.current_recordset, r);
                     self.empty_send.send(Some(prev_rset)).ok(); // error: channel closed is not a problem, happens after calling stop()
                    return Some(Ok((&self.current_recordset, o)));
                }
                Err(e) => return Some(Err(e))
            }
        } else {
            // 'done' signal received
            return None;
        }
    }

    // has to be called before object goes out of scope
    // currently, the signal is delayed (has to wait until it is popped from the
    // 'empty' queue)
    // TODO: should an AtomicBool be used?
    fn stop(self) {
        self.empty_send.send(None).ok();
    }
}

#[macro_export]
macro_rules! parallel_record_impl {
    ($name:ident, $io_r:tt, $rdr:ty, $record:ty, $err:ty) => {

        pub fn $name<$io_r, O, W, F>(parser: $rdr, n_threads: u32, queue_len: usize, work: W, mut func: F) -> Result<(), $err>
            where $io_r: io::Read + Send,
                  O: Default + Send,
                  W: Send + Sync,
                  W: Fn($record, &mut O),
                  F: FnMut($record, &O) -> bool,
        {
            let reader = $crate::parallel::ReusableReader::new(parser);

            $crate::parallel::read_parallel(reader, n_threads, queue_len, |d| {
                let mut iter = d.0.into_iter();
                let out: &mut Vec<O> = &mut d.1;
                for mut x in out.iter_mut().zip(&mut iter) {
                    work(x.1, &mut x.0);
                }
                for i in iter {
                    out.push(O::default());
                    work(i, out.last_mut().unwrap())
                }

            }, |records| {
                while let Some(result) = records.next() {
                    let (r, _) = result?;
                    for x in r.0.into_iter().zip(&r.1) {
                        if ! func(x.0, x.1) {
                            break;
                        }
                    }
                }
                Ok(())
            })
        }
     };
}

parallel_record_impl!(parallel_fasta, R, fasta::Reader<R>, fasta::RefRecord, fasta::ParseError);

parallel_record_impl!(parallel_fastq, R, fastq::Reader<R>, fastq::RefRecord, fastq::ParseError);

/// Wrapper for `parallel::Reader` instances allowing
/// the output to be reused in order to save allocations.
/// Used by `parallel_fasta`/`parallel_fastq`
///
/// ```no_run
/// use seq_io::fastq::{Reader,Record,RecordSet};
/// use seq_io::parallel::{read_parallel,ReusableReader};
///
/// let inner = Reader::from_path("seqs.fastq").unwrap();
/// let reader = ReusableReader::new(inner);
///
/// read_parallel(reader, 4, 2, |&mut (ref record_set, ref mut out): &mut (RecordSet, Vec<bool>)| {
///     out.clear();
///     for record in record_set {
///         let found = record.seq().windows(3).position(|s| s == b"AAA").is_some();
///         out.push(found);
///     }
/// }, |record_sets| {
///     while let Some(result) = record_sets.next() {
///         let &(ref record_set, ref out) = result.unwrap().0;
///         for (record, found) in record_set.into_iter().zip(out) {
///             // ...
///         }
///     }
/// });
/// ```
pub struct ReusableReader<P, O>(P, PhantomData<O>);

impl<P, O> ReusableReader<P, O> {
    pub fn new(p: P) -> ReusableReader<P, O> {
        ReusableReader(p, PhantomData)
    }
}

impl<P, O> Reader for ReusableReader<P, O>
    where P: Reader,
          O: Default + Send {

    type DataSet = (P::DataSet, O);
    type Err = P::Err;
    fn fill_data(&mut self, data: &mut Self::DataSet) -> Option<Result<(), P::Err>> {
        self.0.fill_data(&mut data.0)
    }
}

/// Using this function currently does not work due to a
/// [compiler bug](https://github.com/rust-lang/rust/issues/42950).
///
/// `parallel_fasta`/`parallel_fastq` provide the same functionality for now
/// (implemented using `parallel_record_impl` macro)
pub fn parallel_records<P, O, W, F, Out>(parser: P, n_threads: u32, queue_len: usize, work: W, mut func: F) -> Result<(), P::Err>
    where P: Reader,
          for<'a> &'a P::DataSet: IntoIterator,
          O: Default + Send,
          W: Send + Sync,
          W: Fn(<&P::DataSet as IntoIterator>::Item, &mut O),
          F: FnMut(<&P::DataSet as IntoIterator>::Item, &O) -> bool,
{
    let reader = ReusableReader(parser, PhantomData);

    read_parallel(reader, n_threads, queue_len, |d| {
        let mut iter = d.0.into_iter();
        let out: &mut Vec<O> = &mut d.1;
        for mut x in out.iter_mut().zip(&mut iter) {
            work(x.1, &mut x.0);
        }
        for i in iter {
            out.push(O::default());
            work(i, out.last_mut().unwrap())
        }

    }, |records| {
        while let Some(result) = records.next() {
            let (r, _) = result?;
            for x in r.0.into_iter().zip(&r.1) {
                if ! func(x.0, x.1) {
                    break;
                }
            }
        }
        Ok(())
    })
}

// trait impls

use super::fasta;

impl<R, S> Reader for fasta::Reader<R, S>
    where R: io::Read + Send,
          S: super::strategy::BufGrowStrategy + Send
{
    type DataSet = fasta::RecordSet;
    type Err = fasta::ParseError;
    fn fill_data(&mut self, rset: &mut fasta::RecordSet) -> Option<Result<(), fasta::ParseError>> {
        self.read_record_set(rset)
    }
}

use super::fastq;

impl<R, S> Reader for fastq::Reader<R, S>
    where R: io::Read + Send,
          S: super::strategy::BufGrowStrategy + Send
{
    type DataSet = fastq::RecordSet;
    type Err = fastq::ParseError;
    fn fill_data(&mut self, rset: &mut fastq::RecordSet) -> Option<Result<(), fastq::ParseError>> {
        self.read_record_set(rset)
    }
}
<commit_msg>Change to parallel API: allow return values<commit_after>//! Experiments with parallel processing
//!
//! The provided functions focus on the possibility of
//! returning results while the parser proceeds. Sequences are processesd in
//! batches (`RecordSet`) because sending across channels has a performance
//! impact. FASTA/FASTQ records can be accessed in both the 'worker' function and
//! (after processing) a function running in the main thread.
//!
//! # Search first occurrence of a sequence pattern
//!
//! ```no_run
//! use seq_io::fastq::{Reader,Record};
//! use seq_io::parallel::read_parallel;
//!
//! let reader = Reader::from_path("seqs.fastq").unwrap();
//!
//! read_parallel(reader, 4, 2, |record_set| {
//!     // this function does the heavy work
//!     for (i, record) in record_set.into_iter().enumerate() {
//!         // this is not very efficient code, just for demonstration
//!         if let Some(pos) = record.seq().windows(3).position(|s| s == b"AAA") {
//!             return Some((i, pos));
//!         }
//!     }
//!     None
//! }, |record_sets| {
//!     // This function runs in the main thread. It provides a streaming iterator over
//!     // record sets and the corresponding return values from the worker function
//!     // (not necessarily in the same order as in the file)
//!     while let Some(result) = record_sets.next() {
//!         let (record_set, found) = result.unwrap();
//!         if let Some((i, pos)) = found {
//!             let record = record_set.into_iter().nth(i).unwrap();
//!             println!("Found AAA in record {} at position {}", record.id().unwrap(), pos);
//!              // this will also stop the worker threads, although with some delay
//!             return;
//!         }
//!     }
//! });
//! ```
//!
//! # Per-record processsing
//! The `parallel_fasta` / `parallel_fastq` functions are designed to efficiently pass
//! results for **each record** to the main thread without having to care about record sets.
//! This example filters sequences by the occurrence of a pattern:
//!
//! ```no_run
//! use seq_io::fastq::{Reader,Record};
//! use std::fs::File;
//! use std::io::BufWriter;
//!
//! let reader = Reader::from_path("seqs.fastq").unwrap();
//! let mut writer = BufWriter::new(File::create("filtered.fastq").unwrap());
//!
//! seq_io::parallel::parallel_fastq(reader, 4, 2,
//!     |record, found| { // runs in worker
//!         *found = record.seq().windows(3).position(|s| s == b"AAA").is_some();
//!     },
//!     |record, found| { // runs in main thread
//!         if *found {
//!             record.write(&mut writer).unwrap();
//!         }
//!         // Some(value) will stop the reader, and the value will be returned
//!         None
//! }).unwrap();
//! ```

use std::io;

extern crate crossbeam;
extern crate scoped_threadpool;

use std::sync::mpsc;
use std::marker::PhantomData;

pub trait Reader: Send {
    type DataSet: Default + Send;
    type Err: Send;
    fn fill_data(&mut self, record: &mut Self::DataSet) -> Option<Result<(), Self::Err>>;
}

pub fn read_parallel<P, O, W, F, Out>(mut reader: P, n_threads: u32, queue_len: usize, work: W, func: F) -> Out
    where P: Reader,
          O: Default + Send,
          W: Send + Sync,
          W: Fn(&mut P::DataSet) -> O,
          F: FnOnce(&mut ParallelRecordsets<P::DataSet, P::Err, O>) -> Out,
{

    let (done_send, done_recv) = mpsc::sync_channel(queue_len);
    let (empty_send, empty_recv): (mpsc::SyncSender<Option<P::DataSet>>, _) = mpsc::sync_channel(queue_len);

    crossbeam::scope(|scope| {

        scope.spawn(move || {

            let mut pool = scoped_threadpool::Pool::new(n_threads);

            pool.scoped(|pool_scope| {

                let work = &work;

                loop {
                    // recycle an old DataSet sent back after use by the streaming iterator
                    let mut data =
                        if let Ok(Some(r)) = empty_recv.recv() {
                            r
                        } else {
                            // 'ParallelRecordsets::stop()' called
                            return;
                        };

                    // each time, we need a new reference
                    let done_send = done_send.clone();

                    if let Some(res) = reader.fill_data(&mut data) {
                        match res {
                            Ok(_) => {
                                // expensive work carried out by func()
                                pool_scope.execute(move || {

                                    let out = work(&mut data);

                                    done_send.send(Some(Ok((data, out)))).ok();
                                });
                            }
                            Err(e) => {
                                done_send.send(Some(Err(e))).ok();
                                break;
                            }
                        }
                    } else {
                        break;
                    }
                }

                // make sure that the 'done' signal is only sent after everything else is done
                pool_scope.join_all();

                done_send.send(None).ok();
            });
        });

        for _ in 0..queue_len  {
            empty_send.send(Some(P::DataSet::default())).unwrap();
        }

        let mut records = ParallelRecordsets {
            empty_send: empty_send,
            done_recv: done_recv,
            current_recordset: P::DataSet::default(),
        };

        let out = func(&mut records);

        records.stop();

        out
    })
}

pub struct ParallelRecordsets<R, E, O>
    where R: Default + Send,
          E: Send,
          O: Send,
{
    empty_send: mpsc::SyncSender<Option<R>>,
    done_recv: mpsc::Receiver<Option<Result<(R, O), E>>>,
    current_recordset: R,
}

impl<R, E, O> ParallelRecordsets<R, E, O>
    where R: Default + Send,
          E: Send,
          O: Send,
 {

    pub fn next<'b>(&'b mut self) -> Option<Result<(&'b R, O), E>> {
        if let Some(result) = self.done_recv.recv().unwrap() {
            match result {
                Ok((r, o)) => {
                     let prev_rset = ::std::mem::replace(&mut self.current_recordset, r);
                     self.empty_send.send(Some(prev_rset)).ok(); // error: channel closed is not a problem, happens after calling stop()
                    return Some(Ok((&self.current_recordset, o)));
                }
                Err(e) => return Some(Err(e))
            }
        } else {
            // 'done' signal received
            return None;
        }
    }

    // has to be called before object goes out of scope
    // currently, the signal is delayed (has to wait until it is popped from the
    // 'empty' queue)
    // TODO: should an AtomicBool be used?
    fn stop(self) {
        self.empty_send.send(None).ok();
    }
}

#[macro_export]
macro_rules! parallel_record_impl {
    ($name:ident, $io_r:tt, $rdr:ty, $record:ty, $err:ty) => {

        pub fn $name<$io_r, O, W, F, Out>(parser: $rdr, n_threads: u32, queue_len: usize, work: W, mut func: F) -> Result<Option<Out>, $err>
            where $io_r: io::Read + Send,
                  O: Default + Send,
                  W: Send + Sync,
                  W: Fn($record, &mut O),
                  F: FnMut($record, &O) -> Option<Out>,
        {
            let reader = $crate::parallel::ReusableReader::new(parser);

            $crate::parallel::read_parallel(reader, n_threads, queue_len, |d| {
                let mut iter = d.0.into_iter();
                let out: &mut Vec<O> = &mut d.1;
                for mut x in out.iter_mut().zip(&mut iter) {
                    work(x.1, &mut x.0);
                }
                for i in iter {
                    out.push(O::default());
                    work(i, out.last_mut().unwrap())
                }

            }, |records| {
                while let Some(result) = records.next() {
                    let (r, _) = result?;
                    for x in r.0.into_iter().zip(&r.1) {
                        if let Some(out) = func(x.0, x.1) {
                            return Ok(Some(out));
                        }
                    }
                }
                Ok(None)
            })
        }
     };
}

parallel_record_impl!(parallel_fasta, R, fasta::Reader<R>, fasta::RefRecord, fasta::ParseError);

parallel_record_impl!(parallel_fastq, R, fastq::Reader<R>, fastq::RefRecord, fastq::ParseError);

/// Wrapper for `parallel::Reader` instances allowing
/// the output to be reused in order to save allocations.
/// Used by `parallel_fasta`/`parallel_fastq`
///
/// ```no_run
/// use seq_io::fastq::{Reader,Record,RecordSet};
/// use seq_io::parallel::{read_parallel,ReusableReader};
///
/// let inner = Reader::from_path("seqs.fastq").unwrap();
/// let reader = ReusableReader::new(inner);
///
/// read_parallel(reader, 4, 2, |&mut (ref record_set, ref mut out): &mut (RecordSet, Vec<bool>)| {
///     out.clear();
///     for record in record_set {
///         let found = record.seq().windows(3).position(|s| s == b"AAA").is_some();
///         out.push(found);
///     }
/// }, |record_sets| {
///     while let Some(result) = record_sets.next() {
///         let &(ref record_set, ref out) = result.unwrap().0;
///         for (record, found) in record_set.into_iter().zip(out) {
///             // ...
///         }
///     }
/// });
/// ```
pub struct ReusableReader<P, O>(P, PhantomData<O>);

impl<P, O> ReusableReader<P, O> {
    pub fn new(p: P) -> ReusableReader<P, O> {
        ReusableReader(p, PhantomData)
    }
}

impl<P, O> Reader for ReusableReader<P, O>
    where P: Reader,
          O: Default + Send {

    type DataSet = (P::DataSet, O);
    type Err = P::Err;
    fn fill_data(&mut self, data: &mut Self::DataSet) -> Option<Result<(), P::Err>> {
        self.0.fill_data(&mut data.0)
    }
}

/// Using this function currently does not work due to a
/// [compiler bug](https://github.com/rust-lang/rust/issues/42950).
///
/// `parallel_fasta`/`parallel_fastq` provide the same functionality for now
/// (implemented using `parallel_record_impl` macro)
pub fn parallel_records<P, O, W, F, Out>(parser: P, n_threads: u32, queue_len: usize, work: W, mut func: F) -> Result<Option<Out>, P::Err>
    where P: Reader,
          for<'a> &'a P::DataSet: IntoIterator,
          O: Default + Send,
          W: Send + Sync,
          W: Fn(<&P::DataSet as IntoIterator>::Item, &mut O),
          F: FnMut(<&P::DataSet as IntoIterator>::Item, &O) -> Option<Out>,
{
    let reader = ReusableReader(parser, PhantomData);

    read_parallel(reader, n_threads, queue_len, |d| {
        let mut iter = d.0.into_iter();
        let out: &mut Vec<O> = &mut d.1;
        for mut x in out.iter_mut().zip(&mut iter) {
            work(x.1, &mut x.0);
        }
        for i in iter {
            out.push(O::default());
            work(i, out.last_mut().unwrap())
        }

    }, |records| {
        while let Some(result) = records.next() {
            let (r, _) = result?;
            for x in r.0.into_iter().zip(&r.1) {
                if let Some(out) = func(x.0, x.1) {
                    return Ok(Some(out));
                }
            }
        }
        Ok(None)
    })
}

// trait impls

use super::fasta;

impl<R, S> Reader for fasta::Reader<R, S>
    where R: io::Read + Send,
          S: super::strategy::BufGrowStrategy + Send
{
    type DataSet = fasta::RecordSet;
    type Err = fasta::ParseError;
    fn fill_data(&mut self, rset: &mut fasta::RecordSet) -> Option<Result<(), fasta::ParseError>> {
        self.read_record_set(rset)
    }
}

use super::fastq;

impl<R, S> Reader for fastq::Reader<R, S>
    where R: io::Read + Send,
          S: super::strategy::BufGrowStrategy + Send
{
    type DataSet = fastq::RecordSet;
    type Err = fastq::ParseError;
    fn fill_data(&mut self, rset: &mut fastq::RecordSet) -> Option<Result<(), fastq::ParseError>> {
        self.read_record_set(rset)
    }
}
<|endoftext|>