<commit_before>use std::io::{self, Buffer, Reader, Writer, IoResult, BufferedStream, standard_error};
use std::mem;
use std::collections::BTreeMap;
use std::rand::{thread_rng, Rng};
use std::{u64, u16};
use std::num::{Int, FromPrimitive, ToPrimitive};
use std::slice::SliceConcatExt;
use url::Url;

#[cfg(test)]
use test::Bencher;

use nonce::Nonce;
use message::{WSMessage, WSHeader, WS_MASK, WS_LEN, WS_LEN16, WS_LEN64, WS_OPTERM};
use stream::NetworkStream;

pub struct WebSocket<S = NetworkStream> {
    stream: Option<BufferedStream<S>>,
    pub url: Url,
    hostname: String,
    use_ssl: bool,
    version: uint,
    extensions: Option<Vec<String>>,
    protocols: Option<Vec<String>>
}

impl WebSocket {
    pub fn with_options(url: Url, version: uint, protocols: Option<&[&str]>, extensions: Option<&[&str]>) -> WebSocket {
        let use_ssl = url.scheme[] == "wss";

        let port = match url.port() {
            Some(p) => p,
            None if use_ssl => 443,
            _ => 80
        };

        WebSocket {
            stream: None,
            hostname: format!("{}:{}", url.serialize_host().unwrap(), port),
            url: url,
            use_ssl: use_ssl,
            version: version,
            extensions: extensions.map(|v| v.iter().map(|v| v.to_string()).collect()),
            protocols: protocols.map(|v| v.iter().map(|v| v.to_string()).collect())
        }
    }

    #[inline] pub fn new(url: Url) -> WebSocket {
        WebSocket::with_options(url, 1, None, None)
    }

    fn try_connect(&mut self) -> IoResult<()> {
        self.stream = Some(BufferedStream::new(try!(NetworkStream::connect(self.hostname[], self.use_ssl))));
        Ok(())
    }

    fn write_request(&mut self, nonce: &str) -> IoResult<()> {
        let s = match self.stream { Some(ref mut s) => s, None => return Err(standard_error(io::IoErrorKind::NotConnected)) };

        try!(s.write(format!("GET {} HTTP/1.1\r\n", self.url.serialize_path().unwrap_or("/".to_string())).as_bytes()));
        try!(s.write(format!("Host: {}\r\n", self.url.host().unwrap()).as_bytes()));
        try!(s.write(format!("Origin: {}\r\n", self.url.serialize_no_fragment()).as_bytes()));
        try!(s.write(format!("Sec-WebSocket-Key: {}\r\n", nonce).as_bytes()));

        try!(s.write(b"Upgrade: websocket\r\n"));
        try!(s.write(b"Connection: Upgrade\r\n"));
        try!(s.write(format!("Sec-WebSocket-Version: {}\r\n", self.version).as_bytes()));
        if let Some(ref protos) = self.protocols {
            try!(s.write(format!("Sec-WebSocket-Protocol: {}\r\n", protos.connect(", ")).as_bytes()));
        }
        if let Some(ref exts) = self.extensions {
            try!(s.write(format!("Sec-WebSocket-Extensions: {}\r\n", exts.connect(", ")).as_bytes()));
        }
        try!(s.write(b"\r\n"));

        s.flush()
    }

    fn read_response(&mut self, nonce: &str) -> IoResult<()> {
        let spaces: &[_] = &[' ', '\t', '\r', '\n'];
        let s = match self.stream { Some(ref mut s) => s, None => return Err(standard_error(io::IoErrorKind::NotConnected)) };
        let status = try!(s.read_line())[].splitn(2, ' ').nth(1).and_then(|s| s.parse::<uint>());

        match status {
            Some(101) => (),
            _ => return Err(standard_error(io::InvalidInput))
        }

        let headers = s.lines().map(|r| r.unwrap_or("\r\n".to_string())) .take_while(|l| l[] != "\r\n")
            .map(|s| s[].splitn(1, ':').map(|s| s.trim_matches(spaces).to_string()).collect::<Vec<String>>())
            .map(|p| (p[0].to_string(), p[1].to_string()))
            .collect::<BTreeMap<String, String>>();

        try!(s.flush());

        let response = headers.get("Sec-WebSocket-Accept");
        match response {
            Some(r) if nonce == r[] => (),
            _ => return Err(standard_error(io::InvalidInput))
        }

        Ok(())
    }

    pub fn connect(&mut self) -> IoResult<()> {
        let mut nonce = Nonce::new();

        try!(self.try_connect());
        try!(self.write_request(nonce[]));

        nonce = nonce.encode();
        try!(self.read_response(nonce[]));

        Ok(())
    }

    fn read_header(&mut self) -> IoResult<WSHeader> {
        // XXX: this is a bug, WSHeader should accept u16
        Ok(WSHeader::from_bits_truncate(try!(self.read_be_u16())))
    }

    fn read_length(&mut self, header: &WSHeader) -> IoResult<uint> {
        let wslen = *header & WS_LEN;
        if wslen == WS_LEN16 { self.read_be_u16().map(|v| v as uint) }
        else if wslen == WS_LEN64 { self.read_be_u64().map(|v| v as uint) }
        else { Ok(wslen.bits() as uint) }
    }

    pub fn read_message(&mut self) -> IoResult<WSMessage> {
        let header = try!(self.read_header());
        let mut len = try!(self.read_length(&header));

        let mask = if header.contains(WS_MASK) {
            Some(try!(self.read_be_u32()))
        } else {
            None
        };

        // If this is the terminating frame (close command),
        // first two bytes of data MUST BE u16 status code
        let mut status = if header.contains(WS_OPTERM) {
            // compensate length of status code
            len = len - 2;
            Some(try!(self.read_be_u16()))
        } else {
            None
        };

        let mut data = try!(self.read_exact(len));

        // If we have mask, decrypt data
        if let Some(mut m) = mask {
            // decrypt status if present
            if let Some(s) = status {
                status = Some(s ^ (m & 0xffff) as u16);
                // compensate the usage of two mask bytes
                m = m.rotate_right(16);
            }
            data = WebSocket::mask_data(data[], m);
        }

        Ok(WSMessage { header: header, data: data, status: status.and_then(FromPrimitive::from_u16) })
    }

    fn mask_data(data: &[u8], mask: u32) -> Vec<u8> {
        data.iter().enumerate().map(|(i, b)| *b ^ (mask >> ((i % 4) << 3) & 0xff) as u8).collect::<Vec<u8>>()
    }

    pub fn send_message(&mut self, msg: &WSMessage) -> IoResult<()> {
        let mut len = msg.data.len();
        let mut hdr = msg.header - WS_LEN;

        // If we have status set, the data length is increased by status size
        if msg.status.is_some() {
            len = len + 2;
        }

        // Encode and send length along with header
        if len < u16::MAX as uint {
            hdr = hdr | WSHeader::from_bits_truncate(len as u16);
            try!(self.write_be_u16(hdr.bits()));

        } else if len < u64::MAX as uint {
            hdr = hdr | WS_LEN16;
            try!(self.write_be_u16(hdr.bits()));
            try!(self.write_be_u16(len as u16));

        } else {
            hdr = hdr | WS_LEN64;
            try!(self.write_be_u16(hdr.bits()));
            try!(self.write_be_u64(len as u64));
        }

        // If user required masking, encrypt all data
        if hdr.contains(WS_MASK) {
            // Generate and send random mask
            let mut mask = thread_rng().gen::<u32>();
            try!(self.write_be_u32(mask));

            // Encrypt status code if present
            if let Some(status) = msg.status {
                try!(self.write_be_u16(status.to_u16().unwrap() ^ (mask & 0xffff) as u16));
                // compensate for mask already used for status encryption
                mask = mask.rotate_right(16);
            }

            try!(self.write(WebSocket::mask_data(msg.data[], mask)[]));
        } else {
            // Send status code if present
            if let Some(status) = msg.status {
                try!(self.write_be_u16(status.to_u16().unwrap()));
            }
            try!(self.write(msg.data[]));
        }

        self.flush()
    }

    pub fn iter(&mut self) -> WSMessages {
        WSMessages { sock: self }
    }
}

impl Reader for WebSocket {
    fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> {
        match self.stream {
            Some(ref mut s) => s.read(buf),
            None => Err(standard_error(io::IoErrorKind::NotConnected))
        }
    }
}

impl Writer for WebSocket {
    fn write(&mut self, buf: &[u8]) -> IoResult<()> {
        match self.stream {
            Some(ref mut s) => s.write(buf),
            None => Err(standard_error(io::IoErrorKind::NotConnected))
        }
    }

    fn flush(&mut self) -> IoResult<()> {
        match self.stream {
            Some(ref mut s) => s.flush(),
            None => Err(standard_error(io::IoErrorKind::NotConnected))
        }
    }
}

impl Buffer for WebSocket {
    fn fill_buf<'a>(&'a mut self) -> IoResult<&'a [u8]> {
        match self.stream {
            Some(ref mut s) => s.fill_buf(),
            None => Err(standard_error(io::IoErrorKind::NotConnected))
        }
    }

    fn consume(&mut self, amt: uint) {
        match self.stream {
            Some(ref mut s) => s.consume(amt),
            None => ()
        }
    }
}

pub struct WSMessages<'a> {
    sock: &'a mut WebSocket
}

pub struct WSDefragMessages<'a> {
    underlying: &'a mut WSMessages<'a>,
    buffer: WSMessage
}

impl<'a> WSMessages<'a> {
    pub fn defrag(&'a mut self) -> WSDefragMessages<'a> {
        WSDefragMessages{ underlying: self, buffer: WSMessage{ header: WSHeader::empty(), data: Vec::new(), status: None } }
    }
}

impl<'a> Iterator for WSMessages<'a> {
    type Item = WSMessage;
    fn next(&mut self) -> Option<WSMessage> {
        self.sock.read_message().ok()
    }
}

impl<'a> WSDefragMessages<'a> {
    fn popbuf(&mut self) -> Option<WSMessage> {
        if self.buffer.data.is_empty() {
            None
        } else {
            let mut buf = WSMessage{ header: WSHeader::empty(), data: Vec::new(), status: None };
            mem::swap(&mut self.buffer, &mut buf);
            Some(buf)
        }
    }

    fn swapbuf(&mut self, msg: &mut WSMessage) {
        mem::swap(&mut self.buffer, msg);
    }
}

impl<'a> Iterator for WSDefragMessages<'a> {
    type Item = WSMessage;
    fn next(&mut self) -> Option<WSMessage> {
        loop {
            match self.underlying.next() {
                None => return self.popbuf(),
                Some(mut msg) => {
                    if msg.is_whole() {
                        return Some(msg);
                    } else if msg.is_first() {
                        self.swapbuf(&mut msg);
                    } else if msg.is_more() {
                        self.buffer.push(msg);
                    } else if msg.is_last() {
                        self.buffer.push(msg);
                        return self.popbuf().map(|v| v.last());
                    }
                }
            }
        }
    }
}

<commit_msg>fix send length<commit_after>use std::io::{self, Buffer, Reader, Writer, IoResult, BufferedStream, standard_error};
use std::mem;
use std::collections::BTreeMap;
use std::rand::{thread_rng, Rng};
use std::{u64, u16};
use std::num::{Int, FromPrimitive, ToPrimitive};
use std::slice::SliceConcatExt;
use url::Url;

#[cfg(test)]
use test::Bencher;

use nonce::Nonce;
use message::{WSMessage, WSHeader, WS_MASK, WS_LEN, WS_LEN16, WS_LEN64, WS_OPTERM};
use stream::NetworkStream;

pub struct WebSocket<S = NetworkStream> {
    stream: Option<BufferedStream<S>>,
    pub url: Url,
    hostname: String,
    use_ssl: bool,
    version: uint,
    extensions: Option<Vec<String>>,
    protocols: Option<Vec<String>>
}

impl WebSocket {
    pub fn with_options(url: Url, version: uint, protocols: Option<&[&str]>, extensions: Option<&[&str]>) -> WebSocket {
        let use_ssl = url.scheme[] == "wss";

        let port = match url.port() {
            Some(p) => p,
            None if use_ssl => 443,
            _ => 80
        };

        WebSocket {
            stream: None,
            hostname: format!("{}:{}", url.serialize_host().unwrap(), port),
            url: url,
            use_ssl: use_ssl,
            version: version,
            extensions: extensions.map(|v| v.iter().map(|v| v.to_string()).collect()),
            protocols: protocols.map(|v| v.iter().map(|v| v.to_string()).collect())
        }
    }

    #[inline] pub fn new(url: Url) -> WebSocket {
        WebSocket::with_options(url, 1, None, None)
    }

    fn try_connect(&mut self) -> IoResult<()> {
        self.stream = Some(BufferedStream::new(try!(NetworkStream::connect(self.hostname[], self.use_ssl))));
        Ok(())
    }

    fn write_request(&mut self, nonce: &str) -> IoResult<()> {
        let s = match self.stream { Some(ref mut s) => s, None => return Err(standard_error(io::IoErrorKind::NotConnected)) };

        try!(s.write(format!("GET {} HTTP/1.1\r\n", self.url.serialize_path().unwrap_or("/".to_string())).as_bytes()));
        try!(s.write(format!("Host: {}\r\n", self.url.host().unwrap()).as_bytes()));
        try!(s.write(format!("Origin: {}\r\n", self.url.serialize_no_fragment()).as_bytes()));
        try!(s.write(format!("Sec-WebSocket-Key: {}\r\n", nonce).as_bytes()));

        try!(s.write(b"Upgrade: websocket\r\n"));
        try!(s.write(b"Connection: Upgrade\r\n"));
        try!(s.write(format!("Sec-WebSocket-Version: {}\r\n", self.version).as_bytes()));
        if let Some(ref protos) = self.protocols {
            try!(s.write(format!("Sec-WebSocket-Protocol: {}\r\n", protos.connect(", ")).as_bytes()));
        }
        if let Some(ref exts) = self.extensions {
            try!(s.write(format!("Sec-WebSocket-Extensions: {}\r\n", exts.connect(", ")).as_bytes()));
        }
        try!(s.write(b"\r\n"));

        s.flush()
    }

    fn read_response(&mut self, nonce: &str) -> IoResult<()> {
        let spaces: &[_] = &[' ', '\t', '\r', '\n'];
        let s = match self.stream { Some(ref mut s) => s, None => return Err(standard_error(io::IoErrorKind::NotConnected)) };
        let status = try!(s.read_line())[].splitn(2, ' ').nth(1).and_then(|s| s.parse::<uint>());

        match status {
            Some(101) => (),
            _ => return Err(standard_error(io::InvalidInput))
        }

        let headers = s.lines().map(|r| r.unwrap_or("\r\n".to_string())) .take_while(|l| l[] != "\r\n")
            .map(|s| s[].splitn(1, ':').map(|s| s.trim_matches(spaces).to_string()).collect::<Vec<String>>())
            .map(|p| (p[0].to_string(), p[1].to_string()))
            .collect::<BTreeMap<String, String>>();

        try!(s.flush());

        let response = headers.get("Sec-WebSocket-Accept");
        match response {
            Some(r) if nonce == r[] => (),
            _ => return Err(standard_error(io::InvalidInput))
        }

        Ok(())
    }

    pub fn connect(&mut self) -> IoResult<()> {
        let mut nonce = Nonce::new();

        try!(self.try_connect());
        try!(self.write_request(nonce[]));

        nonce = nonce.encode();
        try!(self.read_response(nonce[]));

        Ok(())
    }

    fn read_header(&mut self) -> IoResult<WSHeader> {
        // XXX: this is a bug, WSHeader should accept u16
        Ok(WSHeader::from_bits_truncate(try!(self.read_be_u16())))
    }

    fn read_length(&mut self, header: &WSHeader) -> IoResult<uint> {
        let wslen = *header & WS_LEN;
        if wslen == WS_LEN16 { self.read_be_u16().map(|v| v as uint) }
        else if wslen == WS_LEN64 { self.read_be_u64().map(|v| v as uint) }
        else { Ok(wslen.bits() as uint) }
    }

    pub fn read_message(&mut self) -> IoResult<WSMessage> {
        let header = try!(self.read_header());
        let mut len = try!(self.read_length(&header));

        let mask = if header.contains(WS_MASK) {
            Some(try!(self.read_be_u32()))
        } else {
            None
        };

        // If this is the terminating frame (close command),
        // first two bytes of data MUST BE u16 status code
        let mut status = if header.contains(WS_OPTERM) {
            // compensate length of status code
            len = len - 2;
            Some(try!(self.read_be_u16()))
        } else {
            None
        };

        let mut data = try!(self.read_exact(len));

        // If we have mask, decrypt data
        if let Some(mut m) = mask {
            // decrypt status if present
            if let Some(s) = status {
                status = Some(s ^ (m & 0xffff) as u16);
                // compensate the usage of two mask bytes
                m = m.rotate_right(16);
            }
            data = WebSocket::mask_data(data[], m);
        }

        Ok(WSMessage { header: header, data: data, status: status.and_then(FromPrimitive::from_u16) })
    }

    fn mask_data(data: &[u8], mask: u32) -> Vec<u8> {
        data.iter().enumerate().map(|(i, b)| *b ^ (mask >> ((i % 4) << 3) & 0xff) as u8).collect::<Vec<u8>>()
    }

    pub fn send_message(&mut self, msg: &WSMessage) -> IoResult<()> {
        let mut len = msg.data.len();
        let mut hdr = msg.header - WS_LEN;

        // If we have status set, the data length is increased by status size
        if msg.status.is_some() {
            len = len + 2;
        }

        // Encode and send length along with header
        if len < WS_LEN16.bits() as uint {
            hdr = hdr | WSHeader::from_bits_truncate(len as u16 & WS_LEN.bits());
            try!(self.write_be_u16(hdr.bits()));

        } else if len < u16::MAX as uint {
            hdr = hdr | WS_LEN16;
            try!(self.write_be_u16(hdr.bits()));
            try!(self.write_be_u16(len as u16));

        } else {
            hdr = hdr | WS_LEN64;
            try!(self.write_be_u16(hdr.bits()));
            try!(self.write_be_u64(len as u64));
        }

        // If user required masking, encrypt all data
        if hdr.contains(WS_MASK) {
            // Generate and send random mask
            let mut mask = thread_rng().gen::<u32>();
            try!(self.write_be_u32(mask));

            // Encrypt status code if present
            if let Some(status) = msg.status {
                try!(self.write_be_u16(status.to_u16().unwrap() ^ (mask & 0xffff) as u16));
                // compensate for mask already used for status encryption
                mask = mask.rotate_right(16);
            }

            try!(self.write(WebSocket::mask_data(msg.data[], mask)[]));
        } else {
            // Send status code if present
            if let Some(status) = msg.status {
                try!(self.write_be_u16(status.to_u16().unwrap()));
            }
            try!(self.write(msg.data[]));
        }

        self.flush()
    }

    pub fn iter(&mut self) -> WSMessages {
        WSMessages { sock: self }
    }
}

impl Reader for WebSocket {
    fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> {
        match self.stream {
            Some(ref mut s) => s.read(buf),
            None => Err(standard_error(io::IoErrorKind::NotConnected))
        }
    }
}

impl Writer for WebSocket {
    fn write(&mut self, buf: &[u8]) -> IoResult<()> {
        match self.stream {
            Some(ref mut s) => s.write(buf),
            None => Err(standard_error(io::IoErrorKind::NotConnected))
        }
    }

    fn flush(&mut self) -> IoResult<()> {
        match self.stream {
            Some(ref mut s) => s.flush(),
            None => Err(standard_error(io::IoErrorKind::NotConnected))
        }
    }
}

impl Buffer for WebSocket {
    fn fill_buf<'a>(&'a mut self) -> IoResult<&'a [u8]> {
        match self.stream {
            Some(ref mut s) => s.fill_buf(),
            None => Err(standard_error(io::IoErrorKind::NotConnected))
        }
    }

    fn consume(&mut self, amt: uint) {
        match self.stream {
            Some(ref mut s) => s.consume(amt),
            None => ()
        }
    }
}

pub struct WSMessages<'a> {
    sock: &'a mut WebSocket
}

pub struct WSDefragMessages<'a> {
    underlying: &'a mut WSMessages<'a>,
    buffer: WSMessage
}

impl<'a> WSMessages<'a> {
    pub fn defrag(&'a mut self) -> WSDefragMessages<'a> {
        WSDefragMessages{ underlying: self, buffer: WSMessage{ header: WSHeader::empty(), data: Vec::new(), status: None } }
    }
}

impl<'a> Iterator for WSMessages<'a> {
    type Item = WSMessage;
    fn next(&mut self) -> Option<WSMessage> {
        self.sock.read_message().ok()
    }
}

impl<'a> WSDefragMessages<'a> {
    fn popbuf(&mut self) -> Option<WSMessage> {
        if self.buffer.data.is_empty() {
            None
        } else {
            let mut buf = WSMessage{ header: WSHeader::empty(), data: Vec::new(), status: None };
            mem::swap(&mut self.buffer, &mut buf);
            Some(buf)
        }
    }

    fn swapbuf(&mut self, msg: &mut WSMessage) {
        mem::swap(&mut self.buffer, msg);
    }
}

impl<'a> Iterator for WSDefragMessages<'a> {
    type Item = WSMessage;
    fn next(&mut self) -> Option<WSMessage> {
        loop {
            match self.underlying.next() {
                None => return self.popbuf(),
                Some(mut msg) => {
                    if msg.is_whole() {
                        return Some(msg);
                    } else if msg.is_first() {
                        self.swapbuf(&mut msg);
                    } else if msg.is_more() {
                        self.buffer.push(msg);
                    } else if msg.is_last() {
                        self.buffer.push(msg);
                        return self.popbuf().map(|v| v.last());
                    }
                }
            }
        }
    }
}

<|endoftext|>


<commit_before>use nom;
use nom::{Err, IResult};

use metadata;

use metadata::{Metadata, StreamInfo, metadata_parser};
use frame::{frame_parser, Frame};
use utility::{ErrorKind, ByteStream, ReadStream, StreamProducer};

use std::io;
use std::fs::File;

enum ParserState {
  Marker,
  Metadata,
  Frame,
}

pub struct Stream {
  info: StreamInfo,
  pub metadata: Vec<Metadata>,
  pub frames: Vec<Frame>,
  state: ParserState,
}

named!(pub stream_parser <&[u8], Stream>,
  chain!(
    blocks: metadata_parser ~
    frames: many1!(apply!(frame_parser, &blocks.0)),
    move|| {
      Stream {
        info: blocks.0,
        metadata: blocks.1,
        frames: frames,
        state: ParserState::Marker,
      }
    }
  )
);

impl Stream {
  pub fn new() -> Stream {
    Stream {
      info: StreamInfo::new(),
      metadata: Vec::new(),
      frames: Vec::new(),
      state: ParserState::Marker,
    }
  }

  pub fn info(&self) -> StreamInfo {
    self.info
  }

  pub fn from_file(filename: &str) -> io::Result<Stream> {
    File::open(filename).and_then(|file| {
      let mut reader   = ReadStream::new(file);
      let mut is_error = false;
      let mut stream   = Stream {
        info: StreamInfo::new(),
        metadata: Vec::new(),
        frames: Vec::new(),
        state: ParserState::Marker,
      };

      loop {
        match stream.handle(&mut reader) {
          Ok(_)                         => break,
          Err(ErrorKind::EndOfInput)    => break,
          Err(ErrorKind::Consumed(_))   => continue,
          Err(ErrorKind::Incomplete(_)) => continue,
          Err(_)                        => {
            is_error = true;

            break;
          }
        }
      }

      if !is_error {
        Ok(stream)
      } else {
        let error_str = format!("parser: couldn't parse the given file {}",
                                filename);

        Err(io::Error::new(io::ErrorKind::InvalidData, error_str))
      }
    })
  }

  fn from_stream_producer<P>(producer: &mut P, error_str: &str)
                             -> io::Result<Stream>
   where P: StreamProducer {
    let mut is_error = false;
    let mut stream   = Stream {
      info: StreamInfo::new(),
      metadata: Vec::new(),
      frames: Vec::new(),
      state: ParserState::Marker,
    };

    loop {
      match stream.handle(producer) {
        Ok(_)                         => break,
        Err(ErrorKind::EndOfInput)    => break,
        Err(ErrorKind::Consumed(_))   => continue,
        Err(ErrorKind::Incomplete(_)) => continue,
        Err(_)                        => {
          is_error = true;

          break;
        }
      }
    }

    if !is_error {
      Ok(stream)
    } else {
      Err(io::Error::new(io::ErrorKind::InvalidData, error_str))
    }
  }

  fn handle_marker<'a>(&mut self, input: &'a [u8]) -> IResult<&'a [u8], ()> {
    let kind = nom::ErrorKind::Custom(0);

    match tag!(input, "fLaC") {
      IResult::Done(i, _)    => {
        self.state = ParserState::Metadata;

        IResult::Error(Err::Position(kind, i))
      }
      IResult::Error(_)      => IResult::Error(Err::Code(kind)),
      IResult::Incomplete(n) => IResult::Incomplete(n),
    }
  }

  fn handle_metadata<'a>(&mut self, input: &'a [u8])
                         -> IResult<&'a [u8], ()> {
    let kind = nom::ErrorKind::Custom(1);

    match metadata::block(input) {
      IResult::Done(i, block) => {
        let is_last = block.is_last;

        if let metadata::Data::StreamInfo(info) = block.data {
          self.info = info;
        } else {
          self.metadata.push(block);
        }

        if is_last {
          self.state = ParserState::Frame;
        }

        IResult::Error(Err::Position(kind, i))
      }
      IResult::Error(_)      => IResult::Error(Err::Code(kind)),
      IResult::Incomplete(n) => IResult::Incomplete(n),
    }
  }

  fn handle_frame<'a>(&mut self, input: &'a [u8]) -> IResult<&'a [u8], ()> {
    let kind = nom::ErrorKind::Custom(2);

    match frame_parser(input, &self.info) {
      IResult::Done(i, frame) => {
        self.frames.push(frame);

        IResult::Error(Err::Position(kind, i))
      }
      IResult::Error(_)      => IResult::Error(Err::Code(kind)),
      IResult::Incomplete(n) => IResult::Incomplete(n),
    }
  }

  pub fn handle<S: StreamProducer>(&mut self, stream: &mut S)
                                   -> Result<(), ErrorKind> {
    stream.parse(|input| {
      match self.state {
        ParserState::Marker   => self.handle_marker(input),
        ParserState::Metadata => self.handle_metadata(input),
        ParserState::Frame    => self.handle_frame(input),
      }
    })
  }
}
<commit_msg>Simplify `from_file` using `from_stream_producer`<commit_after>use nom;
use nom::{Err, IResult};

use metadata;

use metadata::{Metadata, StreamInfo, metadata_parser};
use frame::{frame_parser, Frame};
use utility::{ErrorKind, ByteStream, ReadStream, StreamProducer};

use std::io;
use std::fs::File;

enum ParserState {
  Marker,
  Metadata,
  Frame,
}

pub struct Stream {
  info: StreamInfo,
  pub metadata: Vec<Metadata>,
  pub frames: Vec<Frame>,
  state: ParserState,
}

named!(pub stream_parser <&[u8], Stream>,
  chain!(
    blocks: metadata_parser ~
    frames: many1!(apply!(frame_parser, &blocks.0)),
    move|| {
      Stream {
        info: blocks.0,
        metadata: blocks.1,
        frames: frames,
        state: ParserState::Marker,
      }
    }
  )
);

impl Stream {
  pub fn new() -> Stream {
    Stream {
      info: StreamInfo::new(),
      metadata: Vec::new(),
      frames: Vec::new(),
      state: ParserState::Marker,
    }
  }

  pub fn info(&self) -> StreamInfo {
    self.info
  }

  pub fn from_file(filename: &str) -> io::Result<Stream> {
    File::open(filename).and_then(|file| {
      let mut producer = ReadStream::new(file);
      let error_str    = format!("parser: couldn't parse the given file {}",
                                 filename);

      Stream::from_stream_producer(&mut producer, &error_str)
    })
  }

  fn from_stream_producer<P>(producer: &mut P, error_str: &str)
                             -> io::Result<Stream>
   where P: StreamProducer {
    let mut is_error = false;
    let mut stream   = Stream {
      info: StreamInfo::new(),
      metadata: Vec::new(),
      frames: Vec::new(),
      state: ParserState::Marker,
    };

    loop {
      match stream.handle(producer) {
        Ok(_)                         => break,
        Err(ErrorKind::EndOfInput)    => break,
        Err(ErrorKind::Consumed(_))   => continue,
        Err(ErrorKind::Incomplete(_)) => continue,
        Err(_)                        => {
          is_error = true;

          break;
        }
      }
    }

    if !is_error {
      Ok(stream)
    } else {
      Err(io::Error::new(io::ErrorKind::InvalidData, error_str))
    }
  }

  fn handle_marker<'a>(&mut self, input: &'a [u8]) -> IResult<&'a [u8], ()> {
    let kind = nom::ErrorKind::Custom(0);

    match tag!(input, "fLaC") {
      IResult::Done(i, _)    => {
        self.state = ParserState::Metadata;

        IResult::Error(Err::Position(kind, i))
      }
      IResult::Error(_)      => IResult::Error(Err::Code(kind)),
      IResult::Incomplete(n) => IResult::Incomplete(n),
    }
  }

  fn handle_metadata<'a>(&mut self, input: &'a [u8])
                         -> IResult<&'a [u8], ()> {
    let kind = nom::ErrorKind::Custom(1);

    match metadata::block(input) {
      IResult::Done(i, block) => {
        let is_last = block.is_last;

        if let metadata::Data::StreamInfo(info) = block.data {
          self.info = info;
        } else {
          self.metadata.push(block);
        }

        if is_last {
          self.state = ParserState::Frame;
        }

        IResult::Error(Err::Position(kind, i))
      }
      IResult::Error(_)      => IResult::Error(Err::Code(kind)),
      IResult::Incomplete(n) => IResult::Incomplete(n),
    }
  }

  fn handle_frame<'a>(&mut self, input: &'a [u8]) -> IResult<&'a [u8], ()> {
    let kind = nom::ErrorKind::Custom(2);

    match frame_parser(input, &self.info) {
      IResult::Done(i, frame) => {
        self.frames.push(frame);

        IResult::Error(Err::Position(kind, i))
      }
      IResult::Error(_)      => IResult::Error(Err::Code(kind)),
      IResult::Incomplete(n) => IResult::Incomplete(n),
    }
  }

  pub fn handle<S: StreamProducer>(&mut self, stream: &mut S)
                                   -> Result<(), ErrorKind> {
    stream.parse(|input| {
      match self.state {
        ParserState::Marker   => self.handle_marker(input),
        ParserState::Metadata => self.handle_metadata(input),
        ParserState::Frame    => self.handle_frame(input),
      }
    })
  }
}
<|endoftext|>


<commit_before>use base::*;
use resampler::StreamResampler;
use time::TimeDelta;

pub trait Input: Send {
    fn read(&mut self) -> Result<Option<Frame>>;
    fn min_delay(&self) -> TimeDelta;
}

pub struct InputResampler {
    input: Box<Input>,
    resampler: StreamResampler,
    delay: TimeDelta,
}

impl InputResampler {
    pub fn new(input: Box<Input>, output_rate: f32, window_size: usize) -> InputResampler {
        InputResampler {
            input: input,
            resampler: StreamResampler::new(output_rate, window_size),
            delay: samples_to_timedelta(output_rate, window_size as i64),
        }
    }
}

impl Input for InputResampler {
    fn read(&mut self) -> Result<Option<Frame>> {
        Ok(match try!(self.input.read()) {
            Some(f) => self.resampler.resample(f),
            None => None,
        })
    }

    fn min_delay(&self) -> TimeDelta {
        self.input.min_delay() + self.delay
    }
}
<commit_msg>Simpler delay calculation in input resampler<commit_after>use base::*;
use resampler::StreamResampler;
use time::TimeDelta;

pub trait Input: Send {
    fn read(&mut self) -> Result<Option<Frame>>;
    fn min_delay(&self) -> TimeDelta;
}

pub struct InputResampler {
    input: Box<Input>,
    resampler: StreamResampler,
}

impl InputResampler {
    pub fn new(input: Box<Input>, output_rate: f32, window_size: usize) -> InputResampler {
        InputResampler {
            input: input,
            resampler: StreamResampler::new(output_rate, window_size),
        }
    }
}

impl Input for InputResampler {
    fn read(&mut self) -> Result<Option<Frame>> {
        Ok(match try!(self.input.read()) {
            Some(f) => self.resampler.resample(f),
            None => None,
        })
    }

    fn min_delay(&self) -> TimeDelta {
        self.input.min_delay() + self.resampler.delay()
    }
}
<|endoftext|>


<commit_before>use rustc_serialize::json;
use error::KarmaError;
use chrono::{DateTime, Local};
use dazeus::{DaZeusClient, Scope, Response};
use std::ascii::AsciiExt;

pub const STORE_PREFIX: &'static str = "dazeus_karma.";

#[derive(Debug, Clone, Copy, PartialEq)]
pub struct KarmaChange {
    pub up: u64,
    pub down: u64
}

impl KarmaChange {
    pub fn new(up: u64, down: u64) -> KarmaChange {
        KarmaChange { up: up, down: down }
    }

    pub fn total(&self) -> i64 {
        (self.up as i64) - (self.down as i64)
    }

    pub fn to_string(&self) -> String {
        format!("{} (+{}, -{})", self.total(), self.up, self.down)
    }
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum KarmaStyle {
    Notify,
    Silent,
    Implicit
}

impl KarmaStyle {
    pub fn most_explicit(first: KarmaStyle, second: KarmaStyle) -> KarmaStyle {
        if first == KarmaStyle::Implicit && second == KarmaStyle::Implicit {
            KarmaStyle::Implicit
        } else if first == KarmaStyle::Silent && second != KarmaStyle::Notify {
            KarmaStyle::Silent
        } else if first == KarmaStyle::Notify {
            KarmaStyle::Notify
        } else {
            second
        }
    }
}

#[derive(Debug, Clone, PartialEq)]
pub struct Karma {
    pub term: String,
    pub change: KarmaChange,
    pub style: KarmaStyle
}

#[derive(Debug, Clone, PartialEq)]
pub struct KarmaValue {
    pub term: String,
    pub original_term: String,
    pub votes: KarmaChange,
    pub last_vote: DateTime<Local>,
    pub first_vote: DateTime<Local>
}

impl KarmaValue {
    pub fn new(term: &str) -> KarmaValue {
        KarmaValue {
            term: term.to_ascii_lowercase(),
            original_term: term.to_string(),
            votes: KarmaChange::new(0, 0),
            last_vote: Local::now(),
            first_vote: Local::now()
        }
    }

    pub fn vote(&mut self, karma: &Karma) {
        self.last_vote = Local::now();
        self.votes.up += karma.change.up;
        self.votes.down += karma.change.down;
    }

    pub fn from_json(data: json::Json) -> Result<KarmaValue, Box<::std::error::Error>> {
        macro_rules! get_key {
            ($o:expr, $s:expr, $i_is:ident, $i_as:ident) => (match $o.get($s) {
                Some(m) if m.$i_is() => m.$i_as().unwrap(),
                _ => return Err(From::from(KarmaError::new(&format!("No value or invalid value for key '{}'", $s)[..])))
            });
        }

        if data.is_object() {
            let obj = data.as_object().unwrap();
            let term = get_key!(obj, "term", is_string, as_string);
            let first_vote_str = get_key!(obj, "first_vote", is_string, as_string);
            let last_vote_str = get_key!(obj, "last_vote", is_string, as_string);
            let votes = get_key!(obj, "votes", is_object, as_object);

            let upvotes = match votes.get("up") {
                Some(m) if m.is_u64() => m.as_u64().unwrap(),
                Some(m) if m.is_string() && m.as_string().unwrap().parse::<u64>().is_ok() => m.as_string().unwrap().parse().unwrap(),
                _ => return Err(From::from(KarmaError::new("No value or invalid value for key 'up'")))
            };

            let downvotes = match votes.get("down") {
                Some(m) if m.is_u64() => m.as_u64().unwrap(),
                Some(m) if m.is_string() && m.as_string().unwrap().parse::<u64>().is_ok() => m.as_string().unwrap().parse().unwrap(),
                _ => return Err(From::from(KarmaError::new("No value or invalid value for key 'down'")))
            };

            let first_vote = try!(first_vote_str.parse::<DateTime<Local>>());
            let last_vote = try!(last_vote_str.parse::<DateTime<Local>>());

            Ok(KarmaValue {
                term: term.to_ascii_lowercase(),
                original_term: term.to_string(),
                votes: KarmaChange::new(upvotes, downvotes),
                last_vote: last_vote,
                first_vote: first_vote
            })
        } else {
            Err(From::from(KarmaError::new("Invalid json: not an object")))
        }
    }

    pub fn from_str(s: &str) -> Result<KarmaValue, Box<::std::error::Error>> {
        let data = try!(json::Json::from_str(s));
        let karma = KarmaValue::from_json(data);
        match karma {
            Ok(mut k) => {
                k.original_term = k.term.clone();
                Ok(k)
            },
            o => o
        }
    }

    pub fn from_response(r: &Response) -> Result<KarmaValue, Box<::std::error::Error>> {
        match r.get_str("value") {
            Some(s) => KarmaValue::from_str(s),
            None => Err(From::from(KarmaError::new("No value found in response"))),
        }
    }

    pub fn from_dazeus(dazeus: &DaZeusClient, scope: Scope, term: &str) -> Result<KarmaValue, Box<::std::error::Error>> {
        let property = format!("{}{}", STORE_PREFIX, term);
        let mut karma = KarmaValue::from_response(&dazeus.get_property(&property[..].to_ascii_lowercase(), scope));
        if let Ok(ref mut k) = karma {
            k.original_term = term.to_string();
        }
        karma
    }

    pub fn to_string(&self) -> String {
        match self.votes.total() {
            0 => format!("{} has neutral karma (+{}, -{})", self.original_term, self.votes.up, self.votes.down),
            _ => format!("{} has a karma of {}", self.original_term, self.votes.to_string()),
        }
    }
}

impl json::ToJson for KarmaValue {
    fn to_json(&self) -> json::Json {
        let mut obj = json::Object::new();
        obj.insert("term".to_string(), self.term.to_json());

        let mut votes = json::Object::new();
        votes.insert("up".to_string(), self.votes.up.to_json());
        votes.insert("down".to_string(), self.votes.down.to_json());

        obj.insert("votes".to_string(), votes.to_json());
        obj.insert("first_vote".to_string(), self.first_vote.format("%FT%TZ").to_string().to_json());
        obj.insert("last_vote".to_string(), self.last_vote.format("%FT%TZ").to_string().to_json());

        obj.to_json()
    }
}
<commit_msg>Allow key prefix to be case sensitive<commit_after>use rustc_serialize::json;
use error::KarmaError;
use chrono::{DateTime, Local};
use dazeus::{DaZeusClient, Scope, Response};
use std::ascii::AsciiExt;

pub const STORE_PREFIX: &'static str = "dazeus_karma.";

#[derive(Debug, Clone, Copy, PartialEq)]
pub struct KarmaChange {
    pub up: u64,
    pub down: u64
}

impl KarmaChange {
    pub fn new(up: u64, down: u64) -> KarmaChange {
        KarmaChange { up: up, down: down }
    }

    pub fn total(&self) -> i64 {
        (self.up as i64) - (self.down as i64)
    }

    pub fn to_string(&self) -> String {
        format!("{} (+{}, -{})", self.total(), self.up, self.down)
    }
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum KarmaStyle {
    Notify,
    Silent,
    Implicit
}

impl KarmaStyle {
    pub fn most_explicit(first: KarmaStyle, second: KarmaStyle) -> KarmaStyle {
        if first == KarmaStyle::Implicit && second == KarmaStyle::Implicit {
            KarmaStyle::Implicit
        } else if first == KarmaStyle::Silent && second != KarmaStyle::Notify {
            KarmaStyle::Silent
        } else if first == KarmaStyle::Notify {
            KarmaStyle::Notify
        } else {
            second
        }
    }
}

#[derive(Debug, Clone, PartialEq)]
pub struct Karma {
    pub term: String,
    pub change: KarmaChange,
    pub style: KarmaStyle
}

#[derive(Debug, Clone, PartialEq)]
pub struct KarmaValue {
    pub term: String,
    pub original_term: String,
    pub votes: KarmaChange,
    pub last_vote: DateTime<Local>,
    pub first_vote: DateTime<Local>
}

impl KarmaValue {
    pub fn new(term: &str) -> KarmaValue {
        KarmaValue {
            term: term.to_ascii_lowercase(),
            original_term: term.to_string(),
            votes: KarmaChange::new(0, 0),
            last_vote: Local::now(),
            first_vote: Local::now()
        }
    }

    pub fn vote(&mut self, karma: &Karma) {
        self.last_vote = Local::now();
        self.votes.up += karma.change.up;
        self.votes.down += karma.change.down;
    }

    pub fn from_json(data: json::Json) -> Result<KarmaValue, Box<::std::error::Error>> {
        macro_rules! get_key {
            ($o:expr, $s:expr, $i_is:ident, $i_as:ident) => (match $o.get($s) {
                Some(m) if m.$i_is() => m.$i_as().unwrap(),
                _ => return Err(From::from(KarmaError::new(&format!("No value or invalid value for key '{}'", $s)[..])))
            });
        }

        if data.is_object() {
            let obj = data.as_object().unwrap();
            let term = get_key!(obj, "term", is_string, as_string);
            let first_vote_str = get_key!(obj, "first_vote", is_string, as_string);
            let last_vote_str = get_key!(obj, "last_vote", is_string, as_string);
            let votes = get_key!(obj, "votes", is_object, as_object);

            let upvotes = match votes.get("up") {
                Some(m) if m.is_u64() => m.as_u64().unwrap(),
                Some(m) if m.is_string() && m.as_string().unwrap().parse::<u64>().is_ok() => m.as_string().unwrap().parse().unwrap(),
                _ => return Err(From::from(KarmaError::new("No value or invalid value for key 'up'")))
            };

            let downvotes = match votes.get("down") {
                Some(m) if m.is_u64() => m.as_u64().unwrap(),
                Some(m) if m.is_string() && m.as_string().unwrap().parse::<u64>().is_ok() => m.as_string().unwrap().parse().unwrap(),
                _ => return Err(From::from(KarmaError::new("No value or invalid value for key 'down'")))
            };

            let first_vote = try!(first_vote_str.parse::<DateTime<Local>>());
            let last_vote = try!(last_vote_str.parse::<DateTime<Local>>());

            Ok(KarmaValue {
                term: term.to_ascii_lowercase(),
                original_term: term.to_string(),
                votes: KarmaChange::new(upvotes, downvotes),
                last_vote: last_vote,
                first_vote: first_vote
            })
        } else {
            Err(From::from(KarmaError::new("Invalid json: not an object")))
        }
    }

    pub fn from_str(s: &str) -> Result<KarmaValue, Box<::std::error::Error>> {
        let data = try!(json::Json::from_str(s));
        let karma = KarmaValue::from_json(data);
        match karma {
            Ok(mut k) => {
                k.original_term = k.term.clone();
                Ok(k)
            },
            o => o
        }
    }

    pub fn from_response(r: &Response) -> Result<KarmaValue, Box<::std::error::Error>> {
        match r.get_str("value") {
            Some(s) => KarmaValue::from_str(s),
            None => Err(From::from(KarmaError::new("No value found in response"))),
        }
    }

    pub fn from_dazeus(dazeus: &DaZeusClient, scope: Scope, term: &str) -> Result<KarmaValue, Box<::std::error::Error>> {
        let property = format!("{}{}", STORE_PREFIX, term.to_ascii_lowercase());

        let mut karma = KarmaValue::from_response(&dazeus.get_property(&property[..], scope));
        if let Ok(ref mut k) = karma {
            k.original_term = term.to_string();
        }
        karma
    }

    pub fn to_string(&self) -> String {
        match self.votes.total() {
            0 => format!("{} has neutral karma (+{}, -{})", self.original_term, self.votes.up, self.votes.down),
            _ => format!("{} has a karma of {}", self.original_term, self.votes.to_string()),
        }
    }
}

impl json::ToJson for KarmaValue {
    fn to_json(&self) -> json::Json {
        let mut obj = json::Object::new();
        obj.insert("term".to_string(), self.term.to_json());

        let mut votes = json::Object::new();
        votes.insert("up".to_string(), self.votes.up.to_json());
        votes.insert("down".to_string(), self.votes.down.to_json());

        obj.insert("votes".to_string(), votes.to_json());
        obj.insert("first_vote".to_string(), self.first_vote.format("%FT%TZ").to_string().to_json());
        obj.insert("last_vote".to_string(), self.last_vote.format("%FT%TZ").to_string().to_json());

        obj.to_json()
    }
}
<|endoftext|>


<commit_before>#![feature(const_fn)]

/// Given an array containing the number of codes of each code length,
/// this function generates the huffman codes lengths and their respective
/// code lengths as specified by the JPEG spec.
const fn derive_codes_and_sizes_const(bits: &[u8]) -> ([u8; 256], [u16; 256]) {
    let mut huffsize = [0u8; 256];
    let mut huffcode = [0u16; 256];

    let mut k = 0;

    // Annex C.2
    // Figure C.1
    // Generate table of individual code lengths
    let mut i = 0;
    while i < 16 {
        let mut j = 0;
        while j < bits[i as usize] {
            huffsize[k] = i + 1;
            k += 1;
            j += 1;
        }
        i += 1;
    }

    huffsize[k] = 0;

    // Annex C.2
    // Figure C.2
    // Generate table of huffman codes
    k = 0;
    let mut code = 0u16;
    let mut size = huffsize[0];

    while huffsize[k] != 0 {
        huffcode[k] = code;
        code += 1;
        k += 1;

        if huffsize[k] == size {
            continue;
        }

        // FIXME there is something wrong with this code
        let diff = huffsize[k].wrapping_sub(size);
        code = if diff < 16 { code << diff as usize } else { 0 };

        size = size.wrapping_add(diff);
    }

    (huffsize, huffcode)
}

pub(crate) const fn build_huff_lut_const(bits: &[u8], huffval: &[u8]) -> [(u8, u16); 256] {
    let mut lut = [(17u8, 0u16); 256];
    let (huffsize, huffcode) = derive_codes_and_sizes_const(bits);

    let mut i = 0;
    while i < huffval.len() {
        lut[huffval[i] as usize] = (huffsize[i], huffcode[i]);
        i += 1;
    }

    lut
}

/// Given an array containing the number of codes of each code length,
/// this function generates the huffman codes lengths and their respective
/// code lengths as specified by the JPEG spec.
fn derive_codes_and_sizes(bits: &[u8]) -> (Vec<u8>, Vec<u16>) {
    let mut huffsize = vec![0u8; 256];
    let mut huffcode = vec![0u16; 256];

    let mut k = 0;

    // Annex C.2
    // Figure C.1
    // Generate table of individual code lengths
    for i in 0u8..16 {
        let mut j = 0;

        while j < bits[usize::from(i)] {
            huffsize[k] = i + 1;
            k += 1;
            j += 1;
        }
    }

    huffsize[k] = 0;

    // Annex C.2
    // Figure C.2
    // Generate table of huffman codes
    k = 0;
    let mut code = 0u16;
    let mut size = huffsize[0];

    while huffsize[k] != 0 {
        huffcode[k] = code;
        code += 1;
        k += 1;

        if huffsize[k] == size {
            continue;
        }

        // FIXME there is something wrong with this code
        let diff = huffsize[k].wrapping_sub(size);
        code = if diff < 16 { code << diff as usize } else { 0 };

        size = size.wrapping_add(diff);
    }

    (huffsize, huffcode)
}

pub(crate) fn build_huff_lut(bits: &[u8], huffval: &[u8]) -> Vec<(u8, u16)> {
    let mut lut = vec![(17u8, 0u16); 256];
    let (huffsize, huffcode) = derive_codes_and_sizes(bits);

    for (i, &v) in huffval.iter().enumerate() {
        lut[v as usize] = (huffsize[i], huffcode[i]);
    }

    lut
}

#[cfg(test)]
mod test {
    // Code lengths and values for table K.4
    static STD_CHROMA_DC_CODE_LENGTHS: [u8; 16] = [
        0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
    ];

    static STD_CHROMA_DC_VALUES: [u8; 12] = [
        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
    ];

    use super::{build_huff_lut, build_huff_lut_const};

    #[test]
    fn test_build_huff_lut() {
        let cd = build_huff_lut(&STD_CHROMA_DC_CODE_LENGTHS, &STD_CHROMA_DC_VALUES);
        let cd_const = build_huff_lut_const(&STD_CHROMA_DC_CODE_LENGTHS, &STD_CHROMA_DC_VALUES).to_vec();
        assert_eq!(cd_const, cd);
    }
}<commit_msg>Remove code duplication<commit_after>#![feature(const_fn)]

/// Given an array containing the number of codes of each code length,
/// this function generates the huffman codes lengths and their respective
/// code lengths as specified by the JPEG spec.
const fn derive_codes_and_sizes(bits: &[u8]) -> ([u8; 256], [u16; 256]) {
    let mut huffsize = [0u8; 256];
    let mut huffcode = [0u16; 256];

    let mut k = 0;

    // Annex C.2
    // Figure C.1
    // Generate table of individual code lengths
    let mut i = 0;
    while i < 16 {
        let mut j = 0;
        while j < bits[i as usize] {
            huffsize[k] = i + 1;
            k += 1;
            j += 1;
        }
        i += 1;
    }

    huffsize[k] = 0;

    // Annex C.2
    // Figure C.2
    // Generate table of huffman codes
    k = 0;
    let mut code = 0u16;
    let mut size = huffsize[0];

    while huffsize[k] != 0 {
        huffcode[k] = code;
        code += 1;
        k += 1;

        if huffsize[k] == size {
            continue;
        }

        // FIXME there is something wrong with this code
        let diff = huffsize[k].wrapping_sub(size);
        code = if diff < 16 { code << diff as usize } else { 0 };

        size = size.wrapping_add(diff);
    }

    (huffsize, huffcode)
}

pub(crate) const fn build_huff_lut_const(bits: &[u8], huffval: &[u8]) -> [(u8, u16); 256] {
    let mut lut = [(17u8, 0u16); 256];
    let (huffsize, huffcode) = derive_codes_and_sizes(bits);

    let mut i = 0;
    while i < huffval.len() {
        lut[huffval[i] as usize] = (huffsize[i], huffcode[i]);
        i += 1;
    }

    lut
}
<|endoftext|>


<commit_before>use super::super::{Grid, Coord, FiniteSearchQueue};
use super::*;

#[derive(Clone, Copy)]
struct FieldGrp {
    unmet_num: i32,
    unmet_sum: i32,
    unused: Cand,
}
pub struct Field<'a> {
    dic: &'a Dictionary,
    shape: FieldShape,
    grps: Vec<FieldGrp>,
    val: Vec<i32>,
    cand: Vec<Cand>,
    inconsistent: bool,
    queue: FiniteSearchQueue
}
impl<'a> Field<'a> {
    pub fn new(problem: &Grid<Clue>, dic: &'a Dictionary) -> Field<'a> {
        let n_cells = (problem.height() * problem.width()) as usize;
        let mut has_clue = Grid::new(problem.height(), problem.width(), false);
        for i in 0..n_cells {
            if let Clue::Clue { vertical: _, horizontal: _ } = problem[i] {
                has_clue[i] = true;
            }
        }
        let shape = FieldShape::new(&has_clue);

        let mut grps = vec![FieldGrp {
            unmet_num: 0,
            unmet_sum: 0,
            unused: 0,
        }; shape.group_to_cells.len()];
        let n_groups = grps.len();

        for i in 0..shape.group_to_cells.len() {
            let loc = shape.clue_locations[i];
            let clue_val = match loc {
                ClueLocation::Vertical(v) => match problem[v as usize] {
                    Clue::NoClue => panic!("unexpected condition"),
                    Clue::Clue { vertical: v, horizontal: _ } => v,
                },
                ClueLocation::Horizontal(h) => match problem[h as usize] {
                    Clue::NoClue => panic!("unexpected condition"),
                    Clue::Clue { vertical: _, horizontal: h } => h,
                },
            };
            let mut n_cells = 0;
            for _ in shape.group_to_cells[i] {
                n_cells += 1;
            }
            grps[i] = FieldGrp {
                unmet_num: n_cells,
                unmet_sum: clue_val,
                unused: CAND_ALL,
            };
        }

        Field {
            dic: dic,
            shape: shape,
            grps: grps,
            val: vec![UNDECIDED; n_cells],
            cand: vec![CAND_ALL; n_cells],
            inconsistent: false,
            queue: FiniteSearchQueue::new(n_groups as i32),
        }
    }
    pub fn inconsistent(&self) -> bool {
        self.inconsistent
    }
    pub fn height(&self) -> i32 {
        self.shape.has_clue.height()
    }
    pub fn width(&self) -> i32 {
        self.shape.has_clue.width()
    }
    pub fn val(&self, loc: Coord) -> i32 {
        self.val[self.location(loc)]
    }
    pub fn decide(&mut self, loc: Coord, val: i32) {
        let loc = self.location(loc);

        self.queue.start();
        self.decide_int(loc, val);
        while !self.queue.empty() {
            let g = self.queue.pop();
            self.check_group(g);
        }
        self.queue.finish();

    }
    pub fn check_all(&mut self) {
        self.queue.start();
        for i in 0..(self.grps.len() as i32) {
            self.queue.push(i);
        }
        while !self.queue.empty() {
            let g = self.queue.pop();
            self.check_group(g);
        }
        self.queue.finish();
    }
    fn location(&self, loc: Coord) -> usize {
        self.shape.has_clue.index(loc)
    }
    fn decide_int(&mut self, loc: usize, val: i32) {
        if self.val[loc] != UNDECIDED {
            if self.val[loc] != val {
                self.inconsistent = true;
            }
            return;
        }

        self.val[loc] = val;
        let (g1, g2) = self.shape.cell_to_groups[loc];
        self.grps[g1 as usize].unmet_num -= 1;
        self.grps[g1 as usize].unmet_sum -= val;
        if (self.grps[g1 as usize].unused & (1 << (val - 1) as Cand)) == 0 {
            self.inconsistent = true;
            return;
        }
        self.grps[g1 as usize].unused &= !(1 << (val - 1) as Cand);
        self.grps[g2 as usize].unmet_num -= 1;
        self.grps[g2 as usize].unmet_sum -= val;
        if (self.grps[g2 as usize].unused & (1 << (val - 1) as Cand)) == 0 {
            self.inconsistent = true;
            return;
        }
        self.grps[g2 as usize].unused &= !(1 << (val - 1) as Cand);

        self.eliminate_cand_from_group(g1, val, loc as i32);
        self.eliminate_cand_from_group(g2, val, loc as i32);

        self.queue.push(g1);
        self.queue.push(g2);
    }
    fn eliminate_cand_from_group(&mut self, grp: i32, rem_cand: i32, cur: i32) {
        let cand = !(1 << (rem_cand - 1) as Cand);
        for c in self.shape.group_to_cells[grp as usize] {
            if c != cur {
                self.limit_cand(c as usize, cand);
            }
        }
    }
    fn limit_cand(&mut self, loc: usize, lim: Cand) {
        if self.cand[loc] & lim == self.cand[loc] {
            return;
        }
        self.cand[loc] &= lim;

        let current_cand = self.cand[loc];
        if current_cand == 0 {
            self.inconsistent = true;
            return;
        }
        if current_cand.count_ones() == 1 {
            self.decide_int(loc, (current_cand.trailing_zeros() + 1) as i32);
        }

        let (g1, g2) = self.shape.cell_to_groups[loc];
        self.queue.push(g1);
        self.queue.push(g2);
    }
    fn check_group(&mut self, gid: i32) {
        let grp = self.grps[gid as usize];
        let (imperative, allowed) = self.dic.at(grp.unmet_num, grp.unmet_sum, grp.unused);

        // unique position technique
        if imperative != 0 {
            let mut uniq = 0;
            let mut mult = 0;
            for c in self.shape.group_to_cells[gid as usize] {
                if self.val[c as usize] == UNDECIDED {
                    mult |= uniq & self.cand[c as usize];
                    uniq |= self.cand[c as usize];
                }
            }
            uniq &= imperative & !mult;
            if uniq != 0 {
                for c in self.shape.group_to_cells[gid as usize] {
                    if self.val[c as usize] == UNDECIDED && (self.cand[c as usize] & uniq) != 0 {
                        let val = ((self.cand[c as usize] & uniq).trailing_zeros() + 1) as i32;
                        self.decide_int(c as usize, val);
                    }
                }
            }
        }

        // candidate limitation
        for c in self.shape.group_to_cells[gid as usize] {
            if self.val[c as usize] == UNDECIDED {
                self.limit_cand(c as usize, allowed);
            }
        }

        // two-cells propagation (TODO: improve complexity)
        let grp = self.grps[gid as usize];
        if grp.unmet_num == 2 {
            let mut c1 = -1;
            let mut c2 = -1;
            for c in self.shape.group_to_cells[gid as usize] {
                if self.val[c as usize] == UNDECIDED {
                    if c1 == -1 {
                        c1 = c;
                    } else {
                        c2 = c;
                    }
                }
            }
            for i in 1..(MAX_VAL + 1) {
                if (self.cand[c1 as usize] & (1 << (i - 1) as Cand) == 0) && 1 <= (grp.unmet_sum - i) && (grp.unmet_sum - i) <= MAX_VAL {
                    self.limit_cand(c2 as usize, !(1 << (grp.unmet_sum - i - 1) as Cand));
                }
                if (self.cand[c2 as usize] & (1 << (i - 1) as Cand) == 0) && 1 <= (grp.unmet_sum - i) && (grp.unmet_sum - i) <= MAX_VAL {
                    self.limit_cand(c1 as usize, !(1 << (grp.unmet_sum - i - 1) as Cand));
                }
            }
        }

        // naked pair (TODO: improve complexity)
        for c in self.shape.group_to_cells[gid as usize] {
            if self.val[c as usize] != -1 { continue; }
            for d in self.shape.group_to_cells[gid as usize] {
                if self.val[d as usize] != -1 { continue; }
                if c != d && self.cand[c as usize] == self.cand[d as usize] && self.cand[c as usize].count_ones() == 2 {
                    for e in self.shape.group_to_cells[gid as usize] {
                        if c != e && d != e {
                            let lim = !self.cand[c as usize];
                            self.limit_cand(e as usize, lim);
                        }
                    }
                }
            }
        }

        // min-max method
        let grp = self.grps[gid as usize];
        let mut min_sum = 0;
        let mut max_sum = 0;
        for c in self.shape.group_to_cells[gid as usize] {
            if self.val[c as usize] != -1 { continue; }
            let cand = self.cand[c as usize];
            min_sum += cand.trailing_zeros() + 1;
            max_sum += 32 - cand.leading_zeros();
        }
        for c in self.shape.group_to_cells[gid as usize] {
            if self.val[c as usize] != -1 { continue; }
            let cand = self.cand[c as usize];

            let current_max = grp.unmet_sum - (min_sum - (cand.trailing_zeros() + 1)) as i32;
            let current_min = grp.unmet_sum - (max_sum - (32 - cand.leading_zeros())) as i32;

            let mut lim = CAND_ALL;
            if current_max <= 8 {
                lim &= (1 << current_max as Cand) - 1;
            }
            if current_min >= 2 {
                lim &= !((1 << (current_min as Cand - 1)) - 1);
            }
            if lim != CAND_ALL {
                self.limit_cand(c as usize, lim);
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use common;

    #[test]
    fn test_field() {
        let dic = Dictionary::default();
        let mut problem_base = Grid::new(3, 3, Clue::NoClue);
        problem_base[Coord { y: 0, x: 0 }] = Clue::Clue { horizontal: -1, vertical: -1 };
        problem_base[Coord { y: 0, x: 1 }] = Clue::Clue { horizontal: -1, vertical: 3 };
        problem_base[Coord { y: 0, x: 2 }] = Clue::Clue { horizontal: -1, vertical: 8 };
        problem_base[Coord { y: 1, x: 0 }] = Clue::Clue { horizontal: 4, vertical: -1 };
        problem_base[Coord { y: 2, x: 0 }] = Clue::Clue { horizontal: 7, vertical: -1 };

        let mut field = Field::new(&problem_base, &dic);
        field.check_all();

        assert_eq!(field.val(Coord { y: 1, x: 1 }), 1);
        assert_eq!(field.val(Coord { y: 1, x: 2 }), 3);
        assert_eq!(field.val(Coord { y: 2, x: 1 }), 2);
        assert_eq!(field.val(Coord { y: 2, x: 2 }), 5);
        assert_eq!(field.inconsistent(), false);
    }
}
<commit_msg>Kakuro: strengthen conditions for inconsistency check<commit_after>use super::super::{Grid, Coord, FiniteSearchQueue};
use super::*;

#[derive(Clone, Copy)]
struct FieldGrp {
    unmet_num: i32,
    unmet_sum: i32,
    unused: Cand,
}
pub struct Field<'a> {
    dic: &'a Dictionary,
    shape: FieldShape,
    grps: Vec<FieldGrp>,
    val: Vec<i32>,
    cand: Vec<Cand>,
    inconsistent: bool,
    queue: FiniteSearchQueue
}
impl<'a> Field<'a> {
    pub fn new(problem: &Grid<Clue>, dic: &'a Dictionary) -> Field<'a> {
        let n_cells = (problem.height() * problem.width()) as usize;
        let mut has_clue = Grid::new(problem.height(), problem.width(), false);
        for i in 0..n_cells {
            if let Clue::Clue { vertical: _, horizontal: _ } = problem[i] {
                has_clue[i] = true;
            }
        }
        let shape = FieldShape::new(&has_clue);

        let mut grps = vec![FieldGrp {
            unmet_num: 0,
            unmet_sum: 0,
            unused: 0,
        }; shape.group_to_cells.len()];
        let n_groups = grps.len();

        for i in 0..shape.group_to_cells.len() {
            let loc = shape.clue_locations[i];
            let clue_val = match loc {
                ClueLocation::Vertical(v) => match problem[v as usize] {
                    Clue::NoClue => panic!("unexpected condition"),
                    Clue::Clue { vertical: v, horizontal: _ } => v,
                },
                ClueLocation::Horizontal(h) => match problem[h as usize] {
                    Clue::NoClue => panic!("unexpected condition"),
                    Clue::Clue { vertical: _, horizontal: h } => h,
                },
            };
            let mut n_cells = 0;
            for _ in shape.group_to_cells[i] {
                n_cells += 1;
            }
            grps[i] = FieldGrp {
                unmet_num: n_cells,
                unmet_sum: clue_val,
                unused: CAND_ALL,
            };
        }

        Field {
            dic: dic,
            shape: shape,
            grps: grps,
            val: vec![UNDECIDED; n_cells],
            cand: vec![CAND_ALL; n_cells],
            inconsistent: false,
            queue: FiniteSearchQueue::new(n_groups as i32),
        }
    }
    pub fn inconsistent(&self) -> bool {
        self.inconsistent
    }
    pub fn height(&self) -> i32 {
        self.shape.has_clue.height()
    }
    pub fn width(&self) -> i32 {
        self.shape.has_clue.width()
    }
    pub fn val(&self, loc: Coord) -> i32 {
        self.val[self.location(loc)]
    }
    pub fn decide(&mut self, loc: Coord, val: i32) {
        let loc = self.location(loc);

        self.queue.start();
        self.decide_int(loc, val);
        while !self.queue.empty() {
            let g = self.queue.pop();
            self.check_group(g);
        }
        self.queue.finish();

    }
    pub fn check_all(&mut self) {
        self.queue.start();
        for i in 0..(self.grps.len() as i32) {
            self.queue.push(i);
        }
        while !self.queue.empty() {
            let g = self.queue.pop();
            self.check_group(g);
        }
        self.queue.finish();
    }
    fn location(&self, loc: Coord) -> usize {
        self.shape.has_clue.index(loc)
    }
    fn decide_int(&mut self, loc: usize, val: i32) {
        if self.val[loc] != UNDECIDED {
            if self.val[loc] != val {
                self.inconsistent = true;
            }
            return;
        }

        self.val[loc] = val;
        if (self.cand[loc] & (1 << (val - 1))) == 0 {
            self.inconsistent = true;
            return;
        }
        self.cand[loc] = 1 << (val - 1);

        let (g1, g2) = self.shape.cell_to_groups[loc];
        self.grps[g1 as usize].unmet_num -= 1;
        self.grps[g1 as usize].unmet_sum -= val;
        if (self.grps[g1 as usize].unused & (1 << (val - 1) as Cand)) == 0 {
            self.inconsistent = true;
            return;
        }
        self.grps[g1 as usize].unused &= !(1 << (val - 1) as Cand);
        self.grps[g2 as usize].unmet_num -= 1;
        self.grps[g2 as usize].unmet_sum -= val;
        if (self.grps[g2 as usize].unused & (1 << (val - 1) as Cand)) == 0 {
            self.inconsistent = true;
            return;
        }
        self.grps[g2 as usize].unused &= !(1 << (val - 1) as Cand);

        self.eliminate_cand_from_group(g1, val, loc as i32);
        self.eliminate_cand_from_group(g2, val, loc as i32);

        self.queue.push(g1);
        self.queue.push(g2);
    }
    fn eliminate_cand_from_group(&mut self, grp: i32, rem_cand: i32, cur: i32) {
        let cand = !(1 << (rem_cand - 1) as Cand);
        for c in self.shape.group_to_cells[grp as usize] {
            if c != cur {
                self.limit_cand(c as usize, cand);
            }
        }
    }
    fn limit_cand(&mut self, loc: usize, lim: Cand) {
        if self.cand[loc] & lim == self.cand[loc] {
            return;
        }
        self.cand[loc] &= lim;

        let current_cand = self.cand[loc];
        if current_cand == 0 {
            self.inconsistent = true;
            return;
        }
        if current_cand.count_ones() == 1 {
            self.decide_int(loc, (current_cand.trailing_zeros() + 1) as i32);
        }

        let (g1, g2) = self.shape.cell_to_groups[loc];
        self.queue.push(g1);
        self.queue.push(g2);
    }
    fn check_group(&mut self, gid: i32) {
        let grp = self.grps[gid as usize];
        let (imperative, allowed) = self.dic.at(grp.unmet_num, grp.unmet_sum, grp.unused);
        if (imperative, allowed) == dictionary::IMPOSSIBLE {
            self.inconsistent = true;
            return;
        }

        // unique position technique
        if imperative != 0 {
            let mut uniq = 0;
            let mut mult = 0;
            for c in self.shape.group_to_cells[gid as usize] {
                if self.val[c as usize] == UNDECIDED {
                    mult |= uniq & self.cand[c as usize];
                    uniq |= self.cand[c as usize];
                }
            }
            uniq &= imperative & !mult;
            if uniq != 0 {
                for c in self.shape.group_to_cells[gid as usize] {
                    if self.val[c as usize] == UNDECIDED && (self.cand[c as usize] & uniq) != 0 {
                        let val = ((self.cand[c as usize] & uniq).trailing_zeros() + 1) as i32;
                        self.decide_int(c as usize, val);
                    }
                }
            }
        }

        // candidate limitation
        for c in self.shape.group_to_cells[gid as usize] {
            if self.val[c as usize] == UNDECIDED {
                self.limit_cand(c as usize, allowed);
            }
        }

        // two-cells propagation (TODO: improve complexity)
        let grp = self.grps[gid as usize];
        if grp.unmet_num == 2 {
            let mut c1 = -1;
            let mut c2 = -1;
            for c in self.shape.group_to_cells[gid as usize] {
                if self.val[c as usize] == UNDECIDED {
                    if c1 == -1 {
                        c1 = c;
                    } else {
                        c2 = c;
                    }
                }
            }
            for i in 1..(MAX_VAL + 1) {
                if (self.cand[c1 as usize] & (1 << (i - 1) as Cand) == 0) && 1 <= (grp.unmet_sum - i) && (grp.unmet_sum - i) <= MAX_VAL {
                    self.limit_cand(c2 as usize, !(1 << (grp.unmet_sum - i - 1) as Cand));
                }
                if (self.cand[c2 as usize] & (1 << (i - 1) as Cand) == 0) && 1 <= (grp.unmet_sum - i) && (grp.unmet_sum - i) <= MAX_VAL {
                    self.limit_cand(c1 as usize, !(1 << (grp.unmet_sum - i - 1) as Cand));
                }
            }
        }

        // naked pair (TODO: improve complexity)
        for c in self.shape.group_to_cells[gid as usize] {
            if self.val[c as usize] != -1 { continue; }
            for d in self.shape.group_to_cells[gid as usize] {
                if self.val[d as usize] != -1 { continue; }
                if c != d && self.cand[c as usize] == self.cand[d as usize] && self.cand[c as usize].count_ones() == 2 {
                    for e in self.shape.group_to_cells[gid as usize] {
                        if c != e && d != e {
                            let lim = !self.cand[c as usize];
                            self.limit_cand(e as usize, lim);
                        }
                    }
                }
            }
        }

        // min-max method
        let grp = self.grps[gid as usize];
        let mut min_sum = 0;
        let mut max_sum = 0;
        for c in self.shape.group_to_cells[gid as usize] {
            if self.val[c as usize] != -1 { continue; }
            let cand = self.cand[c as usize];
            min_sum += cand.trailing_zeros() + 1;
            max_sum += 32 - cand.leading_zeros();
        }
        for c in self.shape.group_to_cells[gid as usize] {
            if self.val[c as usize] != -1 { continue; }
            let cand = self.cand[c as usize];

            let current_max = grp.unmet_sum - (min_sum - (cand.trailing_zeros() + 1)) as i32;
            let current_min = grp.unmet_sum - (max_sum - (32 - cand.leading_zeros())) as i32;

            let mut lim = CAND_ALL;
            if current_max <= 8 {
                lim &= (1 << current_max as Cand) - 1;
            }
            if current_min >= 2 {
                lim &= !((1 << (current_min as Cand - 1)) - 1);
            }
            if lim != CAND_ALL {
                self.limit_cand(c as usize, lim);
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use common;

    #[test]
    fn test_field() {
        let dic = Dictionary::default();
        let mut problem_base = Grid::new(3, 3, Clue::NoClue);
        problem_base[Coord { y: 0, x: 0 }] = Clue::Clue { horizontal: -1, vertical: -1 };
        problem_base[Coord { y: 0, x: 1 }] = Clue::Clue { horizontal: -1, vertical: 3 };
        problem_base[Coord { y: 0, x: 2 }] = Clue::Clue { horizontal: -1, vertical: 8 };
        problem_base[Coord { y: 1, x: 0 }] = Clue::Clue { horizontal: 4, vertical: -1 };
        problem_base[Coord { y: 2, x: 0 }] = Clue::Clue { horizontal: 7, vertical: -1 };

        let mut field = Field::new(&problem_base, &dic);
        field.check_all();

        assert_eq!(field.val(Coord { y: 1, x: 1 }), 1);
        assert_eq!(field.val(Coord { y: 1, x: 2 }), 3);
        assert_eq!(field.val(Coord { y: 2, x: 1 }), 2);
        assert_eq!(field.val(Coord { y: 2, x: 2 }), 5);
        assert_eq!(field.inconsistent(), false);
    }
}
<|endoftext|>


<commit_before>//! Assorted testing utilities.
//!
//! Most notable things are:
//!
//! * Rich text comparison, which outputs a diff.
//! * Extracting markup (mainly, `<|>` markers) out of fixture strings.
//! * marks (see the eponymous module).

#[macro_use]
pub mod mark;
mod fixture;

use std::{
    env, fs,
    path::{Path, PathBuf},
};

use serde_json::Value;
use stdx::lines_with_ends;
use text_size::{TextRange, TextSize};

pub use difference::Changeset as __Changeset;
pub use rustc_hash::FxHashMap;

pub use crate::fixture::Fixture;

pub const CURSOR_MARKER: &str = "<|>";

/// Asserts that two strings are equal, otherwise displays a rich diff between them.
///
/// The diff shows changes from the "original" left string to the "actual" right string.
///
/// All arguments starting from and including the 3rd one are passed to
/// `eprintln!()` macro in case of text inequality.
#[macro_export]
macro_rules! assert_eq_text {
    ($left:expr, $right:expr) => {
        assert_eq_text!($left, $right,)
    };
    ($left:expr, $right:expr, $($tt:tt)*) => {{
        let left = $left;
        let right = $right;
        if left != right {
            if left.trim() == right.trim() {
                eprintln!("Left:\n{:?}\n\nRight:\n{:?}\n\nWhitespace difference\n", left, right);
            } else {
                let changeset = $crate::__Changeset::new(left, right, "\n");
                eprintln!("Left:\n{}\n\nRight:\n{}\n\nDiff:\n{}\n", left, right, changeset);
            }
            eprintln!($($tt)*);
            panic!("text differs");
        }
    }};
}

/// Infallible version of `try_extract_offset()`.
pub fn extract_offset(text: &str) -> (TextSize, String) {
    match try_extract_offset(text) {
        None => panic!("text should contain cursor marker"),
        Some(result) => result,
    }
}

/// Returns the offset of the first occurence of `<|>` marker and the copy of `text`
/// without the marker.
fn try_extract_offset(text: &str) -> Option<(TextSize, String)> {
    let cursor_pos = text.find(CURSOR_MARKER)?;
    let mut new_text = String::with_capacity(text.len() - CURSOR_MARKER.len());
    new_text.push_str(&text[..cursor_pos]);
    new_text.push_str(&text[cursor_pos + CURSOR_MARKER.len()..]);
    let cursor_pos = TextSize::from(cursor_pos as u32);
    Some((cursor_pos, new_text))
}

/// Infallible version of `try_extract_range()`.
pub fn extract_range(text: &str) -> (TextRange, String) {
    match try_extract_range(text) {
        None => panic!("text should contain cursor marker"),
        Some(result) => result,
    }
}

/// Returns `TextRange` between the first two markers `<|>...<|>` and the copy
/// of `text` without both of these markers.
fn try_extract_range(text: &str) -> Option<(TextRange, String)> {
    let (start, text) = try_extract_offset(text)?;
    let (end, text) = try_extract_offset(&text)?;
    Some((TextRange::new(start, end), text))
}

#[derive(Clone, Copy)]
pub enum RangeOrOffset {
    Range(TextRange),
    Offset(TextSize),
}

impl From<RangeOrOffset> for TextRange {
    fn from(selection: RangeOrOffset) -> Self {
        match selection {
            RangeOrOffset::Range(it) => it,
            RangeOrOffset::Offset(it) => TextRange::empty(it),
        }
    }
}

/// Extracts `TextRange` or `TextSize` depending on the amount of `<|>` markers
/// found in `text`.
///
/// # Panics
/// Panics if no `<|>` marker is present in the `text`.
pub fn extract_range_or_offset(text: &str) -> (RangeOrOffset, String) {
    if let Some((range, text)) = try_extract_range(text) {
        return (RangeOrOffset::Range(range), text);
    }
    let (offset, text) = extract_offset(text);
    (RangeOrOffset::Offset(offset), text)
}

/// Extracts ranges, marked with `<tag> </tag>` pairs from the `text`
pub fn extract_ranges(mut text: &str, tag: &str) -> (Vec<TextRange>, String) {
    let open = format!("<{}>", tag);
    let close = format!("</{}>", tag);
    let mut ranges = Vec::new();
    let mut res = String::new();
    let mut stack = Vec::new();
    loop {
        match text.find('<') {
            None => {
                res.push_str(text);
                break;
            }
            Some(i) => {
                res.push_str(&text[..i]);
                text = &text[i..];
                if text.starts_with(&open) {
                    text = &text[open.len()..];
                    let from = TextSize::of(&res);
                    stack.push(from);
                } else if text.starts_with(&close) {
                    text = &text[close.len()..];
                    let from = stack.pop().unwrap_or_else(|| panic!("unmatched </{}>", tag));
                    let to = TextSize::of(&res);
                    ranges.push(TextRange::new(from, to));
                }
            }
        }
    }
    assert!(stack.is_empty(), "unmatched <{}>", tag);
    ranges.sort_by_key(|r| (r.start(), r.end()));
    (ranges, res)
}

/// Inserts `<|>` marker into the `text` at `offset`.
pub fn add_cursor(text: &str, offset: TextSize) -> String {
    let offset: usize = offset.into();
    let mut res = String::new();
    res.push_str(&text[..offset]);
    res.push_str("<|>");
    res.push_str(&text[offset..]);
    res
}

/// Extracts `//^ some text` annotations
pub fn extract_annotations(text: &str) -> Vec<(TextRange, String)> {
    let mut res = Vec::new();
    let mut prev_line_start: Option<TextSize> = None;
    let mut line_start: TextSize = 0.into();
    for line in lines_with_ends(text) {
        if let Some(idx) = line.find("//^") {
            let offset = prev_line_start.unwrap() + TextSize::of(&line[..idx + "//".len()]);
            let data = line[idx + "//^".len()..].trim().to_string();
            res.push((TextRange::at(offset, 1.into()), data))
        }
        prev_line_start = Some(line_start);
        line_start += TextSize::of(line);
    }
    res
}

#[test]
fn test_extract_annotations() {
    let text = stdx::trim_indent(
        r#"
fn main() {
    let x = 92;
      //^ def
    z + 1
} //^ i32
    "#,
    );
    let res = extract_annotations(&text)
        .into_iter()
        .map(|(range, ann)| (&text[range], ann))
        .collect::<Vec<_>>();
    assert_eq!(res, vec![("x", "def".into()), ("z", "i32".into()),]);
}

// Comparison functionality borrowed from cargo:

/// Compare a line with an expected pattern.
/// - Use `[..]` as a wildcard to match 0 or more characters on the same line
///   (similar to `.*` in a regex).
pub fn lines_match(expected: &str, actual: &str) -> bool {
    // Let's not deal with / vs \ (windows...)
    // First replace backslash-escaped backslashes with forward slashes
    // which can occur in, for example, JSON output
    let expected = expected.replace(r"\\", "/").replace(r"\", "/");
    let mut actual: &str = &actual.replace(r"\\", "/").replace(r"\", "/");
    for (i, part) in expected.split("[..]").enumerate() {
        match actual.find(part) {
            Some(j) => {
                if i == 0 && j != 0 {
                    return false;
                }
                actual = &actual[j + part.len()..];
            }
            None => return false,
        }
    }
    actual.is_empty() || expected.ends_with("[..]")
}

#[test]
fn lines_match_works() {
    assert!(lines_match("a b", "a b"));
    assert!(lines_match("a[..]b", "a b"));
    assert!(lines_match("a[..]", "a b"));
    assert!(lines_match("[..]", "a b"));
    assert!(lines_match("[..]b", "a b"));

    assert!(!lines_match("[..]b", "c"));
    assert!(!lines_match("b", "c"));
    assert!(!lines_match("b", "cb"));
}

/// Compares JSON object for approximate equality.
/// You can use `[..]` wildcard in strings (useful for OS dependent things such
/// as paths). You can use a `"{...}"` string literal as a wildcard for
/// arbitrary nested JSON. Arrays are sorted before comparison.
pub fn find_mismatch<'a>(expected: &'a Value, actual: &'a Value) -> Option<(&'a Value, &'a Value)> {
    use serde_json::Value::*;
    match (expected, actual) {
        (&Number(ref l), &Number(ref r)) if l == r => None,
        (&Bool(l), &Bool(r)) if l == r => None,
        (&String(ref l), &String(ref r)) if lines_match(l, r) => None,
        (&Array(ref l), &Array(ref r)) => {
            if l.len() != r.len() {
                return Some((expected, actual));
            }

            let mut l = l.iter().collect::<Vec<_>>();
            let mut r = r.iter().collect::<Vec<_>>();

            l.retain(|l| match r.iter().position(|r| find_mismatch(l, r).is_none()) {
                Some(i) => {
                    r.remove(i);
                    false
                }
                None => true,
            });

            if !l.is_empty() {
                assert!(!r.is_empty());
                Some((&l[0], &r[0]))
            } else {
                assert_eq!(r.len(), 0);
                None
            }
        }
        (&Object(ref l), &Object(ref r)) => {
            let same_keys = l.len() == r.len() && l.keys().all(|k| r.contains_key(k));
            if !same_keys {
                return Some((expected, actual));
            }

            l.values().zip(r.values()).filter_map(|(l, r)| find_mismatch(l, r)).next()
        }
        (&Null, &Null) => None,
        // magic string literal "{...}" acts as wildcard for any sub-JSON
        (&String(ref l), _) if l == "{...}" => None,
        _ => Some((expected, actual)),
    }
}

/// Calls callback `f` with input code and file paths for each `.rs` file in `test_data_dir`
/// subdirectories defined by `paths`.
///
/// If the content of the matching output file differs from the output of `f()`
/// the test will fail.
///
/// If there is no matching output file it will be created and filled with the
/// output of `f()`, but the test will fail.
pub fn dir_tests<F>(test_data_dir: &Path, paths: &[&str], outfile_extension: &str, f: F)
where
    F: Fn(&str, &Path) -> String,
{
    for (path, input_code) in collect_rust_files(test_data_dir, paths) {
        let actual = f(&input_code, &path);
        let path = path.with_extension(outfile_extension);
        if !path.exists() {
            println!("\nfile: {}", path.display());
            println!("No .txt file with expected result, creating...\n");
            println!("{}\n{}", input_code, actual);
            fs::write(&path, &actual).unwrap();
            panic!("No expected result");
        }
        let expected = read_text(&path);
        assert_equal_text(&expected, &actual, &path);
    }
}

/// Collects all `.rs` files from `dir` subdirectories defined by `paths`.
pub fn collect_rust_files(root_dir: &Path, paths: &[&str]) -> Vec<(PathBuf, String)> {
    paths
        .iter()
        .flat_map(|path| {
            let path = root_dir.to_owned().join(path);
            rust_files_in_dir(&path).into_iter()
        })
        .map(|path| {
            let text = read_text(&path);
            (path, text)
        })
        .collect()
}

/// Collects paths to all `.rs` files from `dir` in a sorted `Vec<PathBuf>`.
fn rust_files_in_dir(dir: &Path) -> Vec<PathBuf> {
    let mut acc = Vec::new();
    for file in fs::read_dir(&dir).unwrap() {
        let file = file.unwrap();
        let path = file.path();
        if path.extension().unwrap_or_default() == "rs" {
            acc.push(path);
        }
    }
    acc.sort();
    acc
}

/// Returns the path to the root directory of `rust-analyzer` project.
pub fn project_dir() -> PathBuf {
    let dir = env!("CARGO_MANIFEST_DIR");
    PathBuf::from(dir).parent().unwrap().parent().unwrap().to_owned()
}

/// Read file and normalize newlines.
///
/// `rustc` seems to always normalize `\r\n` newlines to `\n`:
///
/// ```
/// let s = "
/// ";
/// assert_eq!(s.as_bytes(), &[10]);
/// ```
///
/// so this should always be correct.
pub fn read_text(path: &Path) -> String {
    fs::read_to_string(path)
        .unwrap_or_else(|_| panic!("File at {:?} should be valid", path))
        .replace("\r\n", "\n")
}

/// Returns `false` if slow tests should not run, otherwise returns `true` and
/// also creates a file at `./target/.slow_tests_cookie` which serves as a flag
/// that slow tests did run.
pub fn skip_slow_tests() -> bool {
    let should_skip = std::env::var("CI").is_err() && std::env::var("RUN_SLOW_TESTS").is_err();
    if should_skip {
        eprintln!("ignoring slow test")
    } else {
        let path = project_dir().join("./target/.slow_tests_cookie");
        fs::write(&path, ".").unwrap();
    }
    should_skip
}

/// Asserts that `expected` and `actual` strings are equal. If they differ only
/// in trailing or leading whitespace the test won't fail and
/// the contents of `actual` will be written to the file located at `path`.
fn assert_equal_text(expected: &str, actual: &str, path: &Path) {
    if expected == actual {
        return;
    }
    let dir = project_dir();
    let pretty_path = path.strip_prefix(&dir).unwrap_or_else(|_| path);
    if expected.trim() == actual.trim() {
        println!("whitespace difference, rewriting");
        println!("file: {}\n", pretty_path.display());
        fs::write(path, actual).unwrap();
        return;
    }
    if env::var("UPDATE_EXPECTATIONS").is_ok() {
        println!("rewriting {}", pretty_path.display());
        fs::write(path, actual).unwrap();
        return;
    }
    assert_eq_text!(expected, actual, "file: {}", pretty_path.display());
}
<commit_msg>Generalize annotations<commit_after>//! Assorted testing utilities.
//!
//! Most notable things are:
//!
//! * Rich text comparison, which outputs a diff.
//! * Extracting markup (mainly, `<|>` markers) out of fixture strings.
//! * marks (see the eponymous module).

#[macro_use]
pub mod mark;
mod fixture;

use std::{
    convert::TryInto,
    env, fs,
    path::{Path, PathBuf},
};

use serde_json::Value;
use stdx::lines_with_ends;
use text_size::{TextRange, TextSize};

pub use difference::Changeset as __Changeset;
pub use rustc_hash::FxHashMap;

pub use crate::fixture::Fixture;

pub const CURSOR_MARKER: &str = "<|>";

/// Asserts that two strings are equal, otherwise displays a rich diff between them.
///
/// The diff shows changes from the "original" left string to the "actual" right string.
///
/// All arguments starting from and including the 3rd one are passed to
/// `eprintln!()` macro in case of text inequality.
#[macro_export]
macro_rules! assert_eq_text {
    ($left:expr, $right:expr) => {
        assert_eq_text!($left, $right,)
    };
    ($left:expr, $right:expr, $($tt:tt)*) => {{
        let left = $left;
        let right = $right;
        if left != right {
            if left.trim() == right.trim() {
                eprintln!("Left:\n{:?}\n\nRight:\n{:?}\n\nWhitespace difference\n", left, right);
            } else {
                let changeset = $crate::__Changeset::new(left, right, "\n");
                eprintln!("Left:\n{}\n\nRight:\n{}\n\nDiff:\n{}\n", left, right, changeset);
            }
            eprintln!($($tt)*);
            panic!("text differs");
        }
    }};
}

/// Infallible version of `try_extract_offset()`.
pub fn extract_offset(text: &str) -> (TextSize, String) {
    match try_extract_offset(text) {
        None => panic!("text should contain cursor marker"),
        Some(result) => result,
    }
}

/// Returns the offset of the first occurence of `<|>` marker and the copy of `text`
/// without the marker.
fn try_extract_offset(text: &str) -> Option<(TextSize, String)> {
    let cursor_pos = text.find(CURSOR_MARKER)?;
    let mut new_text = String::with_capacity(text.len() - CURSOR_MARKER.len());
    new_text.push_str(&text[..cursor_pos]);
    new_text.push_str(&text[cursor_pos + CURSOR_MARKER.len()..]);
    let cursor_pos = TextSize::from(cursor_pos as u32);
    Some((cursor_pos, new_text))
}

/// Infallible version of `try_extract_range()`.
pub fn extract_range(text: &str) -> (TextRange, String) {
    match try_extract_range(text) {
        None => panic!("text should contain cursor marker"),
        Some(result) => result,
    }
}

/// Returns `TextRange` between the first two markers `<|>...<|>` and the copy
/// of `text` without both of these markers.
fn try_extract_range(text: &str) -> Option<(TextRange, String)> {
    let (start, text) = try_extract_offset(text)?;
    let (end, text) = try_extract_offset(&text)?;
    Some((TextRange::new(start, end), text))
}

#[derive(Clone, Copy)]
pub enum RangeOrOffset {
    Range(TextRange),
    Offset(TextSize),
}

impl From<RangeOrOffset> for TextRange {
    fn from(selection: RangeOrOffset) -> Self {
        match selection {
            RangeOrOffset::Range(it) => it,
            RangeOrOffset::Offset(it) => TextRange::empty(it),
        }
    }
}

/// Extracts `TextRange` or `TextSize` depending on the amount of `<|>` markers
/// found in `text`.
///
/// # Panics
/// Panics if no `<|>` marker is present in the `text`.
pub fn extract_range_or_offset(text: &str) -> (RangeOrOffset, String) {
    if let Some((range, text)) = try_extract_range(text) {
        return (RangeOrOffset::Range(range), text);
    }
    let (offset, text) = extract_offset(text);
    (RangeOrOffset::Offset(offset), text)
}

/// Extracts ranges, marked with `<tag> </tag>` pairs from the `text`
pub fn extract_ranges(mut text: &str, tag: &str) -> (Vec<TextRange>, String) {
    let open = format!("<{}>", tag);
    let close = format!("</{}>", tag);
    let mut ranges = Vec::new();
    let mut res = String::new();
    let mut stack = Vec::new();
    loop {
        match text.find('<') {
            None => {
                res.push_str(text);
                break;
            }
            Some(i) => {
                res.push_str(&text[..i]);
                text = &text[i..];
                if text.starts_with(&open) {
                    text = &text[open.len()..];
                    let from = TextSize::of(&res);
                    stack.push(from);
                } else if text.starts_with(&close) {
                    text = &text[close.len()..];
                    let from = stack.pop().unwrap_or_else(|| panic!("unmatched </{}>", tag));
                    let to = TextSize::of(&res);
                    ranges.push(TextRange::new(from, to));
                }
            }
        }
    }
    assert!(stack.is_empty(), "unmatched <{}>", tag);
    ranges.sort_by_key(|r| (r.start(), r.end()));
    (ranges, res)
}

/// Inserts `<|>` marker into the `text` at `offset`.
pub fn add_cursor(text: &str, offset: TextSize) -> String {
    let offset: usize = offset.into();
    let mut res = String::new();
    res.push_str(&text[..offset]);
    res.push_str("<|>");
    res.push_str(&text[offset..]);
    res
}

/// Extracts `//^ some text` annotations
pub fn extract_annotations(text: &str) -> Vec<(TextRange, String)> {
    let mut res = Vec::new();
    let mut prev_line_start: Option<TextSize> = None;
    let mut line_start: TextSize = 0.into();
    for line in lines_with_ends(text) {
        if let Some(idx) = line.find("//^") {
            let offset = prev_line_start.unwrap() + TextSize::of(&line[..idx + "//".len()]);
            let marker_and_data = &line[idx + "//".len()..];
            let len = marker_and_data.chars().take_while(|&it| it == '^').count();
            let data = marker_and_data[len..].trim().to_string();
            res.push((TextRange::at(offset, len.try_into().unwrap()), data))
        }
        prev_line_start = Some(line_start);
        line_start += TextSize::of(line);
    }
    res
}

#[test]
fn test_extract_annotations() {
    let text = stdx::trim_indent(
        r#"
fn main() {
    let x = 92;
      //^ def
    zoo + 1
} //^^^ i32
    "#,
    );
    let res = extract_annotations(&text)
        .into_iter()
        .map(|(range, ann)| (&text[range], ann))
        .collect::<Vec<_>>();
    assert_eq!(res, vec![("x", "def".into()), ("zoo", "i32".into()),]);
}

// Comparison functionality borrowed from cargo:

/// Compare a line with an expected pattern.
/// - Use `[..]` as a wildcard to match 0 or more characters on the same line
///   (similar to `.*` in a regex).
pub fn lines_match(expected: &str, actual: &str) -> bool {
    // Let's not deal with / vs \ (windows...)
    // First replace backslash-escaped backslashes with forward slashes
    // which can occur in, for example, JSON output
    let expected = expected.replace(r"\\", "/").replace(r"\", "/");
    let mut actual: &str = &actual.replace(r"\\", "/").replace(r"\", "/");
    for (i, part) in expected.split("[..]").enumerate() {
        match actual.find(part) {
            Some(j) => {
                if i == 0 && j != 0 {
                    return false;
                }
                actual = &actual[j + part.len()..];
            }
            None => return false,
        }
    }
    actual.is_empty() || expected.ends_with("[..]")
}

#[test]
fn lines_match_works() {
    assert!(lines_match("a b", "a b"));
    assert!(lines_match("a[..]b", "a b"));
    assert!(lines_match("a[..]", "a b"));
    assert!(lines_match("[..]", "a b"));
    assert!(lines_match("[..]b", "a b"));

    assert!(!lines_match("[..]b", "c"));
    assert!(!lines_match("b", "c"));
    assert!(!lines_match("b", "cb"));
}

/// Compares JSON object for approximate equality.
/// You can use `[..]` wildcard in strings (useful for OS dependent things such
/// as paths). You can use a `"{...}"` string literal as a wildcard for
/// arbitrary nested JSON. Arrays are sorted before comparison.
pub fn find_mismatch<'a>(expected: &'a Value, actual: &'a Value) -> Option<(&'a Value, &'a Value)> {
    use serde_json::Value::*;
    match (expected, actual) {
        (&Number(ref l), &Number(ref r)) if l == r => None,
        (&Bool(l), &Bool(r)) if l == r => None,
        (&String(ref l), &String(ref r)) if lines_match(l, r) => None,
        (&Array(ref l), &Array(ref r)) => {
            if l.len() != r.len() {
                return Some((expected, actual));
            }

            let mut l = l.iter().collect::<Vec<_>>();
            let mut r = r.iter().collect::<Vec<_>>();

            l.retain(|l| match r.iter().position(|r| find_mismatch(l, r).is_none()) {
                Some(i) => {
                    r.remove(i);
                    false
                }
                None => true,
            });

            if !l.is_empty() {
                assert!(!r.is_empty());
                Some((&l[0], &r[0]))
            } else {
                assert_eq!(r.len(), 0);
                None
            }
        }
        (&Object(ref l), &Object(ref r)) => {
            let same_keys = l.len() == r.len() && l.keys().all(|k| r.contains_key(k));
            if !same_keys {
                return Some((expected, actual));
            }

            l.values().zip(r.values()).filter_map(|(l, r)| find_mismatch(l, r)).next()
        }
        (&Null, &Null) => None,
        // magic string literal "{...}" acts as wildcard for any sub-JSON
        (&String(ref l), _) if l == "{...}" => None,
        _ => Some((expected, actual)),
    }
}

/// Calls callback `f` with input code and file paths for each `.rs` file in `test_data_dir`
/// subdirectories defined by `paths`.
///
/// If the content of the matching output file differs from the output of `f()`
/// the test will fail.
///
/// If there is no matching output file it will be created and filled with the
/// output of `f()`, but the test will fail.
pub fn dir_tests<F>(test_data_dir: &Path, paths: &[&str], outfile_extension: &str, f: F)
where
    F: Fn(&str, &Path) -> String,
{
    for (path, input_code) in collect_rust_files(test_data_dir, paths) {
        let actual = f(&input_code, &path);
        let path = path.with_extension(outfile_extension);
        if !path.exists() {
            println!("\nfile: {}", path.display());
            println!("No .txt file with expected result, creating...\n");
            println!("{}\n{}", input_code, actual);
            fs::write(&path, &actual).unwrap();
            panic!("No expected result");
        }
        let expected = read_text(&path);
        assert_equal_text(&expected, &actual, &path);
    }
}

/// Collects all `.rs` files from `dir` subdirectories defined by `paths`.
pub fn collect_rust_files(root_dir: &Path, paths: &[&str]) -> Vec<(PathBuf, String)> {
    paths
        .iter()
        .flat_map(|path| {
            let path = root_dir.to_owned().join(path);
            rust_files_in_dir(&path).into_iter()
        })
        .map(|path| {
            let text = read_text(&path);
            (path, text)
        })
        .collect()
}

/// Collects paths to all `.rs` files from `dir` in a sorted `Vec<PathBuf>`.
fn rust_files_in_dir(dir: &Path) -> Vec<PathBuf> {
    let mut acc = Vec::new();
    for file in fs::read_dir(&dir).unwrap() {
        let file = file.unwrap();
        let path = file.path();
        if path.extension().unwrap_or_default() == "rs" {
            acc.push(path);
        }
    }
    acc.sort();
    acc
}

/// Returns the path to the root directory of `rust-analyzer` project.
pub fn project_dir() -> PathBuf {
    let dir = env!("CARGO_MANIFEST_DIR");
    PathBuf::from(dir).parent().unwrap().parent().unwrap().to_owned()
}

/// Read file and normalize newlines.
///
/// `rustc` seems to always normalize `\r\n` newlines to `\n`:
///
/// ```
/// let s = "
/// ";
/// assert_eq!(s.as_bytes(), &[10]);
/// ```
///
/// so this should always be correct.
pub fn read_text(path: &Path) -> String {
    fs::read_to_string(path)
        .unwrap_or_else(|_| panic!("File at {:?} should be valid", path))
        .replace("\r\n", "\n")
}

/// Returns `false` if slow tests should not run, otherwise returns `true` and
/// also creates a file at `./target/.slow_tests_cookie` which serves as a flag
/// that slow tests did run.
pub fn skip_slow_tests() -> bool {
    let should_skip = std::env::var("CI").is_err() && std::env::var("RUN_SLOW_TESTS").is_err();
    if should_skip {
        eprintln!("ignoring slow test")
    } else {
        let path = project_dir().join("./target/.slow_tests_cookie");
        fs::write(&path, ".").unwrap();
    }
    should_skip
}

/// Asserts that `expected` and `actual` strings are equal. If they differ only
/// in trailing or leading whitespace the test won't fail and
/// the contents of `actual` will be written to the file located at `path`.
fn assert_equal_text(expected: &str, actual: &str, path: &Path) {
    if expected == actual {
        return;
    }
    let dir = project_dir();
    let pretty_path = path.strip_prefix(&dir).unwrap_or_else(|_| path);
    if expected.trim() == actual.trim() {
        println!("whitespace difference, rewriting");
        println!("file: {}\n", pretty_path.display());
        fs::write(path, actual).unwrap();
        return;
    }
    if env::var("UPDATE_EXPECTATIONS").is_ok() {
        println!("rewriting {}", pretty_path.display());
        fs::write(path, actual).unwrap();
        return;
    }
    assert_eq_text!(expected, actual, "file: {}", pretty_path.display());
}
<|endoftext|>