<commit_before>//! A "tiny" example of HTTP request/response handling using just tokio-core
//!
//! This example is intended for *learning purposes* to see how various pieces
//! hook up together and how HTTP can get up and running. Note that this example
//! is written with the restriction that it *can't* use any "big" library other
//! than tokio-core, if you'd like a "real world" HTTP library you likely want a
//! crate like Hyper.
//!
//! Code here is based on the `echo-threads` example and implements two paths,
//! the `/plaintext` and `/json` routes to respond with some text and json,
//! respectively. By default this will run I/O on all the cores your system has
//! available, and it doesn't support HTTP request bodies.

extern crate bytes;
extern crate futures;
extern crate http;
extern crate httparse;
extern crate num_cpus;
#[macro_use]
extern crate serde_derive;
extern crate serde_json;
extern crate time;
extern crate tokio_core;
extern crate tokio_io;

use std::env;
use std::fmt;
use std::io;
use std::net::{self, SocketAddr};
use std::thread;

use bytes::BytesMut;
use futures::future;
use futures::sync::mpsc;
use futures::{Stream, Future, Sink};
use http::{Request, Response, StatusCode};
use http::header::HeaderValue;
use tokio_core::net::TcpStream;
use tokio_core::reactor::Core;
use tokio_io::codec::{Encoder, Decoder};
use tokio_io::{AsyncRead};

fn main() {
    // Parse the arguments, bind the TCP socket we'll be listening to, spin up
    // our worker threads, and start shipping sockets to those worker threads.
    let addr = env::args().nth(1).unwrap_or("127.0.0.1:8080".to_string());
    let addr = addr.parse::<SocketAddr>().unwrap();
    let num_threads = env::args().nth(2).and_then(|s| s.parse().ok())
        .unwrap_or(num_cpus::get());

    let listener = net::TcpListener::bind(&addr).expect("failed to bind");
    println!("Listening on: {}", addr);

    let mut channels = Vec::new();
    for _ in 0..num_threads {
        let (tx, rx) = mpsc::unbounded();
        channels.push(tx);
        thread::spawn(|| worker(rx));
    }
    let mut next = 0;
    for socket in listener.incoming() {
        let socket = socket.expect("failed to accept");
        channels[next].unbounded_send(socket).expect("worker thread died");
        next = (next + 1) % channels.len();
    }
}

fn worker(rx: mpsc::UnboundedReceiver<net::TcpStream>) {
    let mut core = Core::new().unwrap();
    let handle = core.handle();

    let done = rx.for_each(move |socket| {
        // Associate each socket we get with our local event loop, and then use
        // the codec support in the tokio-io crate to deal with discrete
        // request/response types instead of bytes. Here we'll just use our
        // framing defined below and then use the `send_all` helper to send the
        // responses back on the socket after we've processed them
        let socket = future::result(TcpStream::from_stream(socket, &handle));
        let req = socket.and_then(|socket| {
            let (tx, rx) = socket.framed(Http).split();
            tx.send_all(rx.and_then(respond))
        });
        handle.spawn(req.then(move |result| {
            drop(result);
            Ok(())
        }));
        Ok(())
    });
    core.run(done).unwrap();
}

/// "Server logic" is implemented in this function.
///
/// This function is a map from and HTTP request to a future of a response and
/// represents the various handling a server might do. Currently the contents
/// here are pretty uninteresting.
fn respond(req: Request<()>)
    -> Box<Future<Item = Response<String>, Error = io::Error>>
{
    let mut ret = Response::builder();
    let body = match req.uri().path() {
        "/plaintext" => {
            ret.header("Content-Type", "text/plain");
            "Hello, World!".to_string()
        }
        "/json" => {
            ret.header("Content-Type", "application/json");

            #[derive(Serialize)]
            struct Message {
                message: &'static str,
            }
            serde_json::to_string(&Message { message: "Hello, World!" })
                .unwrap()
        }
        _ => {
            ret.status(StatusCode::NOT_FOUND);
            String::new()
        }
    };
    Box::new(future::ok(ret.body(body).unwrap()))
}

struct Http;

/// Implementation of encoding an HTTP response into a `BytesMut`, basically
/// just writing out an HTTP/1.1 response.
impl Encoder for Http {
    type Item = Response<String>;
    type Error = io::Error;

    fn encode(&mut self, item: Response<String>, dst: &mut BytesMut) -> io::Result<()> {
        use std::fmt::Write;

        write!(BytesWrite(dst), "\
            HTTP/1.1 {}\r\n\
            Server: Example\r\n\
            Content-Length: {}\r\n\
            Date: {}\r\n\
        ", item.status(), item.body().len(), date::now()).unwrap();

        for (k, v) in item.headers() {
            dst.extend_from_slice(k.as_str().as_bytes());
            dst.extend_from_slice(b": ");
            dst.extend_from_slice(v.as_bytes());
            dst.extend_from_slice(b"\r\n");
        }

        dst.extend_from_slice(b"\r\n");
        dst.extend_from_slice(item.body().as_bytes());

        return Ok(());

        // Right now `write!` on `Vec<u8>` goes through io::Write and is not
        // super speedy, so inline a less-crufty implementation here which
        // doesn't go through io::Error.
        struct BytesWrite<'a>(&'a mut BytesMut);

        impl<'a> fmt::Write for BytesWrite<'a> {
            fn write_str(&mut self, s: &str) -> fmt::Result {
                self.0.extend_from_slice(s.as_bytes());
                Ok(())
            }

            fn write_fmt(&mut self, args: fmt::Arguments) -> fmt::Result {
                fmt::write(self, args)
            }
        }
    }
}

/// Implementation of decoding an HTTP request from the bytes we've read so far.
/// This leverages the `httparse` crate to do the actual parsing and then we use
/// that information to construct an instance of a `http::Request` object,
/// trying to avoid allocations where possible.
impl Decoder for Http {
    type Item = Request<()>;
    type Error = io::Error;

    fn decode(&mut self, src: &mut BytesMut) -> io::Result<Option<Request<()>>> {
        // TODO: we should grow this headers array if parsing fails and asks
        //       for more headers
        let mut headers = [None; 16];
        let (method, path, version, amt) = {
            let mut parsed_headers = [httparse::EMPTY_HEADER; 16];
            let mut r = httparse::Request::new(&mut parsed_headers);
            let status = r.parse(src).map_err(|e| {
                let msg = format!("failed to parse http request: {:?}", e);
                io::Error::new(io::ErrorKind::Other, msg)
            })?;

            let amt = match status {
                httparse::Status::Complete(amt) => amt,
                httparse::Status::Partial => return Ok(None),
            };

            let toslice = |a: &[u8]| {
                let start = a.as_ptr() as usize - src.as_ptr() as usize;
                assert!(start < src.len());
                (start, start + a.len())
            };

            for (i, header) in r.headers.iter().enumerate() {
                let k = toslice(header.name.as_bytes());
                let v = toslice(header.value);
                headers[i] = Some((k, v));
            }

            (toslice(r.method.unwrap().as_bytes()),
             toslice(r.path.unwrap().as_bytes()),
             r.version.unwrap(),
             amt)
        };
        if version != 1 {
            return Err(io::Error::new(io::ErrorKind::Other, "only HTTP/1.1 accepted"))
        }
        let data = src.split_to(amt).freeze();
        let mut ret = Request::builder();
        ret.method(&data[method.0..method.1]);
        ret.uri(data.slice(path.0, path.1));
        ret.version(http::Version::HTTP_11);
        for header in headers.iter() {
            let (k, v) = match *header {
                Some((ref k, ref v)) => (k, v),
                None => break,
            };
            let value = unsafe {
                HeaderValue::from_shared_unchecked(data.slice(v.0, v.1))
            };
            ret.header(&data[k.0..k.1], value);
        }

        let req = ret.body(()).map_err(|e| {
            io::Error::new(io::ErrorKind::Other, e)
        })?;
        Ok(Some(req))
    }
}

mod date {
    use std::cell::RefCell;
    use std::fmt::{self, Write};
    use std::str;

    use time::{self, Duration};

    pub struct Now(());

    /// Returns a struct, which when formatted, renders an appropriate `Date`
    /// header value.
    pub fn now() -> Now {
        Now(())
    }

    // Gee Alex, doesn't this seem like premature optimization. Well you see
    // there Billy, you're absolutely correct! If your server is *bottlenecked*
    // on rendering the `Date` header, well then boy do I have news for you, you
    // don't need this optimization.
    //
    // In all seriousness, though, a simple "hello world" benchmark which just
    // sends back literally "hello world" with standard headers actually is
    // bottlenecked on rendering a date into a byte buffer. Since it was at the
    // top of a profile, and this was done for some competitive benchmarks, this
    // module was written.
    //
    // Just to be clear, though, I was not intending on doing this because it
    // really does seem kinda absurd, but it was done by someone else [1], so I
    // blame them!  :)
    //
    // [1]: https://github.com/rapidoid/rapidoid/blob/f1c55c0555007e986b5d069fe1086e6d09933f7b/rapidoid-commons/src/main/java/org/rapidoid/commons/Dates.java#L48-L66

    struct LastRenderedNow {
        bytes: [u8; 128],
        amt: usize,
        next_update: time::Timespec,
    }

    thread_local!(static LAST: RefCell<LastRenderedNow> = RefCell::new(LastRenderedNow {
        bytes: [0; 128],
        amt: 0,
        next_update: time::Timespec::new(0, 0),
    }));

    impl fmt::Display for Now {
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
            LAST.with(|cache| {
                let mut cache = cache.borrow_mut();
                let now = time::get_time();
                if now > cache.next_update {
                    cache.update(now);
                }
                f.write_str(cache.buffer())
            })
        }
    }

    impl LastRenderedNow {
        fn buffer(&self) -> &str {
            str::from_utf8(&self.bytes[..self.amt]).unwrap()
        }

        fn update(&mut self, now: time::Timespec) {
            self.amt = 0;
            write!(LocalBuffer(self), "{}", time::at(now).rfc822()).unwrap();
            self.next_update = now + Duration::seconds(1);
            self.next_update.nsec = 0;
        }
    }

    struct LocalBuffer<'a>(&'a mut LastRenderedNow);

    impl<'a> fmt::Write for LocalBuffer<'a> {
        fn write_str(&mut self, s: &str) -> fmt::Result {
            let start = self.0.amt;
            let end = start + s.len();
            self.0.bytes[start..end].copy_from_slice(s.as_bytes());
            self.0.amt += s.len();
            Ok(())
        }
    }
}
<commit_msg>Don't unwrap accepted connections<commit_after>//! A "tiny" example of HTTP request/response handling using just tokio-core
//!
//! This example is intended for *learning purposes* to see how various pieces
//! hook up together and how HTTP can get up and running. Note that this example
//! is written with the restriction that it *can't* use any "big" library other
//! than tokio-core, if you'd like a "real world" HTTP library you likely want a
//! crate like Hyper.
//!
//! Code here is based on the `echo-threads` example and implements two paths,
//! the `/plaintext` and `/json` routes to respond with some text and json,
//! respectively. By default this will run I/O on all the cores your system has
//! available, and it doesn't support HTTP request bodies.

extern crate bytes;
extern crate futures;
extern crate http;
extern crate httparse;
extern crate num_cpus;
#[macro_use]
extern crate serde_derive;
extern crate serde_json;
extern crate time;
extern crate tokio_core;
extern crate tokio_io;

use std::env;
use std::fmt;
use std::io;
use std::net::{self, SocketAddr};
use std::thread;

use bytes::BytesMut;
use futures::future;
use futures::sync::mpsc;
use futures::{Stream, Future, Sink};
use http::{Request, Response, StatusCode};
use http::header::HeaderValue;
use tokio_core::net::TcpStream;
use tokio_core::reactor::Core;
use tokio_io::codec::{Encoder, Decoder};
use tokio_io::{AsyncRead};

fn main() {
    // Parse the arguments, bind the TCP socket we'll be listening to, spin up
    // our worker threads, and start shipping sockets to those worker threads.
    let addr = env::args().nth(1).unwrap_or("127.0.0.1:8080".to_string());
    let addr = addr.parse::<SocketAddr>().unwrap();
    let num_threads = env::args().nth(2).and_then(|s| s.parse().ok())
        .unwrap_or(num_cpus::get());

    let listener = net::TcpListener::bind(&addr).expect("failed to bind");
    println!("Listening on: {}", addr);

    let mut channels = Vec::new();
    for _ in 0..num_threads {
        let (tx, rx) = mpsc::unbounded();
        channels.push(tx);
        thread::spawn(|| worker(rx));
    }
    let mut next = 0;
    for socket in listener.incoming() {
        if let Ok(socket) = socket {
            channels[next].unbounded_send(socket).expect("worker thread died");
            next = (next + 1) % channels.len();
        }
    }
}

fn worker(rx: mpsc::UnboundedReceiver<net::TcpStream>) {
    let mut core = Core::new().unwrap();
    let handle = core.handle();

    let done = rx.for_each(move |socket| {
        // Associate each socket we get with our local event loop, and then use
        // the codec support in the tokio-io crate to deal with discrete
        // request/response types instead of bytes. Here we'll just use our
        // framing defined below and then use the `send_all` helper to send the
        // responses back on the socket after we've processed them
        let socket = future::result(TcpStream::from_stream(socket, &handle));
        let req = socket.and_then(|socket| {
            let (tx, rx) = socket.framed(Http).split();
            tx.send_all(rx.and_then(respond))
        });
        handle.spawn(req.then(move |result| {
            drop(result);
            Ok(())
        }));
        Ok(())
    });
    core.run(done).unwrap();
}

/// "Server logic" is implemented in this function.
///
/// This function is a map from and HTTP request to a future of a response and
/// represents the various handling a server might do. Currently the contents
/// here are pretty uninteresting.
fn respond(req: Request<()>)
    -> Box<Future<Item = Response<String>, Error = io::Error>>
{
    let mut ret = Response::builder();
    let body = match req.uri().path() {
        "/plaintext" => {
            ret.header("Content-Type", "text/plain");
            "Hello, World!".to_string()
        }
        "/json" => {
            ret.header("Content-Type", "application/json");

            #[derive(Serialize)]
            struct Message {
                message: &'static str,
            }
            serde_json::to_string(&Message { message: "Hello, World!" })
                .unwrap()
        }
        _ => {
            ret.status(StatusCode::NOT_FOUND);
            String::new()
        }
    };
    Box::new(future::ok(ret.body(body).unwrap()))
}

struct Http;

/// Implementation of encoding an HTTP response into a `BytesMut`, basically
/// just writing out an HTTP/1.1 response.
impl Encoder for Http {
    type Item = Response<String>;
    type Error = io::Error;

    fn encode(&mut self, item: Response<String>, dst: &mut BytesMut) -> io::Result<()> {
        use std::fmt::Write;

        write!(BytesWrite(dst), "\
            HTTP/1.1 {}\r\n\
            Server: Example\r\n\
            Content-Length: {}\r\n\
            Date: {}\r\n\
        ", item.status(), item.body().len(), date::now()).unwrap();

        for (k, v) in item.headers() {
            dst.extend_from_slice(k.as_str().as_bytes());
            dst.extend_from_slice(b": ");
            dst.extend_from_slice(v.as_bytes());
            dst.extend_from_slice(b"\r\n");
        }

        dst.extend_from_slice(b"\r\n");
        dst.extend_from_slice(item.body().as_bytes());

        return Ok(());

        // Right now `write!` on `Vec<u8>` goes through io::Write and is not
        // super speedy, so inline a less-crufty implementation here which
        // doesn't go through io::Error.
        struct BytesWrite<'a>(&'a mut BytesMut);

        impl<'a> fmt::Write for BytesWrite<'a> {
            fn write_str(&mut self, s: &str) -> fmt::Result {
                self.0.extend_from_slice(s.as_bytes());
                Ok(())
            }

            fn write_fmt(&mut self, args: fmt::Arguments) -> fmt::Result {
                fmt::write(self, args)
            }
        }
    }
}

/// Implementation of decoding an HTTP request from the bytes we've read so far.
/// This leverages the `httparse` crate to do the actual parsing and then we use
/// that information to construct an instance of a `http::Request` object,
/// trying to avoid allocations where possible.
impl Decoder for Http {
    type Item = Request<()>;
    type Error = io::Error;

    fn decode(&mut self, src: &mut BytesMut) -> io::Result<Option<Request<()>>> {
        // TODO: we should grow this headers array if parsing fails and asks
        //       for more headers
        let mut headers = [None; 16];
        let (method, path, version, amt) = {
            let mut parsed_headers = [httparse::EMPTY_HEADER; 16];
            let mut r = httparse::Request::new(&mut parsed_headers);
            let status = r.parse(src).map_err(|e| {
                let msg = format!("failed to parse http request: {:?}", e);
                io::Error::new(io::ErrorKind::Other, msg)
            })?;

            let amt = match status {
                httparse::Status::Complete(amt) => amt,
                httparse::Status::Partial => return Ok(None),
            };

            let toslice = |a: &[u8]| {
                let start = a.as_ptr() as usize - src.as_ptr() as usize;
                assert!(start < src.len());
                (start, start + a.len())
            };

            for (i, header) in r.headers.iter().enumerate() {
                let k = toslice(header.name.as_bytes());
                let v = toslice(header.value);
                headers[i] = Some((k, v));
            }

            (toslice(r.method.unwrap().as_bytes()),
             toslice(r.path.unwrap().as_bytes()),
             r.version.unwrap(),
             amt)
        };
        if version != 1 {
            return Err(io::Error::new(io::ErrorKind::Other, "only HTTP/1.1 accepted"))
        }
        let data = src.split_to(amt).freeze();
        let mut ret = Request::builder();
        ret.method(&data[method.0..method.1]);
        ret.uri(data.slice(path.0, path.1));
        ret.version(http::Version::HTTP_11);
        for header in headers.iter() {
            let (k, v) = match *header {
                Some((ref k, ref v)) => (k, v),
                None => break,
            };
            let value = unsafe {
                HeaderValue::from_shared_unchecked(data.slice(v.0, v.1))
            };
            ret.header(&data[k.0..k.1], value);
        }

        let req = ret.body(()).map_err(|e| {
            io::Error::new(io::ErrorKind::Other, e)
        })?;
        Ok(Some(req))
    }
}

mod date {
    use std::cell::RefCell;
    use std::fmt::{self, Write};
    use std::str;

    use time::{self, Duration};

    pub struct Now(());

    /// Returns a struct, which when formatted, renders an appropriate `Date`
    /// header value.
    pub fn now() -> Now {
        Now(())
    }

    // Gee Alex, doesn't this seem like premature optimization. Well you see
    // there Billy, you're absolutely correct! If your server is *bottlenecked*
    // on rendering the `Date` header, well then boy do I have news for you, you
    // don't need this optimization.
    //
    // In all seriousness, though, a simple "hello world" benchmark which just
    // sends back literally "hello world" with standard headers actually is
    // bottlenecked on rendering a date into a byte buffer. Since it was at the
    // top of a profile, and this was done for some competitive benchmarks, this
    // module was written.
    //
    // Just to be clear, though, I was not intending on doing this because it
    // really does seem kinda absurd, but it was done by someone else [1], so I
    // blame them!  :)
    //
    // [1]: https://github.com/rapidoid/rapidoid/blob/f1c55c0555007e986b5d069fe1086e6d09933f7b/rapidoid-commons/src/main/java/org/rapidoid/commons/Dates.java#L48-L66

    struct LastRenderedNow {
        bytes: [u8; 128],
        amt: usize,
        next_update: time::Timespec,
    }

    thread_local!(static LAST: RefCell<LastRenderedNow> = RefCell::new(LastRenderedNow {
        bytes: [0; 128],
        amt: 0,
        next_update: time::Timespec::new(0, 0),
    }));

    impl fmt::Display for Now {
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
            LAST.with(|cache| {
                let mut cache = cache.borrow_mut();
                let now = time::get_time();
                if now > cache.next_update {
                    cache.update(now);
                }
                f.write_str(cache.buffer())
            })
        }
    }

    impl LastRenderedNow {
        fn buffer(&self) -> &str {
            str::from_utf8(&self.bytes[..self.amt]).unwrap()
        }

        fn update(&mut self, now: time::Timespec) {
            self.amt = 0;
            write!(LocalBuffer(self), "{}", time::at(now).rfc822()).unwrap();
            self.next_update = now + Duration::seconds(1);
            self.next_update.nsec = 0;
        }
    }

    struct LocalBuffer<'a>(&'a mut LastRenderedNow);

    impl<'a> fmt::Write for LocalBuffer<'a> {
        fn write_str(&mut self, s: &str) -> fmt::Result {
            let start = self.0.amt;
            let end = start + s.len();
            self.0.bytes[start..end].copy_from_slice(s.as_bytes());
            self.0.amt += s.len();
            Ok(())
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2015-2016 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Import a new corpus into the framework

extern crate glob;
extern crate Archive;

use glob::glob;
// use regex::Regex;
use Archive::*;
use std::path::Path;
use std::path::PathBuf;
use std::fs;
use std::env;
use std::io::Error;
use backend::Backend;
use data::{Task, TaskStatus, Corpus};

/// Struct for performing corpus imports into `CorTeX`
pub struct Importer {
  /// a `Corpus` to be imported, containing all relevant metadata
  pub corpus: Corpus,
  /// a `Backend` on which to persist the import into the Task store
  pub backend: Backend,
  /// the current working directory, to resolve relative paths
  pub cwd: PathBuf,
}
impl Default for Importer {
  fn default() -> Importer {
    let default_backend = Backend::default();
    Importer {
      corpus: default_backend.add(Corpus {
                               id: None,
                               path: ".".to_string(),
                               name: "default".to_string(),
                               complex: false,
                             })
                             .unwrap(),
      backend: default_backend,
      cwd: Importer::cwd(),
    }
  }
}

impl Importer {
  /// Convenience method for (recklessly?) obtaining the current working dir
  pub fn cwd() -> PathBuf {
    env::current_dir().unwrap()
  }
  /// Top-level method for unpacking an arxiv-toplogy corpus from its tar-ed form
  fn unpack(&self) -> Result<(), Error> {
    try!(self.unpack_arxiv_top());
    try!(self.unpack_arxiv_months());
    Ok(())
  }
  fn unpack_extend(&self) -> Result<(), Error> {
    try!(self.unpack_extend_arxiv_top());
    // We can reuse the monthly unpack, as it deletes all unpacked document archives
    // In other words, it always acts as a conservative extension
    try!(self.unpack_arxiv_months());
    Ok(())
  }

  /// Unpack the top-level tar files from an arxiv-topology corpus
  fn unpack_arxiv_top(&self) -> Result<(), Error> {
    println!("-- Starting top-level unpack process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // let base_name = path.file_stem().unwrap().to_str().unwrap();
          // If we wanted fine-grained control, we could infer the dir name:
          // let arxiv_name_re = Regex::new(r"arXiv_src_(\d+)_").unwrap();
          // let captures = arxiv_name_re.captures(base_name).unwrap();
          // let unpack_dirname = match captures.at(1) {
          //   Some(month) => month,
          //   None => base_name
          // };
          // --- but not for now

          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
                                 .unwrap()
                                 .support_filter_all()
                                 .support_format_all()
                                 .open_filename(path.to_str().unwrap(), 10240)
                                 .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                println!("To unpack: {:?}", full_extract_path);
                match e.extract_to(&full_extract_path, Vec::new()) {
                  Ok(_) => {}
                  _ => {
                    println!("Failed to extract {:?}", full_extract_path);
                  }
                }
              }
            }
          }
        }
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }
  /// Top-level extension unpacking for arxiv-topology corpora
  fn unpack_extend_arxiv_top(&self) -> Result<(), Error> {
    // What I am trying to figure out right now is how to avoid a monstrous amount of code duplication
    // I don't want to copy-paste all of unpack_arxiv_top in here ...
    println!("-- Starting top-level unpack-extend process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
                                 .unwrap()
                                 .support_filter_all()
                                 .support_format_all()
                                 .open_filename(path.to_str().unwrap(), 10240)
                                 .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => {}//println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                // Archive entries end in .gz, let's try that as well, to check if the directory is there
                let dir_extract_path = &full_extract_path[0..full_extract_path.len() - 3];
                match fs::metadata(dir_extract_path) {
                  Ok(_) => {}//println!("Directory for {:?} already exists, won't unpack.", e.pathname()),
                  Err(_) => {
                    println!("To unpack: {:?}", full_extract_path);
                    match e.extract_to(&full_extract_path, Vec::new()) {
                      Ok(_) => {}
                      _ => {
                        println!("Failed to extract {:?}", full_extract_path);
                      }
                    }
                  }
                }
              }
            }
          }
        }
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Unpack the monthly sub-archives of an arxiv-topology corpus, into the CorTeX organization
  fn unpack_arxiv_months(&self) -> Result<(), Error> {
    println!("-- Starting to unpack monthly .gz archives");
    let path_str = self.corpus.path.clone();
    let gzs_path = path_str.to_string() + "/*/*.gz";
    for entry in glob(&gzs_path).unwrap() {
      match entry {
        Ok(path) => {
          let entry_path = path.to_str().unwrap();
          let entry_dir = path.parent().unwrap().to_str().unwrap();
          let base_name = path.file_stem().unwrap().to_str().unwrap();
          let entry_cp_dir = entry_dir.to_string() + "/" + base_name;
          fs::create_dir_all(entry_cp_dir.clone()).unwrap_or_else(|reason| {
            println!("Failed to mkdir -p {:?} because: {:?}",
                     entry_cp_dir.clone(),
                     reason.kind());
          });
          // Careful here, some of arXiv's .gz files are really plain-text TeX files (surprise!!!)
          let archive_reader_new = Reader::new()
                                     .unwrap()
                                     .support_filter_all()
                                     .support_format_all()
                                     .open_filename(entry_path, 10240);
          // We'll write out a ZIP file for each entry
          let full_extract_path = entry_cp_dir.to_string() + "/" + base_name + ".zip";
          let mut archive_writer_new = Writer::new().unwrap()
            //.add_filter(ArchiveFilter::Lzip)
            .set_compression(ArchiveFilter::None)
            .set_format(ArchiveFormat::Zip);
          archive_writer_new.open_filename(&full_extract_path.clone()).unwrap();

          match archive_reader_new {
            Err(_) => {
              let raw_reader_new = Reader::new()
                                     .unwrap()
                                     .support_filter_all()
                                     .support_format_raw()
                                     .open_filename(entry_path, 10240);
              match raw_reader_new {
                Ok(raw_reader) => {
                  match raw_reader.next_header() {
                    Ok(_) => {
                      let tex_target = base_name.to_string() + ".tex";
                      // In a "raw" read, we don't know the data size in advance. So we bite the bullet and
                      // read the usually tiny tex file in memory, obtaining a size estimate
                      let mut raw_data = Vec::new();
                      loop {
                        let chunk_data = raw_reader.read_data(10240);
                        match chunk_data {
                          Ok(chunk) => raw_data.extend(chunk.into_iter()),
                          Err(_) => break,
                        };
                      }
                      match archive_writer_new.write_header_new(&tex_target, raw_data.len() as i64) {
                        Ok(_) => {}
                        Err(e) => {
                          println!("Couldn't write header: {:?}", e);
                          break;
                        }
                      }
                      match archive_writer_new.write_data(raw_data) {
                        Ok(_) => {}
                        Err(e) => {
                          println!("Failed to write data to {:?} because {:?}",
                                   tex_target.clone(),
                                   e)
                        }
                      };
                    }
                    Err(_) => println!("No content in archive: {:?}", entry_path),
                  }
                }
                Err(_) => println!("Unrecognizeable archive: {:?}", entry_path),
              }
            }
            Ok(archive_reader) => {
              while let Ok(e) = archive_reader.next_header() {
                match archive_writer_new.write_header(e) {
                  _ => {} // TODO: If we need to print an error message, we can do so later.
                };
                loop {
                  let entry_data = archive_reader.read_data(10240);
                  match entry_data {
                    Ok(chunk) => {
                      archive_writer_new.write_data(chunk).unwrap();
                    }
                    Err(_) => {
                      break;
                    }
                  };
                }
              }
            }
          }
          // Done with this .gz , remove it:
          match fs::remove_file(path.clone()) {
            Ok(_) => {}
            Err(e) => println!("Can't remove source .gz: {:?}", e),
          };
        }
        Err(e) => println!("Failed gz glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Given a CorTeX-topology corpus, walk the file system and import it into the Task store
  pub fn walk_import(&self) -> Result<(), Error> {
    println!("-- Starting import walk");
    let import_extension = if self.corpus.complex {
      "zip"
    } else {
      "tex"
    };
    let mut walk_q: Vec<PathBuf> = vec![Path::new(&self.corpus.path).to_owned()];
    let mut import_q: Vec<Task> = Vec::new();
    let mut import_counter = 0;
    while !walk_q.is_empty() {
      let current_path = walk_q.pop().unwrap();
      // println!("-- current path {:?}", current_path);
      let current_metadata = try!(fs::metadata(current_path.clone()));
      if current_metadata.is_dir() {
        // Ignore files
        // First, test if we just found an entry:
        let current_local_dir = current_path.file_name().unwrap();
        let current_entry = current_local_dir.to_str().unwrap().to_string() + "." + import_extension;
        let current_entry_path = current_path.to_str().unwrap().to_string() + "/" + &current_entry;
        match fs::metadata(current_entry_path.clone()) {
          Ok(_) => {
            // Found the expected file, import this entry:
            // println!("Found entry: {:?}", current_entry_path);
            import_counter += 1;
            import_q.push(self.new_task(current_entry_path));
            if import_q.len() >= 1000 {
              // Flush the import queue to backend:
              println!("Checkpoint backend writer: job {:?}", import_counter);
              self.backend.mark_imported(&import_q).unwrap(); // TODO: Proper Error-handling
              import_q.clear();
            }
          }
          Err(_) => {
            // No such entry found, traversing into the directory:
            for subentry in try!(fs::read_dir(current_path.clone())) {
              let subentry = try!(subentry);
              walk_q.push(subentry.path());
            }
          }
        }
      }
    }
    if !import_q.is_empty() {
      println!("Checkpoint backend writer: job {:?}", import_q.len());
      self.backend.mark_imported(&import_q).unwrap();
    } // TODO: Proper Error-handling
    println!("--- Imported {:?} entries.", import_counter);
    Ok(())
  }

  /// Create a new NoProblem task for the "import" service and the Importer-specified corpus
  pub fn new_task(&self, entry: String) -> Task {
    let abs_entry: String = if Path::new(&entry).is_relative() {
      let mut new_abs = self.cwd.clone();
      new_abs.push(&entry);
      new_abs.to_str().unwrap().to_string()
    } else {
      entry.clone()
    };

    Task {
      id: None,
      entry: abs_entry,
      status: TaskStatus::NoProblem.raw(),
      corpusid: self.corpus.id.unwrap(),
      serviceid: 2,
    }
  }
  /// Top-level import driver, performs an optional unpack, and then an import into the Task store
  pub fn process(&self) -> Result<(), Error> {
    // println!("Greetings from the import processor");
    if self.corpus.complex {
      // Complex setup has an unpack step:
      try!(self.unpack());
    }
    // Walk the directory tree and import the files in the Task store:
    try!(self.walk_import());

    Ok(())
  }

  /// Top-level corpus extension, performs a check for newly added documents and extracts+adds them to the existing corpus tasks
  pub fn extend_corpus(&self) -> Result<(), Error> {
    if self.corpus.complex {
      // Complex setup has an unpack step:
      try!(self.unpack_extend());
    }
    // Use the regular walk_import, at the cost of more database work,
    // the "Backend::mark_imported" ORM method allows us to insert only if new
    try!(self.walk_import());
    Ok(())
  }
}
<commit_msg>outdated comment<commit_after>// Copyright 2015-2016 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Import a new corpus into the framework

extern crate glob;
extern crate Archive;

use glob::glob;
// use regex::Regex;
use Archive::*;
use std::path::Path;
use std::path::PathBuf;
use std::fs;
use std::env;
use std::io::Error;
use backend::Backend;
use data::{Task, TaskStatus, Corpus};

/// Struct for performing corpus imports into `CorTeX`
pub struct Importer {
  /// a `Corpus` to be imported, containing all relevant metadata
  pub corpus: Corpus,
  /// a `Backend` on which to persist the import into the Task store
  pub backend: Backend,
  /// the current working directory, to resolve relative paths
  pub cwd: PathBuf,
}
impl Default for Importer {
  fn default() -> Importer {
    let default_backend = Backend::default();
    Importer {
      corpus: default_backend.add(Corpus {
                               id: None,
                               path: ".".to_string(),
                               name: "default".to_string(),
                               complex: false,
                             })
                             .unwrap(),
      backend: default_backend,
      cwd: Importer::cwd(),
    }
  }
}

impl Importer {
  /// Convenience method for (recklessly?) obtaining the current working dir
  pub fn cwd() -> PathBuf {
    env::current_dir().unwrap()
  }
  /// Top-level method for unpacking an arxiv-toplogy corpus from its tar-ed form
  fn unpack(&self) -> Result<(), Error> {
    try!(self.unpack_arxiv_top());
    try!(self.unpack_arxiv_months());
    Ok(())
  }
  fn unpack_extend(&self) -> Result<(), Error> {
    try!(self.unpack_extend_arxiv_top());
    // We can reuse the monthly unpack, as it deletes all unpacked document archives
    // In other words, it always acts as a conservative extension
    try!(self.unpack_arxiv_months());
    Ok(())
  }

  /// Unpack the top-level tar files from an arxiv-topology corpus
  fn unpack_arxiv_top(&self) -> Result<(), Error> {
    println!("-- Starting top-level unpack process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // let base_name = path.file_stem().unwrap().to_str().unwrap();
          // If we wanted fine-grained control, we could infer the dir name:
          // let arxiv_name_re = Regex::new(r"arXiv_src_(\d+)_").unwrap();
          // let captures = arxiv_name_re.captures(base_name).unwrap();
          // let unpack_dirname = match captures.at(1) {
          //   Some(month) => month,
          //   None => base_name
          // };
          // --- but not for now

          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
                                 .unwrap()
                                 .support_filter_all()
                                 .support_format_all()
                                 .open_filename(path.to_str().unwrap(), 10240)
                                 .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                println!("To unpack: {:?}", full_extract_path);
                match e.extract_to(&full_extract_path, Vec::new()) {
                  Ok(_) => {}
                  _ => {
                    println!("Failed to extract {:?}", full_extract_path);
                  }
                }
              }
            }
          }
        }
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }
  /// Top-level extension unpacking for arxiv-topology corpora
  fn unpack_extend_arxiv_top(&self) -> Result<(), Error> {
    println!("-- Starting top-level unpack-extend process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
                                 .unwrap()
                                 .support_filter_all()
                                 .support_format_all()
                                 .open_filename(path.to_str().unwrap(), 10240)
                                 .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => {}//println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                // Archive entries end in .gz, let's try that as well, to check if the directory is there
                let dir_extract_path = &full_extract_path[0..full_extract_path.len() - 3];
                match fs::metadata(dir_extract_path) {
                  Ok(_) => {}//println!("Directory for {:?} already exists, won't unpack.", e.pathname()),
                  Err(_) => {
                    println!("To unpack: {:?}", full_extract_path);
                    match e.extract_to(&full_extract_path, Vec::new()) {
                      Ok(_) => {}
                      _ => {
                        println!("Failed to extract {:?}", full_extract_path);
                      }
                    }
                  }
                }
              }
            }
          }
        }
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Unpack the monthly sub-archives of an arxiv-topology corpus, into the CorTeX organization
  fn unpack_arxiv_months(&self) -> Result<(), Error> {
    println!("-- Starting to unpack monthly .gz archives");
    let path_str = self.corpus.path.clone();
    let gzs_path = path_str.to_string() + "/*/*.gz";
    for entry in glob(&gzs_path).unwrap() {
      match entry {
        Ok(path) => {
          let entry_path = path.to_str().unwrap();
          let entry_dir = path.parent().unwrap().to_str().unwrap();
          let base_name = path.file_stem().unwrap().to_str().unwrap();
          let entry_cp_dir = entry_dir.to_string() + "/" + base_name;
          fs::create_dir_all(entry_cp_dir.clone()).unwrap_or_else(|reason| {
            println!("Failed to mkdir -p {:?} because: {:?}",
                     entry_cp_dir.clone(),
                     reason.kind());
          });
          // Careful here, some of arXiv's .gz files are really plain-text TeX files (surprise!!!)
          let archive_reader_new = Reader::new()
                                     .unwrap()
                                     .support_filter_all()
                                     .support_format_all()
                                     .open_filename(entry_path, 10240);
          // We'll write out a ZIP file for each entry
          let full_extract_path = entry_cp_dir.to_string() + "/" + base_name + ".zip";
          let mut archive_writer_new = Writer::new().unwrap()
            //.add_filter(ArchiveFilter::Lzip)
            .set_compression(ArchiveFilter::None)
            .set_format(ArchiveFormat::Zip);
          archive_writer_new.open_filename(&full_extract_path.clone()).unwrap();

          match archive_reader_new {
            Err(_) => {
              let raw_reader_new = Reader::new()
                                     .unwrap()
                                     .support_filter_all()
                                     .support_format_raw()
                                     .open_filename(entry_path, 10240);
              match raw_reader_new {
                Ok(raw_reader) => {
                  match raw_reader.next_header() {
                    Ok(_) => {
                      let tex_target = base_name.to_string() + ".tex";
                      // In a "raw" read, we don't know the data size in advance. So we bite the bullet and
                      // read the usually tiny tex file in memory, obtaining a size estimate
                      let mut raw_data = Vec::new();
                      loop {
                        let chunk_data = raw_reader.read_data(10240);
                        match chunk_data {
                          Ok(chunk) => raw_data.extend(chunk.into_iter()),
                          Err(_) => break,
                        };
                      }
                      match archive_writer_new.write_header_new(&tex_target, raw_data.len() as i64) {
                        Ok(_) => {}
                        Err(e) => {
                          println!("Couldn't write header: {:?}", e);
                          break;
                        }
                      }
                      match archive_writer_new.write_data(raw_data) {
                        Ok(_) => {}
                        Err(e) => {
                          println!("Failed to write data to {:?} because {:?}",
                                   tex_target.clone(),
                                   e)
                        }
                      };
                    }
                    Err(_) => println!("No content in archive: {:?}", entry_path),
                  }
                }
                Err(_) => println!("Unrecognizeable archive: {:?}", entry_path),
              }
            }
            Ok(archive_reader) => {
              while let Ok(e) = archive_reader.next_header() {
                match archive_writer_new.write_header(e) {
                  _ => {} // TODO: If we need to print an error message, we can do so later.
                };
                loop {
                  let entry_data = archive_reader.read_data(10240);
                  match entry_data {
                    Ok(chunk) => {
                      archive_writer_new.write_data(chunk).unwrap();
                    }
                    Err(_) => {
                      break;
                    }
                  };
                }
              }
            }
          }
          // Done with this .gz , remove it:
          match fs::remove_file(path.clone()) {
            Ok(_) => {}
            Err(e) => println!("Can't remove source .gz: {:?}", e),
          };
        }
        Err(e) => println!("Failed gz glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Given a CorTeX-topology corpus, walk the file system and import it into the Task store
  pub fn walk_import(&self) -> Result<(), Error> {
    println!("-- Starting import walk");
    let import_extension = if self.corpus.complex {
      "zip"
    } else {
      "tex"
    };
    let mut walk_q: Vec<PathBuf> = vec![Path::new(&self.corpus.path).to_owned()];
    let mut import_q: Vec<Task> = Vec::new();
    let mut import_counter = 0;
    while !walk_q.is_empty() {
      let current_path = walk_q.pop().unwrap();
      // println!("-- current path {:?}", current_path);
      let current_metadata = try!(fs::metadata(current_path.clone()));
      if current_metadata.is_dir() {
        // Ignore files
        // First, test if we just found an entry:
        let current_local_dir = current_path.file_name().unwrap();
        let current_entry = current_local_dir.to_str().unwrap().to_string() + "." + import_extension;
        let current_entry_path = current_path.to_str().unwrap().to_string() + "/" + &current_entry;
        match fs::metadata(current_entry_path.clone()) {
          Ok(_) => {
            // Found the expected file, import this entry:
            // println!("Found entry: {:?}", current_entry_path);
            import_counter += 1;
            import_q.push(self.new_task(current_entry_path));
            if import_q.len() >= 1000 {
              // Flush the import queue to backend:
              println!("Checkpoint backend writer: job {:?}", import_counter);
              self.backend.mark_imported(&import_q).unwrap(); // TODO: Proper Error-handling
              import_q.clear();
            }
          }
          Err(_) => {
            // No such entry found, traversing into the directory:
            for subentry in try!(fs::read_dir(current_path.clone())) {
              let subentry = try!(subentry);
              walk_q.push(subentry.path());
            }
          }
        }
      }
    }
    if !import_q.is_empty() {
      println!("Checkpoint backend writer: job {:?}", import_q.len());
      self.backend.mark_imported(&import_q).unwrap();
    } // TODO: Proper Error-handling
    println!("--- Imported {:?} entries.", import_counter);
    Ok(())
  }

  /// Create a new NoProblem task for the "import" service and the Importer-specified corpus
  pub fn new_task(&self, entry: String) -> Task {
    let abs_entry: String = if Path::new(&entry).is_relative() {
      let mut new_abs = self.cwd.clone();
      new_abs.push(&entry);
      new_abs.to_str().unwrap().to_string()
    } else {
      entry.clone()
    };

    Task {
      id: None,
      entry: abs_entry,
      status: TaskStatus::NoProblem.raw(),
      corpusid: self.corpus.id.unwrap(),
      serviceid: 2,
    }
  }
  /// Top-level import driver, performs an optional unpack, and then an import into the Task store
  pub fn process(&self) -> Result<(), Error> {
    // println!("Greetings from the import processor");
    if self.corpus.complex {
      // Complex setup has an unpack step:
      try!(self.unpack());
    }
    // Walk the directory tree and import the files in the Task store:
    try!(self.walk_import());

    Ok(())
  }

  /// Top-level corpus extension, performs a check for newly added documents and extracts+adds them to the existing corpus tasks
  pub fn extend_corpus(&self) -> Result<(), Error> {
    if self.corpus.complex {
      // Complex setup has an unpack step:
      try!(self.unpack_extend());
    }
    // Use the regular walk_import, at the cost of more database work,
    // the "Backend::mark_imported" ORM method allows us to insert only if new
    try!(self.walk_import());
    Ok(())
  }
}
<|endoftext|>


<commit_before>//! Basic types to build the parsers

use self::Needed::*;
use crate::error::ErrorKind;
use core::num::NonZeroUsize;

/// Holds the result of parsing functions
///
/// It depends on I, the input type, O, the output type, and E, the error type (by default u32)
///
/// The `Ok` side is a pair containing the remainder of the input (the part of the data that
/// was not parsed) and the produced value. The `Err` side contains an instance of `nom::Err`.
///
pub type IResult<I, O, E = (I, ErrorKind)> = Result<(I, O), Err<E>>;

/// Contains information on needed data if a parser returned `Incomplete`
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum Needed {
  /// needs more data, but we do not know how much
  Unknown,
  /// contains the required data size
  Size(NonZeroUsize),
}

impl Needed {
  /// creates Needed instance, returns `Needed::Unknown` if the argument is zero
  pub fn new(s: usize) -> Self {
    match NonZeroUsize::new(s) {
      Some(sz) => Needed::Size(sz),
      None => Needed::Unknown,
    }
  }

  /// indicates if we know how many bytes we need
  pub fn is_known(&self) -> bool {
    *self != Unknown
  }

  /// Maps a `Needed` to `Needed` by applying a function to a contained `Size` value.
  #[inline]
  pub fn map<F: Fn(NonZeroUsize) -> usize>(self, f: F) -> Needed {
    match self {
      Unknown => Unknown,
      Size(n) => Needed::new(f(n)),
    }
  }
}

/// The `Err` enum indicates the parser was not successful
///
/// It has three cases:
///
/// * `Incomplete` indicates that more data is needed to decide. The `Needed` enum
/// can contain how many additional bytes are necessary. If you are sure your parser
/// is working on full data, you can wrap your parser with the `complete` combinator
/// to transform that case in `Error`
/// * `Error` means some parser did not succeed, but another one might (as an example,
/// when testing different branches of an `alt` combinator)
/// * `Failure` indicates an unrecoverable error. As an example, if you recognize a prefix
/// to decide on the next parser to apply, and that parser fails, you know there's no need
/// to try other parsers, you were already in the right branch, so the data is invalid
///
#[derive(Debug, Clone, PartialEq)]
pub enum Err<E> {
  /// There was not enough data
  Incomplete(Needed),
  /// The parser had an error (recoverable)
  Error(E),
  /// The parser had an unrecoverable error: we got to the right
  /// branch and we know other branches won't work, so backtrack
  /// as fast as possible
  Failure(E),
}

impl<E> Err<E> {
  /// tests if the result is Incomplete
  pub fn is_incomplete(&self) -> bool {
    if let Err::Incomplete(_) = self {
      true
    } else {
      false
    }
  }

  /// Applies the given function to the inner error
  pub fn map<E2, F>(self, f: F) -> Err<E2>
  where
    F: FnOnce(E) -> E2,
  {
    match self {
      Err::Incomplete(n) => Err::Incomplete(n),
      Err::Failure(t) => Err::Failure(f(t)),
      Err::Error(t) => Err::Error(f(t)),
    }
  }

  /// automatically converts between errors if the underlying type supports it
  pub fn convert<F>(e: Err<F>) -> Self
  where
    E: From<F>,
  {
    e.map(Into::into)
  }
}

impl<T> Err<(T, ErrorKind)> {
  /// maps `Err<(T, ErrorKind)>` to `Err<(U, ErrorKind)>` with the given F: T -> U
  pub fn map_input<U, F>(self, f: F) -> Err<(U, ErrorKind)>
  where
    F: FnOnce(T) -> U,
  {
    match self {
      Err::Incomplete(n) => Err::Incomplete(n),
      Err::Failure((input, k)) => Err::Failure((f(input), k)),
      Err::Error((input, k)) => Err::Error((f(input), k)),
    }
  }
}

#[cfg(feature = "std")]
impl Err<(&[u8], ErrorKind)> {
  /// Obtaining ownership
  pub fn to_owned(self) -> Err<(Vec<u8>, ErrorKind)> {
    self.map_input(ToOwned::to_owned)
  }
}

#[cfg(feature = "std")]
impl Err<(&str, ErrorKind)> {
  /// automatically converts between errors if the underlying type supports it
  pub fn to_owned(self) -> Err<(String, ErrorKind)> {
    self.map_input(ToOwned::to_owned)
  }
}

impl<E: Eq> Eq for Err<E> {}

#[cfg(feature = "std")]
use std::fmt;

#[cfg(feature = "std")]
impl<E> fmt::Display for Err<E>
where
  E: fmt::Debug,
{
  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
    match self {
      Err::Incomplete(Needed::Size(u)) => write!(f, "Parsing requires {} bytes/chars", u),
      Err::Incomplete(Needed::Unknown) => write!(f, "Parsing requires more data"),
      Err::Failure(c) => write!(f, "Parsing Failure: {:?}", c),
      Err::Error(c) => write!(f, "Parsing Error: {:?}", c),
    }
  }
}

#[cfg(feature = "std")]
use std::error::Error;

#[cfg(feature = "std")]
impl<E> Error for Err<E>
where
  E: fmt::Debug,
{
  fn source(&self) -> Option<&(dyn Error + 'static)> {
    None // no underlying error
  }
}

/// all nom parsers implement ths trait
pub trait Parser<I, O, E> {
  /// a parser takes in input type, and returns a `Result` containing
  /// either the remaining input and the output value, or an error
  fn parse(&mut self, input: I) -> IResult<I, O, E>;

  /// maps a function over the result of a parser
  fn map<G, O2>(self, g: G) -> Mapper<Self, G, O>
  where
    G: Fn(O) -> O2,
    Self: std::marker::Sized,
  {
    Mapper {
      f: self,
      g,
      phantom: std::marker::PhantomData,
    }
  }
}

impl<'a, I, O, E, F> Parser<I, O, E> for F
where
  F: FnMut(I) -> IResult<I, O, E> + 'a,
{
  fn parse(&mut self, i: I) -> IResult<I, O, E> {
    self(i)
  }
}

/// implementation of parser mapping
pub struct Mapper<F, G, O1> {
  f: F,
  g: G,
  phantom: std::marker::PhantomData<O1>,
}

impl<'a, I, O1, O2, E, F: Parser<I, O1, E>, G: Fn(O1) -> O2> Parser<I, O2, E> for Mapper<F, G, O1> {
  fn parse(&mut self, i: I) -> IResult<I, O2, E> {
    match self.f.parse(i) {
      Err(e) => Err(e),
      Ok((i, o)) => Ok((i, (self.g)(o))),
    }
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use crate::error::ErrorKind;

  #[doc(hidden)]
  #[macro_export]
  macro_rules! assert_size (
    ($t:ty, $sz:expr) => (
      assert_eq!(crate::lib::std::mem::size_of::<$t>(), $sz);
    );
  );

  #[test]
  #[cfg(target_pointer_width = "64")]
  fn size_test() {
    assert_size!(IResult<&[u8], &[u8], (&[u8], u32)>, 40);
    assert_size!(IResult<&str, &str, u32>, 40);
    assert_size!(Needed, 8);
    assert_size!(Err<u32>, 16);
    assert_size!(ErrorKind, 1);
  }

  #[test]
  fn err_map_test() {
    let e = Err::Error(1);
    assert_eq!(e.map(|v| v + 1), Err::Error(2));
  }
}
<commit_msg>add flat_map, and, and_then, or methods to the Parser trait<commit_after>//! Basic types to build the parsers

use self::Needed::*;
use crate::error::ErrorKind;
use core::num::NonZeroUsize;

/// Holds the result of parsing functions
///
/// It depends on I, the input type, O, the output type, and E, the error type (by default u32)
///
/// The `Ok` side is a pair containing the remainder of the input (the part of the data that
/// was not parsed) and the produced value. The `Err` side contains an instance of `nom::Err`.
///
pub type IResult<I, O, E = (I, ErrorKind)> = Result<(I, O), Err<E>>;

/// Contains information on needed data if a parser returned `Incomplete`
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum Needed {
  /// needs more data, but we do not know how much
  Unknown,
  /// contains the required data size
  Size(NonZeroUsize),
}

impl Needed {
  /// creates Needed instance, returns `Needed::Unknown` if the argument is zero
  pub fn new(s: usize) -> Self {
    match NonZeroUsize::new(s) {
      Some(sz) => Needed::Size(sz),
      None => Needed::Unknown,
    }
  }

  /// indicates if we know how many bytes we need
  pub fn is_known(&self) -> bool {
    *self != Unknown
  }

  /// Maps a `Needed` to `Needed` by applying a function to a contained `Size` value.
  #[inline]
  pub fn map<F: Fn(NonZeroUsize) -> usize>(self, f: F) -> Needed {
    match self {
      Unknown => Unknown,
      Size(n) => Needed::new(f(n)),
    }
  }
}

/// The `Err` enum indicates the parser was not successful
///
/// It has three cases:
///
/// * `Incomplete` indicates that more data is needed to decide. The `Needed` enum
/// can contain how many additional bytes are necessary. If you are sure your parser
/// is working on full data, you can wrap your parser with the `complete` combinator
/// to transform that case in `Error`
/// * `Error` means some parser did not succeed, but another one might (as an example,
/// when testing different branches of an `alt` combinator)
/// * `Failure` indicates an unrecoverable error. As an example, if you recognize a prefix
/// to decide on the next parser to apply, and that parser fails, you know there's no need
/// to try other parsers, you were already in the right branch, so the data is invalid
///
#[derive(Debug, Clone, PartialEq)]
pub enum Err<E> {
  /// There was not enough data
  Incomplete(Needed),
  /// The parser had an error (recoverable)
  Error(E),
  /// The parser had an unrecoverable error: we got to the right
  /// branch and we know other branches won't work, so backtrack
  /// as fast as possible
  Failure(E),
}

impl<E> Err<E> {
  /// tests if the result is Incomplete
  pub fn is_incomplete(&self) -> bool {
    if let Err::Incomplete(_) = self {
      true
    } else {
      false
    }
  }

  /// Applies the given function to the inner error
  pub fn map<E2, F>(self, f: F) -> Err<E2>
  where
    F: FnOnce(E) -> E2,
  {
    match self {
      Err::Incomplete(n) => Err::Incomplete(n),
      Err::Failure(t) => Err::Failure(f(t)),
      Err::Error(t) => Err::Error(f(t)),
    }
  }

  /// automatically converts between errors if the underlying type supports it
  pub fn convert<F>(e: Err<F>) -> Self
  where
    E: From<F>,
  {
    e.map(Into::into)
  }
}

impl<T> Err<(T, ErrorKind)> {
  /// maps `Err<(T, ErrorKind)>` to `Err<(U, ErrorKind)>` with the given F: T -> U
  pub fn map_input<U, F>(self, f: F) -> Err<(U, ErrorKind)>
  where
    F: FnOnce(T) -> U,
  {
    match self {
      Err::Incomplete(n) => Err::Incomplete(n),
      Err::Failure((input, k)) => Err::Failure((f(input), k)),
      Err::Error((input, k)) => Err::Error((f(input), k)),
    }
  }
}

#[cfg(feature = "std")]
impl Err<(&[u8], ErrorKind)> {
  /// Obtaining ownership
  pub fn to_owned(self) -> Err<(Vec<u8>, ErrorKind)> {
    self.map_input(ToOwned::to_owned)
  }
}

#[cfg(feature = "std")]
impl Err<(&str, ErrorKind)> {
  /// automatically converts between errors if the underlying type supports it
  pub fn to_owned(self) -> Err<(String, ErrorKind)> {
    self.map_input(ToOwned::to_owned)
  }
}

impl<E: Eq> Eq for Err<E> {}

#[cfg(feature = "std")]
use std::fmt;

#[cfg(feature = "std")]
impl<E> fmt::Display for Err<E>
where
  E: fmt::Debug,
{
  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
    match self {
      Err::Incomplete(Needed::Size(u)) => write!(f, "Parsing requires {} bytes/chars", u),
      Err::Incomplete(Needed::Unknown) => write!(f, "Parsing requires more data"),
      Err::Failure(c) => write!(f, "Parsing Failure: {:?}", c),
      Err::Error(c) => write!(f, "Parsing Error: {:?}", c),
    }
  }
}

#[cfg(feature = "std")]
use std::error::Error;

#[cfg(feature = "std")]
impl<E> Error for Err<E>
where
  E: fmt::Debug,
{
  fn source(&self) -> Option<&(dyn Error + 'static)> {
    None // no underlying error
  }
}

/// all nom parsers implement ths trait
pub trait Parser<I, O, E> {
  /// a parser takes in input type, and returns a `Result` containing
  /// either the remaining input and the output value, or an error
  fn parse(&mut self, input: I) -> IResult<I, O, E>;

  /// maps a function over the result of a parser
  fn map<G, O2>(self, g: G) -> Map<Self, G, O>
  where
    G: Fn(O) -> O2,
    Self: std::marker::Sized,
  {
    Map {
      f: self,
      g,
      phantom: std::marker::PhantomData,
    }
  }

  /// creates a second parser from the output of the first one, then apply over the rest of the input
  fn flat_map<G, H, O2>(self, g: G) -> FlatMap<Self, G, O>
  where
    G: Fn(O) -> H,
    H: Parser<I, O2, E>,
    Self: std::marker::Sized,
  {
    FlatMap {
      f: self,
      g,
      phantom: std::marker::PhantomData,
    }
  }

  /// applies a second parser over the output of the first one
  fn and_then<G, O2>(self, g: G) -> AndThen<Self, G, O>
  where
    G: Parser<O, O2, E>,
    Self: std::marker::Sized,
  {
    AndThen {
      f: self,
      g,
      phantom: std::marker::PhantomData,
    }
  }

  /// applies a second parser after the first one, return their results as a tuple
  fn and<G, O2>(self, g: G) -> And<Self, G>
  where
    G: Parser<I, O2, E>,
    Self: std::marker::Sized,
  {
    And { f: self, g }
  }

  /// applies a second parser over the input if the first one failed
  fn or<G>(self, g: G) -> Or<Self, G>
  where
    G: Parser<I, O, E>,
    Self: std::marker::Sized,
  {
    Or { f: self, g }
  }
}

impl<'a, I, O, E, F> Parser<I, O, E> for F
where
  F: FnMut(I) -> IResult<I, O, E> + 'a,
{
  fn parse(&mut self, i: I) -> IResult<I, O, E> {
    self(i)
  }
}

/// implementation of Parser:::map
pub struct Map<F, G, O1> {
  f: F,
  g: G,
  phantom: std::marker::PhantomData<O1>,
}

impl<'a, I, O1, O2, E, F: Parser<I, O1, E>, G: Fn(O1) -> O2> Parser<I, O2, E> for Map<F, G, O1> {
  fn parse(&mut self, i: I) -> IResult<I, O2, E> {
    match self.f.parse(i) {
      Err(e) => Err(e),
      Ok((i, o)) => Ok((i, (self.g)(o))),
    }
  }
}

/// implementation of Parser::flat_map
pub struct FlatMap<F, G, O1> {
  f: F,
  g: G,
  phantom: std::marker::PhantomData<O1>,
}

impl<'a, I, O1, O2, E, F: Parser<I, O1, E>, G: Fn(O1) -> H, H: Parser<I, O2, E>> Parser<I, O2, E>
  for FlatMap<F, G, O1>
{
  fn parse(&mut self, i: I) -> IResult<I, O2, E> {
    let (i, o1) = self.f.parse(i)?;
    (self.g)(o1).parse(i)
  }
}

/// implementation of Parser::and_then
pub struct AndThen<F, G, O1> {
  f: F,
  g: G,
  phantom: std::marker::PhantomData<O1>,
}

impl<'a, I, O1, O2, E, F: Parser<I, O1, E>, G: Parser<O1, O2, E>> Parser<I, O2, E>
  for AndThen<F, G, O1>
{
  fn parse(&mut self, i: I) -> IResult<I, O2, E> {
    let (i, o1) = self.f.parse(i)?;
    let (_, o2) = self.g.parse(o1)?;
    Ok((i, o2))
  }
}

/// implementation of Parser::and
pub struct And<F, G> {
  f: F,
  g: G,
}

impl<'a, I, O1, O2, E, F: Parser<I, O1, E>, G: Parser<I, O2, E>> Parser<I, (O1, O2), E>
  for And<F, G>
{
  fn parse(&mut self, i: I) -> IResult<I, (O1, O2), E> {
    let (i, o1) = self.f.parse(i)?;
    let (i, o2) = self.g.parse(i)?;
    Ok((i, (o1, o2)))
  }
}

/// implementation of Parser::or
pub struct Or<F, G> {
  f: F,
  g: G,
}

impl<'a, I: Clone, O, E: crate::error::ParseError<I>, F: Parser<I, O, E>, G: Parser<I, O, E>>
  Parser<I, O, E> for Or<F, G>
{
  fn parse(&mut self, i: I) -> IResult<I, O, E> {
    match self.f.parse(i.clone()) {
      Err(Err::Error(e1)) => match self.g.parse(i) {
        Err(Err::Error(e2)) => Err(Err::Error(e1.or(e2))),
        res => res,
      },
      res => res,
    }
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use crate::error::ErrorKind;

  #[doc(hidden)]
  #[macro_export]
  macro_rules! assert_size (
    ($t:ty, $sz:expr) => (
      assert_eq!(crate::lib::std::mem::size_of::<$t>(), $sz);
    );
  );

  #[test]
  #[cfg(target_pointer_width = "64")]
  fn size_test() {
    assert_size!(IResult<&[u8], &[u8], (&[u8], u32)>, 40);
    assert_size!(IResult<&str, &str, u32>, 40);
    assert_size!(Needed, 8);
    assert_size!(Err<u32>, 16);
    assert_size!(ErrorKind, 1);
  }

  #[test]
  fn err_map_test() {
    let e = Err::Error(1);
    assert_eq!(e.map(|v| v + 1), Err::Error(2));
  }
}
<|endoftext|>


<commit_before>// Kevin Cantu
// Boyer-Moore string searching

use std;

// Function: findn
//
// Boyer-Moore string search, which returns
// up to `nn` byte positions of matched substrings
fn findn (needle: str, haystack: str, nn: uint) -> [uint] {

   let results = [];

   let nlen = str::len(needle);
   let hlen = str::len(haystack);

   // empty haystack
   if hlen == 0u {
      ret results;
   }

   // empty needle
   if nlen == 0u {
      vec::push(results, 0u);
      ret results;
   }

   // needle too large
   if hlen < nlen {
      ret results;
   }

   // generate the tables
   let ct = char_table(needle);
   let pt = prefix_table(needle);

   // simplify the table referencing
   let getShift = fn@(pos: uint) -> uint {
      let charShift, prefShift; // also: ct, pt, needle

      assert needle != "";
      assert pos >= 0u;
      assert pos < nlen;

      alt ct.find(needle[pos] as uint) {
         option::none    { charShift = nlen;}
         option::some(x) { charShift = x;}
      }

      let offset = nlen - pos;

      assert offset >= 0u;
      assert offset < nlen;

      alt pt.find(offset) {
         option::none { fail "something is out of range" }
         option::some(x) { prefShift = x;}
      }

      //std::io::println(#fmt("<charShift: %u, prefShift: %u>", charShift, prefShift));
      ret greaterOf(charShift, prefShift);
   };

   // step up through the haystack
   let outerii = 0u;
   while outerii < hlen - nlen + 1u {

      // step back through needle
      let windowii = nlen;
      while 0u < windowii
            && (outerii + (windowii - 1u)) < hlen  // don't exceed haystack
      {
         windowii -= 1u;

         //std::io::println(#fmt("%02x.@%u =? %02x.@%u",
         //                 needle[windowii] as uint,
         //                 windowii,
         //                 haystack[outerii+windowii] as uint,
         //                 outerii + windowii));

         // if still matching
         if needle[windowii] == haystack[outerii+windowii] {

            // if full match
            if windowii == 0u {
               //std::io::println(#fmt("[pushing %u]", outerii));
               vec::push(results, outerii);

               if vec::len(results) >= nn {
                  ret results;
               } else {
                  outerii += nlen;
               }
            }
         } else {
            outerii +=
               if windowii == nlen - 1u {
                  // not matching yet
                  1u
               } else {
                  // was partial match
                  getShift(nlen - windowii)
               };

            break;
         }
      }
   }

   ret results;
}

fn char_table (needle: str) -> std::map::map<uint, uint> {
   let mm: std::map::map<uint, uint> = std::map::new_uint_hash();
   let len = str::len(needle);

   let jj = len - 1u; // drop the last byte

   assert jj >= 0u;
   assert jj < str::len(needle);

   // from last-1 to first
   while jj > 0u {
      jj -= 1u;
      if ! mm.contains_key(needle[jj] as uint) {
         // store reverse indexes
         mm.insert(needle[jj] as uint, len - 1u - jj);
      }
   }

   ret mm;
}

#[test]
fn test_char_table () {
   let ct = char_table("ANPANMAN");
   assert 1u == ct.get('A' as uint);
   assert 2u == ct.get('M' as uint);
   assert 3u == ct.get('N' as uint);
   assert 5u == ct.get('P' as uint);
   assert option::none == ct.find('z' as uint);
}

fn prefix_table (needle: str) -> std::map::map<uint, uint> {
   let needle_ = str::bytes(needle);
   let mm: std::map::map<uint, uint> = std::map::new_uint_hash();

   // WAIT, HOW IS THIS ALLOWED TO MUTATE mm?
   let fill = fn@(kk: uint, vv: uint) {
      if ! mm.contains_key(kk) {
         mm.insert(kk, vv);
         //std::io::println(#fmt("%u => %u", kk, vv));
      }
   };

   let lim   = vec::len(needle_);
   assert 0u < lim;

   // step to larger suffixes
   let sii = 0u;
   while sii < lim {

      // tail of the needle we seek
      let suffix      = vec::slice(needle_, lim - sii,      lim);
      let suffix_plus = vec::slice(needle_, lim - sii - 1u, lim);
      let slen = vec::len(suffix);

      // step to smaller prefixes
      let pii = lim - 1u;
      while pii > 0u {

         // a prefix of the needle that might be matched by
         // a partial match of a suffix
         // (which we might want to jump to)
         let prefix = vec::slice(needle_, 0u, pii);
         let plen = vec::len(prefix);

         //let msg  = "<"+suffix+"("+#fmt("%u",sii)+")";
         //let msg2 = prefix+"("+#fmt("%u",pii)+")>";
         //let msg  = "<suf("+#fmt("%u",sii)+")";
         //let msg2 = "pref("+#fmt("%u",pii)+")>";
         //std::io::println(msg + " × " + msg2);

         // suffix might be fully matched
         let is_suffix_matched =
            vec::ends_with(prefix, suffix)
            && !vec::ends_with(prefix, suffix_plus)
            &&  slen < plen;

         // prefix is bigger than suffix, only tail can match
         let is_suffix_partially_matched =
            plen <= slen
            && vec::ends_with(suffix, prefix);

         if is_suffix_matched || is_suffix_partially_matched {
            fill(sii, lim-pii);
         }

         pii -= 1u;
      }

      // no matching prefix
      fill(sii, lim-pii);

      sii += 1u;
   }

   ret mm;
}

#[test]
fn test_prefix_table_ascii() {
   let pt = prefix_table("ANPANMAN");
                            //      ... 8

   assert 1u == pt.get(0u); //        (n)
   assert 8u == pt.get(1u); //       (a)n
   assert 3u == pt.get(2u); //      (m)an
   assert 6u == pt.get(3u); //     (n)man
   assert 6u == pt.get(4u); //    (a)nman
   assert 6u == pt.get(5u); //   (p)anman
   assert 6u == pt.get(6u); //  (n)panman
   assert 6u == pt.get(7u); // (a)npanman
   //assert 0u == pt.get(8u); // fail
}

#[test]
fn test_prefix_table_utf8() {
   let pt = prefix_table("ประเ");

   assert  1u == pt.get(0u);
   assert 12u == pt.get(3u);
   assert 12u == pt.get(6u);
   assert 12u == pt.get(9u);
}

fn greaterOf(a: uint, b: uint) -> uint {
   if a > b { ret a; } else { ret b; }
}

#[test]
fn test_greaterOf() {
   assert greaterOf(15u, 17u) == 17u;
   assert greaterOf(17u, 15u) == 17u;
}

fn find_(needle: str, haystack: str) -> option<uint> {
   let found = findn(needle, haystack, 1u);

   alt vec::len(found) {
      0u { option::none }
      xx { option::some(found[0u]) }
   }
}

#[test]
fn test_findn() {
   assert [] == findn("banana", "apple pie", 1u);
   assert (findn("abc", "abcxxxxxx", 1u) == [0u]);
   assert (findn("abc", "xxxabcxxx", 1u) == [3u]);
   assert (findn("abc", "xxxxxxabc", 1u) == [6u]);
   assert (findn("abc", "xxxabcabc", 1u) == [3u]);
   assert (findn("abc", "xxxabcabc", 5u) == [3u, 6u]);
   assert (findn("abc", "xxxabcxabc", 5u) == [3u, 7u]);
   assert (findn("abc", "xxxabcxxabc", 5u) == [3u, 8u]);
}

#[test]
fn test_find_ascii() {
  assert (find_("banana", "apple pie") == option::none);
  assert (find_("", "") == option::none);
  assert (find_("abc", "abcxxxxxx") == option::some(0u));
  assert (find_("abc", "xxxabcxxx") == option::some(3u));
  assert (find_("abc", "xxxxxxabc") == option::some(6u));
}

#[test]
fn test_find_utf8() {
  let data = "ประเทศไทย中华Việt Nam";
  assert (find_("ไท华", data) == option::none);
  assert (find_("", data)     == option::some( 0u));
  assert (find_("ประเ", data) == option::some( 0u));
  assert (find_("ะเ", data)   == option::some( 6u));
  assert (find_("ระ", data) == option::some(3u));
  assert (find_("ศไทย中华", data) == option::some(15u));
  assert (find_("ไทย中华", data) == option::some(18u));
  assert (find_("ย中华", data) == option::some(24u));
  assert (find_("中华", data) == option::some(27u));
}

<commit_msg>sep. file for prefixes, suffixes<commit_after>// Kevin Cantu
// Boyer-Moore string searching

use std;

// Function: findn
//
// Boyer-Moore string search, which returns
// up to `nn` byte positions of matched substrings
fn findn (needle: str, haystack: str, nn: uint) -> [uint] {

   let results = [];

   let nlen = str::len(needle);
   let hlen = str::len(haystack);

   // empty haystack
   if hlen == 0u {
      ret results;
   }

   // empty needle
   if nlen == 0u {
      vec::push(results, 0u);
      ret results;
   }

   // needle too large
   if hlen < nlen {
      ret results;
   }

   // generate the tables
   let ct = char_table(needle);
   let pt = prefix_table(needle);

   // simplify the table referencing
   let getShift = fn@(pos: uint) -> uint {
      let charShift, prefShift; // also: ct, pt, needle

      assert needle != "";
      assert 0u <= pos;
      assert       pos < nlen;

      alt ct.find(needle[pos] as uint) {
         option::none    { charShift = nlen;}
         option::some(x) { charShift = x;}
      }

      let offset = nlen - pos;

      assert 0u <= offset;
      assert       offset < nlen;

      alt pt.find(offset) {
         option::none { fail "something is out of range" }
         option::some(x) { prefShift = x;}
      }

      //std::io::println(#fmt("<charShift: %u, prefShift: %u>", charShift, prefShift));
      ret greaterOf(charShift, prefShift);
   };

   // step up through the haystack
   let outerii = 0u;
   while outerii < hlen - nlen + 1u {

      // step back through needle
      let windowii = nlen;
      while 0u < windowii
            && (outerii + (windowii - 1u)) < hlen  // don't exceed haystack
      {
         windowii -= 1u;

         //std::io::println(#fmt("%02x.@%u =? %02x.@%u",
         //                 needle[windowii] as uint,
         //                 windowii,
         //                 haystack[outerii+windowii] as uint,
         //                 outerii + windowii));

         // if still matching
         if needle[windowii] == haystack[outerii+windowii] {

            // if full match
            if windowii == 0u {
               //std::io::println(#fmt("[pushing %u]", outerii));
               vec::push(results, outerii);

               if vec::len(results) >= nn {
                  ret results;
               } else {
                  outerii += nlen;
               }
            }
         } else {
            outerii +=
               if windowii == nlen - 1u {
                  // not matching yet
                  1u
               } else {
                  // was partial match
                  getShift(nlen - windowii)
               };

            break;
         }
      }
   }

   ret results;
}

fn char_table (needle: str) -> std::map::map<uint, uint> {
   let mm: std::map::map<uint, uint> = std::map::new_uint_hash();
   let len = str::len(needle);

   let jj = len - 1u; // drop the last byte

   assert jj >= 0u;
   assert jj < str::len(needle);

   // from last-1 to first
   while jj > 0u {
      jj -= 1u;
      if ! mm.contains_key(needle[jj] as uint) {
         // store reverse indexes
         mm.insert(needle[jj] as uint, len - 1u - jj);
      }
   }

   ret mm;
}

#[test]
fn test_char_table () {
   let ct = char_table("ANPANMAN");
   assert 1u == ct.get('A' as uint);
   assert 2u == ct.get('M' as uint);
   assert 3u == ct.get('N' as uint);
   assert 5u == ct.get('P' as uint);
   assert option::none == ct.find('z' as uint);
}

fn prefix_table (needle: str) -> std::map::map<uint, uint> {
   let needle_ = str::bytes(needle);
   let mm: std::map::map<uint, uint> = std::map::new_uint_hash();

   // WAIT, HOW IS THIS ALLOWED TO MUTATE mm?
   let fill = fn@(kk: uint, vv: uint) {
      if ! mm.contains_key(kk) {
         mm.insert(kk, vv);
         //std::io::println(#fmt("%u => %u", kk, vv));
      }
   };

   let lim   = vec::len(needle_);
   assert 0u < lim;

   // step to larger suffixes
   let sii = 0u;
   while sii < lim {

      // tail of the needle we seek
      let suffix      = vec::slice(needle_, lim - sii,      lim);
      let suffix_plus = vec::slice(needle_, lim - sii - 1u, lim);
      let slen = vec::len(suffix);

      // step to smaller prefixes
      let pii = lim - 1u;
      while pii > 0u {

         // a prefix of the needle that might be matched by
         // a partial match of a suffix
         // (which we might want to jump to)
         let prefix = vec::slice(needle_, 0u, pii);
         let plen = vec::len(prefix);

         //let msg  = "<"+suffix+"("+#fmt("%u",sii)+")";
         //let msg2 = prefix+"("+#fmt("%u",pii)+")>";
         //let msg  = "<suf("+#fmt("%u",sii)+")";
         //let msg2 = "pref("+#fmt("%u",pii)+")>";
         //std::io::println(msg + " × " + msg2);

         // suffix might be fully matched
         let is_suffix_matched =
            vec::ends_with(prefix, suffix)
            && !vec::ends_with(prefix, suffix_plus)
            &&  slen < plen;

         // prefix is bigger than suffix, only tail can match
         let is_suffix_partially_matched =
            plen <= slen
            && vec::ends_with(suffix, prefix);

         if is_suffix_matched || is_suffix_partially_matched {
            fill(sii, lim-pii);
         }

         pii -= 1u;
      }

      // no matching prefix
      fill(sii, lim-pii);

      sii += 1u;
   }

   ret mm;
}

#[test]
fn test_prefix_table_ascii() {
   let pt = prefix_table("ANPANMAN");
                            //      ... 8

   assert 1u == pt.get(0u); //        (n)
   assert 8u == pt.get(1u); //       (a)n
   assert 3u == pt.get(2u); //      (m)an
   assert 6u == pt.get(3u); //     (n)man
   assert 6u == pt.get(4u); //    (a)nman
   assert 6u == pt.get(5u); //   (p)anman
   assert 6u == pt.get(6u); //  (n)panman
   assert 6u == pt.get(7u); // (a)npanman
   //assert 0u == pt.get(8u); // fail
}

#[test]
fn test_prefix_table_utf8() {
   let pt = prefix_table("ประเ");

   assert  1u == pt.get(0u);
   assert 12u == pt.get(3u);
   assert 12u == pt.get(6u);
   assert 12u == pt.get(9u);
}

fn greaterOf(a: uint, b: uint) -> uint {
   if a > b { ret a; } else { ret b; }
}

#[test]
fn test_greaterOf() {
   assert greaterOf(15u, 17u) == 17u;
   assert greaterOf(17u, 15u) == 17u;
}

fn find_(needle: str, haystack: str) -> option<uint> {
   let found = findn(needle, haystack, 1u);

   alt vec::len(found) {
      0u { option::none }
      xx { option::some(found[0u]) }
   }
}

#[test]
fn test_findn() {
   assert [] == findn("banana", "apple pie", 1u);
   assert (findn("abc", "abcxxxxxx", 1u) == [0u]);
   assert (findn("abc", "xxxabcxxx", 1u) == [3u]);
   assert (findn("abc", "xxxxxxabc", 1u) == [6u]);
   assert (findn("abc", "xxxabcabc", 1u) == [3u]);
   assert (findn("abc", "xxxabcabc", 5u) == [3u, 6u]);
   assert (findn("abc", "xxxabcxabc", 5u) == [3u, 7u]);
   assert (findn("abc", "xxxabcxxabc", 5u) == [3u, 8u]);
}

#[test]
fn test_find_ascii() {
  assert (find_("banana", "apple pie") == option::none);
  assert (find_("", "") == option::none);
  assert (find_("abc", "abcxxxxxx") == option::some(0u));
  assert (find_("abc", "xxxabcxxx") == option::some(3u));
  assert (find_("abc", "xxxxxxabc") == option::some(6u));
}

#[test]
fn test_find_utf8() {
  let data = "ประเทศไทย中华Việt Nam";
  assert (find_("ไท华", data) == option::none);
  assert (find_("", data)     == option::some( 0u));
  assert (find_("ประเ", data) == option::some( 0u));
  assert (find_("ะเ", data)   == option::some( 6u));
  assert (find_("ระ", data) == option::some(3u));
  assert (find_("ศไทย中华", data) == option::some(15u));
  assert (find_("ไทย中华", data) == option::some(18u));
  assert (find_("ย中华", data) == option::some(24u));
  assert (find_("中华", data) == option::some(27u));
}

<|endoftext|>