<commit_before>use std::path::Path;
use rustc_serialize::json::{self, Json};

use git2::build::RepoBuilder;
use git2::{Object, BranchType, Oid, Branch, Reference, Delta, DiffFormat, ObjectType, Tree,
           Repository, ErrorClass, Error as GitError};
use std::str;

static INDEX_GIT_URL: &'static str = "https://github.com/rust-lang/crates.io-index";
static LAST_SEEN_REFNAME: &'static str = "crates-index-diff_last-seen";
static EMPTY_TREE_HASH: &'static str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";

pub struct Index {
    pub seen_ref_name: &'static str,
    repo: Repository,
}

#[derive(RustcEncodable, Ord, PartialOrd, Eq, PartialEq, Debug)]
pub enum ChangeType {
    Added,
    Yanked,
}

#[derive(RustcEncodable, Ord, PartialOrd, Eq, PartialEq, Debug)]
pub struct Crate {
    pub name: String,
    pub state: ChangeType,
    pub version: String,
}

#[derive(PartialOrd, PartialEq, Debug)]
enum CrateDecodeError {
    InvalidTopology { json: Json },
    MissingFieldError {
        object: json::Object,
        field: &'static str,
    },
    StringExpected { json: Json },
    BoolExpected { json: Json },
}

use self::CrateDecodeError::*;

impl Crate {
    fn from_crates_diff_json(value: Json) -> Result<Crate, CrateDecodeError> {
        fn extract<'a>(o: &'a json::Object,
                       field: &'static str)
                       -> Result<&'a Json, CrateDecodeError> {
            o.get(field).ok_or_else(|| {
                MissingFieldError {
                    object: o.clone(),
                    field: field,
                }
            })
        }

        fn into_string(value: &Json) -> Result<String, CrateDecodeError> {
            value.as_string()
                .ok_or_else(|| StringExpected { json: value.clone() })
                .map(Into::into)
        }

        fn into_bool(value: &Json) -> Result<bool, CrateDecodeError> {
            value.as_boolean()
                .ok_or_else(|| BoolExpected { json: value.clone() })
                .map(Into::into)
        }

        let o = value.as_object().ok_or_else(|| InvalidTopology { json: value.clone() })?;
        let name = extract(o, "name").and_then(into_string)?;
        let version = extract(o, "vers").and_then(into_string)?;
        let yanked = extract(o, "yanked").and_then(into_bool)?;

        Ok(Crate {
            name: name,
            state: if yanked {
                ChangeType::Yanked
            } else {
                ChangeType::Added
            },
            version: version,
        })
    }
}

impl Index {
    pub fn repository(&self) -> &Repository {
        &self.repo
    }

    pub fn last_seen_reference(&self) -> Result<Reference, GitError> {
        self.repo.find_branch(self.seen_ref_name, BranchType::Local).map(Branch::into_reference)
    }

    pub fn from_path_or_cloned<P>(path: P) -> Result<Index, GitError>
        where P: AsRef<Path>
    {
        let repo =
            Repository::open(path.as_ref()).or_else(|err| if err.class() == ErrorClass::Repository {
                    RepoBuilder::new().bare(true).clone(INDEX_GIT_URL, path.as_ref())
                } else {
                    Err(err)
                })?;

        Ok(Index {
            repo: repo,
            seen_ref_name: LAST_SEEN_REFNAME,
        })
    }

    pub fn fetch_changes(&self) -> Result<Vec<Crate>, GitError> {
        let from = self.last_seen_reference()
            .and_then(|r| {
                r.target().ok_or_else(|| {
                    GitError::from_str("last-seen reference did not have a valid target")
                })
            })
            .or_else(|_| Oid::from_str(EMPTY_TREE_HASH))?;
        let to = {
            self.repo
                .find_remote("origin")
                .and_then(|mut r| r.fetch(&["refs/heads/*:refs/remotes/origin/*"], None, None))?;
            let latest_fetched_commit_oid = self.repo.refname_to_id("refs/remotes/origin/master")?;
            self.last_seen_reference()
                .and_then(|mut seen_ref| {
                    seen_ref.set_target(latest_fetched_commit_oid,
                                        "updating seen-ref head to latest fetched commit")
                })
                .or_else(|_err| {
                    self.repo
                        .find_commit(latest_fetched_commit_oid)
                        .and_then(|commit| {
                            self.repo
                                .branch(self.seen_ref_name, &commit, true)
                                .map(Branch::into_reference)
                        })
                })?;
            latest_fetched_commit_oid
        };
        self.changes_from_objects(self.repo.find_object(from, None)?,
                                  self.repo.find_object(to, None)?)
    }

    pub fn changes<S1, S2>(&self, from: S1, to: S2) -> Result<Vec<Crate>, GitError>
        where S1: AsRef<str>,
              S2: AsRef<str>
    {
        self.changes_from_objects(self.repo.revparse_single(from.as_ref())?,
                                  self.repo.revparse_single(to.as_ref())?)
    }

    pub fn changes_from_objects(&self, from: Object, to: Object) -> Result<Vec<Crate>, GitError> {
        fn into_tree<'a>(repo: &'a Repository, obj: Object) -> Result<Tree<'a>, GitError> {
            repo.find_tree(match obj.kind() {
                Some(ObjectType::Commit)
                    => obj.as_commit().expect("object of kind commit yields commit").tree_id(),
                _ => /* let it possibly fail later */ obj.id()
            })
        }
        let diff = self.repo
            .diff_tree_to_tree(Some(&into_tree(&self.repo, from)?),
                               Some(&into_tree(&self.repo, to)?),
                               None)?;
        let mut res = Vec::new();
        diff.print(DiffFormat::Patch, |delta, _, diffline| -> bool {
                if !match delta.status() {
                    Delta::Added | Delta::Modified => true,
                    _ => false,
                } {
                    return true;
                }

                let content = match str::from_utf8(diffline.content()) {
                    Ok(c) => c,
                    Err(_) => return true,
                };
                if diffline.origin() != '+' {
                    return true;
                }

                if let Some(c) = Json::from_str(content)
                    .ok()
                    .and_then(|json| Crate::from_crates_diff_json(json).ok()) {
                    res.push(c)
                }
                return true;
            })?;

        Ok(res)
    }
}
<commit_msg>refactor(index): better diff.print implementation<commit_after>use std::path::Path;
use rustc_serialize::json::{self, Json};

use git2::build::RepoBuilder;
use git2::{Object, BranchType, Oid, Branch, Reference, Delta, DiffFormat, ObjectType, Tree,
           Repository, ErrorClass, Error as GitError};
use std::str;

static INDEX_GIT_URL: &'static str = "https://github.com/rust-lang/crates.io-index";
static LAST_SEEN_REFNAME: &'static str = "crates-index-diff_last-seen";
static EMPTY_TREE_HASH: &'static str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
static LINE_ADDED_INDICATOR: char = '+';

pub struct Index {
    pub seen_ref_name: &'static str,
    repo: Repository,
}

#[derive(RustcEncodable, Ord, PartialOrd, Eq, PartialEq, Debug)]
pub enum ChangeType {
    Added,
    Yanked,
}

#[derive(RustcEncodable, Ord, PartialOrd, Eq, PartialEq, Debug)]
pub struct Crate {
    pub name: String,
    pub state: ChangeType,
    pub version: String,
}

#[derive(PartialOrd, PartialEq, Debug)]
enum CrateDecodeError {
    InvalidTopology { json: Json },
    MissingFieldError {
        object: json::Object,
        field: &'static str,
    },
    StringExpected { json: Json },
    BoolExpected { json: Json },
}

use self::CrateDecodeError::*;

impl Crate {
    fn from_crates_diff_json(value: Json) -> Result<Crate, CrateDecodeError> {
        fn extract<'a>(o: &'a json::Object,
                       field: &'static str)
                       -> Result<&'a Json, CrateDecodeError> {
            o.get(field).ok_or_else(|| {
                MissingFieldError {
                    object: o.clone(),
                    field: field,
                }
            })
        }

        fn into_string(value: &Json) -> Result<String, CrateDecodeError> {
            value.as_string()
                .ok_or_else(|| StringExpected { json: value.clone() })
                .map(Into::into)
        }

        fn into_bool(value: &Json) -> Result<bool, CrateDecodeError> {
            value.as_boolean()
                .ok_or_else(|| BoolExpected { json: value.clone() })
                .map(Into::into)
        }

        let o = value.as_object().ok_or_else(|| InvalidTopology { json: value.clone() })?;
        let name = extract(o, "name").and_then(into_string)?;
        let version = extract(o, "vers").and_then(into_string)?;
        let yanked = extract(o, "yanked").and_then(into_bool)?;

        Ok(Crate {
            name: name,
            state: if yanked {
                ChangeType::Yanked
            } else {
                ChangeType::Added
            },
            version: version,
        })
    }
}

impl Index {
    pub fn repository(&self) -> &Repository {
        &self.repo
    }

    pub fn last_seen_reference(&self) -> Result<Reference, GitError> {
        self.repo.find_branch(self.seen_ref_name, BranchType::Local).map(Branch::into_reference)
    }

    pub fn from_path_or_cloned<P>(path: P) -> Result<Index, GitError>
        where P: AsRef<Path>
    {
        let repo =
            Repository::open(path.as_ref()).or_else(|err| if err.class() == ErrorClass::Repository {
                    RepoBuilder::new().bare(true).clone(INDEX_GIT_URL, path.as_ref())
                } else {
                    Err(err)
                })?;

        Ok(Index {
            repo: repo,
            seen_ref_name: LAST_SEEN_REFNAME,
        })
    }

    pub fn fetch_changes(&self) -> Result<Vec<Crate>, GitError> {
        let from = self.last_seen_reference()
            .and_then(|r| {
                r.target().ok_or_else(|| {
                    GitError::from_str("last-seen reference did not have a valid target")
                })
            })
            .or_else(|_| Oid::from_str(EMPTY_TREE_HASH))?;
        let to = {
            self.repo
                .find_remote("origin")
                .and_then(|mut r| r.fetch(&["refs/heads/*:refs/remotes/origin/*"], None, None))?;
            let latest_fetched_commit_oid = self.repo.refname_to_id("refs/remotes/origin/master")?;
            self.last_seen_reference()
                .and_then(|mut seen_ref| {
                    seen_ref.set_target(latest_fetched_commit_oid,
                                        "updating seen-ref head to latest fetched commit")
                })
                .or_else(|_err| {
                    self.repo
                        .find_commit(latest_fetched_commit_oid)
                        .and_then(|commit| {
                            self.repo
                                .branch(self.seen_ref_name, &commit, true)
                                .map(Branch::into_reference)
                        })
                })?;
            latest_fetched_commit_oid
        };
        self.changes_from_objects(&self.repo.find_object(from, None)?,
                                  &self.repo.find_object(to, None)?)
    }

    pub fn changes<S1, S2>(&self, from: S1, to: S2) -> Result<Vec<Crate>, GitError>
        where S1: AsRef<str>,
              S2: AsRef<str>
    {
        self.changes_from_objects(&self.repo.revparse_single(from.as_ref())?,
                                  &self.repo.revparse_single(to.as_ref())?)
    }

    pub fn changes_from_objects(&self, from: &Object, to: &Object) -> Result<Vec<Crate>, GitError> {
        fn into_tree<'a>(repo: &'a Repository, obj: &Object) -> Result<Tree<'a>, GitError> {
            repo.find_tree(match obj.kind() {
                Some(ObjectType::Commit)
                    => obj.as_commit().expect("object of kind commit yields commit").tree_id(),
                _ => /* let it possibly fail later */ obj.id()
            })
        }
        let diff = self.repo
            .diff_tree_to_tree(Some(&into_tree(&self.repo, from)?),
                               Some(&into_tree(&self.repo, to)?),
                               None)?;
        let mut res = Vec::new();
        diff.print(DiffFormat::Patch, |delta, _, diffline| {
                if diffline.origin() != LINE_ADDED_INDICATOR {
                    return true;
                }

                if !match delta.status() {
                    Delta::Added | Delta::Modified => true,
                    _ => false,
                } {
                    return true;
                }

                let content = match str::from_utf8(diffline.content()) {
                    Ok(c) => c,
                    Err(_) => return true,
                };

                if let Some(c) = Json::from_str(content)
                    .ok()
                    .and_then(|json| Crate::from_crates_diff_json(json).ok()) {
                    res.push(c)
                }
                return true;
            })
            .map(|_| res)
    }
}
<|endoftext|>


<commit_before>use super::CrateVersion;
use serde_json;
use std::path::Path;

use git2::{
    build::RepoBuilder, Delta, DiffFormat, Error as GitError, ErrorClass, Object, ObjectType, Oid,
    Reference, Repository, Tree,
};
use std::str;

static INDEX_GIT_URL: &str = "https://github.com/rust-lang/crates.io-index";
static LAST_SEEN_REFNAME: &str = "refs/heads/crates-index-diff_last-seen";
static EMPTY_TREE_HASH: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
static LINE_ADDED_INDICATOR: char = '+';

/// A wrapper for a repository of the crates.io index.
pub struct Index {
    /// The name and path of the reference used to keep track of the last seen state of the
    /// crates.io repository. The default value is `refs/heads/crates-index-diff_last-seen`.
    pub seen_ref_name: &'static str,
    /// The crates.io repository.
    repo: Repository,
}

/// Options for use in `Index::from_path_or_cloned_with_options`
#[non_exhaustive]
pub struct CloneOptions {
    pub repository_url: String,
}

impl Default for CloneOptions {
    fn default() -> Self {
        CloneOptions {
            repository_url: INDEX_GIT_URL.into(),
        }
    }
}

impl Index {
    /// Return the crates.io repository.
    pub fn repository(&self) -> &Repository {
        &self.repo
    }

    /// Return the reference pointing to the state we have seen after calling `fetch_changes()`.
    pub fn last_seen_reference(&self) -> Result<Reference, GitError> {
        self.repo.find_reference(self.seen_ref_name)
    }

    /// Return a new `Index` instance from the given `path`, which should contain a bare or non-bare
    /// clone of the `crates.io` index.
    /// If the directory does not contain the repository or does not exist, it will be cloned from
    /// the official location automatically (with complete history).
    ///
    /// An error will occour if the repository exists and the remote URL does not match the given repository URL.
    ///
    /// # Examples
    ///
    /// ```
    /// use crates_index_diff::{CloneOptions, Index};
    ///
    /// # let path = tempdir::TempDir::new("index").unwrap();
    /// let mut options = CloneOptions::default();
    /// options.repository_url = "https://github.com/rust-lang/staging.crates.io-index".into();
    ///
    /// let index = Index::from_path_or_cloned_with_options(path, options)?;
    /// # Ok::<(), git2::Error>(())
    /// ```
    pub fn from_path_or_cloned_with_options(
        path: impl AsRef<Path>,
        options: CloneOptions,
    ) -> Result<Index, GitError> {
        let mut repo_did_exist = true;
        let repo = Repository::open(path.as_ref()).or_else(|err| {
            if err.class() == ErrorClass::Repository {
                repo_did_exist = false;
                RepoBuilder::new()
                    .bare(true)
                    .clone(&options.repository_url, path.as_ref())
            } else {
                Err(err)
            }
        })?;

        if repo_did_exist {
            let remote = repo.find_remote("origin")?;
            let actual_remote_url = remote
                .url()
                .ok_or_else(|| GitError::from_str("did not obtain URL of remote named 'origin'"))?;
            if actual_remote_url != options.repository_url {
                return Err(GitError::from_str(&format!(
                    "Actual 'origin' remote url {:#?} did not match desired one at {:#?}",
                    actual_remote_url, options.repository_url
                )));
            }
        }

        Ok(Index {
            repo,
            seen_ref_name: LAST_SEEN_REFNAME,
        })
    }

    /// Return a new `Index` instance from the given `path`, which should contain a bare or non-bare
    /// clone of the `crates.io` index.
    /// If the directory does not contain the repository or does not exist, it will be cloned from
    /// the official location automatically (with complete history).
    pub fn from_path_or_cloned(path: impl AsRef<Path>) -> Result<Index, GitError> {
        Index::from_path_or_cloned_with_options(path, CloneOptions::default())
    }

    /// As `peek_changes_with_options`, but without the options.
    pub fn peek_changes(&self) -> Result<(Vec<CrateVersion>, git2::Oid), GitError> {
        self.peek_changes_with_options(None)
    }

    /// Return all `CrateVersion`s that are observed between the last time `fetch_changes(…)` was called
    /// and the latest state of the `crates.io` index repository, which is obtained by fetching
    /// the remote called `origin`.
    /// The `last_seen_reference()` will not be created or updated.
    /// The second field in the returned tuple is the commit object to which the changes were provided.
    /// If one would set the `last_seen_reference()` to that object, the effect is exactly the same
    /// as if `fetch_changes(…)` had been called.
    pub fn peek_changes_with_options(
        &self,
        options: Option<&mut git2::FetchOptions<'_>>,
    ) -> Result<(Vec<CrateVersion>, git2::Oid), GitError> {
        let from = self
            .last_seen_reference()
            .and_then(|r| {
                r.target().ok_or_else(|| {
                    GitError::from_str("last-seen reference did not have a valid target")
                })
            })
            .or_else(|_| Oid::from_str(EMPTY_TREE_HASH))?;
        let to = {
            self.repo.find_remote("origin").and_then(|mut r| {
                r.fetch(&["refs/heads/*:refs/remotes/origin/*"], options, None)
            })?;
            let latest_fetched_commit_oid =
                self.repo.refname_to_id("refs/remotes/origin/master")?;
            latest_fetched_commit_oid
        };

        Ok((
            self.changes_from_objects(
                &self.repo.find_object(from, None)?,
                &self.repo.find_object(to, None)?,
            )?,
            to,
        ))
    }

    /// As `fetch_changes_with_options`, but without the options.
    pub fn fetch_changes(&self) -> Result<Vec<CrateVersion>, GitError> {
        self.fetch_changes_with_options(None)
    }

    /// Return all `CrateVersion`s that are observed between the last time this method was called
    /// and the latest state of the `crates.io` index repository, which is obtained by fetching
    /// the remote called `origin`.
    /// The `last_seen_reference()` will be created or adjusted to point to the latest fetched
    /// state, which causes this method to have a different result each time it is called.
    pub fn fetch_changes_with_options(
        &self,
        options: Option<&mut git2::FetchOptions<'_>>,
    ) -> Result<Vec<CrateVersion>, GitError> {
        let (changes, to) = self.peek_changes_with_options(options)?;
        self.set_last_seen_reference(to)?;
        Ok(changes)
    }

    /// Set the last seen reference to the given Oid. It will be created if it does not yet exists.
    pub fn set_last_seen_reference(&self, to: Oid) -> Result<(), GitError> {
        self.last_seen_reference()
            .and_then(|mut seen_ref| {
                seen_ref.set_target(to, "updating seen-ref head to latest fetched commit")
            })
            .or_else(|_err| {
                self.repo.reference(
                    self.seen_ref_name,
                    to,
                    true,
                    "creating seen-ref at latest fetched commit",
                )
            })?;
        Ok(())
    }

    /// Return all `CreateVersion`s observed between `from` and `to`. Both parameter are ref-specs
    /// pointing to either a commit or a tree.
    /// Learn more about specifying revisions
    /// in the
    /// [official documentation](https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html)
    pub fn changes(
        &self,
        from: impl AsRef<str>,
        to: impl AsRef<str>,
    ) -> Result<Vec<CrateVersion>, GitError> {
        self.changes_from_objects(
            &self.repo.revparse_single(from.as_ref())?,
            &self.repo.revparse_single(to.as_ref())?,
        )
    }

    /// Similar to `changes()`, but requires `from` and `to` objects to be provided. They may point
    /// to either `Commit`s or `Tree`s.
    pub fn changes_from_objects(
        &self,
        from: &Object,
        to: &Object,
    ) -> Result<Vec<CrateVersion>, GitError> {
        fn into_tree<'a>(repo: &'a Repository, obj: &Object) -> Result<Tree<'a>, GitError> {
            repo.find_tree(match obj.kind() {
                Some(ObjectType::Commit) => obj
                    .as_commit()
                    .expect("object of kind commit yields commit")
                    .tree_id(),
                _ =>
                /* let it possibly fail later */
                {
                    obj.id()
                }
            })
        }
        let diff = self.repo.diff_tree_to_tree(
            Some(&into_tree(&self.repo, from)?),
            Some(&into_tree(&self.repo, to)?),
            None,
        )?;
        let mut res: Vec<CrateVersion> = Vec::new();
        diff.print(DiffFormat::Patch, |delta, _, diffline| {
            if diffline.origin() != LINE_ADDED_INDICATOR {
                return true;
            }

            if !match delta.status() {
                Delta::Added | Delta::Modified => true,
                _ => false,
            } {
                return true;
            }

            if let Ok(c) = serde_json::from_slice(diffline.content()) {
                res.push(c)
            }
            true
        })
        .map(|_| res)
    }
}
<commit_msg>Avoid cloning the crates.io index as part of the doctests; convenience<commit_after>use super::CrateVersion;
use serde_json;
use std::path::Path;

use git2::{
    build::RepoBuilder, Delta, DiffFormat, Error as GitError, ErrorClass, Object, ObjectType, Oid,
    Reference, Repository, Tree,
};
use std::str;

static INDEX_GIT_URL: &str = "https://github.com/rust-lang/crates.io-index";
static LAST_SEEN_REFNAME: &str = "refs/heads/crates-index-diff_last-seen";
static EMPTY_TREE_HASH: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
static LINE_ADDED_INDICATOR: char = '+';

/// A wrapper for a repository of the crates.io index.
pub struct Index {
    /// The name and path of the reference used to keep track of the last seen state of the
    /// crates.io repository. The default value is `refs/heads/crates-index-diff_last-seen`.
    pub seen_ref_name: &'static str,
    /// The crates.io repository.
    repo: Repository,
}

/// Options for use in `Index::from_path_or_cloned_with_options`
pub struct CloneOptions {
    pub repository_url: String,
}

impl Default for CloneOptions {
    fn default() -> Self {
        CloneOptions {
            repository_url: INDEX_GIT_URL.into(),
        }
    }
}

impl Index {
    /// Return the crates.io repository.
    pub fn repository(&self) -> &Repository {
        &self.repo
    }

    /// Return the reference pointing to the state we have seen after calling `fetch_changes()`.
    pub fn last_seen_reference(&self) -> Result<Reference, GitError> {
        self.repo.find_reference(self.seen_ref_name)
    }

    /// Return a new `Index` instance from the given `path`, which should contain a bare or non-bare
    /// clone of the `crates.io` index.
    /// If the directory does not contain the repository or does not exist, it will be cloned from
    /// the official location automatically (with complete history).
    ///
    /// An error will occour if the repository exists and the remote URL does not match the given repository URL.
    ///
    /// # Examples
    ///
    /// ```no_run
    /// use crates_index_diff::{CloneOptions, Index};
    ///
    /// # let path = tempdir::TempDir::new("index").unwrap();
    /// let mut options = CloneOptions {
    ///   repository_url: "https://github.com/rust-lang/staging.crates.io-index".into(),
    ///   ..Default::default()
    /// };
    ///
    ///
    /// let index = Index::from_path_or_cloned_with_options(path, options)?;
    /// # Ok::<(), git2::Error>(())
    /// ```
    pub fn from_path_or_cloned_with_options(
        path: impl AsRef<Path>,
        options: CloneOptions,
    ) -> Result<Index, GitError> {
        let mut repo_did_exist = true;
        let repo = Repository::open(path.as_ref()).or_else(|err| {
            if err.class() == ErrorClass::Repository {
                repo_did_exist = false;
                RepoBuilder::new()
                    .bare(true)
                    .clone(&options.repository_url, path.as_ref())
            } else {
                Err(err)
            }
        })?;

        if repo_did_exist {
            let remote = repo.find_remote("origin")?;
            let actual_remote_url = remote
                .url()
                .ok_or_else(|| GitError::from_str("did not obtain URL of remote named 'origin'"))?;
            if actual_remote_url != options.repository_url {
                return Err(GitError::from_str(&format!(
                    "Actual 'origin' remote url {:#?} did not match desired one at {:#?}",
                    actual_remote_url, options.repository_url
                )));
            }
        }

        Ok(Index {
            repo,
            seen_ref_name: LAST_SEEN_REFNAME,
        })
    }

    /// Return a new `Index` instance from the given `path`, which should contain a bare or non-bare
    /// clone of the `crates.io` index.
    /// If the directory does not contain the repository or does not exist, it will be cloned from
    /// the official location automatically (with complete history).
    pub fn from_path_or_cloned(path: impl AsRef<Path>) -> Result<Index, GitError> {
        Index::from_path_or_cloned_with_options(path, CloneOptions::default())
    }

    /// As `peek_changes_with_options`, but without the options.
    pub fn peek_changes(&self) -> Result<(Vec<CrateVersion>, git2::Oid), GitError> {
        self.peek_changes_with_options(None)
    }

    /// Return all `CrateVersion`s that are observed between the last time `fetch_changes(…)` was called
    /// and the latest state of the `crates.io` index repository, which is obtained by fetching
    /// the remote called `origin`.
    /// The `last_seen_reference()` will not be created or updated.
    /// The second field in the returned tuple is the commit object to which the changes were provided.
    /// If one would set the `last_seen_reference()` to that object, the effect is exactly the same
    /// as if `fetch_changes(…)` had been called.
    pub fn peek_changes_with_options(
        &self,
        options: Option<&mut git2::FetchOptions<'_>>,
    ) -> Result<(Vec<CrateVersion>, git2::Oid), GitError> {
        let from = self
            .last_seen_reference()
            .and_then(|r| {
                r.target().ok_or_else(|| {
                    GitError::from_str("last-seen reference did not have a valid target")
                })
            })
            .or_else(|_| Oid::from_str(EMPTY_TREE_HASH))?;
        let to = {
            self.repo.find_remote("origin").and_then(|mut r| {
                r.fetch(&["refs/heads/*:refs/remotes/origin/*"], options, None)
            })?;
            let latest_fetched_commit_oid =
                self.repo.refname_to_id("refs/remotes/origin/master")?;
            latest_fetched_commit_oid
        };

        Ok((
            self.changes_from_objects(
                &self.repo.find_object(from, None)?,
                &self.repo.find_object(to, None)?,
            )?,
            to,
        ))
    }

    /// As `fetch_changes_with_options`, but without the options.
    pub fn fetch_changes(&self) -> Result<Vec<CrateVersion>, GitError> {
        self.fetch_changes_with_options(None)
    }

    /// Return all `CrateVersion`s that are observed between the last time this method was called
    /// and the latest state of the `crates.io` index repository, which is obtained by fetching
    /// the remote called `origin`.
    /// The `last_seen_reference()` will be created or adjusted to point to the latest fetched
    /// state, which causes this method to have a different result each time it is called.
    pub fn fetch_changes_with_options(
        &self,
        options: Option<&mut git2::FetchOptions<'_>>,
    ) -> Result<Vec<CrateVersion>, GitError> {
        let (changes, to) = self.peek_changes_with_options(options)?;
        self.set_last_seen_reference(to)?;
        Ok(changes)
    }

    /// Set the last seen reference to the given Oid. It will be created if it does not yet exists.
    pub fn set_last_seen_reference(&self, to: Oid) -> Result<(), GitError> {
        self.last_seen_reference()
            .and_then(|mut seen_ref| {
                seen_ref.set_target(to, "updating seen-ref head to latest fetched commit")
            })
            .or_else(|_err| {
                self.repo.reference(
                    self.seen_ref_name,
                    to,
                    true,
                    "creating seen-ref at latest fetched commit",
                )
            })?;
        Ok(())
    }

    /// Return all `CreateVersion`s observed between `from` and `to`. Both parameter are ref-specs
    /// pointing to either a commit or a tree.
    /// Learn more about specifying revisions
    /// in the
    /// [official documentation](https://www.kernel.org/pub/software/scm/git/docs/gitrevisions.html)
    pub fn changes(
        &self,
        from: impl AsRef<str>,
        to: impl AsRef<str>,
    ) -> Result<Vec<CrateVersion>, GitError> {
        self.changes_from_objects(
            &self.repo.revparse_single(from.as_ref())?,
            &self.repo.revparse_single(to.as_ref())?,
        )
    }

    /// Similar to `changes()`, but requires `from` and `to` objects to be provided. They may point
    /// to either `Commit`s or `Tree`s.
    pub fn changes_from_objects(
        &self,
        from: &Object,
        to: &Object,
    ) -> Result<Vec<CrateVersion>, GitError> {
        fn into_tree<'a>(repo: &'a Repository, obj: &Object) -> Result<Tree<'a>, GitError> {
            repo.find_tree(match obj.kind() {
                Some(ObjectType::Commit) => obj
                    .as_commit()
                    .expect("object of kind commit yields commit")
                    .tree_id(),
                _ =>
                /* let it possibly fail later */
                {
                    obj.id()
                }
            })
        }
        let diff = self.repo.diff_tree_to_tree(
            Some(&into_tree(&self.repo, from)?),
            Some(&into_tree(&self.repo, to)?),
            None,
        )?;
        let mut res: Vec<CrateVersion> = Vec::new();
        diff.print(DiffFormat::Patch, |delta, _, diffline| {
            if diffline.origin() != LINE_ADDED_INDICATOR {
                return true;
            }

            if !match delta.status() {
                Delta::Added | Delta::Modified => true,
                _ => false,
            } {
                return true;
            }

            if let Ok(c) = serde_json::from_slice(diffline.content()) {
                res.push(c)
            }
            true
        })
        .map(|_| res)
    }
}
<|endoftext|>


<commit_before>//! User input.

use std::io::{self, Read, Write};
use std::ops;

use event::{self, Event, Key};
use raw::IntoRawMode;

/// An iterator over input keys.
pub struct Keys<R> {
    iter: Events<R>,
}

impl<R: Read> Iterator for Keys<R> {
    type Item = Result<Key, io::Error>;

    fn next(&mut self) -> Option<Result<Key, io::Error>> {
        loop {
            match self.iter.next() {
                Some(Ok(Event::Key(k))) => return Some(Ok(k)),
                Some(Ok(_)) => continue,
                e @ Some(Err(_)) => e,
                None => return None,
            };
        }
    }
}

/// An iterator over input events.
pub struct Events<R>  {
    inner: EventsAndRaw<R>
}

impl<R: Read> Iterator for Events<R> {
    type Item = Result<Event, io::Error>;

    fn next(&mut self) -> Option<Result<Event, io::Error>> {
        self.inner.next().map(|tuple| tuple.map(|(event, _raw)| event))
    }
}

/// An iterator over input events and the bytes that define them.
pub struct EventsAndRaw<R> {
    source: R,
    leftover: Option<u8>,
}

impl<R: Read> Iterator for EventsAndRaw<R> {
    type Item = Result<(Event, Vec<u8>), io::Error>;

    fn next(&mut self) -> Option<Result<(Event, Vec<u8>), io::Error>> {
        let mut source = &mut self.source;

        if let Some(c) = self.leftover {
            // we have a leftover byte, use it
            self.leftover = None;
            return Some(parse_event(c, &mut source.bytes()));
        }

        // Here we read two bytes at a time. We need to distinguish between single ESC key presses,
        // and escape sequences (which start with ESC or a x1B byte). The idea is that if this is
        // an escape sequence, we will read multiple bytes (the first byte being ESC) but if this
        // is a single ESC keypress, we will only read a single byte.
        let mut buf = [0u8; 2];
        let res = match source.read(&mut buf) {
            Ok(0) => return None,
            Ok(1) => {
                match buf[0] {
                    b'\x1B' => Ok((Event::Key(Key::Esc), vec![b'\x1B'])),
                    c => parse_event(c, &mut source.bytes()),
                }
            }
            Ok(2) => {
                let mut option_iter = &mut Some(buf[1]).into_iter();
                let result = {
                    let mut iter = option_iter.map(|c| Ok(c)).chain(source.bytes());
                    parse_event(buf[0], &mut iter)
                };
                // If the option_iter wasn't consumed, keep the byte for later.
                self.leftover = option_iter.next();
                result
            }
            Ok(_) => unreachable!(),
            Err(e) => Err(e),
        };

        Some(res)
    }
}

fn parse_event<I>(item: u8, iter: &mut I) -> Result<(Event, Vec<u8>), io::Error>
    where I: Iterator<Item = Result<u8, io::Error>>
{
    let mut buf = vec![item];
    let result = {
        let mut iter = iter.inspect(|byte| if let &Ok(byte) = byte {
                                        buf.push(byte);
                                    });
        event::parse_event(item, &mut iter)
    };
    result.or(Ok(Event::Unsupported(buf.clone()))).map(|e| (e, buf))
}

/// Extension to `Read` trait.
pub trait TermRead {
    /// An iterator over input events.
    fn events(self) -> Events<Self> where Self: Sized;

    /// An iterator over key inputs.
    fn keys(self) -> Keys<Self> where Self: Sized;

    /// Read a line.
    ///
    /// EOT and ETX will abort the prompt, returning `None`. Newline or carriage return will
    /// complete the input.
    fn read_line(&mut self) -> io::Result<Option<String>>;

    /// Read a password.
    ///
    /// EOT and ETX will abort the prompt, returning `None`. Newline or carriage return will
    /// complete the input.
    fn read_passwd<W: Write>(&mut self, writer: &mut W) -> io::Result<Option<String>> {
        let _raw = try!(writer.into_raw_mode());
        self.read_line()
    }
}

impl<R: Read + TermReadEventsAndRaw> TermRead for R {
    fn events(self) -> Events<Self> {
        Events {
            inner: self.events_and_raw()
        }
    }
    fn keys(self) -> Keys<Self> {
        Keys { iter: self.events() }
    }

    fn read_line(&mut self) -> io::Result<Option<String>> {
        let mut buf = Vec::with_capacity(30);

        for c in self.bytes() {
            match c {
                Err(e) => return Err(e),
                Ok(0) | Ok(3) | Ok(4) => return Ok(None),
                Ok(0x7f) => {
                    buf.pop();
                }
                Ok(b'\n') | Ok(b'\r') => break,
                Ok(c) => buf.push(c),
            }
        }

        let string = try!(String::from_utf8(buf)
            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)));
        Ok(Some(string))
    }
}

/// Extension to `TermRead` trait. A separate trait in order to maintain backwards compatibility.
pub trait TermReadEventsAndRaw {
    /// An iterator over input events and the bytes that define them.
    fn events_and_raw(self) -> EventsAndRaw<Self> where Self: Sized;
}

impl<R: Read> TermReadEventsAndRaw for R {
    fn events_and_raw(self) -> EventsAndRaw<Self> {
        EventsAndRaw {
            source: self,
            leftover: None,
        }
    }
}

/// A sequence of escape codes to enable terminal mouse support.
const ENTER_MOUSE_SEQUENCE: &'static str = csi!("?1000h\x1b[?1002h\x1b[?1015h\x1b[?1006h");

/// A sequence of escape codes to disable terminal mouse support.
const EXIT_MOUSE_SEQUENCE: &'static str = csi!("?1006l\x1b[?1015l\x1b[?1002l\x1b[?1000l");

/// A terminal with added mouse support.
///
/// This can be obtained through the `From` implementations.
pub struct MouseTerminal<W: Write> {
    term: W,
}

impl<W: Write> From<W> for MouseTerminal<W> {
    fn from(mut from: W) -> MouseTerminal<W> {
        from.write_all(ENTER_MOUSE_SEQUENCE.as_bytes()).unwrap();

        MouseTerminal { term: from }
    }
}

impl<W: Write> Drop for MouseTerminal<W> {
    fn drop(&mut self) {
        self.term.write_all(EXIT_MOUSE_SEQUENCE.as_bytes()).unwrap();
    }
}

impl<W: Write> ops::Deref for MouseTerminal<W> {
    type Target = W;

    fn deref(&self) -> &W {
        &self.term
    }
}

impl<W: Write> ops::DerefMut for MouseTerminal<W> {
    fn deref_mut(&mut self) -> &mut W {
        &mut self.term
    }
}

impl<W: Write> Write for MouseTerminal<W> {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        self.term.write(buf)
    }

    fn flush(&mut self) -> io::Result<()> {
        self.term.flush()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::io;
    use event::{Key, Event, MouseEvent, MouseButton};

    #[test]
    fn test_keys() {
        let mut i = b"\x1Bayo\x7F\x1B[D".keys();

        assert_eq!(i.next().unwrap().unwrap(), Key::Alt('a'));
        assert_eq!(i.next().unwrap().unwrap(), Key::Char('y'));
        assert_eq!(i.next().unwrap().unwrap(), Key::Char('o'));
        assert_eq!(i.next().unwrap().unwrap(), Key::Backspace);
        assert_eq!(i.next().unwrap().unwrap(), Key::Left);
        assert!(i.next().is_none());
    }

    #[test]
    fn test_events() {
        let mut i =
            b"\x1B[\x00bc\x7F\x1B[D\
                    \x1B[M\x00\x22\x24\x1B[<0;2;4;M\x1B[32;2;4M\x1B[<0;2;4;m\x1B[35;2;4Mb"
                    .events();

        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Unsupported(vec![0x1B, b'[', 0x00]));
        assert_eq!(i.next().unwrap().unwrap(), Event::Key(Key::Char('b')));
        assert_eq!(i.next().unwrap().unwrap(), Event::Key(Key::Char('c')));
        assert_eq!(i.next().unwrap().unwrap(), Event::Key(Key::Backspace));
        assert_eq!(i.next().unwrap().unwrap(), Event::Key(Key::Left));
        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Mouse(MouseEvent::Press(MouseButton::WheelUp, 2, 4)));
        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Mouse(MouseEvent::Press(MouseButton::Left, 2, 4)));
        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Mouse(MouseEvent::Press(MouseButton::Left, 2, 4)));
        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Mouse(MouseEvent::Release(2, 4)));
        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Mouse(MouseEvent::Release(2, 4)));
        assert_eq!(i.next().unwrap().unwrap(), Event::Key(Key::Char('b')));
        assert!(i.next().is_none());
    }

    #[test]
    fn test_events_and_raw() {
        let input = b"\x1B[\x00bc\x7F\x1B[D\
                    \x1B[M\x00\x22\x24\x1B[<0;2;4;M\x1B[32;2;4M\x1B[<0;2;4;m\x1B[35;2;4Mb";
        let mut output = Vec::<u8>::new();
        {
            let mut i = input.events_and_raw().map(|res| res.unwrap())
                .inspect(|&(_, ref raw)| { output.extend(raw); }).map(|(event, _)| event);

            assert_eq!(i.next().unwrap(),
            Event::Unsupported(vec![0x1B, b'[', 0x00]));
            assert_eq!(i.next().unwrap(), Event::Key(Key::Char('b')));
            assert_eq!(i.next().unwrap(), Event::Key(Key::Char('c')));
            assert_eq!(i.next().unwrap(), Event::Key(Key::Backspace));
            assert_eq!(i.next().unwrap(), Event::Key(Key::Left));
            assert_eq!(i.next().unwrap(),
            Event::Mouse(MouseEvent::Press(MouseButton::WheelUp, 2, 4)));
            assert_eq!(i.next().unwrap(),
            Event::Mouse(MouseEvent::Press(MouseButton::Left, 2, 4)));
            assert_eq!(i.next().unwrap(),
            Event::Mouse(MouseEvent::Press(MouseButton::Left, 2, 4)));
            assert_eq!(i.next().unwrap(),
            Event::Mouse(MouseEvent::Release(2, 4)));
            assert_eq!(i.next().unwrap(),
            Event::Mouse(MouseEvent::Release(2, 4)));
            assert_eq!(i.next().unwrap(), Event::Key(Key::Char('b')));
            assert!(i.next().is_none());
        }

        assert_eq!(input.iter().map(|b| *b).collect::<Vec<u8>>(), output)
    }

    #[test]
    fn test_function_keys() {
        let mut st = b"\x1BOP\x1BOQ\x1BOR\x1BOS".keys();
        for i in 1..5 {
            assert_eq!(st.next().unwrap().unwrap(), Key::F(i));
        }

        let mut st = b"\x1B[11~\x1B[12~\x1B[13~\x1B[14~\x1B[15~\
        \x1B[17~\x1B[18~\x1B[19~\x1B[20~\x1B[21~\x1B[23~\x1B[24~"
                .keys();
        for i in 1..13 {
            assert_eq!(st.next().unwrap().unwrap(), Key::F(i));
        }
    }

    #[test]
    fn test_special_keys() {
        let mut st = b"\x1B[2~\x1B[H\x1B[7~\x1B[5~\x1B[3~\x1B[F\x1B[8~\x1B[6~".keys();
        assert_eq!(st.next().unwrap().unwrap(), Key::Insert);
        assert_eq!(st.next().unwrap().unwrap(), Key::Home);
        assert_eq!(st.next().unwrap().unwrap(), Key::Home);
        assert_eq!(st.next().unwrap().unwrap(), Key::PageUp);
        assert_eq!(st.next().unwrap().unwrap(), Key::Delete);
        assert_eq!(st.next().unwrap().unwrap(), Key::End);
        assert_eq!(st.next().unwrap().unwrap(), Key::End);
        assert_eq!(st.next().unwrap().unwrap(), Key::PageDown);
        assert!(st.next().is_none());
    }

    #[test]
    fn test_esc_key() {
        let mut st = b"\x1B".keys();
        assert_eq!(st.next().unwrap().unwrap(), Key::Esc);
        assert!(st.next().is_none());
    }

    fn line_match(a: &str, b: Option<&str>) {
        let mut sink = io::sink();

        let line = a.as_bytes().read_line().unwrap();
        let pass = a.as_bytes().read_passwd(&mut sink).unwrap();

        // godammit rustc

        assert_eq!(line, pass);

        if let Some(l) = line {
            assert_eq!(Some(l.as_str()), b);
        } else {
            assert!(b.is_none());
        }
    }

    #[test]
    fn test_read() {
        let test1 = "this is the first test";
        let test2 = "this is the second test";

        line_match(test1, Some(test1));
        line_match(test2, Some(test2));
    }

    #[test]
    fn test_backspace() {
        line_match("this is the\x7f first\x7f\x7f test",
                   Some("this is th fir test"));
        line_match("this is the seco\x7fnd test\x7f",
                   Some("this is the secnd tes"));
    }

    #[test]
    fn test_end() {
        line_match("abc\nhttps://www.youtube.com/watch?v=dQw4w9WgXcQ",
                   Some("abc"));
        line_match("hello\rhttps://www.youtube.com/watch?v=yPYZpwSpKmA",
                   Some("hello"));
    }

    #[test]
    fn test_abort() {
        line_match("abc\x03https://www.youtube.com/watch?v=dQw4w9WgXcQ", None);
        line_match("hello\x04https://www.youtube.com/watch?v=yPYZpwSpKmA", None);
    }

}
<commit_msg>Remove unused 'mut' qualifier in src/input.rs (#143)<commit_after>//! User input.

use std::io::{self, Read, Write};
use std::ops;

use event::{self, Event, Key};
use raw::IntoRawMode;

/// An iterator over input keys.
pub struct Keys<R> {
    iter: Events<R>,
}

impl<R: Read> Iterator for Keys<R> {
    type Item = Result<Key, io::Error>;

    fn next(&mut self) -> Option<Result<Key, io::Error>> {
        loop {
            match self.iter.next() {
                Some(Ok(Event::Key(k))) => return Some(Ok(k)),
                Some(Ok(_)) => continue,
                e @ Some(Err(_)) => e,
                None => return None,
            };
        }
    }
}

/// An iterator over input events.
pub struct Events<R>  {
    inner: EventsAndRaw<R>
}

impl<R: Read> Iterator for Events<R> {
    type Item = Result<Event, io::Error>;

    fn next(&mut self) -> Option<Result<Event, io::Error>> {
        self.inner.next().map(|tuple| tuple.map(|(event, _raw)| event))
    }
}

/// An iterator over input events and the bytes that define them.
pub struct EventsAndRaw<R> {
    source: R,
    leftover: Option<u8>,
}

impl<R: Read> Iterator for EventsAndRaw<R> {
    type Item = Result<(Event, Vec<u8>), io::Error>;

    fn next(&mut self) -> Option<Result<(Event, Vec<u8>), io::Error>> {
        let source = &mut self.source;

        if let Some(c) = self.leftover {
            // we have a leftover byte, use it
            self.leftover = None;
            return Some(parse_event(c, &mut source.bytes()));
        }

        // Here we read two bytes at a time. We need to distinguish between single ESC key presses,
        // and escape sequences (which start with ESC or a x1B byte). The idea is that if this is
        // an escape sequence, we will read multiple bytes (the first byte being ESC) but if this
        // is a single ESC keypress, we will only read a single byte.
        let mut buf = [0u8; 2];
        let res = match source.read(&mut buf) {
            Ok(0) => return None,
            Ok(1) => {
                match buf[0] {
                    b'\x1B' => Ok((Event::Key(Key::Esc), vec![b'\x1B'])),
                    c => parse_event(c, &mut source.bytes()),
                }
            }
            Ok(2) => {
                let mut option_iter = &mut Some(buf[1]).into_iter();
                let result = {
                    let mut iter = option_iter.map(|c| Ok(c)).chain(source.bytes());
                    parse_event(buf[0], &mut iter)
                };
                // If the option_iter wasn't consumed, keep the byte for later.
                self.leftover = option_iter.next();
                result
            }
            Ok(_) => unreachable!(),
            Err(e) => Err(e),
        };

        Some(res)
    }
}

fn parse_event<I>(item: u8, iter: &mut I) -> Result<(Event, Vec<u8>), io::Error>
    where I: Iterator<Item = Result<u8, io::Error>>
{
    let mut buf = vec![item];
    let result = {
        let mut iter = iter.inspect(|byte| if let &Ok(byte) = byte {
                                        buf.push(byte);
                                    });
        event::parse_event(item, &mut iter)
    };
    result.or(Ok(Event::Unsupported(buf.clone()))).map(|e| (e, buf))
}

/// Extension to `Read` trait.
pub trait TermRead {
    /// An iterator over input events.
    fn events(self) -> Events<Self> where Self: Sized;

    /// An iterator over key inputs.
    fn keys(self) -> Keys<Self> where Self: Sized;

    /// Read a line.
    ///
    /// EOT and ETX will abort the prompt, returning `None`. Newline or carriage return will
    /// complete the input.
    fn read_line(&mut self) -> io::Result<Option<String>>;

    /// Read a password.
    ///
    /// EOT and ETX will abort the prompt, returning `None`. Newline or carriage return will
    /// complete the input.
    fn read_passwd<W: Write>(&mut self, writer: &mut W) -> io::Result<Option<String>> {
        let _raw = try!(writer.into_raw_mode());
        self.read_line()
    }
}

impl<R: Read + TermReadEventsAndRaw> TermRead for R {
    fn events(self) -> Events<Self> {
        Events {
            inner: self.events_and_raw()
        }
    }
    fn keys(self) -> Keys<Self> {
        Keys { iter: self.events() }
    }

    fn read_line(&mut self) -> io::Result<Option<String>> {
        let mut buf = Vec::with_capacity(30);

        for c in self.bytes() {
            match c {
                Err(e) => return Err(e),
                Ok(0) | Ok(3) | Ok(4) => return Ok(None),
                Ok(0x7f) => {
                    buf.pop();
                }
                Ok(b'\n') | Ok(b'\r') => break,
                Ok(c) => buf.push(c),
            }
        }

        let string = try!(String::from_utf8(buf)
            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)));
        Ok(Some(string))
    }
}

/// Extension to `TermRead` trait. A separate trait in order to maintain backwards compatibility.
pub trait TermReadEventsAndRaw {
    /// An iterator over input events and the bytes that define them.
    fn events_and_raw(self) -> EventsAndRaw<Self> where Self: Sized;
}

impl<R: Read> TermReadEventsAndRaw for R {
    fn events_and_raw(self) -> EventsAndRaw<Self> {
        EventsAndRaw {
            source: self,
            leftover: None,
        }
    }
}

/// A sequence of escape codes to enable terminal mouse support.
const ENTER_MOUSE_SEQUENCE: &'static str = csi!("?1000h\x1b[?1002h\x1b[?1015h\x1b[?1006h");

/// A sequence of escape codes to disable terminal mouse support.
const EXIT_MOUSE_SEQUENCE: &'static str = csi!("?1006l\x1b[?1015l\x1b[?1002l\x1b[?1000l");

/// A terminal with added mouse support.
///
/// This can be obtained through the `From` implementations.
pub struct MouseTerminal<W: Write> {
    term: W,
}

impl<W: Write> From<W> for MouseTerminal<W> {
    fn from(mut from: W) -> MouseTerminal<W> {
        from.write_all(ENTER_MOUSE_SEQUENCE.as_bytes()).unwrap();

        MouseTerminal { term: from }
    }
}

impl<W: Write> Drop for MouseTerminal<W> {
    fn drop(&mut self) {
        self.term.write_all(EXIT_MOUSE_SEQUENCE.as_bytes()).unwrap();
    }
}

impl<W: Write> ops::Deref for MouseTerminal<W> {
    type Target = W;

    fn deref(&self) -> &W {
        &self.term
    }
}

impl<W: Write> ops::DerefMut for MouseTerminal<W> {
    fn deref_mut(&mut self) -> &mut W {
        &mut self.term
    }
}

impl<W: Write> Write for MouseTerminal<W> {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        self.term.write(buf)
    }

    fn flush(&mut self) -> io::Result<()> {
        self.term.flush()
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::io;
    use event::{Key, Event, MouseEvent, MouseButton};

    #[test]
    fn test_keys() {
        let mut i = b"\x1Bayo\x7F\x1B[D".keys();

        assert_eq!(i.next().unwrap().unwrap(), Key::Alt('a'));
        assert_eq!(i.next().unwrap().unwrap(), Key::Char('y'));
        assert_eq!(i.next().unwrap().unwrap(), Key::Char('o'));
        assert_eq!(i.next().unwrap().unwrap(), Key::Backspace);
        assert_eq!(i.next().unwrap().unwrap(), Key::Left);
        assert!(i.next().is_none());
    }

    #[test]
    fn test_events() {
        let mut i =
            b"\x1B[\x00bc\x7F\x1B[D\
                    \x1B[M\x00\x22\x24\x1B[<0;2;4;M\x1B[32;2;4M\x1B[<0;2;4;m\x1B[35;2;4Mb"
                    .events();

        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Unsupported(vec![0x1B, b'[', 0x00]));
        assert_eq!(i.next().unwrap().unwrap(), Event::Key(Key::Char('b')));
        assert_eq!(i.next().unwrap().unwrap(), Event::Key(Key::Char('c')));
        assert_eq!(i.next().unwrap().unwrap(), Event::Key(Key::Backspace));
        assert_eq!(i.next().unwrap().unwrap(), Event::Key(Key::Left));
        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Mouse(MouseEvent::Press(MouseButton::WheelUp, 2, 4)));
        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Mouse(MouseEvent::Press(MouseButton::Left, 2, 4)));
        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Mouse(MouseEvent::Press(MouseButton::Left, 2, 4)));
        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Mouse(MouseEvent::Release(2, 4)));
        assert_eq!(i.next().unwrap().unwrap(),
                   Event::Mouse(MouseEvent::Release(2, 4)));
        assert_eq!(i.next().unwrap().unwrap(), Event::Key(Key::Char('b')));
        assert!(i.next().is_none());
    }

    #[test]
    fn test_events_and_raw() {
        let input = b"\x1B[\x00bc\x7F\x1B[D\
                    \x1B[M\x00\x22\x24\x1B[<0;2;4;M\x1B[32;2;4M\x1B[<0;2;4;m\x1B[35;2;4Mb";
        let mut output = Vec::<u8>::new();
        {
            let mut i = input.events_and_raw().map(|res| res.unwrap())
                .inspect(|&(_, ref raw)| { output.extend(raw); }).map(|(event, _)| event);

            assert_eq!(i.next().unwrap(),
            Event::Unsupported(vec![0x1B, b'[', 0x00]));
            assert_eq!(i.next().unwrap(), Event::Key(Key::Char('b')));
            assert_eq!(i.next().unwrap(), Event::Key(Key::Char('c')));
            assert_eq!(i.next().unwrap(), Event::Key(Key::Backspace));
            assert_eq!(i.next().unwrap(), Event::Key(Key::Left));
            assert_eq!(i.next().unwrap(),
            Event::Mouse(MouseEvent::Press(MouseButton::WheelUp, 2, 4)));
            assert_eq!(i.next().unwrap(),
            Event::Mouse(MouseEvent::Press(MouseButton::Left, 2, 4)));
            assert_eq!(i.next().unwrap(),
            Event::Mouse(MouseEvent::Press(MouseButton::Left, 2, 4)));
            assert_eq!(i.next().unwrap(),
            Event::Mouse(MouseEvent::Release(2, 4)));
            assert_eq!(i.next().unwrap(),
            Event::Mouse(MouseEvent::Release(2, 4)));
            assert_eq!(i.next().unwrap(), Event::Key(Key::Char('b')));
            assert!(i.next().is_none());
        }

        assert_eq!(input.iter().map(|b| *b).collect::<Vec<u8>>(), output)
    }

    #[test]
    fn test_function_keys() {
        let mut st = b"\x1BOP\x1BOQ\x1BOR\x1BOS".keys();
        for i in 1..5 {
            assert_eq!(st.next().unwrap().unwrap(), Key::F(i));
        }

        let mut st = b"\x1B[11~\x1B[12~\x1B[13~\x1B[14~\x1B[15~\
        \x1B[17~\x1B[18~\x1B[19~\x1B[20~\x1B[21~\x1B[23~\x1B[24~"
                .keys();
        for i in 1..13 {
            assert_eq!(st.next().unwrap().unwrap(), Key::F(i));
        }
    }

    #[test]
    fn test_special_keys() {
        let mut st = b"\x1B[2~\x1B[H\x1B[7~\x1B[5~\x1B[3~\x1B[F\x1B[8~\x1B[6~".keys();
        assert_eq!(st.next().unwrap().unwrap(), Key::Insert);
        assert_eq!(st.next().unwrap().unwrap(), Key::Home);
        assert_eq!(st.next().unwrap().unwrap(), Key::Home);
        assert_eq!(st.next().unwrap().unwrap(), Key::PageUp);
        assert_eq!(st.next().unwrap().unwrap(), Key::Delete);
        assert_eq!(st.next().unwrap().unwrap(), Key::End);
        assert_eq!(st.next().unwrap().unwrap(), Key::End);
        assert_eq!(st.next().unwrap().unwrap(), Key::PageDown);
        assert!(st.next().is_none());
    }

    #[test]
    fn test_esc_key() {
        let mut st = b"\x1B".keys();
        assert_eq!(st.next().unwrap().unwrap(), Key::Esc);
        assert!(st.next().is_none());
    }

    fn line_match(a: &str, b: Option<&str>) {
        let mut sink = io::sink();

        let line = a.as_bytes().read_line().unwrap();
        let pass = a.as_bytes().read_passwd(&mut sink).unwrap();

        // godammit rustc

        assert_eq!(line, pass);

        if let Some(l) = line {
            assert_eq!(Some(l.as_str()), b);
        } else {
            assert!(b.is_none());
        }
    }

    #[test]
    fn test_read() {
        let test1 = "this is the first test";
        let test2 = "this is the second test";

        line_match(test1, Some(test1));
        line_match(test2, Some(test2));
    }

    #[test]
    fn test_backspace() {
        line_match("this is the\x7f first\x7f\x7f test",
                   Some("this is th fir test"));
        line_match("this is the seco\x7fnd test\x7f",
                   Some("this is the secnd tes"));
    }

    #[test]
    fn test_end() {
        line_match("abc\nhttps://www.youtube.com/watch?v=dQw4w9WgXcQ",
                   Some("abc"));
        line_match("hello\rhttps://www.youtube.com/watch?v=yPYZpwSpKmA",
                   Some("hello"));
    }

    #[test]
    fn test_abort() {
        line_match("abc\x03https://www.youtube.com/watch?v=dQw4w9WgXcQ", None);
        line_match("hello\x04https://www.youtube.com/watch?v=yPYZpwSpKmA", None);
    }

}
<|endoftext|>


<commit_before>#![allow(dead_code)]

use std::collections::HashMap;
use regex::Regex;
use token::{Token, TokenType};

/// Lexes a string slice into an vector of tokens
pub fn lex_str<'a>(text: &'a str) -> Vec<Token<'a>> {
    Lexer::new(text).lex()
}

////////////////////////////////////////////////
struct Lexer<'a> {
    text: &'a str,
    current_line: u32,
    current_column: u32,
    current_byte_offset: usize,
    tokens: Vec<Token<'a>>,
}

impl<'a> Lexer<'a> {
    fn new(text: &'a str) -> Lexer<'a> {
        Lexer {
            text: text,
            current_line: 0,
            current_column: 0,
            current_byte_offset: 0,
            tokens: vec!(),
        }
    }

    // Lexing consumes the lexer
    fn lex(mut self) -> Vec<Token<'a>> {
        //==================================
        // Build our regexes
        // TODO: can we do this outside of the lex function, perhaps
        // at program initialization?

        // Non-newline whitespace
        let re_whitespace = Regex::new(r"[ \t]+").unwrap();

        // A single newline
        let re_newline = Regex::new(r"(\r\n|\r|\n)").unwrap();

        // A comment
        let re_comment = Regex::new(r"#[^\r\n]*").unwrap();

        // A doc comment
        let re_doc_comment = Regex::new(r"#:[^\r\n]*").unwrap();

        // Literals
        let re_int = Regex::new(r"[0-9]+").unwrap();
        let re_real = Regex::new(r"[0-9]+\.[0-9]+").unwrap();

        // Identifiers
        let re_ident_or_keyword = Regex::new(r"[a-zA-Z][a-zA-Z0-9_]+").unwrap();
        let re_ident_generic = Regex::new(r"_[a-zA-Z][a-zA-Z0-9_]+").unwrap();

        // Operators
        let re_operator = Regex::new(r"[-+/*%|&!~]+").unwrap();

        //===================================
        // Set up map for single-character tokens
        let mut single_byte_tokens = HashMap::new();
        single_byte_tokens.insert("(", TokenType::LParen);
        single_byte_tokens.insert(")", TokenType::RParen);
        single_byte_tokens.insert("[", TokenType::LSquare);
        single_byte_tokens.insert("]", TokenType::RSquare);
        single_byte_tokens.insert("{", TokenType::LCurly);
        single_byte_tokens.insert("}", TokenType::RCurly);
        single_byte_tokens.insert(",", TokenType::Comma);
        single_byte_tokens.insert(":", TokenType::Colon);
        single_byte_tokens.insert("@", TokenType::At);
        single_byte_tokens.insert(".", TokenType::Period);
        single_byte_tokens.insert("`", TokenType::BackTick);
        single_byte_tokens.insert("$", TokenType::Dollar);

        //==================================
        //  Do the lexing
        loop {
            let mut bytes_consumed;

            // End of file
            if self.text.len() == 0 {
                self.tokens.push(Token {
                    token_type: TokenType::EOF,
                    text: self.text,
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
                break;
            }

            // Newline
            else if let Some((0, n)) = re_newline.find(self.text) {
                bytes_consumed = n;
                self.tokens.push(Token {
                    token_type: TokenType::NewLine,
                    text: &self.text[0..n],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });

                // Handle state updates specially
                self.current_line += 1;
                self.current_column = 0;
                self.current_byte_offset += bytes_consumed;
                self.text = &self.text[bytes_consumed..];
                continue;
            }

            // Doc comment
            else if let Some((0, n)) = re_doc_comment.find(self.text) {
                bytes_consumed = n;
                self.tokens.push(Token {
                    token_type: TokenType::DocComment,
                    text: &self.text[0..1],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Comment
            else if let Some((0, n)) = re_comment.find(self.text) {
                bytes_consumed = n;
            }

            // White space
            else if let Some((0, n)) = re_whitespace.find(self.text) {
                bytes_consumed = n;
            }

            // Punctuation
            else if let Some(tt) = single_byte_tokens.get(&self.text[0..1]) {
                bytes_consumed = 1;
                self.tokens.push(Token {
                    token_type: *tt,
                    text: &self.text[0..1],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Operators
            else if let Some((0, n)) = re_operator.find(self.text) {
                bytes_consumed = n;
                self.tokens.push(Token {
                    token_type: TokenType::Operator,
                    text: &self.text[0..1],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Real number literal
            else if let Some((0, n)) = re_real.find(self.text) {
                bytes_consumed = n;
                self.tokens.push(Token {
                    token_type: TokenType::LIT_Real,
                    text: &self.text[0..1],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Integer literal
            else if let Some((0, n)) = re_int.find(self.text) {
                bytes_consumed = n;
                self.tokens.push(Token {
                    token_type: TokenType::LIT_Int,
                    text: &self.text[0..1],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Identifier or keyword
            else if let Some((0, n)) = re_ident_or_keyword.find(self.text) {
                bytes_consumed = n;

                let tt = match &self.text[0..n] {
                    "namespace" => TokenType::KEY_Namespace,
                    "pub" => TokenType::KEY_Pub,
                    "unsafe" => TokenType::KEY_Unsafe,
                    "const" => TokenType::KEY_Const,
                    "val" => TokenType::KEY_Val,
                    "var" => TokenType::KEY_Var,
                    "mut" => TokenType::KEY_Mut,
                    "ref" => TokenType::KEY_Ref,
                    "fn" => TokenType::KEY_Fn,
                    "struct" => TokenType::KEY_Struct,
                    "enum" => TokenType::KEY_Enum,
                    "union" => TokenType::KEY_Union,
                    "trait" => TokenType::KEY_Trait,
                    "is" => TokenType::KEY_Is,
                    "if" => TokenType::KEY_If,
                    "else" => TokenType::KEY_Else,
                    "loop" => TokenType::KEY_Loop,
                    "while" => TokenType::KEY_While,
                    "until" => TokenType::KEY_Until,
                    "for" => TokenType::KEY_For,
                    "in" => TokenType::KEY_In,
                    "break" => TokenType::KEY_Break,
                    "continue" => TokenType::KEY_Continue,
                    "return" => TokenType::KEY_Return,
                    "as" => TokenType::KEY_As,
                    "alias" => TokenType::KEY_Alias,
                    "type" => TokenType::KEY_Type,

                    _ => TokenType::Identifier,
                };

                self.tokens.push(Token {
                    token_type: tt,
                    text: &self.text[0..n],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Identifier of a generic parameter
            else if let Some((0, n)) = re_ident_generic.find(self.text) {
                bytes_consumed = n;

                self.tokens.push(Token {
                    token_type: TokenType::IdentifierGeneric,
                    text: &self.text[0..n],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Unknown input text
            else {
                println!("{:?}", self.text);
                panic!("Error: unknown text!");
            }

            // Update state
            self.current_column += bytes_consumed as u32; // TODO: actually base this on grapheme count
            self.current_byte_offset += bytes_consumed;
            self.text = &self.text[bytes_consumed..];
        }

        return self.tokens;
    }

}

#[cfg(test)]
mod tests {
    use super::*;
    use token::TokenType;

    #[test]
    fn idents_and_keywords() {
        let tokens = lex_str("var hello");

        assert_eq!(tokens[0].token_type, TokenType::KEY_Var);
        assert_eq!(tokens[1].token_type, TokenType::Identifier);
        assert_eq!(tokens[2].token_type, TokenType::EOF);
    }

    #[test]
    fn newlines() {
        let tokens = lex_str("var\n \n   \n hello");

        assert_eq!(tokens[0].token_type, TokenType::KEY_Var);
        assert_eq!(tokens[1].token_type, TokenType::NewLine);
        assert_eq!(tokens[2].token_type, TokenType::NewLine);
        assert_eq!(tokens[3].token_type, TokenType::NewLine);
        assert_eq!(tokens[4].token_type, TokenType::Identifier);
        assert_eq!(tokens[5].token_type, TokenType::EOF);
    }

    #[test]
    fn punctuation() {
        let tokens = lex_str("{}()[]@.,:`$");

        assert_eq!(tokens[0].token_type, TokenType::LCurly);
        assert_eq!(tokens[1].token_type, TokenType::RCurly);
        assert_eq!(tokens[2].token_type, TokenType::LParen);
        assert_eq!(tokens[3].token_type, TokenType::RParen);
        assert_eq!(tokens[4].token_type, TokenType::LSquare);
        assert_eq!(tokens[5].token_type, TokenType::RSquare);
        assert_eq!(tokens[6].token_type, TokenType::At);
        assert_eq!(tokens[7].token_type, TokenType::Period);
        assert_eq!(tokens[8].token_type, TokenType::Comma);
        assert_eq!(tokens[9].token_type, TokenType::Colon);
        assert_eq!(tokens[10].token_type, TokenType::BackTick);
        assert_eq!(tokens[11].token_type, TokenType::Dollar);
        assert_eq!(tokens[12].token_type, TokenType::EOF);
    }

    #[test]
    fn operator() {
        let tokens = lex_str("- + / * % | & ! ~ ++-*&|%");

        assert_eq!(tokens[0].token_type, TokenType::Operator);
        assert_eq!(tokens[1].token_type, TokenType::Operator);
        assert_eq!(tokens[2].token_type, TokenType::Operator);
        assert_eq!(tokens[3].token_type, TokenType::Operator);
        assert_eq!(tokens[4].token_type, TokenType::Operator);
        assert_eq!(tokens[5].token_type, TokenType::Operator);
        assert_eq!(tokens[6].token_type, TokenType::Operator);
        assert_eq!(tokens[7].token_type, TokenType::Operator);
        assert_eq!(tokens[8].token_type, TokenType::Operator);
        assert_eq!(tokens[9].token_type, TokenType::Operator);
        assert_eq!(tokens[10].token_type, TokenType::EOF);
    }

    #[test]
    fn ints_and_reals() {
        let tokens = lex_str("123 12.3");

        assert_eq!(tokens[0].token_type, TokenType::LIT_Int);
        assert_eq!(tokens[1].token_type, TokenType::LIT_Real);
        assert_eq!(tokens[2].token_type, TokenType::EOF);
    }

    #[test]
    fn comments() {
        let tokens = lex_str("var hello# How's it going?\n");

        assert_eq!(tokens[0].token_type, TokenType::KEY_Var);
        assert_eq!(tokens[1].token_type, TokenType::Identifier);
        assert_eq!(tokens[2].token_type, TokenType::NewLine);
        assert_eq!(tokens[3].token_type, TokenType::EOF);
    }

    #[test]
    fn doc_comments() {
        let tokens = lex_str("var hello#: How's it going?\n");

        assert_eq!(tokens[0].token_type, TokenType::KEY_Var);
        assert_eq!(tokens[1].token_type, TokenType::Identifier);
        assert_eq!(tokens[2].token_type, TokenType::DocComment);
        assert_eq!(tokens[3].token_type, TokenType::NewLine);
        assert_eq!(tokens[4].token_type, TokenType::EOF);
    }
}<commit_msg>Added lexing of string literals and raw string literals.<commit_after>#![allow(dead_code)]

use std::collections::HashMap;
use regex::Regex;
use token::{Token, TokenType};

/// Lexes a string slice into an vector of tokens
pub fn lex_str<'a>(text: &'a str) -> Vec<Token<'a>> {
    Lexer::new(text).lex()
}

////////////////////////////////////////////////
struct Lexer<'a> {
    text: &'a str,
    current_line: u32,
    current_column: u32,
    current_byte_offset: usize,
    tokens: Vec<Token<'a>>,
}

impl<'a> Lexer<'a> {
    fn new(text: &'a str) -> Lexer<'a> {
        Lexer {
            text: text,
            current_line: 0,
            current_column: 0,
            current_byte_offset: 0,
            tokens: vec!(),
        }
    }

    // Lexing consumes the lexer
    fn lex(mut self) -> Vec<Token<'a>> {
        //==================================
        // Build our regexes
        // TODO: can we do this outside of the lex function, perhaps
        // at program initialization?

        // Non-newline whitespace
        let re_whitespace = Regex::new(r"[ \t]+").unwrap();

        // A single newline
        let re_newline = Regex::new(r"(\r\n|\r|\n)").unwrap();

        // A comment
        let re_comment = Regex::new(r"#[^\r\n]*").unwrap();

        // A doc comment
        let re_doc_comment = Regex::new(r"#:[^\r\n]*").unwrap();

        // Literals
        let re_int = Regex::new(r"[0-9]+").unwrap();
        let re_real = Regex::new(r"[0-9]+\.[0-9]+").unwrap();
        let re_raw_string_start = Regex::new("'+\"").unwrap();

        // Identifiers
        let re_ident_or_keyword = Regex::new(r"[a-zA-Z][a-zA-Z0-9_]+").unwrap();
        let re_ident_generic = Regex::new(r"_[a-zA-Z][a-zA-Z0-9_]+").unwrap();

        // Operators
        let re_operator = Regex::new(r"[-+/*%|&!~]+").unwrap();

        //===================================
        // Set up map for single-character tokens
        let mut single_byte_tokens = HashMap::new();
        single_byte_tokens.insert("(", TokenType::LParen);
        single_byte_tokens.insert(")", TokenType::RParen);
        single_byte_tokens.insert("[", TokenType::LSquare);
        single_byte_tokens.insert("]", TokenType::RSquare);
        single_byte_tokens.insert("{", TokenType::LCurly);
        single_byte_tokens.insert("}", TokenType::RCurly);
        single_byte_tokens.insert(",", TokenType::Comma);
        single_byte_tokens.insert(":", TokenType::Colon);
        single_byte_tokens.insert("@", TokenType::At);
        single_byte_tokens.insert(".", TokenType::Period);
        single_byte_tokens.insert("`", TokenType::BackTick);
        single_byte_tokens.insert("$", TokenType::Dollar);

        //==================================
        //  Do the lexing
        loop {
            let mut bytes_consumed;

            // End of file
            if self.text.len() == 0 {
                self.tokens.push(Token {
                    token_type: TokenType::EOF,
                    text: self.text,
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
                break;
            }

            // Newline
            else if let Some((0, n)) = re_newline.find(self.text) {
                bytes_consumed = n;
                self.tokens.push(Token {
                    token_type: TokenType::NewLine,
                    text: &self.text[0..n],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });

                // Handle state updates specially
                self.current_line += 1;
                self.current_column = 0;
                self.current_byte_offset += bytes_consumed;
                self.text = &self.text[bytes_consumed..];
                continue;
            }

            // Doc comment
            else if let Some((0, n)) = re_doc_comment.find(self.text) {
                bytes_consumed = n;
                self.tokens.push(Token {
                    token_type: TokenType::DocComment,
                    text: &self.text[0..1],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Comment
            else if let Some((0, n)) = re_comment.find(self.text) {
                bytes_consumed = n;
            }

            // White space
            else if let Some((0, n)) = re_whitespace.find(self.text) {
                bytes_consumed = n;
            }

            // Punctuation
            else if let Some(tt) = single_byte_tokens.get(&self.text[0..1]) {
                bytes_consumed = 1;
                self.tokens.push(Token {
                    token_type: *tt,
                    text: &self.text[0..1],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Operators
            else if let Some((0, n)) = re_operator.find(self.text) {
                bytes_consumed = n;
                self.tokens.push(Token {
                    token_type: TokenType::Operator,
                    text: &self.text[0..1],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Real number literal
            else if let Some((0, n)) = re_real.find(self.text) {
                bytes_consumed = n;
                self.tokens.push(Token {
                    token_type: TokenType::LIT_Real,
                    text: &self.text[0..1],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Integer literal
            else if let Some((0, n)) = re_int.find(self.text) {
                bytes_consumed = n;
                self.tokens.push(Token {
                    token_type: TokenType::LIT_Int,
                    text: &self.text[0..1],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // String literal
            else if self.text.starts_with("\"") {
                let (newline_count, trailing_txt, txt) = self.lex_string_literal();
                bytes_consumed = txt.len();
                self.tokens.push(Token {
                    token_type: TokenType::LIT_String,
                    text: &self.text[0..bytes_consumed],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });

                // Handle state updates specially
                self.current_line += newline_count;
                self.current_column = trailing_txt.len() as u32; // TODO: actually base this on grapheme count
                self.current_byte_offset += bytes_consumed;
                self.text = &self.text[bytes_consumed..];
                continue;
            }

            // Raw string literal
            else if let Some((0, _)) = re_raw_string_start.find(self.text) {
                let (newline_count, trailing_txt, txt) = self.lex_raw_string_literal();
                bytes_consumed = txt.len();
                self.tokens.push(Token {
                    token_type: TokenType::LIT_RawString,
                    text: &self.text[0..bytes_consumed],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });

                // Handle state updates specially
                self.current_line += newline_count;
                self.current_column = trailing_txt.len() as u32; // TODO: actually base this on grapheme count
                self.current_byte_offset += bytes_consumed;
                self.text = &self.text[bytes_consumed..];
                continue;
            }

            // Identifier or keyword
            else if let Some((0, n)) = re_ident_or_keyword.find(self.text) {
                bytes_consumed = n;

                let tt = match &self.text[0..n] {
                    "namespace" => TokenType::KEY_Namespace,
                    "pub" => TokenType::KEY_Pub,
                    "unsafe" => TokenType::KEY_Unsafe,
                    "const" => TokenType::KEY_Const,
                    "val" => TokenType::KEY_Val,
                    "var" => TokenType::KEY_Var,
                    "mut" => TokenType::KEY_Mut,
                    "ref" => TokenType::KEY_Ref,
                    "fn" => TokenType::KEY_Fn,
                    "struct" => TokenType::KEY_Struct,
                    "enum" => TokenType::KEY_Enum,
                    "union" => TokenType::KEY_Union,
                    "trait" => TokenType::KEY_Trait,
                    "is" => TokenType::KEY_Is,
                    "if" => TokenType::KEY_If,
                    "else" => TokenType::KEY_Else,
                    "loop" => TokenType::KEY_Loop,
                    "while" => TokenType::KEY_While,
                    "until" => TokenType::KEY_Until,
                    "for" => TokenType::KEY_For,
                    "in" => TokenType::KEY_In,
                    "break" => TokenType::KEY_Break,
                    "continue" => TokenType::KEY_Continue,
                    "return" => TokenType::KEY_Return,
                    "as" => TokenType::KEY_As,
                    "alias" => TokenType::KEY_Alias,
                    "type" => TokenType::KEY_Type,

                    _ => TokenType::Identifier,
                };

                self.tokens.push(Token {
                    token_type: tt,
                    text: &self.text[0..n],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Identifier of a generic parameter
            else if let Some((0, n)) = re_ident_generic.find(self.text) {
                bytes_consumed = n;

                self.tokens.push(Token {
                    token_type: TokenType::IdentifierGeneric,
                    text: &self.text[0..n],
                    line: self.current_line,
                    column: self.current_column,
                    byte_offset: self.current_byte_offset,
                });
            }

            // Unknown input text
            else {
                println!("{:?}", self.text);
                panic!("Error: unknown text at line {} row {}", self.current_line+1, self.current_column);
            }

            // Update state
            self.current_column += bytes_consumed as u32; // TODO: actually base this on grapheme count
            self.current_byte_offset += bytes_consumed;
            self.text = &self.text[bytes_consumed..];
        }

        return self.tokens;
    }

    // TODO: error out when the string is not properly closed (i.e. reached EOF)
    fn lex_string_literal(&mut self) -> (u32, &'a str, &'a str) {
        // Find extent of string literal
        let mut last_was_esc = true;
        let mut ending_byte = 0;
        for (b, c) in self.text.char_indices() {
            ending_byte = b;
            if last_was_esc == true {
                last_was_esc = false;
            }
            else {
                if c == '\\' {
                    last_was_esc = true;
                }
                else if c == '"' {
                    break;
                }
            }
        }
        ending_byte += 1;

        // Figure out how many newlines are in the string literal,
        // and what the trailing string is.
        let re_newline = Regex::new(r"(\r\n|\r|\n)").unwrap(); // TODO: re-use same RE from lex().
        let string_text = &self.text[0..ending_byte];
        let newline_count = re_newline.find_iter(string_text).count() as u32;
        let trailing_text = if let Some(t) = re_newline.split(string_text).last() {t} else {string_text};

        return (newline_count, trailing_text, string_text);
    }

    // TODO: error out when the string is not properly closed (i.e. reached EOF)
    fn lex_raw_string_literal(&mut self) -> (u32, &'a str, &'a str) {
        // Find extent of string literal
        let mut stage = 0i32;
        let mut start_tick_count = 0;
        let mut tick_count = 0;
        let mut ending_byte = 0;
        for (b, c) in self.text.char_indices() {
            ending_byte = b;
            if stage == 0 {
                // Get the starting tick count
                if c == '\'' {
                    start_tick_count += 1;
                }
                else {
                    stage = 1;
                }
            }
            else if stage == 1 {
                // Look for double-quotes
                if c == '"' {
                    stage = 2;
                }
            }
            else if stage == 2 {
                // Count closing ticks
                if c == '\'' {
                    tick_count += 1;
                    if tick_count == start_tick_count {
                        break;
                    }
                }
                else {
                    tick_count = 0;
                    stage = 1;
                }
            }
        }

        ending_byte += 1;

        // Figure out how many newlines are in the string literal,
        // and what the trailing string is.
        let re_newline = Regex::new(r"(\r\n|\r|\n)").unwrap(); // TODO: re-use same RE from lex().
        let string_text = &self.text[0..ending_byte];
        let newline_count = re_newline.find_iter(string_text).count() as u32;
        let trailing_text = if let Some(t) = re_newline.split(string_text).last() {t} else {string_text};

        return (newline_count, trailing_text, string_text);
    }

}

#[cfg(test)]
mod tests {
    use super::*;
    use token::TokenType;

    #[test]
    fn idents_and_keywords() {
        let tokens = lex_str("var hello");

        assert_eq!(tokens[0].token_type, TokenType::KEY_Var);
        assert_eq!(tokens[1].token_type, TokenType::Identifier);
        assert_eq!(tokens[2].token_type, TokenType::EOF);
    }

    #[test]
    fn newlines() {
        let tokens = lex_str("var\n \n   \n hello");

        assert_eq!(tokens[0].token_type, TokenType::KEY_Var);
        assert_eq!(tokens[1].token_type, TokenType::NewLine);
        assert_eq!(tokens[2].token_type, TokenType::NewLine);
        assert_eq!(tokens[3].token_type, TokenType::NewLine);
        assert_eq!(tokens[4].token_type, TokenType::Identifier);
        assert_eq!(tokens[5].token_type, TokenType::EOF);
    }

    #[test]
    fn punctuation() {
        let tokens = lex_str("{}()[]@.,:`$");

        assert_eq!(tokens[0].token_type, TokenType::LCurly);
        assert_eq!(tokens[1].token_type, TokenType::RCurly);
        assert_eq!(tokens[2].token_type, TokenType::LParen);
        assert_eq!(tokens[3].token_type, TokenType::RParen);
        assert_eq!(tokens[4].token_type, TokenType::LSquare);
        assert_eq!(tokens[5].token_type, TokenType::RSquare);
        assert_eq!(tokens[6].token_type, TokenType::At);
        assert_eq!(tokens[7].token_type, TokenType::Period);
        assert_eq!(tokens[8].token_type, TokenType::Comma);
        assert_eq!(tokens[9].token_type, TokenType::Colon);
        assert_eq!(tokens[10].token_type, TokenType::BackTick);
        assert_eq!(tokens[11].token_type, TokenType::Dollar);
        assert_eq!(tokens[12].token_type, TokenType::EOF);
    }

    #[test]
    fn operator() {
        let tokens = lex_str("- + / * % | & ! ~ ++-*&|%");

        assert_eq!(tokens[0].token_type, TokenType::Operator);
        assert_eq!(tokens[1].token_type, TokenType::Operator);
        assert_eq!(tokens[2].token_type, TokenType::Operator);
        assert_eq!(tokens[3].token_type, TokenType::Operator);
        assert_eq!(tokens[4].token_type, TokenType::Operator);
        assert_eq!(tokens[5].token_type, TokenType::Operator);
        assert_eq!(tokens[6].token_type, TokenType::Operator);
        assert_eq!(tokens[7].token_type, TokenType::Operator);
        assert_eq!(tokens[8].token_type, TokenType::Operator);
        assert_eq!(tokens[9].token_type, TokenType::Operator);
        assert_eq!(tokens[10].token_type, TokenType::EOF);
    }

    #[test]
    fn ints_and_reals() {
        let tokens = lex_str("123 12.3");

        assert_eq!(tokens[0].token_type, TokenType::LIT_Int);
        assert_eq!(tokens[1].token_type, TokenType::LIT_Real);
        assert_eq!(tokens[2].token_type, TokenType::EOF);
    }

    #[test]
    fn comments() {
        let tokens = lex_str("var hello# How's it going?\n");

        assert_eq!(tokens[0].token_type, TokenType::KEY_Var);
        assert_eq!(tokens[1].token_type, TokenType::Identifier);
        assert_eq!(tokens[2].token_type, TokenType::NewLine);
        assert_eq!(tokens[3].token_type, TokenType::EOF);
    }

    #[test]
    fn doc_comments() {
        let tokens = lex_str("var hello#: How's it going?\n");

        assert_eq!(tokens[0].token_type, TokenType::KEY_Var);
        assert_eq!(tokens[1].token_type, TokenType::Identifier);
        assert_eq!(tokens[2].token_type, TokenType::DocComment);
        assert_eq!(tokens[3].token_type, TokenType::NewLine);
        assert_eq!(tokens[4].token_type, TokenType::EOF);
    }

    #[test]
    fn string_literal_1() {
        let tokens = lex_str("var\"Suddenly there's \\\"a string!\"hello");

        assert_eq!(tokens[0].token_type, TokenType::KEY_Var);
        assert_eq!(tokens[1].token_type, TokenType::LIT_String);
        assert_eq!(tokens[2].token_type, TokenType::Identifier);
        assert_eq!(tokens[3].token_type, TokenType::EOF);
    }

    #[test]
    fn raw_string_literal_1() {
        let tokens = lex_str("var'\"Suddenly there's \"a raw string!\"'hello");

        assert_eq!(tokens[0].token_type, TokenType::KEY_Var);
        assert_eq!(tokens[1].token_type, TokenType::LIT_RawString);
        assert_eq!(tokens[2].token_type, TokenType::Identifier);
        assert_eq!(tokens[3].token_type, TokenType::EOF);
    }

    #[test]
    fn raw_string_literal_2() {
        let tokens = lex_str("var''\"Suddenly there's \"'a raw string!\"''hello");

        assert_eq!(tokens[0].token_type, TokenType::KEY_Var);
        assert_eq!(tokens[1].token_type, TokenType::LIT_RawString);
        assert_eq!(tokens[2].token_type, TokenType::Identifier);
        assert_eq!(tokens[3].token_type, TokenType::EOF);
    }
}<|endoftext|>