<commit_before>use std::error;
use std::path::{Path, PathBuf};
use std::fmt::{self, Display};
use std::io;
use temp;
use utils;
use walkdir;
use toml;
use notify::{NotificationLevel, Notifyable};
use manifest::Component;

#[derive(Debug)]
pub enum Notification<'a> {
    Utils(utils::Notification<'a>),
    Temp(temp::Notification<'a>),

    Extracting(&'a Path, &'a Path),
    UpdateHashMatches,
    ComponentAlreadyInstalled(&'a Component),
    CantReadUpdateHash(&'a Path),
    NoUpdateHash(&'a Path),
    ChecksumValid(&'a str),
    SignatureValid(&'a str),
    RollingBack,
    ExtensionNotInstalled(&'a Component),
    NonFatalError(&'a Error),
    MissingInstalledComponent(&'a str),
    InstallingToolchain(&'a str),
    DownloadingComponent(&'a str, &'a str),
    InstallingComponent(&'a str),
    DownloadingManifest,
    DownloadingLegacyManifest,
}

#[derive(Debug)]
pub enum Error {
    Utils(utils::Error),
    Temp(temp::Error),

    InvalidFileExtension,
    InvalidInstaller,
    InvalidToolchainName(String),
    NotInstalledHere,
    UnsupportedHost(String),
    ChecksumFailed {
        url: String,
        expected: String,
        calculated: String,
    },
    ComponentConflict {
        name: String,
        path: PathBuf,
    },
    ComponentMissingFile {
        name: String,
        path: PathBuf,
    },
    ComponentMissingDir {
        name: String,
        path: PathBuf,
    },
    CorruptComponent(String),
    ExtractingPackage(io::Error),
    ExtensionNotFound(Component),
    InvalidChangeSet,
    NoGPG,
    BadInstallerVersion(String),
    BadInstalledMetadataVersion(String),
    ComponentDirPermissionsFailed(walkdir::Error),
    ComponentFilePermissionsFailed(io::Error),
    ComponentDownloadFailed(Component, utils::Error),
    ObsoleteDistManifest,
    Parsing(Vec<toml::ParserError>),
    MissingKey(String),
    ExpectedType(&'static str, String),
    PackageNotFound(String),
    TargetNotFound(String),
    MissingRoot,
    UnsupportedVersion(String),
    MissingPackageForComponent(Component),
    RequestedComponentsUnavailable(Vec<Component>),
}

pub type Result<T> = ::std::result::Result<T, Error>;
pub type NotifyHandler<'a> = ::notify::NotifyHandler<'a, for<'b> Notifyable<Notification<'b>>>;
pub type SharedNotifyHandler =
    ::notify::SharedNotifyHandler<for<'b> Notifyable<Notification<'b>>>;

extend_error!(Error: temp::Error, e => Error::Temp(e));
extend_error!(Error: utils::Error, e => Error::Utils(e));

extend_notification!(Notification: utils::Notification, n => Notification::Utils(n));
extend_notification!(Notification: temp::Notification, n => Notification::Temp(n));

impl<'a> Notification<'a> {
    pub fn level(&self) -> NotificationLevel {
        use self::Notification::*;
        match *self {
            Temp(ref n) => n.level(),
            Utils(ref n) => n.level(),
            NoUpdateHash(_) => NotificationLevel::Verbose,
            Extracting(_, _) | ChecksumValid(_) | SignatureValid(_)  |
            InstallingToolchain(_) | DownloadingComponent(_, _) |
            InstallingComponent(_) |
            UpdateHashMatches | ComponentAlreadyInstalled(_)  |
            RollingBack | DownloadingManifest |
            DownloadingLegacyManifest => NotificationLevel::Info,
            CantReadUpdateHash(_) | ExtensionNotInstalled(_) |
            MissingInstalledComponent(_) => NotificationLevel::Warn,
            NonFatalError(_) => NotificationLevel::Error,
        }
    }
}

impl<'a> Display for Notification<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> ::std::result::Result<(), fmt::Error> {
        use self::Notification::*;
        match *self {
            Temp(ref n) => n.fmt(f),
            Utils(ref n) => n.fmt(f),
            Extracting(_, _) => write!(f, "extracting..."),
            UpdateHashMatches => {
                write!(f, "toolchain is already up to date")
            }
            ComponentAlreadyInstalled(ref c) => {
                write!(f, "component '{}' for target '{}' is up to date",
                       c.pkg, c.target)
            }
            CantReadUpdateHash(path) => {
                write!(f,
                       "can't read update hash file: '{}', can't skip update...",
                       path.display())
            }
            NoUpdateHash(path) => write!(f, "no update hash at: '{}'", path.display()),
            ChecksumValid(_) => write!(f, "checksum passed"),
            SignatureValid(_) => write!(f, "signature valid"),
            RollingBack => write!(f, "rolling back changes"),
            ExtensionNotInstalled(c) => {
                write!(f, "extension '{}-{}' was not installed", c.pkg, c.target)
            }
            NonFatalError(e) => write!(f, "{}", e),
            MissingInstalledComponent(c) => write!(f, "during uninstall component {} was not found", c),
            InstallingToolchain(t) => write!(f, "installing toolchain '{}'", t),
            DownloadingComponent(c, u) => write!(f, "downloading component '{}' from '{}'", c, u),
            InstallingComponent(c) => write!(f, "installing component '{}'", c),
            DownloadingManifest => write!(f, "downloading manifest"),
            DownloadingLegacyManifest => write!(f, "manifest not found. trying legacy manifest"),
        }
    }
}

impl error::Error for Error {
    fn description(&self) -> &str {
        use self::Error::*;
        match *self {
            Utils(ref e) => error::Error::description(e),
            Temp(ref e) => error::Error::description(e),
            InvalidFileExtension => "invalid file extension",
            InvalidInstaller => "invalid installer",
            InvalidToolchainName(_) => "invalid custom toolchain name",
            NotInstalledHere => "not installed here",
            UnsupportedHost(_) => "binary package not provided for host",
            ChecksumFailed {..} => "checksum failed",
            ComponentConflict {..} => "conflicting component",
            ComponentMissingFile {..} => "missing file in component",
            ComponentMissingDir {..} => "missing directory in component",
            CorruptComponent(_) => "corrupt component manifest",
            ExtractingPackage(_) => "failed to extract package",
            ExtensionNotFound(_) => "could not find extension",
            InvalidChangeSet => "invalid change-set",
            NoGPG => "could not find 'gpg' on PATH",
            BadInstallerVersion(_) => "unsupported installer version",
            BadInstalledMetadataVersion(_) => "unsupported metadata version in existing installation",
            ComponentDirPermissionsFailed(_) => "I/O error walking directory during install",
            ComponentFilePermissionsFailed(_) => "error setting file permissions during install",
            ComponentDownloadFailed(_, _) => "component download failed",
            ObsoleteDistManifest => "the server unexpectedly provided an obsolete version of the distribution manifest",
            Parsing(_) => "error parsing manifest",
            MissingKey(_) => "missing key",
            ExpectedType(_, _) => "expected type",
            PackageNotFound(_) => "package not found",
            TargetNotFound(_) => "target not found",
            MissingRoot => "manifest has no root package",
            UnsupportedVersion(_) => "unsupported manifest version",
            MissingPackageForComponent(_) => "missing package for component",
            RequestedComponentsUnavailable(_) => "some requested components are unavailable to download",
        }
    }

    fn cause(&self) -> Option<&error::Error> {
        use self::Error::*;
        match *self {
            Utils(ref e) => Some(e),
            Temp(ref e) => Some(e),
            ComponentFilePermissionsFailed(ref e) => Some(e),
            ComponentDirPermissionsFailed(ref e) => Some(e),
            ExtractingPackage(ref e) => Some(e),
            ComponentDownloadFailed(_, ref e) => Some(e),
            InvalidFileExtension |
            InvalidInstaller |
            InvalidToolchainName(_) |
            NotInstalledHere |
            UnsupportedHost(_) |
            ChecksumFailed {..} |
            ComponentConflict {..} |
            ComponentMissingFile {..} |
            ComponentMissingDir {..} |
            CorruptComponent(_) |
            ExtensionNotFound(_) |
            InvalidChangeSet |
            NoGPG |
            BadInstallerVersion(_) |
            BadInstalledMetadataVersion(_) |
            ObsoleteDistManifest |
            Parsing(_) |
            MissingKey(_) |
            ExpectedType(_, _) |
            PackageNotFound(_) |
            TargetNotFound(_) |
            MissingRoot |
            UnsupportedVersion(_) |
            MissingPackageForComponent(_) |
            RequestedComponentsUnavailable(_) => None
        }
    }
}

impl Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> ::std::result::Result<(), fmt::Error> {
        use self::Error::*;
        match *self {
            Temp(ref n) => n.fmt(f),
            Utils(ref n) => n.fmt(f),

            InvalidFileExtension => write!(f, "invalid file extension"),
            InvalidInstaller => write!(f, "invalid installer"),
            InvalidToolchainName(ref s) => write!(f, "invalid custom toolchain name: '{}'", s),
            NotInstalledHere => write!(f, "not installed here"),
            UnsupportedHost(ref spec) => {
                write!(f, "a binary package was not provided for: '{}'", spec)
            }
            ChecksumFailed { url: _, ref expected, ref calculated } => {
                write!(f,
                       "checksum failed, expected: '{}', calculated: '{}'",
                       expected,
                       calculated)
            }
            ComponentConflict { ref name, ref path } => {
                write!(f,
                       "failed to install component: '{}', detected conflict: '{:?}'",
                       name,
                       path)
            }
            ComponentMissingFile { ref name, ref path } => {
                write!(f,
                       "failure removing component '{}', file does not exist: '{:?}'",
                       name,
                       path)
            }
            ComponentMissingDir { ref name, ref path } => {
                write!(f,
                       "failure removing component '{}', directory does not exist: '{:?}'",
                       name,
                       path)
            }
            CorruptComponent(ref name) => write!(f, "component manifest for '{}' is corrupt", name),
            ExtractingPackage(ref error) => write!(f, "failed to extract package: {}", error),
            ExtensionNotFound(ref c) => {
                write!(f, "could not find extension: '{}-{}'", c.pkg, c.target)
            }
            InvalidChangeSet => write!(f, "invalid change-set"),
            NoGPG => {
                write!(f,
                       "could not find 'gpg': ensure it is on PATH or disable GPG verification")
            }
            BadInstallerVersion(ref v) => write!(f, "unsupported installer version: {}", v),
            BadInstalledMetadataVersion(ref v) => {
                write!(f,
                       "unsupported metadata version in existing installation: {}",
                       v)
            }
            ComponentDirPermissionsFailed(ref e) => {
                write!(f, "I/O error walking directory during install: {}", e)
            }
            ComponentFilePermissionsFailed(ref e) => {
                write!(f, "error setting file permissions during install: {}", e)
            }
            ComponentDownloadFailed(ref component, ref e) => {
                write!(f, "component download failed for {}-{}: {}", component.pkg, component.target, e)
            }
            ObsoleteDistManifest => {
                write!(f, "the server unexpectedly provided an obsolete version of the distribution manifest")
            },
            Parsing(ref n) => {
                for e in n {
                    try!(e.fmt(f));
                    try!(writeln!(f, ""));
                }
                Ok(())
            }
            MissingKey(ref n) => write!(f, "missing key: '{}'", n),
            ExpectedType(ref t, ref n) => write!(f, "expected type: '{}' for '{}'", t, n),
            PackageNotFound(ref n) => write!(f, "package not found: '{}'", n),
            TargetNotFound(ref n) => write!(f, "target not found: '{}'", n),
            MissingRoot => write!(f, "manifest has no root package"),
            UnsupportedVersion(ref v) => write!(f, "manifest version '{}' is not supported", v),
            MissingPackageForComponent(ref c) => write!(f,"manifest missing package for component {}", c.name()),
            RequestedComponentsUnavailable(ref cs) => {
                assert!(!cs.is_empty());
                if cs.len() == 1 {
                    write!(f, "component '{}' for '{}' is unavailable for download",
                           cs[0].pkg, cs[0].target)
                } else {
                    use itertools::Itertools;
                    let same_target = cs.iter().all(|c| c.target == cs[0].target);
                    if same_target {
                        let mut cs_strs = cs.iter().map(|c| format!("'{}'", c.pkg));
                        let cs_str = cs_strs.join(", ");
                        write!(f, "some components unavailable for download: {}",
                               cs_str)
                    } else {
                        let mut cs_strs = cs.iter().map(|c| format!("'{}' for '{}'", c.pkg, c.target));
                        let cs_str = cs_strs.join(", ");
                        write!(f, "some components unavailable for download: {}",
                               cs_str)
                    }
                }
            }
        }
    }
}
<commit_msg>Make ChecksumValid a verbose notification<commit_after>use std::error;
use std::path::{Path, PathBuf};
use std::fmt::{self, Display};
use std::io;
use temp;
use utils;
use walkdir;
use toml;
use notify::{NotificationLevel, Notifyable};
use manifest::Component;

#[derive(Debug)]
pub enum Notification<'a> {
    Utils(utils::Notification<'a>),
    Temp(temp::Notification<'a>),

    Extracting(&'a Path, &'a Path),
    UpdateHashMatches,
    ComponentAlreadyInstalled(&'a Component),
    CantReadUpdateHash(&'a Path),
    NoUpdateHash(&'a Path),
    ChecksumValid(&'a str),
    SignatureValid(&'a str),
    RollingBack,
    ExtensionNotInstalled(&'a Component),
    NonFatalError(&'a Error),
    MissingInstalledComponent(&'a str),
    InstallingToolchain(&'a str),
    DownloadingComponent(&'a str, &'a str),
    InstallingComponent(&'a str),
    DownloadingManifest,
    DownloadingLegacyManifest,
}

#[derive(Debug)]
pub enum Error {
    Utils(utils::Error),
    Temp(temp::Error),

    InvalidFileExtension,
    InvalidInstaller,
    InvalidToolchainName(String),
    NotInstalledHere,
    UnsupportedHost(String),
    ChecksumFailed {
        url: String,
        expected: String,
        calculated: String,
    },
    ComponentConflict {
        name: String,
        path: PathBuf,
    },
    ComponentMissingFile {
        name: String,
        path: PathBuf,
    },
    ComponentMissingDir {
        name: String,
        path: PathBuf,
    },
    CorruptComponent(String),
    ExtractingPackage(io::Error),
    ExtensionNotFound(Component),
    InvalidChangeSet,
    NoGPG,
    BadInstallerVersion(String),
    BadInstalledMetadataVersion(String),
    ComponentDirPermissionsFailed(walkdir::Error),
    ComponentFilePermissionsFailed(io::Error),
    ComponentDownloadFailed(Component, utils::Error),
    ObsoleteDistManifest,
    Parsing(Vec<toml::ParserError>),
    MissingKey(String),
    ExpectedType(&'static str, String),
    PackageNotFound(String),
    TargetNotFound(String),
    MissingRoot,
    UnsupportedVersion(String),
    MissingPackageForComponent(Component),
    RequestedComponentsUnavailable(Vec<Component>),
}

pub type Result<T> = ::std::result::Result<T, Error>;
pub type NotifyHandler<'a> = ::notify::NotifyHandler<'a, for<'b> Notifyable<Notification<'b>>>;
pub type SharedNotifyHandler =
    ::notify::SharedNotifyHandler<for<'b> Notifyable<Notification<'b>>>;

extend_error!(Error: temp::Error, e => Error::Temp(e));
extend_error!(Error: utils::Error, e => Error::Utils(e));

extend_notification!(Notification: utils::Notification, n => Notification::Utils(n));
extend_notification!(Notification: temp::Notification, n => Notification::Temp(n));

impl<'a> Notification<'a> {
    pub fn level(&self) -> NotificationLevel {
        use self::Notification::*;
        match *self {
            Temp(ref n) => n.level(),
            Utils(ref n) => n.level(),
            ChecksumValid(_) => NotificationLevel::Verbose,
            NoUpdateHash(_) => NotificationLevel::Verbose,
            Extracting(_, _) | SignatureValid(_)  |
            InstallingToolchain(_) | DownloadingComponent(_, _) |
            InstallingComponent(_) |
            UpdateHashMatches | ComponentAlreadyInstalled(_)  |
            RollingBack | DownloadingManifest |
            DownloadingLegacyManifest => NotificationLevel::Info,
            CantReadUpdateHash(_) | ExtensionNotInstalled(_) |
            MissingInstalledComponent(_) => NotificationLevel::Warn,
            NonFatalError(_) => NotificationLevel::Error,
        }
    }
}

impl<'a> Display for Notification<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> ::std::result::Result<(), fmt::Error> {
        use self::Notification::*;
        match *self {
            Temp(ref n) => n.fmt(f),
            Utils(ref n) => n.fmt(f),
            Extracting(_, _) => write!(f, "extracting..."),
            UpdateHashMatches => {
                write!(f, "toolchain is already up to date")
            }
            ComponentAlreadyInstalled(ref c) => {
                write!(f, "component '{}' for target '{}' is up to date",
                       c.pkg, c.target)
            }
            CantReadUpdateHash(path) => {
                write!(f,
                       "can't read update hash file: '{}', can't skip update...",
                       path.display())
            }
            NoUpdateHash(path) => write!(f, "no update hash at: '{}'", path.display()),
            ChecksumValid(_) => write!(f, "checksum passed"),
            SignatureValid(_) => write!(f, "signature valid"),
            RollingBack => write!(f, "rolling back changes"),
            ExtensionNotInstalled(c) => {
                write!(f, "extension '{}-{}' was not installed", c.pkg, c.target)
            }
            NonFatalError(e) => write!(f, "{}", e),
            MissingInstalledComponent(c) => write!(f, "during uninstall component {} was not found", c),
            InstallingToolchain(t) => write!(f, "installing toolchain '{}'", t),
            DownloadingComponent(c, u) => write!(f, "downloading component '{}' from '{}'", c, u),
            InstallingComponent(c) => write!(f, "installing component '{}'", c),
            DownloadingManifest => write!(f, "downloading manifest"),
            DownloadingLegacyManifest => write!(f, "manifest not found. trying legacy manifest"),
        }
    }
}

impl error::Error for Error {
    fn description(&self) -> &str {
        use self::Error::*;
        match *self {
            Utils(ref e) => error::Error::description(e),
            Temp(ref e) => error::Error::description(e),
            InvalidFileExtension => "invalid file extension",
            InvalidInstaller => "invalid installer",
            InvalidToolchainName(_) => "invalid custom toolchain name",
            NotInstalledHere => "not installed here",
            UnsupportedHost(_) => "binary package not provided for host",
            ChecksumFailed {..} => "checksum failed",
            ComponentConflict {..} => "conflicting component",
            ComponentMissingFile {..} => "missing file in component",
            ComponentMissingDir {..} => "missing directory in component",
            CorruptComponent(_) => "corrupt component manifest",
            ExtractingPackage(_) => "failed to extract package",
            ExtensionNotFound(_) => "could not find extension",
            InvalidChangeSet => "invalid change-set",
            NoGPG => "could not find 'gpg' on PATH",
            BadInstallerVersion(_) => "unsupported installer version",
            BadInstalledMetadataVersion(_) => "unsupported metadata version in existing installation",
            ComponentDirPermissionsFailed(_) => "I/O error walking directory during install",
            ComponentFilePermissionsFailed(_) => "error setting file permissions during install",
            ComponentDownloadFailed(_, _) => "component download failed",
            ObsoleteDistManifest => "the server unexpectedly provided an obsolete version of the distribution manifest",
            Parsing(_) => "error parsing manifest",
            MissingKey(_) => "missing key",
            ExpectedType(_, _) => "expected type",
            PackageNotFound(_) => "package not found",
            TargetNotFound(_) => "target not found",
            MissingRoot => "manifest has no root package",
            UnsupportedVersion(_) => "unsupported manifest version",
            MissingPackageForComponent(_) => "missing package for component",
            RequestedComponentsUnavailable(_) => "some requested components are unavailable to download",
        }
    }

    fn cause(&self) -> Option<&error::Error> {
        use self::Error::*;
        match *self {
            Utils(ref e) => Some(e),
            Temp(ref e) => Some(e),
            ComponentFilePermissionsFailed(ref e) => Some(e),
            ComponentDirPermissionsFailed(ref e) => Some(e),
            ExtractingPackage(ref e) => Some(e),
            ComponentDownloadFailed(_, ref e) => Some(e),
            InvalidFileExtension |
            InvalidInstaller |
            InvalidToolchainName(_) |
            NotInstalledHere |
            UnsupportedHost(_) |
            ChecksumFailed {..} |
            ComponentConflict {..} |
            ComponentMissingFile {..} |
            ComponentMissingDir {..} |
            CorruptComponent(_) |
            ExtensionNotFound(_) |
            InvalidChangeSet |
            NoGPG |
            BadInstallerVersion(_) |
            BadInstalledMetadataVersion(_) |
            ObsoleteDistManifest |
            Parsing(_) |
            MissingKey(_) |
            ExpectedType(_, _) |
            PackageNotFound(_) |
            TargetNotFound(_) |
            MissingRoot |
            UnsupportedVersion(_) |
            MissingPackageForComponent(_) |
            RequestedComponentsUnavailable(_) => None
        }
    }
}

impl Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> ::std::result::Result<(), fmt::Error> {
        use self::Error::*;
        match *self {
            Temp(ref n) => n.fmt(f),
            Utils(ref n) => n.fmt(f),

            InvalidFileExtension => write!(f, "invalid file extension"),
            InvalidInstaller => write!(f, "invalid installer"),
            InvalidToolchainName(ref s) => write!(f, "invalid custom toolchain name: '{}'", s),
            NotInstalledHere => write!(f, "not installed here"),
            UnsupportedHost(ref spec) => {
                write!(f, "a binary package was not provided for: '{}'", spec)
            }
            ChecksumFailed { url: _, ref expected, ref calculated } => {
                write!(f,
                       "checksum failed, expected: '{}', calculated: '{}'",
                       expected,
                       calculated)
            }
            ComponentConflict { ref name, ref path } => {
                write!(f,
                       "failed to install component: '{}', detected conflict: '{:?}'",
                       name,
                       path)
            }
            ComponentMissingFile { ref name, ref path } => {
                write!(f,
                       "failure removing component '{}', file does not exist: '{:?}'",
                       name,
                       path)
            }
            ComponentMissingDir { ref name, ref path } => {
                write!(f,
                       "failure removing component '{}', directory does not exist: '{:?}'",
                       name,
                       path)
            }
            CorruptComponent(ref name) => write!(f, "component manifest for '{}' is corrupt", name),
            ExtractingPackage(ref error) => write!(f, "failed to extract package: {}", error),
            ExtensionNotFound(ref c) => {
                write!(f, "could not find extension: '{}-{}'", c.pkg, c.target)
            }
            InvalidChangeSet => write!(f, "invalid change-set"),
            NoGPG => {
                write!(f,
                       "could not find 'gpg': ensure it is on PATH or disable GPG verification")
            }
            BadInstallerVersion(ref v) => write!(f, "unsupported installer version: {}", v),
            BadInstalledMetadataVersion(ref v) => {
                write!(f,
                       "unsupported metadata version in existing installation: {}",
                       v)
            }
            ComponentDirPermissionsFailed(ref e) => {
                write!(f, "I/O error walking directory during install: {}", e)
            }
            ComponentFilePermissionsFailed(ref e) => {
                write!(f, "error setting file permissions during install: {}", e)
            }
            ComponentDownloadFailed(ref component, ref e) => {
                write!(f, "component download failed for {}-{}: {}", component.pkg, component.target, e)
            }
            ObsoleteDistManifest => {
                write!(f, "the server unexpectedly provided an obsolete version of the distribution manifest")
            },
            Parsing(ref n) => {
                for e in n {
                    try!(e.fmt(f));
                    try!(writeln!(f, ""));
                }
                Ok(())
            }
            MissingKey(ref n) => write!(f, "missing key: '{}'", n),
            ExpectedType(ref t, ref n) => write!(f, "expected type: '{}' for '{}'", t, n),
            PackageNotFound(ref n) => write!(f, "package not found: '{}'", n),
            TargetNotFound(ref n) => write!(f, "target not found: '{}'", n),
            MissingRoot => write!(f, "manifest has no root package"),
            UnsupportedVersion(ref v) => write!(f, "manifest version '{}' is not supported", v),
            MissingPackageForComponent(ref c) => write!(f,"manifest missing package for component {}", c.name()),
            RequestedComponentsUnavailable(ref cs) => {
                assert!(!cs.is_empty());
                if cs.len() == 1 {
                    write!(f, "component '{}' for '{}' is unavailable for download",
                           cs[0].pkg, cs[0].target)
                } else {
                    use itertools::Itertools;
                    let same_target = cs.iter().all(|c| c.target == cs[0].target);
                    if same_target {
                        let mut cs_strs = cs.iter().map(|c| format!("'{}'", c.pkg));
                        let cs_str = cs_strs.join(", ");
                        write!(f, "some components unavailable for download: {}",
                               cs_str)
                    } else {
                        let mut cs_strs = cs.iter().map(|c| format!("'{}' for '{}'", c.pkg, c.target));
                        let cs_str = cs_strs.join(", ");
                        write!(f, "some components unavailable for download: {}",
                               cs_str)
                    }
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>extern crate walkdir;
extern crate phf_codegen;
extern crate flate2;

use std::{env, fmt, io};
use std::borrow::{Borrow, Cow};
use std::collections::HashMap;
use std::fs::{self, File};
use std::io::{BufReader, BufWriter, Write};
use std::path::{Path, PathBuf};

use walkdir::WalkDir;

use flate2::write::GzEncoder;

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum Compression {
    None,
    Gzip
}

impl fmt::Display for Compression {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match *self {
            Compression::None => fmt.write_str("None"),
            Compression::Gzip => fmt.write_str("Gzip"),
        }
    }
}

pub struct IncludeDir {
    files: HashMap<String, (Compression, PathBuf)>,
    name: String,
    manifest_dir: PathBuf
}

pub fn start(static_name: &str) -> IncludeDir {
    IncludeDir {
        files: HashMap::new(),
        name: static_name.to_owned(),
        manifest_dir: Path::new(&env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not set")).to_owned()
    }
}

#[cfg(windows)]
fn as_key(path: &str) -> Cow<str> {
    Cow::Owned(path.replace("\\", "/"))
}

#[cfg(not(windows))]
fn as_key(path: &str) -> Cow<str> {
    Cow::Borrowed(path)
}

impl IncludeDir {
    /// Add a single file to the binary.
    /// With Gzip compression, the file will be encoded to OUT_DIR first.
    /// For chaining, it's not sensible to return a Result. If any to-be-included
    /// files can't be found, or encoded, this function will panic!.
    pub fn file<P: AsRef<Path>>(mut self, path: P, comp: Compression) -> IncludeDir {
        self.add_file(path, comp).unwrap();
        self
    }

    /// ## Panics
    ///
    /// This function panics when CARGO_MANIFEST_DIR or OUT_DIR are not defined.
    pub fn add_file<P: AsRef<Path>>(&mut self, path: P, comp: Compression) -> io::Result<()> {
        let key = path.as_ref().to_string_lossy();
        println!("cargo:rerun-if-changed={}", path.as_ref().display());

        match comp {
            Compression::None => {
                self.files.insert(as_key(key.borrow()).into_owned(),
                                  (comp, path.as_ref().to_owned()));
            }
            Compression::Gzip => {
                // gzip encode file to OUT_DIR
                let in_path = self.manifest_dir.join(&path);
                let mut in_file = BufReader::new(File::open(&in_path)?);

                let out_path = Path::new(&env::var("OUT_DIR").unwrap()).join(&path);
                fs::create_dir_all(&out_path.parent().unwrap())?;
                let out_file = BufWriter::new(File::create(&out_path)?);
                let mut encoder = GzEncoder::new(out_file, flate2::Compression::best());

                io::copy(&mut in_file, &mut encoder)?;

                self.files.insert(as_key(key.borrow()).into_owned(),
                                  (comp, out_path.to_owned()));
            }
        }
        Ok(())
    }

    /// Add a whole directory recursively to the binary.
    /// This function calls `file`, and therefore will panic! on missing files.
    pub fn dir<P: AsRef<Path>>(mut self, path: P, comp: Compression) -> IncludeDir {
        self.add_dir(path, comp).unwrap();
        self
    }

    /// ## Panics
    ///
    /// This function panics when CARGO_MANIFEST_DIR or OUT_DIR are not defined.
    pub fn add_dir<P: AsRef<Path>>(&mut self, path: P, comp: Compression) -> io::Result<()> {
        for entry in WalkDir::new(path).follow_links(true).into_iter() {
            match entry {
                Ok(ref e) if !e.file_type().is_dir() => {
                    self.add_file(e.path(), comp)?;
                }
                _ => (),
            }
        }
        Ok(())
    }

    pub fn build(self, out_name: &str) -> io::Result<()> {
        let out_path = Path::new(&env::var("OUT_DIR").unwrap()).join(out_name);
        let mut out_file = BufWriter::new(File::create(&out_path)?);

        write!(&mut out_file,
               "pub static {}: ::includedir::Files = ::includedir::Files {{\n\
                    files:  ",
                self.name)?;

        let mut map: phf_codegen::Map<String> = phf_codegen::Map::new();

        for (name, (compression, path)) in self.files {
            let include_path = format!("{}", self.manifest_dir.join(path).display());

            map.entry(as_key(&name).into_owned(),
                      &format!("(::includedir::Compression::{}, \
                                include_bytes!(\"{}\") as &'static [u8])",
                               compression, as_key(&include_path)));
        }

        map.build(&mut out_file)?;

        write!(&mut out_file, ", passthrough: ::std::sync::atomic::AtomicBool::new(false)")?;
        write!(&mut out_file, "\n}};\n")?;
        Ok(())
    }
}
<commit_msg>fix clippy warning regarding unreadable_literal in generated code<commit_after>extern crate walkdir;
extern crate phf_codegen;
extern crate flate2;

use std::{env, fmt, io};
use std::borrow::{Borrow, Cow};
use std::collections::HashMap;
use std::fs::{self, File};
use std::io::{BufReader, BufWriter, Write};
use std::path::{Path, PathBuf};

use walkdir::WalkDir;

use flate2::write::GzEncoder;

#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum Compression {
    None,
    Gzip
}

impl fmt::Display for Compression {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match *self {
            Compression::None => fmt.write_str("None"),
            Compression::Gzip => fmt.write_str("Gzip"),
        }
    }
}

pub struct IncludeDir {
    files: HashMap<String, (Compression, PathBuf)>,
    name: String,
    manifest_dir: PathBuf
}

pub fn start(static_name: &str) -> IncludeDir {
    IncludeDir {
        files: HashMap::new(),
        name: static_name.to_owned(),
        manifest_dir: Path::new(&env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not set")).to_owned()
    }
}

#[cfg(windows)]
fn as_key(path: &str) -> Cow<str> {
    Cow::Owned(path.replace("\\", "/"))
}

#[cfg(not(windows))]
fn as_key(path: &str) -> Cow<str> {
    Cow::Borrowed(path)
}

impl IncludeDir {
    /// Add a single file to the binary.
    /// With Gzip compression, the file will be encoded to OUT_DIR first.
    /// For chaining, it's not sensible to return a Result. If any to-be-included
    /// files can't be found, or encoded, this function will panic!.
    pub fn file<P: AsRef<Path>>(mut self, path: P, comp: Compression) -> IncludeDir {
        self.add_file(path, comp).unwrap();
        self
    }

    /// ## Panics
    ///
    /// This function panics when CARGO_MANIFEST_DIR or OUT_DIR are not defined.
    pub fn add_file<P: AsRef<Path>>(&mut self, path: P, comp: Compression) -> io::Result<()> {
        let key = path.as_ref().to_string_lossy();
        println!("cargo:rerun-if-changed={}", path.as_ref().display());

        match comp {
            Compression::None => {
                self.files.insert(as_key(key.borrow()).into_owned(),
                                  (comp, path.as_ref().to_owned()));
            }
            Compression::Gzip => {
                // gzip encode file to OUT_DIR
                let in_path = self.manifest_dir.join(&path);
                let mut in_file = BufReader::new(File::open(&in_path)?);

                let out_path = Path::new(&env::var("OUT_DIR").unwrap()).join(&path);
                fs::create_dir_all(&out_path.parent().unwrap())?;
                let out_file = BufWriter::new(File::create(&out_path)?);
                let mut encoder = GzEncoder::new(out_file, flate2::Compression::best());

                io::copy(&mut in_file, &mut encoder)?;

                self.files.insert(as_key(key.borrow()).into_owned(),
                                  (comp, out_path.to_owned()));
            }
        }
        Ok(())
    }

    /// Add a whole directory recursively to the binary.
    /// This function calls `file`, and therefore will panic! on missing files.
    pub fn dir<P: AsRef<Path>>(mut self, path: P, comp: Compression) -> IncludeDir {
        self.add_dir(path, comp).unwrap();
        self
    }

    /// ## Panics
    ///
    /// This function panics when CARGO_MANIFEST_DIR or OUT_DIR are not defined.
    pub fn add_dir<P: AsRef<Path>>(&mut self, path: P, comp: Compression) -> io::Result<()> {
        for entry in WalkDir::new(path).follow_links(true).into_iter() {
            match entry {
                Ok(ref e) if !e.file_type().is_dir() => {
                    self.add_file(e.path(), comp)?;
                }
                _ => (),
            }
        }
        Ok(())
    }

    pub fn build(self, out_name: &str) -> io::Result<()> {
        let out_path = Path::new(&env::var("OUT_DIR").unwrap()).join(out_name);
        let mut out_file = BufWriter::new(File::create(&out_path)?);

        writeln!(&mut out_file, "#[allow(clippy::unreadable_literal)]")?;

        write!(&mut out_file,
               "pub static {}: ::includedir::Files = ::includedir::Files {{\n\
                    files:  ",
                self.name)?;

        let mut map: phf_codegen::Map<String> = phf_codegen::Map::new();

        for (name, (compression, path)) in self.files {
            let include_path = format!("{}", self.manifest_dir.join(path).display());

            map.entry(as_key(&name).into_owned(),
                      &format!("(::includedir::Compression::{}, \
                                include_bytes!(\"{}\") as &'static [u8])",
                               compression, as_key(&include_path)));
        }

        map.build(&mut out_file)?;

        write!(&mut out_file, ", passthrough: ::std::sync::atomic::AtomicBool::new(false)")?;
        write!(&mut out_file, "\n}};\n")?;
        Ok(())
    }
}
<|endoftext|>


<commit_before>use defs::Word;
use term::lterm::LTerm;

pub struct ConsPtrMut {
  p: *mut Word,
}

impl ConsPtrMut {
  pub fn from_pointer(p: *mut Word) -> ConsPtrMut {
    ConsPtrMut { p }
  }

  pub unsafe fn set_hd(&self, val: LTerm) {
    *self.p = val.raw()
  }

  pub unsafe fn set_tl(&self, val: LTerm) {
    *self.p.offset(1) = val.raw()
  }

  pub fn make_cons(&self) -> LTerm {
    LTerm::make_cons(self.p)
  }

//  /// Peek into the cons, and get head value.
//  pub unsafe fn hd(&self) -> LTerm {
//    LTerm::from_raw(*self.p)
//  }

//  /// Peek into the cons, and get tail value.
//  pub unsafe fn tl(&self) -> LTerm {
//    LTerm::from_raw(*(self.p.offset(1)))
//  }
}

pub struct ConsPtr {
  p: *const Word,
}

impl ConsPtr {
  pub fn from_pointer(p: *const Word) -> ConsPtr {
    ConsPtr { p }
  }

//  pub fn raw_pointer(&self) -> *const Word { self.p }

//  pub fn make_cons(&self) -> LTerm {
//    LTerm::make_cons(self.p)
//  }

  /// Peek into the cons, and get head value.
  pub unsafe fn hd(&self) -> LTerm {
    LTerm::from_raw(*self.p)
  }

  /// Peek into the cons, and get tail value.
  pub unsafe fn tl(&self) -> LTerm {
    LTerm::from_raw(*(self.p.offset(1)))
  }
}
<commit_msg>RawCons is now an enum<commit_after>use defs::Word;
use term::lterm::LTerm;

pub enum ConsPtrMut { Ptr(*mut Word) }

pub enum ConsPtr { Ptr(*const Word) }

impl ConsPtrMut {

  #[inline]
  pub fn from_pointer(p: *mut Word) -> ConsPtrMut {
    ConsPtrMut::Ptr(p)
  }

  pub unsafe fn set_hd(&self, val: LTerm) {
    let ConsPtrMut::Ptr(p) = *self;
    *p = val.raw()
  }

  pub unsafe fn set_tl(&self, val: LTerm) {
    let ConsPtrMut::Ptr(p) = *self;
    *p.offset(1) = val.raw()
  }

  pub fn make_cons(&self) -> LTerm {
    let ConsPtrMut::Ptr(p) = *self;
    LTerm::make_cons(p)
  }

//  /// Peek into the cons, and get head value.
//  pub unsafe fn hd(&self) -> LTerm {
//    LTerm::from_raw(*self.p)
//  }

//  /// Peek into the cons, and get tail value.
//  pub unsafe fn tl(&self) -> LTerm {
//    LTerm::from_raw(*(self.p.offset(1)))
//  }
}

impl ConsPtr {
  #[inline]
  pub fn from_pointer(p: *const Word) -> ConsPtr {
    ConsPtr::Ptr(p)
  }

//  pub fn raw_pointer(&self) -> *const Word { self.p }

//  pub fn make_cons(&self) -> LTerm {
//    LTerm::make_cons(self.p)
//  }

  /// Peek into the cons, and get head value.
  pub unsafe fn hd(&self) -> LTerm {
    let ConsPtr::Ptr(p) = *self;
    LTerm::from_raw(*p)
  }

  /// Peek into the cons, and get tail value.
  pub unsafe fn tl(&self) -> LTerm {
    let ConsPtr::Ptr(p) = *self;
    LTerm::from_raw(*p.offset(1))
  }
}
<|endoftext|>


<commit_before>use std::mem;

fn main() {
    let b = Box::new(0);
    let x = &*b as *const i32;
    // We cannot compare this with a non-NULL integer. After all, these *could* be equal (with the right base address).
    assert!(x != mem::align_of::<i32>() as *const i32); //~ ERROR invalid arithmetic on pointers
}
<commit_msg>fix compile-fail test<commit_after>fn main() {
    let b = Box::new(0);
    let x = &*b as *const i32;
    // We cannot compare this with a non-NULL integer. After all, these *could* be equal (with the right base address).
    assert!(x != 64 as *const i32); //~ ERROR invalid arithmetic on pointers
}
<|endoftext|>


<commit_before>use std::collections::HashMap;
use std::str::FromStr;

use hyper::Uri;

use client::{self, ApiClient};

static DATASTORE_ROOT: &str = "https://datastore.googleapis.com/v1";

pub struct DatastoreService {}
pub type Hub<'a> = client::Hub<'a, DatastoreService>;

pub type ValueMap = HashMap<String, Value>;

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct BeginTransactionRequest {}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct BeginTransactionResponse {
    pub transaction: String,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct CommitRequest {
    pub transaction: String,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub mode: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub mutations: Option<Vec<Mutation>>,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct CommitResponse {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub mutation_results: Option<Vec<MutationResult>>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub index_updates: Option<i32>,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
struct RunQueryRequest {
    partition_id: PartitionId,
    read_options: ReadOptions,

    #[serde(skip_serializing_if = "Option::is_none")]
    gql_query: Option<GqlQuery>,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
struct GqlQuery {
    query_string: String,
    allow_literals: bool,
    named_bindings: HashMap<String, GqlQueryParameter>,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
struct GqlQueryParameter {
    value: Value,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct RunQueryResponse {
    pub batch: QueryResultBatch,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct QueryResultBatch {
    pub entity_results: Option<Vec<EntityResult>>,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Mutation {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub insert: Option<Entity>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub upsert: Option<Entity>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub update: Option<Entity>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub base_version: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub delete: Option<Key>,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct MutationResult {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub version: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub conflict_detected: Option<bool>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub key: Option<Key>,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct LookupRequest {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub keys: Option<Vec<Key>>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub read_options: Option<ReadOptions>,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct LookupResponse {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub found: Option<Vec<EntityResult>>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub missing: Option<Vec<EntityResult>>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub deferred: Option<Vec<Key>>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ReadOptions {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub transaction: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub read_consistency: Option<String>,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct EntityResult {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub version: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub entity: Option<Entity>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Entity {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub properties: Option<HashMap<String, Value>>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub key: Option<Key>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Key {
    pub path: Vec<PathElement>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub partition_id: Option<PartitionId>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct PathElement {
    pub kind: String,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub id: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub name: Option<String>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct PartitionId {
    pub project_id: String,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub namespace_id: Option<String>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Value {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub entity_value: Option<Entity>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub timestamp_value: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub string_value: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub double_value: Option<f64>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub meaning: Option<i32>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub exclude_from_indexes: Option<bool>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub blob_value: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub key_value: Option<Key>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub boolean_value: Option<bool>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub integer_value: Option<i64>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub null_value: Option<String>,

    //#[serde(rename="geoPointValue")]
    //#[serde(skip_serializing_if = "Option::is_none")]
    //pub geo_point_value: Option<LatLng>,

    //#[serde(rename="arrayValue")]
    //#[serde(skip_serializing_if = "Option::is_none")]
    //pub array_value: Option<ArrayValue>,
}

impl<'a> Hub<'a> {
    //
    // api-level operations

    pub fn begin_transaction(&self) -> client::Result<String> {
        let uri = self.mk_uri("beginTransaction");
        let req = BeginTransactionRequest::default();
        self.post::<_, BeginTransactionResponse>(&uri, req, &[])
            .map(|r| r.transaction)
    }

    pub fn commit(&self, req: CommitRequest) -> client::Result<CommitResponse> {
        debug_assert!(!req.transaction.is_empty());
        let uri = self.mk_uri("commit");
        self.post(&uri, req, &[])
    }

    //
    // high-level operations

    pub fn insert_entity_auto_id(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        props: ValueMap,
    ) -> client::Result<String> {
        let key = self.mk_key(kind, Some(ns), ancestors, None, None);
        let res = self.insert(key, props)?;
        let mut results = res.mutation_results.expect("mutations to be valid");

        let mut key = results.remove(0).key.expect("key to be valid");
        Ok(key.path.pop().expect("path to be non-empty").id.expect(
            "id to be valid",
        ))
    }

    pub fn insert_entity_by_name(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        name: &str,
        props: ValueMap,
    ) -> client::Result<()> {
        let key = self.mk_key(kind, Some(ns), ancestors, Some(name), None);
        self.insert(key, props)?;
        Ok(())
    }

    fn insert(&self, key: Key, props: ValueMap) -> client::Result<CommitResponse> {
        let entity = Entity {
            key: Some(key),
            properties: Some(props),
        };

        let insert = Mutation {
            insert: Some(entity),
            ..Default::default()
        };

        let req = CommitRequest {
            transaction: self.begin_transaction()?,
            mutations: Some(vec![insert]),
            ..Default::default()
        };

        self.commit(req)
    }

    pub fn lookup_by_id(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        id: &str,
        txn: Option<&str>,
    ) -> client::Result<Option<ValueMap>> {
        let key = self.mk_key(kind, Some(ns), ancestors, None, Some(id));
        self.lookup_one(key, txn)
    }

    pub fn lookup_by_name(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        name: &str,
        txn: Option<&str>,
    ) -> client::Result<Option<ValueMap>> {
        let key = self.mk_key(kind, Some(ns), ancestors, Some(name), None);
        self.lookup_one(key, txn)
    }

    pub fn gql<B>(
        &self,
        ns: &str,
        q: &str,
        txn: Option<&str>,
        bindings: B,
    ) -> client::Result<RunQueryResponse>
    where
        B: IntoIterator<Item = (String, Value)>,
    {
        let query = GqlQuery {
            query_string: q.to_string(),
            allow_literals: false,
            named_bindings: bindings
                .into_iter()
                .map(|(k, v)| (k, GqlQueryParameter { value: v }))
                .collect(),
        };

        let req = RunQueryRequest {
            partition_id: PartitionId {
                project_id: self.project_id().to_string(),
                namespace_id: Some(ns.to_string()),
            },
            read_options: ReadOptions {
                transaction: txn.map(|t| t.to_string()),
                ..Default::default()
            },
            gql_query: Some(query),
        };

        let uri = self.mk_uri("runQuery");
        self.post::<_, RunQueryResponse>(&uri, req, &[])
    }

    // Lookup a key using default read options:
    // https://cloud.google.com/datastore/docs/reference/rest/v1/ReadOptions
    fn lookup_one(&self, key: Key, txn: Option<&str>) -> client::Result<Option<ValueMap>> {
        let req = LookupRequest {
            keys: Some(vec![key]),
            read_options: Some(ReadOptions {
                transaction: txn.map(|t| t.to_string()),
                ..Default::default()
            }),
        };

        let uri = self.mk_uri("lookup");
        let res = self.post::<_, LookupResponse>(&uri, req, &[])?;

        Ok(
            res.found
                .and_then(|mut f| if f.len() != 1 {
                    None
                } else {
                    f.remove(0).entity
                })
                .and_then(|e| e.properties),
        )
    }

    pub fn update_by_id(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        id: &str,
        props: ValueMap,
    ) -> client::Result<()> {
        let key = self.mk_key(kind, Some(ns), ancestors, None, Some(id));

        let entity = Entity {
            key: Some(key),
            properties: Some(props),
        };

        let update = Mutation {
            update: Some(entity),
            ..Default::default()
        };

        let req = CommitRequest {
            transaction: self.begin_transaction()?,
            mutations: Some(vec![update]),
            ..Default::default()
        };

        self.commit(req).map(|_| ())
    }

    pub fn delete_by_id(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        id: &str,
    ) -> client::Result<()> {
        let key = self.mk_key(kind, Some(ns), ancestors, None, Some(id));

        let delete = Mutation {
            delete: Some(key),
            ..Default::default()
        };

        let req = CommitRequest {
            transaction: self.begin_transaction()?,
            mutations: Some(vec![delete]),
            ..Default::default()
        };

        self.commit(req).map(|_| ())
    }

    fn mk_uri(&self, action: &str) -> Uri {
        let path = format!(
            "{}/projects/{}:{}",
            DATASTORE_ROOT,
            self.project_id(),
            action
        );
        Uri::from_str(&path).expect("uri to be valid")
    }

    fn mk_key(
        &self,
        kind: &str,
        ns: Option<&str>,
        mut path: Vec<PathElement>,
        name: Option<&str>,
        id: Option<&str>,
    ) -> Key {
        let partition_id = PartitionId {
            project_id: self.project_id().to_string(),
            namespace_id: ns.map(|ns| ns.to_string()),
        };

        path.push(PathElement {
            kind: kind.to_string(),
            name: name.map(|name| name.to_string()),
            id: id.map(|id| id.to_string()),
        });

        Key {
            path: path,
            partition_id: Some(partition_id),
        }
    }
}
<commit_msg>add rollback api<commit_after>use std::collections::HashMap;
use std::str::FromStr;

use hyper::Uri;

use client::{self, ApiClient};

static DATASTORE_ROOT: &str = "https://datastore.googleapis.com/v1";

pub struct DatastoreService {}
pub type Hub<'a> = client::Hub<'a, DatastoreService>;

pub type ValueMap = HashMap<String, Value>;

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct BeginTransactionRequest {}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct BeginTransactionResponse {
    pub transaction: String,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct RollbackTransactionRequest {
    pub transaction: String,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct RollbackTransactionResponse {}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct CommitRequest {
    pub transaction: String,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub mode: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub mutations: Option<Vec<Mutation>>,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct CommitResponse {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub mutation_results: Option<Vec<MutationResult>>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub index_updates: Option<i32>,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
struct RunQueryRequest {
    partition_id: PartitionId,
    read_options: ReadOptions,

    #[serde(skip_serializing_if = "Option::is_none")]
    gql_query: Option<GqlQuery>,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
struct GqlQuery {
    query_string: String,
    allow_literals: bool,
    named_bindings: HashMap<String, GqlQueryParameter>,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
struct GqlQueryParameter {
    value: Value,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct RunQueryResponse {
    pub batch: QueryResultBatch,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct QueryResultBatch {
    pub entity_results: Option<Vec<EntityResult>>,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Mutation {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub insert: Option<Entity>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub upsert: Option<Entity>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub update: Option<Entity>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub base_version: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub delete: Option<Key>,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct MutationResult {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub version: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub conflict_detected: Option<bool>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub key: Option<Key>,
}

#[derive(Serialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct LookupRequest {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub keys: Option<Vec<Key>>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub read_options: Option<ReadOptions>,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct LookupResponse {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub found: Option<Vec<EntityResult>>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub missing: Option<Vec<EntityResult>>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub deferred: Option<Vec<Key>>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ReadOptions {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub transaction: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub read_consistency: Option<String>,
}

#[derive(Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct EntityResult {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub cursor: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub version: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub entity: Option<Entity>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Entity {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub properties: Option<HashMap<String, Value>>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub key: Option<Key>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Key {
    pub path: Vec<PathElement>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub partition_id: Option<PartitionId>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct PathElement {
    pub kind: String,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub id: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub name: Option<String>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct PartitionId {
    pub project_id: String,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub namespace_id: Option<String>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Value {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub entity_value: Option<Entity>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub timestamp_value: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub string_value: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub double_value: Option<f64>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub meaning: Option<i32>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub exclude_from_indexes: Option<bool>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub blob_value: Option<String>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub key_value: Option<Key>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub boolean_value: Option<bool>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub integer_value: Option<i64>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub null_value: Option<String>,

    //#[serde(rename="geoPointValue")]
    //#[serde(skip_serializing_if = "Option::is_none")]
    //pub geo_point_value: Option<LatLng>,

    //#[serde(rename="arrayValue")]
    //#[serde(skip_serializing_if = "Option::is_none")]
    //pub array_value: Option<ArrayValue>,
}

impl<'a> Hub<'a> {
    //
    // api-level operations

    pub fn begin_transaction(&self) -> client::Result<String> {
        let uri = self.mk_uri("beginTransaction");
        let req = BeginTransactionRequest::default();
        self.post::<_, BeginTransactionResponse>(&uri, req, &[])
            .map(|r| r.transaction)
    }

    pub fn rollback(&self, txn: &str) -> client::Result<()> {
        let uri = self.mk_uri("rollback");
        let req = RollbackTransactionRequest { transaction: txn.to_string() };
        self.post::<_, RollbackTransactionResponse>(&uri, req, &[])
            .map(|_| ())
    }

    pub fn commit(&self, req: CommitRequest) -> client::Result<CommitResponse> {
        debug_assert!(!req.transaction.is_empty());
        let uri = self.mk_uri("commit");
        self.post(&uri, req, &[])
    }

    //
    // high-level operations

    pub fn insert_entity_auto_id(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        props: ValueMap,
    ) -> client::Result<String> {
        let key = self.mk_key(kind, Some(ns), ancestors, None, None);
        let res = self.insert(key, props)?;
        let mut results = res.mutation_results.expect("mutations to be valid");

        let mut key = results.remove(0).key.expect("key to be valid");
        Ok(key.path.pop().expect("path to be non-empty").id.expect(
            "id to be valid",
        ))
    }

    pub fn insert_entity_by_name(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        name: &str,
        props: ValueMap,
    ) -> client::Result<()> {
        let key = self.mk_key(kind, Some(ns), ancestors, Some(name), None);
        self.insert(key, props)?;
        Ok(())
    }

    fn insert(&self, key: Key, props: ValueMap) -> client::Result<CommitResponse> {
        let entity = Entity {
            key: Some(key),
            properties: Some(props),
        };

        let insert = Mutation {
            insert: Some(entity),
            ..Default::default()
        };

        let req = CommitRequest {
            transaction: self.begin_transaction()?,
            mutations: Some(vec![insert]),
            ..Default::default()
        };

        self.commit(req)
    }

    pub fn lookup_by_id(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        id: &str,
        txn: Option<&str>,
    ) -> client::Result<Option<ValueMap>> {
        let key = self.mk_key(kind, Some(ns), ancestors, None, Some(id));
        self.lookup_one(key, txn)
    }

    pub fn lookup_by_name(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        name: &str,
        txn: Option<&str>,
    ) -> client::Result<Option<ValueMap>> {
        let key = self.mk_key(kind, Some(ns), ancestors, Some(name), None);
        self.lookup_one(key, txn)
    }

    pub fn gql<B>(
        &self,
        ns: &str,
        q: &str,
        txn: Option<&str>,
        bindings: B,
    ) -> client::Result<RunQueryResponse>
    where
        B: IntoIterator<Item = (String, Value)>,
    {
        let query = GqlQuery {
            query_string: q.to_string(),
            allow_literals: false,
            named_bindings: bindings
                .into_iter()
                .map(|(k, v)| (k, GqlQueryParameter { value: v }))
                .collect(),
        };

        let req = RunQueryRequest {
            partition_id: PartitionId {
                project_id: self.project_id().to_string(),
                namespace_id: Some(ns.to_string()),
            },
            read_options: ReadOptions {
                transaction: txn.map(|t| t.to_string()),
                ..Default::default()
            },
            gql_query: Some(query),
        };

        let uri = self.mk_uri("runQuery");
        self.post::<_, RunQueryResponse>(&uri, req, &[])
    }

    // Lookup a key using default read options:
    // https://cloud.google.com/datastore/docs/reference/rest/v1/ReadOptions
    fn lookup_one(&self, key: Key, txn: Option<&str>) -> client::Result<Option<ValueMap>> {
        let req = LookupRequest {
            keys: Some(vec![key]),
            read_options: Some(ReadOptions {
                transaction: txn.map(|t| t.to_string()),
                ..Default::default()
            }),
        };

        let uri = self.mk_uri("lookup");
        let res = self.post::<_, LookupResponse>(&uri, req, &[])?;

        Ok(
            res.found
                .and_then(|mut f| if f.len() != 1 {
                    None
                } else {
                    f.remove(0).entity
                })
                .and_then(|e| e.properties),
        )
    }

    pub fn update_by_id(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        id: &str,
        props: ValueMap,
    ) -> client::Result<()> {
        let key = self.mk_key(kind, Some(ns), ancestors, None, Some(id));

        let entity = Entity {
            key: Some(key),
            properties: Some(props),
        };

        let update = Mutation {
            update: Some(entity),
            ..Default::default()
        };

        let req = CommitRequest {
            transaction: self.begin_transaction()?,
            mutations: Some(vec![update]),
            ..Default::default()
        };

        self.commit(req).map(|_| ())
    }

    pub fn delete_by_id(
        &self,
        kind: &str,
        ns: &str,
        ancestors: Vec<PathElement>,
        id: &str,
    ) -> client::Result<()> {
        let key = self.mk_key(kind, Some(ns), ancestors, None, Some(id));

        let delete = Mutation {
            delete: Some(key),
            ..Default::default()
        };

        let req = CommitRequest {
            transaction: self.begin_transaction()?,
            mutations: Some(vec![delete]),
            ..Default::default()
        };

        self.commit(req).map(|_| ())
    }

    fn mk_uri(&self, action: &str) -> Uri {
        let path = format!(
            "{}/projects/{}:{}",
            DATASTORE_ROOT,
            self.project_id(),
            action
        );
        Uri::from_str(&path).expect("uri to be valid")
    }

    fn mk_key(
        &self,
        kind: &str,
        ns: Option<&str>,
        mut path: Vec<PathElement>,
        name: Option<&str>,
        id: Option<&str>,
    ) -> Key {
        let partition_id = PartitionId {
            project_id: self.project_id().to_string(),
            namespace_id: ns.map(|ns| ns.to_string()),
        };

        path.push(PathElement {
            kind: kind.to_string(),
            name: name.map(|name| name.to_string()),
            id: id.map(|id| id.to_string()),
        });

        Key {
            path: path,
            partition_id: Some(partition_id),
        }
    }
}
<|endoftext|>


<commit_before>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # Tagged stream
//!
//! A tagged stream recursively applies tags to regions of an input stream. A stream consisting of a single tag forms a tree,
//! so this stream type is useful for representing the parsed represention of an input.
//!
//! The simplest form of tagged stream is one that's just a set of untagged symbols. This can be created using the `TaggedStream::from_reader`
//! method.
//!
//! ```
//! # use concordance::*;
//! #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
//! enum SomeTag {
//!     Identifier
//! };
//!
//! let simple_tagged = TaggedStream::from_reader(&mut "Hello, world".read_symbols());
//! let with_tag      = simple_tagged.with_tags(vec![(0..5, SomeTag::Identifier)].iter().cloned());
//! ```
//!

// TODO: a possibly better way to design this would be to make it so that there's an underlying stream we read from when generating tag
// symbols so that we don't need to store the contents of the stream in memory

use std::slice::Iter;
use std::ops::Index;
use std::ops::Range;

use super::symbol_reader::*;
use super::tokenizer::*;
use super::symbol_range_dfa::*;

///
/// Represents a symbol in a tagged stream.
///
/// A symbol can either be a sequence of untagged base symbols, or it can be a tag (which itself can contain any number of tagged symbol)
///
#[derive(Clone, PartialEq, Eq, Debug)]
pub enum TagSymbol<Base: Clone+Ord, Tag: Clone+Ord> {
    /// An untagged element in this stream
    Untagged(Base),

    /// A tagged region of the stream
    Tagged(Tag, TaggedStream<Base, Tag>)
}

use TagSymbol::*;

///
/// Represents a stream of tagged symbols
///
#[derive(Clone, PartialEq, Eq, Debug)]
pub struct TaggedStream<Base: Clone+Ord, Tag: Clone+Ord> {
    /// The data in this stream
    data: Vec<TagSymbol<Base, Tag>>
}

impl<Base: Ord+Clone, Tag: Ord+Clone> TaggedStream<Base, Tag> {
    ///
    /// Creates a basic tagged stream from a source of the base symbol
    ///
    pub fn from_reader(reader: &mut SymbolReader<Base>) -> TaggedStream<Base, Tag> {
        // Read from the reader into an array
        let mut symbols = vec![];

        while let Some(next_symbol) = reader.next_symbol() {
            symbols.push(Untagged(next_symbol));
        }

        // Generate a simple tagged stream from the result
        TaggedStream { data: symbols }
    }

    ///
    /// The number of symbols in this stream
    ///
    pub fn len(&self) -> usize {
        self.data.len()
    }

    ///
    /// Replaces a range in this stream with a tag
    ///
    pub fn tag(&mut self, tag: Tag, range: Range<usize>) {
        // Create a tag to replace the range
        let replaced_symbols    = self.data[range.clone()].to_vec();
        let tag_symbol          = Tagged(tag, TaggedStream { data: replaced_symbols });

        // Draining seems to be for reading a range but does double duty for deleting a range?
        // I don't think rust has a way to replace a range in a vector, or at least not one that's easy to find in the docs.
        self.data.drain(range.clone());

        // Draining then inserting is inefficient compared to flat out replacing items :-/ This may be possible but the vec docs aren't very easy to read
        self.data.insert(range.start, tag_symbol);
    }

    ///
    /// Creates a tag symbol by tagging a particular range within this stream
    ///
    pub fn tag_range(&self, range: Range<usize>, tag: Tag) -> TagSymbol<Base, Tag> {
        let tag_data = self.data[range].to_vec();

        Tagged(tag, TaggedStream { data: tag_data })
    }

    ///
    /// Applies tags to the specified ranges in this stream
    ///
    /// The tag ranges must be in ascending order and must not overlap
    ///
    pub fn with_tags<I>(&self, tags: I) -> TaggedStream<Base, Tag>
        where I : Iterator<Item=(Range<usize>, Tag)> {
        // The data that will make up the new stream
        let mut new_stream = vec![];

        // The last processed range (we haven't processed any range yet, so this starts at 0..0)
        let mut last_range = 0..0;

        // Build up the result from the tag list and the last range
        for tag in tags {
            // Fetch the information for this tag and check ordering
            let (next_range, next_tag)  = tag;
            let end                     = last_range.end;

            if end > next_range.start {
                panic!("Tags for with_tags must be sorted in order");
            }

            // Sometimes need to leave some data from this stream untagged
            if end < next_range.start {
                new_stream.extend(self.data[end..next_range.start].iter().cloned());
            }

            // Push the tag
            new_stream.push(self.tag_range(next_range.clone(), next_tag));

            // Update state
            last_range = next_range;
        }

        // Append anything left in the stream
        if last_range.end < self.data.len() {
            new_stream.extend(self.data[last_range.end..self.data.len()].iter().cloned());
        }

        // Final result
        TaggedStream { data: new_stream }
    }

    ///
    /// Runs the current values of this tagged stream through a tokenizer and tags anything it matches
    ///
    /// This takes a mapping function to describe how symbols in this stream map to symbols in the DFA. Note that every symbol
    /// (tagged or untagged) must be mapped to a DFA symbol, so if only tagged or untagged symbols are being used it's necessary
    /// to decide how the other symbols are mapped (eg, to an unused symbol)
    ///
    pub fn tokenize<DfaSymbol: Ord, MapFn>(&self, dfa: &SymbolRangeDfa<DfaSymbol, Tag>, map_symbol: MapFn) -> TaggedStream<Base, Tag>
        where MapFn: Fn(TagSymbol<Base, Tag>) -> DfaSymbol {
        unimplemented!();
    }
}

impl<Base: Clone+Ord, Tag: Clone+Ord> Index<usize> for TaggedStream<Base, Tag> {
    type Output = TagSymbol<Base, Tag>;

    fn index(&self, index: usize) -> &TagSymbol<Base, Tag> {
        &self.data[index]
    }
}

impl<'a, Base: Clone+Ord, Tag: Clone+Ord> SymbolSource<'a, TagSymbol<Base, Tag>> for &'a TaggedStream<Base, Tag> {
    type SymbolReader = Iter<'a, TagSymbol<Base, Tag>>;

    /// Returns a new object that can read the symbols from this one
    fn read_symbols(self) -> Self::SymbolReader {
        self.data.read_symbols()
    }
}

#[cfg(test)]
mod test {
    use super::super::*;
    use super::super::tagged_stream::TagSymbol::*;

    #[test]
    fn can_tag_range() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let mut tagged: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());

        tagged.tag(Tags::Hello, 0..5);
        tagged.tag(Tags::World, 1..6);

        assert!(tagged.len() == 2);

        if let Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('H'));
            assert!(stream[1] == Untagged('e'));
            assert!(stream[2] == Untagged('l'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('o'));
        } else {
            assert!(false);
        }

        if let Tagged(ref tag, ref stream) = tagged[1] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('W'));
            assert!(stream[1] == Untagged('o'));
            assert!(stream[2] == Untagged('r'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn can_tag_everything_with_tags() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..5, Tags::Hello), (5..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 2);

        if let Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('H'));
            assert!(stream[1] == Untagged('e'));
            assert!(stream[2] == Untagged('l'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('o'));
        } else {
            assert!(false);
        }

        if let Tagged(ref tag, ref stream) = tagged[1] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('W'));
            assert!(stream[1] == Untagged('o'));
            assert!(stream[2] == Untagged('r'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn with_tags_preserves_middle() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..4, Tags::Hello), (6..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 4);
        assert!(tagged[1] == Untagged('o'));
        assert!(tagged[2] == Untagged('W'));

        if let Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 4);
            assert!(stream[0] == Untagged('H'));
            assert!(stream[1] == Untagged('e'));
            assert!(stream[2] == Untagged('l'));
            assert!(stream[3] == Untagged('l'));
        } else {
            assert!(false);
        }

        if let Tagged(ref tag, ref stream) = tagged[3] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 4);
            assert!(stream[0] == Untagged('o'));
            assert!(stream[1] == Untagged('r'));
            assert!(stream[2] == Untagged('l'));
            assert!(stream[3] == Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn with_tags_preserves_end() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..5, Tags::Hello)].iter().cloned());

        assert!(tagged.len() == 6);

        if let Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('H'));
            assert!(stream[1] == Untagged('e'));
            assert!(stream[2] == Untagged('l'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('o'));
        } else {
            assert!(false);
        }

        assert!(tagged[1] == Untagged('W'));
        assert!(tagged[2] == Untagged('o'));
        assert!(tagged[3] == Untagged('r'));
        assert!(tagged[4] == Untagged('l'));
        assert!(tagged[5] == Untagged('d'));
    }

    #[test]
    fn with_tags_preserves_start() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(5..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 6);

        if let Tagged(ref tag, ref stream) = tagged[5] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('W'));
            assert!(stream[1] == Untagged('o'));
            assert!(stream[2] == Untagged('r'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('d'));
        } else {
            assert!(false);
        }

        assert!(tagged[0] == Untagged('H'));
        assert!(tagged[1] == Untagged('e'));
        assert!(tagged[2] == Untagged('l'));
        assert!(tagged[3] == Untagged('l'));
        assert!(tagged[4] == Untagged('o'));
    }
}
<commit_msg>First pass at implementing the tokenize method<commit_after>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

//!
//! # Tagged stream
//!
//! A tagged stream recursively applies tags to regions of an input stream. A stream consisting of a single tag forms a tree,
//! so this stream type is useful for representing the parsed represention of an input.
//!
//! The simplest form of tagged stream is one that's just a set of untagged symbols. This can be created using the `TaggedStream::from_reader`
//! method.
//!
//! ```
//! # use concordance::*;
//! #[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
//! enum SomeTag {
//!     Identifier
//! };
//!
//! let simple_tagged = TaggedStream::from_reader(&mut "Hello, world".read_symbols());
//! let with_tag      = simple_tagged.with_tags(vec![(0..5, SomeTag::Identifier)].iter().cloned());
//! ```
//!

// TODO: a possibly better way to design this would be to make it so that there's an underlying stream we read from when generating tag
// symbols so that we don't need to store the contents of the stream in memory

use std::slice::Iter;
use std::ops::Index;
use std::ops::Range;

use super::countable::*;
use super::symbol_reader::*;
use super::tokenizer::*;
use super::symbol_range_dfa::*;

///
/// Represents a symbol in a tagged stream.
///
/// A symbol can either be a sequence of untagged base symbols, or it can be a tag (which itself can contain any number of tagged symbol)
///
#[derive(Clone, PartialEq, Eq, Debug)]
pub enum TagSymbol<Base: Clone+Ord, Tag: Clone+Ord> {
    /// An untagged element in this stream
    Untagged(Base),

    /// A tagged region of the stream
    Tagged(Tag, TaggedStream<Base, Tag>)
}

use TagSymbol::*;

///
/// Represents a stream of tagged symbols
///
#[derive(Clone, PartialEq, Eq, Debug)]
pub struct TaggedStream<Base: Clone+Ord, Tag: Clone+Ord> {
    /// The data in this stream
    data: Vec<TagSymbol<Base, Tag>>
}

impl<Base: Ord+Clone, Tag: Ord+Clone+'static> TaggedStream<Base, Tag> {
    ///
    /// Creates a basic tagged stream from a source of the base symbol
    ///
    pub fn from_reader(reader: &mut SymbolReader<Base>) -> TaggedStream<Base, Tag> {
        // Read from the reader into an array
        let mut symbols = vec![];

        while let Some(next_symbol) = reader.next_symbol() {
            symbols.push(Untagged(next_symbol));
        }

        // Generate a simple tagged stream from the result
        TaggedStream { data: symbols }
    }

    ///
    /// The number of symbols in this stream
    ///
    pub fn len(&self) -> usize {
        self.data.len()
    }

    ///
    /// Replaces a range in this stream with a tag
    ///
    pub fn tag(&mut self, tag: Tag, range: Range<usize>) {
        // Create a tag to replace the range
        let replaced_symbols    = self.data[range.clone()].to_vec();
        let tag_symbol          = Tagged(tag, TaggedStream { data: replaced_symbols });

        // Draining seems to be for reading a range but does double duty for deleting a range?
        // I don't think rust has a way to replace a range in a vector, or at least not one that's easy to find in the docs.
        self.data.drain(range.clone());

        // Draining then inserting is inefficient compared to flat out replacing items :-/ This may be possible but the vec docs aren't very easy to read
        self.data.insert(range.start, tag_symbol);
    }

    ///
    /// Creates a tag symbol by tagging a particular range within this stream
    ///
    pub fn tag_range(&self, range: Range<usize>, tag: Tag) -> TagSymbol<Base, Tag> {
        let tag_data = self.data[range].to_vec();

        Tagged(tag, TaggedStream { data: tag_data })
    }

    ///
    /// Applies tags to the specified ranges in this stream
    ///
    /// The tag ranges must be in ascending order and must not overlap
    ///
    pub fn with_tags<I>(&self, tags: I) -> TaggedStream<Base, Tag>
        where I : Iterator<Item=(Range<usize>, Tag)> {
        // The data that will make up the new stream
        let mut new_stream = vec![];

        // The last processed range (we haven't processed any range yet, so this starts at 0..0)
        let mut last_range = 0..0;

        // Build up the result from the tag list and the last range
        for tag in tags {
            // Fetch the information for this tag and check ordering
            let (next_range, next_tag)  = tag;
            let end                     = last_range.end;

            if end > next_range.start {
                panic!("Tags for with_tags must be sorted in order");
            }

            // Sometimes need to leave some data from this stream untagged
            if end < next_range.start {
                new_stream.extend(self.data[end..next_range.start].iter().cloned());
            }

            // Push the tag
            new_stream.push(self.tag_range(next_range.clone(), next_tag));

            // Update state
            last_range = next_range;
        }

        // Append anything left in the stream
        if last_range.end < self.data.len() {
            new_stream.extend(self.data[last_range.end..self.data.len()].iter().cloned());
        }

        // Final result
        TaggedStream { data: new_stream }
    }

    ///
    /// Runs the current values of this tagged stream through a tokenizer and tags anything it matches
    ///
    /// This takes a mapping function to describe how symbols in this stream map to symbols in the DFA. Note that every symbol
    /// (tagged or untagged) must be mapped to a DFA symbol, so if only tagged or untagged symbols are being used it's necessary
    /// to decide how the other symbols are mapped (eg, to an unused symbol)
    ///
    pub fn tokenize<DfaSymbol: Ord+Countable+Clone, MapFn>(&self, token_matcher: &SymbolRangeDfa<DfaSymbol, Tag>, map_symbol: MapFn) -> TaggedStream<Base, Tag>
        where MapFn: FnMut(TagSymbol<Base, Tag>) -> DfaSymbol {
        // Generate a symbol reader with the mapping function
        let reader = self.data.read_symbols().map_symbols(map_symbol);

        // Tokenize it
        let tokenizer = Tokenizer::new_prepared(reader, token_matcher);

        // Use the tokenizer to generate the result
        self.with_tags(tokenizer)
    }
}

impl<Base: Clone+Ord, Tag: Clone+Ord> Index<usize> for TaggedStream<Base, Tag> {
    type Output = TagSymbol<Base, Tag>;

    fn index(&self, index: usize) -> &TagSymbol<Base, Tag> {
        &self.data[index]
    }
}

impl<'a, Base: Clone+Ord, Tag: Clone+Ord> SymbolSource<'a, TagSymbol<Base, Tag>> for &'a TaggedStream<Base, Tag> {
    type SymbolReader = Iter<'a, TagSymbol<Base, Tag>>;

    /// Returns a new object that can read the symbols from this one
    fn read_symbols(self) -> Self::SymbolReader {
        self.data.read_symbols()
    }
}

#[cfg(test)]
mod test {
    use super::super::*;
    use super::super::tagged_stream::TagSymbol::*;

    #[test]
    fn can_tag_range() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let mut tagged: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());

        tagged.tag(Tags::Hello, 0..5);
        tagged.tag(Tags::World, 1..6);

        assert!(tagged.len() == 2);

        if let Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('H'));
            assert!(stream[1] == Untagged('e'));
            assert!(stream[2] == Untagged('l'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('o'));
        } else {
            assert!(false);
        }

        if let Tagged(ref tag, ref stream) = tagged[1] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('W'));
            assert!(stream[1] == Untagged('o'));
            assert!(stream[2] == Untagged('r'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn can_tag_everything_with_tags() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..5, Tags::Hello), (5..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 2);

        if let Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('H'));
            assert!(stream[1] == Untagged('e'));
            assert!(stream[2] == Untagged('l'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('o'));
        } else {
            assert!(false);
        }

        if let Tagged(ref tag, ref stream) = tagged[1] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('W'));
            assert!(stream[1] == Untagged('o'));
            assert!(stream[2] == Untagged('r'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn with_tags_preserves_middle() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello,
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..4, Tags::Hello), (6..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 4);
        assert!(tagged[1] == Untagged('o'));
        assert!(tagged[2] == Untagged('W'));

        if let Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 4);
            assert!(stream[0] == Untagged('H'));
            assert!(stream[1] == Untagged('e'));
            assert!(stream[2] == Untagged('l'));
            assert!(stream[3] == Untagged('l'));
        } else {
            assert!(false);
        }

        if let Tagged(ref tag, ref stream) = tagged[3] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 4);
            assert!(stream[0] == Untagged('o'));
            assert!(stream[1] == Untagged('r'));
            assert!(stream[2] == Untagged('l'));
            assert!(stream[3] == Untagged('d'));
        } else {
            assert!(false);
        }
    }

    #[test]
    fn with_tags_preserves_end() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            Hello
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(0..5, Tags::Hello)].iter().cloned());

        assert!(tagged.len() == 6);

        if let Tagged(ref tag, ref stream) = tagged[0] {
            assert!(*tag == Tags::Hello);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('H'));
            assert!(stream[1] == Untagged('e'));
            assert!(stream[2] == Untagged('l'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('o'));
        } else {
            assert!(false);
        }

        assert!(tagged[1] == Untagged('W'));
        assert!(tagged[2] == Untagged('o'));
        assert!(tagged[3] == Untagged('r'));
        assert!(tagged[4] == Untagged('l'));
        assert!(tagged[5] == Untagged('d'));
    }

    #[test]
    fn with_tags_preserves_start() {
        #[derive(Clone, PartialEq, Eq, Copy, PartialOrd, Ord)]
        enum Tags {
            World
        }

        let original: TaggedStream<char, Tags> = TaggedStream::from_reader(&mut "HelloWorld".read_symbols());
        let tagged = original.with_tags(vec![(5..10, Tags::World)].iter().cloned());

        assert!(tagged.len() == 6);

        if let Tagged(ref tag, ref stream) = tagged[5] {
            assert!(*tag == Tags::World);
            assert!(stream.len() == 5);
            assert!(stream[0] == Untagged('W'));
            assert!(stream[1] == Untagged('o'));
            assert!(stream[2] == Untagged('r'));
            assert!(stream[3] == Untagged('l'));
            assert!(stream[4] == Untagged('d'));
        } else {
            assert!(false);
        }

        assert!(tagged[0] == Untagged('H'));
        assert!(tagged[1] == Untagged('e'));
        assert!(tagged[2] == Untagged('l'));
        assert!(tagged[3] == Untagged('l'));
        assert!(tagged[4] == Untagged('o'));
    }
}
<|endoftext|>