<commit_before>use std::fmt::{self, Display};
use std::path::{Path, PathBuf};

use crate::errors::*;

use crate::dist::temp;
use crate::utils::notify::NotificationLevel;

#[derive(Debug)]
pub enum Notification<'a> {
    Install(crate::dist::Notification<'a>),
    Utils(crate::utils::Notification<'a>),
    Temp(temp::Notification<'a>),

    SetDefaultToolchain(&'a str),
    SetOverrideToolchain(&'a Path, &'a str),
    SetProfile(&'a str),
    LookingForToolchain(&'a str),
    ToolchainDirectory(&'a Path, &'a str),
    UpdatingToolchain(&'a str),
    InstallingToolchain(&'a str),
    InstalledToolchain(&'a str),
    UsingExistingToolchain(&'a str),
    UninstallingToolchain(&'a str),
    UninstalledToolchain(&'a str),
    ToolchainNotInstalled(&'a str),
    UpdateHashMatches,
    UpgradingMetadata(&'a str, &'a str),
    MetadataUpgradeNotNeeded(&'a str),
    WritingMetadataVersion(&'a str),
    ReadMetadataVersion(&'a str),
    NonFatalError(&'a Error),
    UpgradeRemovesToolchains,
    MissingFileDuringSelfUninstall(PathBuf),
}

impl<'a> From<crate::dist::Notification<'a>> for Notification<'a> {
    fn from(n: crate::dist::Notification<'a>) -> Self {
        Notification::Install(n)
    }
}
impl<'a> From<crate::utils::Notification<'a>> for Notification<'a> {
    fn from(n: crate::utils::Notification<'a>) -> Self {
        Notification::Utils(n)
    }
}
impl<'a> From<temp::Notification<'a>> for Notification<'a> {
    fn from(n: temp::Notification<'a>) -> Self {
        Notification::Temp(n)
    }
}

impl<'a> Notification<'a> {
    pub fn level(&self) -> NotificationLevel {
        use self::Notification::*;
        match self {
            Install(n) => n.level(),
            Utils(n) => n.level(),
            Temp(n) => n.level(),
            ToolchainDirectory(_, _)
            | LookingForToolchain(_)
            | WritingMetadataVersion(_)
            | InstallingToolchain(_)
            | UpdatingToolchain(_)
            | ReadMetadataVersion(_)
            | InstalledToolchain(_)
            | UpdateHashMatches => NotificationLevel::Verbose,
            SetDefaultToolchain(_)
            | SetOverrideToolchain(_, _)
            | SetProfile(_)
            | UsingExistingToolchain(_)
            | UninstallingToolchain(_)
            | UninstalledToolchain(_)
            | ToolchainNotInstalled(_)
            | UpgradingMetadata(_, _)
            | MetadataUpgradeNotNeeded(_) => NotificationLevel::Info,
            NonFatalError(_) => NotificationLevel::Error,
            UpgradeRemovesToolchains | MissingFileDuringSelfUninstall(_) => NotificationLevel::Warn,
        }
    }
}

impl<'a> Display for Notification<'a> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::result::Result<(), fmt::Error> {
        use self::Notification::*;
        match self {
            Install(n) => n.fmt(f),
            Utils(n) => n.fmt(f),
            Temp(n) => n.fmt(f),
            SetDefaultToolchain(name) => write!(f, "default toolchain set to '{}'", name),
            SetOverrideToolchain(path, name) => write!(
                f,
                "override toolchain for '{}' set to '{}'",
                path.display(),
                name
            ),
            SetProfile(name) => write!(f, "profile set to '{}'", name),
            LookingForToolchain(name) => write!(f, "looking for installed toolchain '{}'", name),
            ToolchainDirectory(path, _) => write!(f, "toolchain directory: '{}'", path.display()),
            UpdatingToolchain(name) => write!(f, "updating existing install for '{}'", name),
            InstallingToolchain(name) => write!(f, "installing toolchain '{}'", name),
            InstalledToolchain(name) => write!(f, "toolchain '{}' installed", name),
            UsingExistingToolchain(name) => write!(f, "using existing install for '{}'", name),
            UninstallingToolchain(name) => write!(f, "uninstalling toolchain '{}'", name),
            UninstalledToolchain(name) => write!(f, "toolchain '{}' uninstalled", name),
            ToolchainNotInstalled(name) => write!(f, "no toolchain installed for '{}'", name),
            UpdateHashMatches => write!(f, "toolchain is already up to date"),
            UpgradingMetadata(from_ver, to_ver) => write!(
                f,
                "upgrading metadata version from '{}' to '{}'",
                from_ver, to_ver
            ),
            MetadataUpgradeNotNeeded(ver) => write!(
                f,
                "nothing to upgrade: metadata version is already '{}'",
                ver
            ),
            WritingMetadataVersion(ver) => write!(f, "writing metadata version: '{}'", ver),
            ReadMetadataVersion(ver) => write!(f, "read metadata version: '{}'", ver),
            NonFatalError(e) => write!(f, "{}", e),
            UpgradeRemovesToolchains => write!(
                f,
                "this upgrade will remove all existing toolchains. you will need to reinstall them"
            ),
            MissingFileDuringSelfUninstall(p) => write!(
                f,
                "expected file does not exist to uninstall: {}",
                p.display()
            ),
        }
    }
}
<commit_msg>notifications: Add PlainVerboseMessage notification<commit_after>use std::fmt::{self, Display};
use std::path::{Path, PathBuf};

use crate::errors::*;

use crate::dist::temp;
use crate::utils::notify::NotificationLevel;

#[derive(Debug)]
pub enum Notification<'a> {
    Install(crate::dist::Notification<'a>),
    Utils(crate::utils::Notification<'a>),
    Temp(temp::Notification<'a>),

    SetDefaultToolchain(&'a str),
    SetOverrideToolchain(&'a Path, &'a str),
    SetProfile(&'a str),
    LookingForToolchain(&'a str),
    ToolchainDirectory(&'a Path, &'a str),
    UpdatingToolchain(&'a str),
    InstallingToolchain(&'a str),
    InstalledToolchain(&'a str),
    UsingExistingToolchain(&'a str),
    UninstallingToolchain(&'a str),
    UninstalledToolchain(&'a str),
    ToolchainNotInstalled(&'a str),
    UpdateHashMatches,
    UpgradingMetadata(&'a str, &'a str),
    MetadataUpgradeNotNeeded(&'a str),
    WritingMetadataVersion(&'a str),
    ReadMetadataVersion(&'a str),
    NonFatalError(&'a Error),
    UpgradeRemovesToolchains,
    MissingFileDuringSelfUninstall(PathBuf),
    PlainVerboseMessage(&'a str),
}

impl<'a> From<crate::dist::Notification<'a>> for Notification<'a> {
    fn from(n: crate::dist::Notification<'a>) -> Self {
        Notification::Install(n)
    }
}
impl<'a> From<crate::utils::Notification<'a>> for Notification<'a> {
    fn from(n: crate::utils::Notification<'a>) -> Self {
        Notification::Utils(n)
    }
}
impl<'a> From<temp::Notification<'a>> for Notification<'a> {
    fn from(n: temp::Notification<'a>) -> Self {
        Notification::Temp(n)
    }
}

impl<'a> Notification<'a> {
    pub fn level(&self) -> NotificationLevel {
        use self::Notification::*;
        match self {
            Install(n) => n.level(),
            Utils(n) => n.level(),
            Temp(n) => n.level(),
            ToolchainDirectory(_, _)
            | LookingForToolchain(_)
            | WritingMetadataVersion(_)
            | InstallingToolchain(_)
            | UpdatingToolchain(_)
            | ReadMetadataVersion(_)
            | InstalledToolchain(_)
            | PlainVerboseMessage(_)
            | UpdateHashMatches => NotificationLevel::Verbose,
            SetDefaultToolchain(_)
            | SetOverrideToolchain(_, _)
            | SetProfile(_)
            | UsingExistingToolchain(_)
            | UninstallingToolchain(_)
            | UninstalledToolchain(_)
            | ToolchainNotInstalled(_)
            | UpgradingMetadata(_, _)
            | MetadataUpgradeNotNeeded(_) => NotificationLevel::Info,
            NonFatalError(_) => NotificationLevel::Error,
            UpgradeRemovesToolchains | MissingFileDuringSelfUninstall(_) => NotificationLevel::Warn,
        }
    }
}

impl<'a> Display for Notification<'a> {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::result::Result<(), fmt::Error> {
        use self::Notification::*;
        match self {
            Install(n) => n.fmt(f),
            Utils(n) => n.fmt(f),
            Temp(n) => n.fmt(f),
            SetDefaultToolchain(name) => write!(f, "default toolchain set to '{}'", name),
            SetOverrideToolchain(path, name) => write!(
                f,
                "override toolchain for '{}' set to '{}'",
                path.display(),
                name
            ),
            SetProfile(name) => write!(f, "profile set to '{}'", name),
            LookingForToolchain(name) => write!(f, "looking for installed toolchain '{}'", name),
            ToolchainDirectory(path, _) => write!(f, "toolchain directory: '{}'", path.display()),
            UpdatingToolchain(name) => write!(f, "updating existing install for '{}'", name),
            InstallingToolchain(name) => write!(f, "installing toolchain '{}'", name),
            InstalledToolchain(name) => write!(f, "toolchain '{}' installed", name),
            UsingExistingToolchain(name) => write!(f, "using existing install for '{}'", name),
            UninstallingToolchain(name) => write!(f, "uninstalling toolchain '{}'", name),
            UninstalledToolchain(name) => write!(f, "toolchain '{}' uninstalled", name),
            ToolchainNotInstalled(name) => write!(f, "no toolchain installed for '{}'", name),
            UpdateHashMatches => write!(f, "toolchain is already up to date"),
            UpgradingMetadata(from_ver, to_ver) => write!(
                f,
                "upgrading metadata version from '{}' to '{}'",
                from_ver, to_ver
            ),
            MetadataUpgradeNotNeeded(ver) => write!(
                f,
                "nothing to upgrade: metadata version is already '{}'",
                ver
            ),
            WritingMetadataVersion(ver) => write!(f, "writing metadata version: '{}'", ver),
            ReadMetadataVersion(ver) => write!(f, "read metadata version: '{}'", ver),
            NonFatalError(e) => write!(f, "{}", e),
            UpgradeRemovesToolchains => write!(
                f,
                "this upgrade will remove all existing toolchains. you will need to reinstall them"
            ),
            MissingFileDuringSelfUninstall(p) => write!(
                f,
                "expected file does not exist to uninstall: {}",
                p.display()
            ),
            PlainVerboseMessage(r) => write!(f, "{}", r),
        }
    }
}
<|endoftext|>


<commit_before>use std::time::Duration;
use std::sync::{Arc, RwLock};
use std::thread;

use tikv::raftserver::store::*;
use kvproto::raftpb::ConfChangeType;
use kvproto::metapb;
use tikv::pd::PdClient;
use tikv::util::HandyRwLock;

use super::cluster::{Cluster, Simulator};
use super::node::new_node_cluster;
use super::server::new_server_cluster;
use super::util::*;
use super::pd::TestPdClient;

fn test_simple_conf_change<T: Simulator>(cluster: &mut Cluster<T>) {
    // init_env_log();
    let r1 = cluster.bootstrap_conf_change();

    cluster.start();

    // Now region 1 only has peer (1, 1, 1);
    let (key, value) = (b"a1", b"v1");

    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    let engine_2 = cluster.get_engine(2);
    assert!(engine_2.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    // add peer (2,2,2) to region 1.
    cluster.change_peer(r1, ConfChangeType::AddNode, new_peer(2, 2, 2));

    let (key, value) = (b"a2", b"v2");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    // now peer 2 must have v1 and v2;
    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a1")).unwrap().unwrap(),
               b"v1");
    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a2")).unwrap().unwrap(),
               b"v2");

    let epoch = cluster.pd_client
                       .rl()
                       .get_region_by_id(cluster.id(), 1)
                       .unwrap()
                       .get_region_epoch()
                       .clone();
    let change_peer = new_admin_request(1,
                                        new_change_peer_cmd(ConfChangeType::AddNode,
                                                            new_peer(2, 2, 2),
                                                            &epoch));
    let resp = cluster.call_command_on_leader(1, change_peer, Duration::from_secs(3)).unwrap();
    assert!(resp.get_header().has_error(),
            "we can't add same peer twice");

    // add peer (3, 3, 3) to region 1.
    cluster.change_peer(r1, ConfChangeType::AddNode, new_peer(3, 3, 3));
    // Remove peer (2, 2, 2) from region 1.
    cluster.change_peer(r1, ConfChangeType::RemoveNode, new_peer(2, 2, 2));

    let (key, value) = (b"a3", b"v3");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));
    // now peer 3 must have v1, v2 and v3
    let engine_3 = cluster.get_engine(3);
    assert_eq!(&*engine_3.get_value(&keys::data_key(b"a1")).unwrap().unwrap(),
               b"v1");
    assert_eq!(&*engine_3.get_value(&keys::data_key(b"a2")).unwrap().unwrap(),
               b"v2");
    assert_eq!(&*engine_3.get_value(&keys::data_key(b"a3")).unwrap().unwrap(),
               b"v3");

    // peer 2 has nothing
    assert!(engine_2.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    assert!(engine_2.get_value(&keys::data_key(b"a2")).unwrap().is_none());

    let epoch = cluster.pd_client
                       .rl()
                       .get_region_by_id(cluster.id(), 1)
                       .unwrap()
                       .get_region_epoch()
                       .clone();
    let change_peer = new_admin_request(1,
                                        new_change_peer_cmd(ConfChangeType::RemoveNode,
                                                            new_peer(2, 2, 2),
                                                            &epoch));
    let resp = cluster.call_command_on_leader(1, change_peer, Duration::from_secs(3)).unwrap();
    assert!(resp.get_header().has_error(),
            "we can't remove same peer twice");

    // add peer 2 then remove it again.
    cluster.change_peer(r1, ConfChangeType::AddNode, new_peer(2, 2, 2));

    // Force update a2 to check whether peer 2 added ok and received the snapshot.
    let (key, value) = (b"a2", b"v2");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    let engine_2 = cluster.get_engine(2);
    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a1")).unwrap().unwrap(),
               b"v1");
    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a2")).unwrap().unwrap(),
               b"v2");
    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a3")).unwrap().unwrap(),
               b"v3");

    // Remove peer (2, 2, 2) from region 1.
    cluster.change_peer(r1, ConfChangeType::RemoveNode, new_peer(2, 2, 2));

    // add peer (2, 2, 4) to region 1.
    cluster.change_peer(r1, ConfChangeType::AddNode, new_peer(2, 2, 4));
    // Remove peer (3, 3, 3) from region 1.
    cluster.change_peer(r1, ConfChangeType::RemoveNode, new_peer(3, 3, 3));

    let (key, value) = (b"a4", b"v4");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));
    // now peer 4 in store 2 must have v1, v2, v3, v4, we check v1 and v4 here.
    let engine_2 = cluster.get_engine(2);

    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a1")).unwrap().unwrap(),
               b"v1");
    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a4")).unwrap().unwrap(),
               b"v4");

    // peer 3 has nothing, we check v1 and v4 here.
    assert!(engine_3.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    assert!(engine_3.get_value(&keys::data_key(b"a4")).unwrap().is_none());

    // TODO: add more tests.
}

fn new_conf_change_peer(store: &metapb::Store,
                        pd_client: &Arc<RwLock<TestPdClient>>)
                        -> metapb::Peer {
    let peer_id = pd_client.wl().alloc_id().unwrap();
    new_peer(store.get_node_id(), store.get_store_id(), peer_id)
}

fn test_pd_conf_change<T: Simulator>(cluster: &mut Cluster<T>) {
    // init_env_log();
    cluster.start();

    let cluster_id = cluster.id();
    let pd_client = cluster.pd_client.clone();
    let region = &pd_client.rl().get_region(cluster_id, b"").unwrap();
    let region_id = region.get_region_id();

    let mut stores = pd_client.rl().get_stores(cluster_id).unwrap();

    // Must have only one peer
    assert_eq!(region.get_peers().len(), 1);

    let peer = &region.get_peers()[0];

    let i = stores.iter().position(|store| store.get_store_id() == peer.get_store_id()).unwrap();
    stores.swap(0, i);

    // Now the first store has first region. others have none.

    let (key, value) = (b"a1", b"v1");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    let peer2 = new_conf_change_peer(&stores[1], &pd_client);
    let engine_2 = cluster.get_engine(peer2.get_node_id());
    assert!(engine_2.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    // add new peer to first region.
    cluster.change_peer(region_id, ConfChangeType::AddNode, peer2.clone());

    let (key, value) = (b"a2", b"v2");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    // now peer 2 must have v1 and v2;
    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a1")).unwrap().unwrap(),
               b"v1");
    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a2")).unwrap().unwrap(),
               b"v2");

    // add new peer to first region.
    let peer3 = new_conf_change_peer(&stores[2], &pd_client);
    cluster.change_peer(region_id, ConfChangeType::AddNode, peer3.clone());
    // Remove peer2 from first region.
    cluster.change_peer(region_id, ConfChangeType::RemoveNode, peer2.clone());

    let (key, value) = (b"a3", b"v3");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));
    // now peer 3 must have v1, v2 and v3
    let engine_3 = cluster.get_engine(peer3.get_node_id());
    assert_eq!(&*engine_3.get_value(&keys::data_key(b"a1")).unwrap().unwrap(),
               b"v1");
    assert_eq!(&*engine_3.get_value(&keys::data_key(b"a2")).unwrap().unwrap(),
               b"v2");
    assert_eq!(&*engine_3.get_value(&keys::data_key(b"a3")).unwrap().unwrap(),
               b"v3");

    // peer 2 has nothing
    assert!(engine_2.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    assert!(engine_2.get_value(&keys::data_key(b"a2")).unwrap().is_none());
    // add peer4 to first region 1.
    let peer4 = new_conf_change_peer(&stores[1], &pd_client);
    cluster.change_peer(region_id, ConfChangeType::AddNode, peer4.clone());
    // Remove peer3 from first region.
    cluster.change_peer(region_id, ConfChangeType::RemoveNode, peer3.clone());

    let (key, value) = (b"a4", b"v4");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));
    // now peer4 must have v1, v2, v3, v4, we check v1 and v4 here.
    let engine_2 = cluster.get_engine(peer4.get_node_id());

    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a1")).unwrap().unwrap(),
               b"v1");
    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a4")).unwrap().unwrap(),
               b"v4");

    // peer 3 has nothing, we check v1 and v4 here.
    assert!(engine_3.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    assert!(engine_3.get_value(&keys::data_key(b"a4")).unwrap().is_none());

    // TODO: add more tests.
}

#[test]
fn test_node_simple_conf_change() {
    let count = 5;
    let mut cluster = new_node_cluster(0, count);
    test_simple_conf_change(&mut cluster);
}

#[test]
fn test_server_simple_conf_change() {
    let count = 5;
    let mut cluster = new_server_cluster(0, count);
    test_simple_conf_change(&mut cluster);
}

#[test]
fn test_node_pd_conf_change() {
    let count = 5;
    let mut cluster = new_node_cluster(0, count);
    test_pd_conf_change(&mut cluster);
}

#[test]
fn test_server_pd_conf_change() {
    let count = 5;
    let mut cluster = new_server_cluster(0, count);
    test_pd_conf_change(&mut cluster);
}

fn test_auto_adjust_replica<T: Simulator>(cluster: &mut Cluster<T>) {
    cluster.cfg.store_cfg.replica_check_tick_interval = 200;
    cluster.start();

    let cluster_id = cluster.id();
    let pd_client = cluster.pd_client.clone();
    let region = pd_client.rl().get_region(cluster_id, b"").unwrap();
    let region_id = region.get_region_id();

    let stores = pd_client.rl().get_stores(cluster_id).unwrap();

    // Must have only one peer
    assert_eq!(region.get_peers().len(), 1);

    thread::sleep(Duration::from_secs(1));

    let region = pd_client.rl().get_region_by_id(cluster_id, region_id).unwrap();
    // default replica is 5.
    assert_eq!(region.get_peers().len(), 5);

    let (key, value) = (b"a1", b"v1");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    let i = stores.iter()
                  .position(|s| {
                      region.get_peers().iter().all(|p| s.get_store_id() != p.get_store_id())
                  })
                  .unwrap();

    let peer = new_conf_change_peer(&stores[i], &pd_client);
    let engine = cluster.get_engine(peer.get_node_id());
    assert!(engine.get_value(&keys::data_key(b"a1")).unwrap().is_none());

    cluster.change_peer(region_id, ConfChangeType::AddNode, peer.clone());

    let region = pd_client.rl().get_region_by_id(cluster_id, region_id).unwrap();
    assert_eq!(region.get_peers().len(), 6);

    thread::sleep(Duration::from_millis(300));

    let region = pd_client.rl().get_region_by_id(cluster_id, region_id).unwrap();
    assert_eq!(region.get_peers().len(), 5);

    let peer = region.get_peers().get(1).unwrap().clone();

    cluster.change_peer(region_id, ConfChangeType::RemoveNode, peer);
    let region = pd_client.rl().get_region_by_id(cluster_id, region_id).unwrap();
    assert_eq!(region.get_peers().len(), 4);

    thread::sleep(Duration::from_millis(300));

    let region = pd_client.rl().get_region_by_id(cluster_id, region_id).unwrap();
    assert_eq!(region.get_peers().len(), 5);
}

#[test]
fn test_node_auto_adjust_replica() {
    let count = 7;
    let mut cluster = new_node_cluster(0, count);
    test_auto_adjust_replica(&mut cluster);
}

#[test]
fn test_server_auto_adjust_replica() {
    let count = 7;
    let mut cluster = new_server_cluster(0, count);
    test_auto_adjust_replica(&mut cluster);
}
<commit_msg>tests: Fix tests and minor clean up.<commit_after>use std::time::Duration;
use std::sync::{Arc, RwLock};
use std::{thread, time};
use rocksdb::DB;

use tikv::raftserver::store::*;
use kvproto::raftpb::ConfChangeType;
use kvproto::metapb;
use tikv::pd::PdClient;
use tikv::util::HandyRwLock;

use super::cluster::{Cluster, Simulator};
use super::node::new_node_cluster;
use super::server::new_server_cluster;
use super::util::*;
use super::pd::TestPdClient;

fn must_get_equal(engine: &Arc<DB>, key: &[u8], value: &[u8]) {
    for _ in 1..100 {
        if let Ok(res) = engine.get_value(&keys::data_key(key)) {
            if let Some(val) = res {
                assert_eq!(&*val, value);
                return;
            }
            thread::sleep(time::Duration::from_millis(10));
        }
    }
    assert!(false);
}

fn test_simple_conf_change<T: Simulator>(cluster: &mut Cluster<T>) {
    // init_env_log();
    let r1 = cluster.bootstrap_conf_change();

    cluster.start();

    // Now region 1 only has peer (1, 1, 1);
    let (key, value) = (b"a1", b"v1");

    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    let engine_2 = cluster.get_engine(2);
    assert!(engine_2.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    // add peer (2,2,2) to region 1.
    cluster.change_peer(r1, ConfChangeType::AddNode, new_peer(2, 2, 2));

    let (key, value) = (b"a2", b"v2");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    // now peer 2 must have v1 and v2;
    must_get_equal(&engine_2, b"a1", b"v1");
    assert_eq!(&*engine_2.get_value(&keys::data_key(b"a2")).unwrap().unwrap(),
               b"v2");

    let epoch = cluster.pd_client
                       .rl()
                       .get_region_by_id(cluster.id(), 1)
                       .unwrap()
                       .get_region_epoch()
                       .clone();
    let change_peer = new_admin_request(1,
                                        new_change_peer_cmd(ConfChangeType::AddNode,
                                                            new_peer(2, 2, 2),
                                                            &epoch));
    let resp = cluster.call_command_on_leader(1, change_peer, Duration::from_secs(3)).unwrap();
    assert!(resp.get_header().has_error(),
            "we can't add same peer twice");

    // add peer (3, 3, 3) to region 1.
    cluster.change_peer(r1, ConfChangeType::AddNode, new_peer(3, 3, 3));
    // Remove peer (2, 2, 2) from region 1.
    cluster.change_peer(r1, ConfChangeType::RemoveNode, new_peer(2, 2, 2));

    let (key, value) = (b"a3", b"v3");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));
    // now peer 3 must have v1, v2 and v3
    let engine_3 = cluster.get_engine(3);
    must_get_equal(&engine_3, b"a1", b"v1");
    must_get_equal(&engine_3, b"a2", b"v2");
    must_get_equal(&engine_3, b"a3", b"v3");

    // peer 2 has nothing
    assert!(engine_2.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    assert!(engine_2.get_value(&keys::data_key(b"a2")).unwrap().is_none());

    let epoch = cluster.pd_client
                       .rl()
                       .get_region_by_id(cluster.id(), 1)
                       .unwrap()
                       .get_region_epoch()
                       .clone();
    let change_peer = new_admin_request(1,
                                        new_change_peer_cmd(ConfChangeType::RemoveNode,
                                                            new_peer(2, 2, 2),
                                                            &epoch));
    let resp = cluster.call_command_on_leader(1, change_peer, Duration::from_secs(3)).unwrap();
    assert!(resp.get_header().has_error(),
            "we can't remove same peer twice");

    // add peer 2 then remove it again.
    cluster.change_peer(r1, ConfChangeType::AddNode, new_peer(2, 2, 2));

    // Force update a2 to check whether peer 2 added ok and received the snapshot.
    let (key, value) = (b"a2", b"v2");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    let engine_2 = cluster.get_engine(2);

    must_get_equal(&engine_2, b"a1", b"v1");
    must_get_equal(&engine_2, b"a2", b"v2");
    must_get_equal(&engine_2, b"a3", b"v3");

    // Remove peer (2, 2, 2) from region 1.
    cluster.change_peer(r1, ConfChangeType::RemoveNode, new_peer(2, 2, 2));

    // add peer (2, 2, 4) to region 1.
    cluster.change_peer(r1, ConfChangeType::AddNode, new_peer(2, 2, 4));
    // Remove peer (3, 3, 3) from region 1.
    cluster.change_peer(r1, ConfChangeType::RemoveNode, new_peer(3, 3, 3));

    let (key, value) = (b"a4", b"v4");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));
    // now peer 4 in store 2 must have v1, v2, v3, v4, we check v1 and v4 here.
    let engine_2 = cluster.get_engine(2);

    must_get_equal(&engine_2, b"a1", b"v1");
    must_get_equal(&engine_2, b"a4", b"v4");

    // peer 3 has nothing, we check v1 and v4 here.
    assert!(engine_3.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    assert!(engine_3.get_value(&keys::data_key(b"a4")).unwrap().is_none());

    // TODO: add more tests.
}

fn new_conf_change_peer(store: &metapb::Store,
                        pd_client: &Arc<RwLock<TestPdClient>>)
                        -> metapb::Peer {
    let peer_id = pd_client.wl().alloc_id().unwrap();
    new_peer(store.get_node_id(), store.get_store_id(), peer_id)
}

fn test_pd_conf_change<T: Simulator>(cluster: &mut Cluster<T>) {
    // init_env_log();
    cluster.start();

    let cluster_id = cluster.id();
    let pd_client = cluster.pd_client.clone();
    let region = &pd_client.rl().get_region(cluster_id, b"").unwrap();
    let region_id = region.get_region_id();

    let mut stores = pd_client.rl().get_stores(cluster_id).unwrap();

    // Must have only one peer
    assert_eq!(region.get_peers().len(), 1);

    let peer = &region.get_peers()[0];

    let i = stores.iter().position(|store| store.get_store_id() == peer.get_store_id()).unwrap();
    stores.swap(0, i);

    // Now the first store has first region. others have none.

    let (key, value) = (b"a1", b"v1");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    let peer2 = new_conf_change_peer(&stores[1], &pd_client);
    let engine_2 = cluster.get_engine(peer2.get_node_id());
    assert!(engine_2.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    // add new peer to first region.
    cluster.change_peer(region_id, ConfChangeType::AddNode, peer2.clone());

    let (key, value) = (b"a2", b"v2");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    // now peer 2 must have v1 and v2;
    must_get_equal(&engine_2, b"a1", b"v1");
    must_get_equal(&engine_2, b"a2", b"v2");

    // add new peer to first region.
    let peer3 = new_conf_change_peer(&stores[2], &pd_client);
    cluster.change_peer(region_id, ConfChangeType::AddNode, peer3.clone());
    // Remove peer2 from first region.
    cluster.change_peer(region_id, ConfChangeType::RemoveNode, peer2.clone());

    let (key, value) = (b"a3", b"v3");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));
    // now peer 3 must have v1, v2 and v3
    let engine_3 = cluster.get_engine(peer3.get_node_id());
    must_get_equal(&engine_3, b"a1", b"v1");
    must_get_equal(&engine_3, b"a2", b"v2");
    must_get_equal(&engine_3, b"a3", b"v3");

    // peer 2 has nothing
    assert!(engine_2.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    assert!(engine_2.get_value(&keys::data_key(b"a2")).unwrap().is_none());
    // add peer4 to first region 1.
    let peer4 = new_conf_change_peer(&stores[1], &pd_client);
    cluster.change_peer(region_id, ConfChangeType::AddNode, peer4.clone());
    // Remove peer3 from first region.
    cluster.change_peer(region_id, ConfChangeType::RemoveNode, peer3.clone());

    let (key, value) = (b"a4", b"v4");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));
    // now peer4 must have v1, v2, v3, v4, we check v1 and v4 here.
    let engine_2 = cluster.get_engine(peer4.get_node_id());

    must_get_equal(&engine_2, b"a1", b"v1");
    must_get_equal(&engine_2, b"a4", b"v4");

    // peer 3 has nothing, we check v1 and v4 here.
    assert!(engine_3.get_value(&keys::data_key(b"a1")).unwrap().is_none());
    assert!(engine_3.get_value(&keys::data_key(b"a4")).unwrap().is_none());

    // TODO: add more tests.
}

#[test]
fn test_node_simple_conf_change() {
    let count = 5;
    let mut cluster = new_node_cluster(0, count);
    test_simple_conf_change(&mut cluster);
}

#[test]
fn test_server_simple_conf_change() {
    let count = 5;
    let mut cluster = new_server_cluster(0, count);
    test_simple_conf_change(&mut cluster);
}

#[test]
fn test_node_pd_conf_change() {
    let count = 5;
    let mut cluster = new_node_cluster(0, count);
    test_pd_conf_change(&mut cluster);
}

#[test]
fn test_server_pd_conf_change() {
    let count = 5;
    let mut cluster = new_server_cluster(0, count);
    test_pd_conf_change(&mut cluster);
}

fn test_auto_adjust_replica<T: Simulator>(cluster: &mut Cluster<T>) {
    cluster.cfg.store_cfg.replica_check_tick_interval = 200;
    cluster.start();

    let cluster_id = cluster.id();
    let pd_client = cluster.pd_client.clone();
    let region = pd_client.rl().get_region(cluster_id, b"").unwrap();
    let region_id = region.get_region_id();

    let stores = pd_client.rl().get_stores(cluster_id).unwrap();

    // Must have only one peer
    assert_eq!(region.get_peers().len(), 1);

    thread::sleep(Duration::from_secs(1));

    let region = pd_client.rl().get_region_by_id(cluster_id, region_id).unwrap();
    // default replica is 5.
    assert_eq!(region.get_peers().len(), 5);

    let (key, value) = (b"a1", b"v1");
    cluster.put(key, value);
    assert_eq!(cluster.get(key), Some(value.to_vec()));

    let i = stores.iter()
                  .position(|s| {
                      region.get_peers().iter().all(|p| s.get_store_id() != p.get_store_id())
                  })
                  .unwrap();

    let peer = new_conf_change_peer(&stores[i], &pd_client);
    let engine = cluster.get_engine(peer.get_node_id());
    assert!(engine.get_value(&keys::data_key(b"a1")).unwrap().is_none());

    cluster.change_peer(region_id, ConfChangeType::AddNode, peer.clone());

    let region = pd_client.rl().get_region_by_id(cluster_id, region_id).unwrap();
    assert_eq!(region.get_peers().len(), 6);

    thread::sleep(Duration::from_millis(300));

    let region = pd_client.rl().get_region_by_id(cluster_id, region_id).unwrap();
    assert_eq!(region.get_peers().len(), 5);

    let peer = region.get_peers().get(1).unwrap().clone();

    cluster.change_peer(region_id, ConfChangeType::RemoveNode, peer);
    let region = pd_client.rl().get_region_by_id(cluster_id, region_id).unwrap();
    assert_eq!(region.get_peers().len(), 4);

    thread::sleep(Duration::from_millis(300));

    let region = pd_client.rl().get_region_by_id(cluster_id, region_id).unwrap();
    assert_eq!(region.get_peers().len(), 5);
}

#[test]
fn test_node_auto_adjust_replica() {
    let count = 7;
    let mut cluster = new_node_cluster(0, count);
    test_auto_adjust_replica(&mut cluster);
}

#[test]
fn test_server_auto_adjust_replica() {
    let count = 7;
    let mut cluster = new_server_cluster(0, count);
    test_auto_adjust_replica(&mut cluster);
}
<|endoftext|>


<commit_before>#![recursion_limit = "80"]

use std::collections::LinkedList;
use std::collections::BTreeMap;

use super::ast;
use super::errors::*;
use pest::prelude::*;

/// Decode an escaped string.
fn decode_escaped_string(input: &str) -> Result<String> {
    let mut out = String::new();
    let mut it = input.chars().skip(1).peekable();

    loop {
        let c = match it.next() {
            None => break,
            Some(c) => c,
        };

        // strip end quote
        if it.peek().is_none() {
            break;
        }

        if c == '\\' {
            let escaped = match it.next().ok_or("expected character")? {
                'n' => '\n',
                'r' => '\r',
                't' => '\t',
                'u' => decode_unicode4(&mut it)?,
                _ => return Err(ErrorKind::InvalidEscape.into()),
            };

            out.push(escaped);
            continue;
        }

        out.push(c);
    }

    Ok(out)
}

/// Decode the next four characters as a unicode escape sequence.
fn decode_unicode4(it: &mut Iterator<Item = char>) -> Result<char> {
    let mut res = 0u32;

    for x in 0..4u32 {
        let c = it.next().ok_or("expected hex character")?.to_string();
        let c = u32::from_str_radix(&c, 16)?;
        res += c << (4 * (3 - x));
    }

    Ok(::std::char::from_u32(res).ok_or("expected valid character")?)
}

impl_rdp! {
    grammar! {
        file = _{ package_decl ~ use_decl* ~ decl* ~ eoi }
        decl = { message_decl | interface_decl | type_decl }

        use_decl = { ["use"] ~ package_ident ~ use_as? ~ [";"] }
        use_as = { ["as"] ~ ident }
        package_decl = { ["package"] ~ package_ident ~ [";"] }
        message_decl = { ["message"] ~ ident ~ ["{"] ~ option_decl* ~ message_member* ~ ["}"] }
        interface_decl = { ["interface"] ~ ident ~ ["{"] ~ option_decl* ~ interface_member* ~ sub_type_decl* ~ ["}"] }
        type_decl = { ["type"] ~ ident ~ ["="] ~ type_spec ~ [";"] }
        sub_type_decl = { sub_type }
        sub_type = { ident ~ ["{"] ~ option_decl* ~ sub_type_member* ~ ["}"] }

        message_member = { field }
        interface_member = { field }
        sub_type_member = { field }

        field = { ident ~ modifier? ~ [":"] ~ type_spec ~ [";"] }
        modifier = { ["?"] }

        type_spec = { array | tuple | used_type | type_literal }
        type_literal = { ident }
        used_type = { ident ~ ["."] ~ ident }
        tuple = { ["("] ~ ( tuple_element ~ ([","] ~ tuple_element)* ) ~ [")"] }
        tuple_element = { type_spec }
        array = { ["["] ~ array_argument ~ ["]"] }
        array_argument = { type_spec }

        option_decl = { ident ~ (option_value ~ ([","] ~ option_value)*) ~ [";"] }

        option_value = { string | number }

        package_ident = @{ ident ~ (["."] ~ ident)* }
        ident =  @{ (['a'..'z'] | ['A'..'Z']) ~ (['0'..'9'] | ['a'..'z'] | ['A'..'Z'])* }

        string  = @{ ["\""] ~ (escape | !(["\""] | ["\\"]) ~ any)* ~ ["\""] }
        escape  =  _{ ["\\"] ~ (["\""] | ["\\"] | ["/"] | ["n"] | ["r"] | ["t"] | unicode) }
        unicode =  _{ ["u"] ~ hex ~ hex ~ hex ~ hex }
        hex     =  _{ ['0'..'9'] | ['a'..'f'] }

        number = @{ ["-"]? ~ int ~ (["."] ~ ['0'..'9']+ ~ exp? | exp)? }
        int    =  _{ ["0"] | ['1'..'9'] ~ ['0'..'9']* }
        exp    =  _{ (["E"] | ["e"]) ~ (["+"] | ["-"])? ~ int }

        whitespace = _{ [" "] | ["\t"] | ["\r"] | ["\n"] }

        comment = _{
            // line comment
            ( ["//"] ~ (!(["\r"] | ["\n"]) ~ any)* ~ (["\n"] | ["\r\n"] | ["\r"] | eoi) ) |
            // block comment
            ( ["/*"] ~ (!(["*/"]) ~ any)* ~ ["*/"] )
        }
    }

    process! {
        _file(&self) -> Result<ast::File> {
            (_: package_decl, package: _package(), imports: _use_list(), decls: _decl_list()) => {
                let imports = imports.into_iter().collect();
                let decls = decls.into_iter().collect();
                Ok(ast::File::new(package, imports, decls))
            },
        }

        _use_list(&self) -> LinkedList<ast::UseDecl> {
            (_: use_decl, package: _package(), alias: _use_as(), mut tail: _use_list()) => {
                tail.push_front(ast::UseDecl::new(package, alias));
                tail
            },

            () => LinkedList::new(),
        }

        _use_as(&self) -> Option<String> {
            (_: use_as, &alias: ident) => {
                Some(alias.to_owned())
            },

            () => None,
        }

        _package(&self) -> ast::Package {
            (_: package_ident, idents: _ident_list()) => {
                ast::Package::new(idents.into_iter().collect())
            },
        }

        _decl_list(&self) -> LinkedList<ast::Decl> {
            (_: decl, first: _decl(), mut tail: _decl_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _decl(&self) -> ast::Decl {
            (token: message_decl, &name: ident, options: _option_list(), members: _message_member_list()) => {
                let options = ast::Options::new(options.into_iter().collect());
                let members = members.into_iter().collect();
                let pos = (token.start, token.end);
                let message_decl = ast::MessageDecl::new(name.to_owned(), options, members, pos);
                ast::Decl::Message(message_decl)
            },

            (
                token: interface_decl,
                &name: ident,
                options: _option_list(),
                members: _interface_member_list(),
                sub_types: _sub_type_list()
            ) => {
                let options = ast::Options::new(options.into_iter().collect());
                let members = members.into_iter().collect();
                let pos = (token.start, token.end);
                let interface_decl = ast::InterfaceDecl::new(name.to_owned(), options, members, sub_types, pos);
                ast::Decl::Interface(interface_decl)
            },

            (token: type_decl, &name: ident, type_spec: _type_spec()) => {
                let pos = (token.start, token.end);
                let type_decl = ast::TypeDecl::new(name.to_owned(), type_spec, pos);
                ast::Decl::Type(type_decl)
            },
        }

        _message_member_list(&self) -> LinkedList<ast::MessageMember> {
            (_: message_member, first: _message_member(), mut tail: _message_member_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _message_member(&self) -> ast::MessageMember {
            (token: field, field: _field()) => {
                let pos = (token.start, token.end);
                ast::MessageMember::Field(field, pos)
            },
        }

        _option_list(&self) -> LinkedList<ast::OptionDecl> {
            (_: option_decl, first: _option_decl(), mut tail: _option_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _option_decl(&self) -> ast::OptionDecl {
            (&name: ident, values: _option_value_list()) => {
                let values = values.into_iter().collect();
                ast::OptionDecl::new(name.to_owned(), values)
            },
        }

        _option_value_list(&self) -> LinkedList<ast::OptionValue> {
            (_: option_value, first: _option_value(), mut tail: _option_value_list()) => {
                tail.push_front(first);
                tail
            },

            () => {
                LinkedList::new()
            },
        }

        _option_value(&self) -> ast::OptionValue {
            (&string: string) => {
                ast::OptionValue::String(decode_escaped_string(string).unwrap())
            },
        }

        _sub_type_member_list(&self) -> LinkedList<ast::SubTypeMember> {
            (_: sub_type_member, first: _sub_type_member(), mut tail: _sub_type_member_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _sub_type_member(&self) -> ast::SubTypeMember {
            (_: field, field: _field()) => {
                ast::SubTypeMember::Field(field)
            },
        }

        _sub_type_list(&self) -> BTreeMap<String, ast::SubType> {
            (_: sub_type_decl, first: _sub_type(), mut tail: _sub_type_list()) => {
                tail.insert(first.name.clone(), first);
                tail
            },

            () => {
                BTreeMap::new()
            },
        }

        _sub_type(&self) -> ast::SubType {
            (token: sub_type, &name: ident, options: _option_list(), members: _sub_type_member_list()) => {
                let name = name.to_owned();
                let options = ast::Options::new(options.into_iter().collect());
                let members = members.into_iter().collect();
                let pos = (token.start, token.end);
                ast::SubType::new(name, options, members, pos)
            },
        }

        _interface_member_list(&self) -> LinkedList<ast::InterfaceMember> {
            (_: interface_member, first: _interface_member(), mut tail: _interface_member_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _interface_member(&self) -> ast::InterfaceMember {
            (token: field, field: _field()) => {
                let pos = (token.start, token.end);
                ast::InterfaceMember::Field(field, pos)
            },
        }

        _field(&self) -> ast::Field {
            (&name: ident, modifier: _modifier(), type_spec: _type_spec()) => {
                ast::Field::new(modifier, name.to_owned(), type_spec, 0)
            },
        }

        _type_spec(&self) -> ast::Type {
            (_: type_spec, _: type_literal, &value: ident) => {
                match value {
                    "double" => ast::Type::Double,
                    "float" => ast::Type::Float,
                    "i32" => ast::Type::I32,
                    "i64" => ast::Type::I64,
                    "u32" => ast::Type::U32,
                    "u64" => ast::Type::U64,
                    "bool" => ast::Type::Bool,
                    "string" => ast::Type::String,
                    "bytes" => ast::Type::Bytes,
                    "any" => ast::Type::Any,
                    name => ast::Type::Custom(name.to_owned()),
                }
            },

            (_: type_spec, _: used_type, &used: ident, &value: ident) => {
                ast::Type::UsedType(used.to_owned(), value.to_owned())
            },

            (_: type_spec, _: tuple, arguments: _tuple_element_list()) => {
                let arguments = arguments.into_iter().collect();
                ast::Type::Tuple(arguments)
            },

            (_: type_spec, _: array, _: array_argument, argument: _type_spec()) => {
                ast::Type::Array(Box::new(argument))
            },
        }

        _tuple_element_list(&self) -> LinkedList<ast::Type> {
            (_: tuple_element, first: _type_spec(), mut tail: _tuple_element_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _modifier(&self) -> ast::Modifier {
            (_: modifier) => ast::Modifier::Optional,
            () => ast::Modifier::Required,
        }

        _ident_list(&self) -> LinkedList<String> {
            (&first: ident, mut tail: _ident_list()) => {
                tail.push_front(first.to_owned());
                tail
            },

            () => LinkedList::new(),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    const FILE1: &[u8] = include_bytes!("tests/file1.reproto");

    #[test]
    fn test_file1() {
        let input = ::std::str::from_utf8(FILE1).unwrap();
        let mut parser = Rdp::new(StringInput::new(input));

        assert!(parser.file());
        assert!(parser.end());

        let file = parser._file().unwrap();

        let package = ast::Package::new(vec!["foo".to_owned(), "bar".to_owned(), "baz".to_owned()]);

        assert_eq!(package, file.package);
        assert_eq!(4, file.decls.len());
    }

    #[test]
    fn test_array() {
        let mut parser = Rdp::new(StringInput::new("[(string, string)]"));

        assert!(parser.type_spec());
        assert!(parser.end());

        parser._type_spec();
    }

    #[test]
    fn test_block_comment() {
        let mut parser = Rdp::new(StringInput::new("/* hello \n world */"));

        assert!(parser.comment());
    }

    #[test]
    fn test_line_comment() {
        let mut parser = Rdp::new(StringInput::new("// hello world\n"));

        assert!(parser.comment());
    }
}
<commit_msg>fix ident<commit_after>#![recursion_limit = "80"]

use std::collections::LinkedList;
use std::collections::BTreeMap;

use super::ast;
use super::errors::*;
use pest::prelude::*;

/// Decode an escaped string.
fn decode_escaped_string(input: &str) -> Result<String> {
    let mut out = String::new();
    let mut it = input.chars().skip(1).peekable();

    loop {
        let c = match it.next() {
            None => break,
            Some(c) => c,
        };

        // strip end quote
        if it.peek().is_none() {
            break;
        }

        if c == '\\' {
            let escaped = match it.next().ok_or("expected character")? {
                'n' => '\n',
                'r' => '\r',
                't' => '\t',
                'u' => decode_unicode4(&mut it)?,
                _ => return Err(ErrorKind::InvalidEscape.into()),
            };

            out.push(escaped);
            continue;
        }

        out.push(c);
    }

    Ok(out)
}

/// Decode the next four characters as a unicode escape sequence.
fn decode_unicode4(it: &mut Iterator<Item = char>) -> Result<char> {
    let mut res = 0u32;

    for x in 0..4u32 {
        let c = it.next().ok_or("expected hex character")?.to_string();
        let c = u32::from_str_radix(&c, 16)?;
        res += c << (4 * (3 - x));
    }

    Ok(::std::char::from_u32(res).ok_or("expected valid character")?)
}

impl_rdp! {
    grammar! {
        file = _{ package_decl ~ use_decl* ~ decl* ~ eoi }
        decl = { message_decl | interface_decl | type_decl }

        use_decl = { ["use"] ~ package_ident ~ use_as? ~ [";"] }
        use_as = { ["as"] ~ ident }
        package_decl = { ["package"] ~ package_ident ~ [";"] }
        message_decl = { ["message"] ~ ident ~ ["{"] ~ option_decl* ~ message_member* ~ ["}"] }
        interface_decl = { ["interface"] ~ ident ~ ["{"] ~ option_decl* ~ interface_member* ~ sub_type_decl* ~ ["}"] }
        type_decl = { ["type"] ~ ident ~ ["="] ~ type_spec ~ [";"] }
        sub_type_decl = { sub_type }
        sub_type = { ident ~ ["{"] ~ option_decl* ~ sub_type_member* ~ ["}"] }

        message_member = { field }
        interface_member = { field }
        sub_type_member = { field }

        field = { ident ~ modifier? ~ [":"] ~ type_spec ~ [";"] }
        modifier = { ["?"] }

        type_spec = { array | tuple | used_type | type_literal }
        type_literal = { ident }
        used_type = { ident ~ ["."] ~ ident }
        tuple = { ["("] ~ ( tuple_element ~ ([","] ~ tuple_element)* ) ~ [")"] }
        tuple_element = { type_spec }
        array = { ["["] ~ array_argument ~ ["]"] }
        array_argument = { type_spec }

        option_decl = { ident ~ (option_value ~ ([","] ~ option_value)*) ~ [";"] }

        option_value = { string | number }

        package_ident = @{ ident ~ (["."] ~ ident)* }
        ident =  @{ (['a'..'z'] | ['A'..'Z'] | ["_"]) ~ (['0'..'9'] | ['a'..'z'] | ['A'..'Z'] | ["_"])* }

        string  = @{ ["\""] ~ (escape | !(["\""] | ["\\"]) ~ any)* ~ ["\""] }
        escape  =  _{ ["\\"] ~ (["\""] | ["\\"] | ["/"] | ["n"] | ["r"] | ["t"] | unicode) }
        unicode =  _{ ["u"] ~ hex ~ hex ~ hex ~ hex }
        hex     =  _{ ['0'..'9'] | ['a'..'f'] }

        number = @{ ["-"]? ~ int ~ (["."] ~ ['0'..'9']+ ~ exp? | exp)? }
        int    =  _{ ["0"] | ['1'..'9'] ~ ['0'..'9']* }
        exp    =  _{ (["E"] | ["e"]) ~ (["+"] | ["-"])? ~ int }

        whitespace = _{ [" "] | ["\t"] | ["\r"] | ["\n"] }

        comment = _{
            // line comment
            ( ["//"] ~ (!(["\r"] | ["\n"]) ~ any)* ~ (["\n"] | ["\r\n"] | ["\r"] | eoi) ) |
            // block comment
            ( ["/*"] ~ (!(["*/"]) ~ any)* ~ ["*/"] )
        }
    }

    process! {
        _file(&self) -> Result<ast::File> {
            (_: package_decl, package: _package(), imports: _use_list(), decls: _decl_list()) => {
                let imports = imports.into_iter().collect();
                let decls = decls.into_iter().collect();
                Ok(ast::File::new(package, imports, decls))
            },
        }

        _use_list(&self) -> LinkedList<ast::UseDecl> {
            (_: use_decl, package: _package(), alias: _use_as(), mut tail: _use_list()) => {
                tail.push_front(ast::UseDecl::new(package, alias));
                tail
            },

            () => LinkedList::new(),
        }

        _use_as(&self) -> Option<String> {
            (_: use_as, &alias: ident) => {
                Some(alias.to_owned())
            },

            () => None,
        }

        _package(&self) -> ast::Package {
            (_: package_ident, idents: _ident_list()) => {
                ast::Package::new(idents.into_iter().collect())
            },
        }

        _decl_list(&self) -> LinkedList<ast::Decl> {
            (_: decl, first: _decl(), mut tail: _decl_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _decl(&self) -> ast::Decl {
            (token: message_decl, &name: ident, options: _option_list(), members: _message_member_list()) => {
                let options = ast::Options::new(options.into_iter().collect());
                let members = members.into_iter().collect();
                let pos = (token.start, token.end);
                let message_decl = ast::MessageDecl::new(name.to_owned(), options, members, pos);
                ast::Decl::Message(message_decl)
            },

            (
                token: interface_decl,
                &name: ident,
                options: _option_list(),
                members: _interface_member_list(),
                sub_types: _sub_type_list()
            ) => {
                let options = ast::Options::new(options.into_iter().collect());
                let members = members.into_iter().collect();
                let pos = (token.start, token.end);
                let interface_decl = ast::InterfaceDecl::new(name.to_owned(), options, members, sub_types, pos);
                ast::Decl::Interface(interface_decl)
            },

            (token: type_decl, &name: ident, type_spec: _type_spec()) => {
                let pos = (token.start, token.end);
                let type_decl = ast::TypeDecl::new(name.to_owned(), type_spec, pos);
                ast::Decl::Type(type_decl)
            },
        }

        _message_member_list(&self) -> LinkedList<ast::MessageMember> {
            (_: message_member, first: _message_member(), mut tail: _message_member_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _message_member(&self) -> ast::MessageMember {
            (token: field, field: _field()) => {
                let pos = (token.start, token.end);
                ast::MessageMember::Field(field, pos)
            },
        }

        _option_list(&self) -> LinkedList<ast::OptionDecl> {
            (_: option_decl, first: _option_decl(), mut tail: _option_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _option_decl(&self) -> ast::OptionDecl {
            (&name: ident, values: _option_value_list()) => {
                let values = values.into_iter().collect();
                ast::OptionDecl::new(name.to_owned(), values)
            },
        }

        _option_value_list(&self) -> LinkedList<ast::OptionValue> {
            (_: option_value, first: _option_value(), mut tail: _option_value_list()) => {
                tail.push_front(first);
                tail
            },

            () => {
                LinkedList::new()
            },
        }

        _option_value(&self) -> ast::OptionValue {
            (&string: string) => {
                ast::OptionValue::String(decode_escaped_string(string).unwrap())
            },
        }

        _sub_type_member_list(&self) -> LinkedList<ast::SubTypeMember> {
            (_: sub_type_member, first: _sub_type_member(), mut tail: _sub_type_member_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _sub_type_member(&self) -> ast::SubTypeMember {
            (_: field, field: _field()) => {
                ast::SubTypeMember::Field(field)
            },
        }

        _sub_type_list(&self) -> BTreeMap<String, ast::SubType> {
            (_: sub_type_decl, first: _sub_type(), mut tail: _sub_type_list()) => {
                tail.insert(first.name.clone(), first);
                tail
            },

            () => {
                BTreeMap::new()
            },
        }

        _sub_type(&self) -> ast::SubType {
            (token: sub_type, &name: ident, options: _option_list(), members: _sub_type_member_list()) => {
                let name = name.to_owned();
                let options = ast::Options::new(options.into_iter().collect());
                let members = members.into_iter().collect();
                let pos = (token.start, token.end);
                ast::SubType::new(name, options, members, pos)
            },
        }

        _interface_member_list(&self) -> LinkedList<ast::InterfaceMember> {
            (_: interface_member, first: _interface_member(), mut tail: _interface_member_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _interface_member(&self) -> ast::InterfaceMember {
            (token: field, field: _field()) => {
                let pos = (token.start, token.end);
                ast::InterfaceMember::Field(field, pos)
            },
        }

        _field(&self) -> ast::Field {
            (&name: ident, modifier: _modifier(), type_spec: _type_spec()) => {
                ast::Field::new(modifier, name.to_owned(), type_spec, 0)
            },
        }

        _type_spec(&self) -> ast::Type {
            (_: type_spec, _: type_literal, &value: ident) => {
                match value {
                    "double" => ast::Type::Double,
                    "float" => ast::Type::Float,
                    "i32" => ast::Type::I32,
                    "i64" => ast::Type::I64,
                    "u32" => ast::Type::U32,
                    "u64" => ast::Type::U64,
                    "bool" => ast::Type::Bool,
                    "string" => ast::Type::String,
                    "bytes" => ast::Type::Bytes,
                    "any" => ast::Type::Any,
                    name => ast::Type::Custom(name.to_owned()),
                }
            },

            (_: type_spec, _: used_type, &used: ident, &value: ident) => {
                ast::Type::UsedType(used.to_owned(), value.to_owned())
            },

            (_: type_spec, _: tuple, arguments: _tuple_element_list()) => {
                let arguments = arguments.into_iter().collect();
                ast::Type::Tuple(arguments)
            },

            (_: type_spec, _: array, _: array_argument, argument: _type_spec()) => {
                ast::Type::Array(Box::new(argument))
            },
        }

        _tuple_element_list(&self) -> LinkedList<ast::Type> {
            (_: tuple_element, first: _type_spec(), mut tail: _tuple_element_list()) => {
                tail.push_front(first);
                tail
            },

            () => LinkedList::new(),
        }

        _modifier(&self) -> ast::Modifier {
            (_: modifier) => ast::Modifier::Optional,
            () => ast::Modifier::Required,
        }

        _ident_list(&self) -> LinkedList<String> {
            (&first: ident, mut tail: _ident_list()) => {
                tail.push_front(first.to_owned());
                tail
            },

            () => LinkedList::new(),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    const FILE1: &[u8] = include_bytes!("tests/file1.reproto");

    #[test]
    fn test_file1() {
        let input = ::std::str::from_utf8(FILE1).unwrap();
        let mut parser = Rdp::new(StringInput::new(input));

        assert!(parser.file());
        assert!(parser.end());

        let file = parser._file().unwrap();

        let package = ast::Package::new(vec!["foo".to_owned(), "bar".to_owned(), "baz".to_owned()]);

        assert_eq!(package, file.package);
        assert_eq!(4, file.decls.len());
    }

    #[test]
    fn test_array() {
        let mut parser = Rdp::new(StringInput::new("[(string, string)]"));

        assert!(parser.type_spec());
        assert!(parser.end());

        parser._type_spec();
    }

    #[test]
    fn test_block_comment() {
        let mut parser = Rdp::new(StringInput::new("/* hello \n world */"));

        assert!(parser.comment());
    }

    #[test]
    fn test_line_comment() {
        let mut parser = Rdp::new(StringInput::new("// hello world\n"));

        assert!(parser.comment());
    }
}
<|endoftext|>


<commit_before><commit_msg>Predicate can also be standalone, not just with op<commit_after><|endoftext|>


<commit_before>use itertools::Itertools;

use crate::ram::schema::{Field, Schema};
use crate::ram::types;
use crate::ram::types::{bool_io, u32_io, SharedMap, SharedValue, Type};

use super::writer::{ARRAY_TYPE_MASK, NULL_PLACEHOLDER};
use dovahkiin::types::key_hash;
use std::collections::HashMap;

fn read_field(
    base_ptr: usize,
    field: &Field,
    selected: Option<&[u64]>,
    is_var: bool,
    tail_offset: &mut usize,
) -> SharedValue {
    let mut rec_field_offset = field.offset.unwrap_or(0);
    let field_offset = if is_var {
        // Is inside size variable field, read directly from the address
        tail_offset
    } else if !field.is_var() {
        // Is not inside var and field not var, read from the offset in field
        &mut rec_field_offset
    } else {
        // Is not inside var and field is var, read the pointer and direct to it
        *tail_offset = *u32_io::read(base_ptr + field.offset.unwrap()) as usize;
        tail_offset
    };
    if field.nullable {
        let null_byte = *bool_io::read(base_ptr + *field_offset);
        *field_offset += 1;
        if null_byte {
            return SharedValue::Null;
        }
    }
    if field.is_array {
        let len = *u32_io::read(base_ptr + *field_offset);
        trace!("Got array length {}", len);
        let mut sub_field = field.clone();
        sub_field.is_array = false;
        *field_offset += u32_io::type_size();
        if field.sub_fields.is_none() {
            // maybe primitive array
            let mut ptr = base_ptr + *field_offset;
            let val = types::get_shared_prim_array_val(field.data_type, len as usize, &mut ptr);
            *field_offset = ptr - base_ptr;
            if let Some(prim_arr) = val {
                SharedValue::PrimArray(prim_arr)
            } else {
                panic!(
                    "type cannot been convert to prim array: {:?}",
                    field.data_type
                )
            }
        } else {
            let mut vals = Vec::<SharedValue>::new();
            for _ in 0..len {
                let nxt_val = read_field(base_ptr, &sub_field, None, true, field_offset);
                vals.push(nxt_val);
            }
            SharedValue::Array(vals)
        }
    } else if let Some(ref subs) = field.sub_fields {
        let mut map = SharedMap::new();
        let mut selected_pos = 0;
        for sub in subs {
            let cval = read_field(base_ptr, &sub, selected, is_var, field_offset);
            map.insert_key_id(sub.name_id, cval);
            match selected {
                None => {}
                Some(field_ids) => {
                    if field_ids[selected_pos] == sub.name_id {
                        selected_pos += 1;
                        if field_ids.len() <= selected_pos {
                            return SharedValue::Map(map);
                        }
                    }
                }
            }
        }
        map.fields = subs.iter().map(|sub| &sub.name).cloned().collect();
        SharedValue::Map(map)
    } else {
        let field_ptr = base_ptr + *field_offset;
        *field_offset += types::get_size(field.data_type, field_ptr);
        types::get_shared_val(field.data_type, field_ptr)
    }
}

pub fn read_attach_dynamic_part(mut tail_ptr: usize, dest: &mut SharedValue) {
    let src = read_dynamic_value(&mut tail_ptr);
    if let &mut SharedValue::Map(ref mut map_dest) = dest {
        if let SharedValue::Map(mut map_src) = src {
            map_dest.fields.append(&mut map_src.fields);
            for (k, v) in map_src.map.into_iter() {
                map_dest.insert_key_id(k, v);
            }
        }
    }
}

const MAP_TYPE_ID: u8 = Type::Map.id();

fn read_dynamic_value(ptr: &mut usize) -> SharedValue {
    let type_id = types::get_shared_val(Type::U8, *ptr).u8().unwrap();
    let is_array = type_id & ARRAY_TYPE_MASK == ARRAY_TYPE_MASK;
    *ptr += types::u8_io::type_size();
    if is_array {
        let base_type = type_id & (!ARRAY_TYPE_MASK);
        let len = types::get_shared_val(Type::U32, *ptr).u32().unwrap();
        *ptr += types::u32_io::type_size();
        if base_type != MAP_TYPE_ID {
            // Primitive array
            if let Some(prim_arr) =
                types::get_shared_prim_array_val(Type::from_id(base_type), *len as usize, ptr)
            {
                // ptr have been moved by `get_shared_prim_array_val`
                return SharedValue::PrimArray(prim_arr);
            } else {
                panic!("Cannot read prim array for dynamic field");
            }
        } else {
            let array = (0..*len).map(|_| read_dynamic_value(ptr)).collect();
            // ptr have been moved by recursion
            return SharedValue::Array(array);
        }
    } else if *type_id == MAP_TYPE_ID {
        // Map
        let len = types::get_shared_val(Type::U32, *ptr).u32().unwrap();
        *ptr += types::u32_io::type_size();
        let field_value_pair = (0..*len)
            .map(|_| {
                let name = types::get_shared_val(Type::String, *ptr).string().unwrap();
                *ptr += types::string_io::size_at(*ptr);
                let value = read_dynamic_value(ptr);
                (name, value)
            })
            .collect_vec();
        let mut fields = Vec::with_capacity(field_value_pair.len());
        let mut map = HashMap::with_capacity(field_value_pair.len());
        for (name, value) in field_value_pair {
            let id = key_hash(&name);
            fields.push(name.to_owned());
            map.insert(id, value);
        }
        return SharedValue::Map(SharedMap { fields, map });
    } else if *type_id == NULL_PLACEHOLDER {
        return SharedValue::Null;
    } else {
        let ty = Type::from_id(*type_id);
        let value = types::get_shared_val(ty, *ptr);
        *ptr += types::get_size(ty, *ptr);
        return value;
    }
}

pub fn read_by_schema(ptr: usize, schema: &Schema) -> SharedValue {
    let mut tail_offset = schema.static_bound;
    let mut schema_value = read_field(ptr, &schema.fields, None, false, &mut tail_offset);
    if schema.is_dynamic {
        read_attach_dynamic_part(ptr + tail_offset, &mut schema_value)
    }
    schema_value
}

pub fn read_by_schema_selected(ptr: usize, schema: &Schema, fields: &[u64]) -> SharedValue {
    let mut tail_offset = schema.static_bound;
    read_field(ptr, &schema.fields, Some(fields), false, &mut tail_offset)
}
<commit_msg>no unecessary data field read<commit_after>use itertools::Itertools;

use crate::ram::schema::{Field, Schema};
use crate::ram::types;
use crate::ram::types::{bool_io, u32_io, SharedMap, SharedValue, Type};

use super::writer::{ARRAY_TYPE_MASK, NULL_PLACEHOLDER};
use dovahkiin::types::key_hash;
use std::collections::HashMap;

fn read_field(
    base_ptr: usize,
    field: &Field,
    selected: Option<&[u64]>,
    is_var: bool,
    tail_offset: &mut usize,
) -> SharedValue {
    let mut rec_field_offset = field.offset.unwrap_or(0);
    let field_offset = if is_var {
        // Is inside size variable field, read directly from the address
        tail_offset
    } else if !field.is_var() {
        // Is not inside var and field not var, read from the offset in field
        &mut rec_field_offset
    } else {
        // Is not inside var and field is var, read the pointer and direct to it
        *tail_offset = *u32_io::read(base_ptr + field.offset.unwrap()) as usize;
        tail_offset
    };
    if field.nullable {
        let null_byte = *bool_io::read(base_ptr + *field_offset);
        *field_offset += 1;
        if null_byte {
            return SharedValue::Null;
        }
    }
    if field.is_array {
        let len = *u32_io::read(base_ptr + *field_offset);
        trace!("Got array length {}", len);
        let mut sub_field = field.clone();
        sub_field.is_array = false;
        *field_offset += u32_io::type_size();
        if field.sub_fields.is_none() {
            // maybe primitive array
            let mut ptr = base_ptr + *field_offset;
            let val = types::get_shared_prim_array_val(field.data_type, len as usize, &mut ptr);
            *field_offset = ptr - base_ptr;
            if let Some(prim_arr) = val {
                SharedValue::PrimArray(prim_arr)
            } else {
                panic!(
                    "type cannot been convert to prim array: {:?}",
                    field.data_type
                )
            }
        } else {
            let mut vals = Vec::<SharedValue>::new();
            for _ in 0..len {
                let nxt_val = read_field(base_ptr, &sub_field, None, true, field_offset);
                vals.push(nxt_val);
            }
            SharedValue::Array(vals)
        }
    } else if let Some(ref subs) = field.sub_fields {
        let mut map = SharedMap::new();
        let mut read_sub = |sub: &Field| {
            map.insert_key_id(sub.name_id, read_field(base_ptr, &sub, selected, is_var, field_offset));
        };
        if let Some(field_ids) = selected {
            for sub in subs {
                let mut selected_pos = 0;
                if field_ids[selected_pos] == sub.name_id {
                    read_sub(sub);
                    selected_pos += 1;
                    if field_ids.len() <= selected_pos {
                        return SharedValue::Map(map);
                    }
                }
            }
        } else {
            for sub in subs {
                read_sub(sub);
            }
        }
        map.fields = subs.iter().map(|sub| &sub.name).cloned().collect();
        SharedValue::Map(map)
    } else {
        let field_ptr = base_ptr + *field_offset;
        *field_offset += types::get_size(field.data_type, field_ptr);
        types::get_shared_val(field.data_type, field_ptr)
    }
}

pub fn read_attach_dynamic_part(mut tail_ptr: usize, dest: &mut SharedValue) {
    let src = read_dynamic_value(&mut tail_ptr);
    if let &mut SharedValue::Map(ref mut map_dest) = dest {
        if let SharedValue::Map(mut map_src) = src {
            map_dest.fields.append(&mut map_src.fields);
            for (k, v) in map_src.map.into_iter() {
                map_dest.insert_key_id(k, v);
            }
        }
    }
}

const MAP_TYPE_ID: u8 = Type::Map.id();

fn read_dynamic_value(ptr: &mut usize) -> SharedValue {
    let type_id = types::get_shared_val(Type::U8, *ptr).u8().unwrap();
    let is_array = type_id & ARRAY_TYPE_MASK == ARRAY_TYPE_MASK;
    *ptr += types::u8_io::type_size();
    if is_array {
        let base_type = type_id & (!ARRAY_TYPE_MASK);
        let len = types::get_shared_val(Type::U32, *ptr).u32().unwrap();
        *ptr += types::u32_io::type_size();
        if base_type != MAP_TYPE_ID {
            // Primitive array
            if let Some(prim_arr) =
                types::get_shared_prim_array_val(Type::from_id(base_type), *len as usize, ptr)
            {
                // ptr have been moved by `get_shared_prim_array_val`
                return SharedValue::PrimArray(prim_arr);
            } else {
                panic!("Cannot read prim array for dynamic field");
            }
        } else {
            let array = (0..*len).map(|_| read_dynamic_value(ptr)).collect();
            // ptr have been moved by recursion
            return SharedValue::Array(array);
        }
    } else if *type_id == MAP_TYPE_ID {
        // Map
        let len = types::get_shared_val(Type::U32, *ptr).u32().unwrap();
        *ptr += types::u32_io::type_size();
        let field_value_pair = (0..*len)
            .map(|_| {
                let name = types::get_shared_val(Type::String, *ptr).string().unwrap();
                *ptr += types::string_io::size_at(*ptr);
                let value = read_dynamic_value(ptr);
                (name, value)
            })
            .collect_vec();
        let mut fields = Vec::with_capacity(field_value_pair.len());
        let mut map = HashMap::with_capacity(field_value_pair.len());
        for (name, value) in field_value_pair {
            let id = key_hash(&name);
            fields.push(name.to_owned());
            map.insert(id, value);
        }
        return SharedValue::Map(SharedMap { fields, map });
    } else if *type_id == NULL_PLACEHOLDER {
        return SharedValue::Null;
    } else {
        let ty = Type::from_id(*type_id);
        let value = types::get_shared_val(ty, *ptr);
        *ptr += types::get_size(ty, *ptr);
        return value;
    }
}

pub fn read_by_schema(ptr: usize, schema: &Schema) -> SharedValue {
    let mut tail_offset = schema.static_bound;
    let mut schema_value = read_field(ptr, &schema.fields, None, false, &mut tail_offset);
    if schema.is_dynamic {
        read_attach_dynamic_part(ptr + tail_offset, &mut schema_value)
    }
    schema_value
}

pub fn read_by_schema_selected(ptr: usize, schema: &Schema, fields: &[u64]) -> SharedValue {
    let mut tail_offset = schema.static_bound;
    read_field(ptr, &schema.fields, Some(fields), false, &mut tail_offset)
}
<|endoftext|>


<commit_before>use std::io;
use std::io::{Write, Read};
use std::ffi::OsStr;

use radix_trie::Trie;

use rsh::State;
use rsh::builtins;

pub struct Input {
    completions: Trie<String, String>,
}

impl Input {
    pub fn from(s: &State) -> Input {
        let mut i = Input { completions: Trie::new() };
        let mut strs: Vec<String> = Vec::new();

        // we don't have to worry about directories here anything in the
        // path should be an executable
        for path in s.exec_paths().iter() {
            let items = path.iter().collect::<Vec<&OsStr>>();
            let mut strings = items.iter()
                .map(|x| x.to_str().unwrap_or(""))
                .map(|x| x.to_string())
                .collect::<Vec<String>>();

            strs.append(&mut strings);
        }

        let builtins = builtins::load();
        let mut keys: Vec<String> = builtins.keys()
            .map(|x| x.clone())
            .collect();

        strs.append(&mut keys);

        for s in strs {
            let mut chars = s.chars();
            let mut current_char = chars.next();
            let mut next_char = chars.next();

            while current_char.is_some() && next_char.is_some() {
                i.completions.insert(current_char.unwrap().to_string(),
                                     next_char.unwrap().to_string());
                current_char = next_char.clone();
                next_char = chars.next();
            }
        }

        i
    }

    pub fn prompt(&self, prompt: String) -> String {
        print!("{}", prompt);

        // this forces the prompt to print
        io::stdout().flush().expect("unable to flush stdout");

        // read the user input
        let mut input = String::new();

        for byt in io::stdin().bytes() {
            match byt {
                Ok(b) => {
                    // TODO this is naive it ignores multi-byte utf-8 characters
                    let c = b as char;

                    input.push(c);

                    if c == '\n' {
                        break;
                    }
                }
                Err(e) => println!("Error reading from stdin: {}", e),
            };
        }

        input
    }
}
<commit_msg>removed useless map<commit_after>use std::io;
use std::io::{Write, Read};
use std::ffi::OsStr;

use radix_trie::Trie;

use rsh::State;
use rsh::builtins;

pub struct Input {
    completions: Trie<String, String>,
}

impl Input {
    pub fn from(s: &State) -> Input {
        let mut i = Input { completions: Trie::new() };
        let mut strs: Vec<String> = Vec::new();

        // we don't have to worry about directories here anything in the
        // path should be an executable
        for path in s.exec_paths().iter() {
            let items = path.iter().collect::<Vec<&OsStr>>();
            let mut strings = items.iter()
                .map(|x| x.to_str().unwrap_or("").to_string())
                .collect::<Vec<String>>();

            strs.append(&mut strings);
        }

        let builtins = builtins::load();
        let mut keys: Vec<String> = builtins.keys()
            .map(|x| x.clone())
            .collect();

        strs.append(&mut keys);

        for s in strs {
            let mut chars = s.chars();
            let mut current_char = chars.next();
            let mut next_char = chars.next();

            while current_char.is_some() && next_char.is_some() {
                i.completions.insert(current_char.unwrap().to_string(),
                                     next_char.unwrap().to_string());
                current_char = next_char.clone();
                next_char = chars.next();
            }
        }

        i
    }

    pub fn prompt(&self, prompt: String) -> String {
        print!("{}", prompt);

        // this forces the prompt to print
        io::stdout().flush().expect("unable to flush stdout");

        // read the user input
        let mut input = String::new();

        for byt in io::stdin().bytes() {
            match byt {
                Ok(b) => {
                    // TODO this is naive it ignores multi-byte utf-8 characters
                    let c = b as char;

                    input.push(c);

                    if c == '\n' {
                        break;
                    }
                }
                Err(e) => println!("Error reading from stdin: {}", e),
            };
        }

        input
    }
}
<|endoftext|>


<commit_before>use std::iter::Peekable;

/// An iterator that performs a lookahead of 1, utilizing the existing Peakable Iterator
/// Can be stacked to perform an even greater lookahead
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
#[derive(Clone)]
pub struct Peeking<I, A> where
    A: Clone,
    I: Iterator<Item=A>,
{
    iter: Peekable<I>,
}

impl<I, A> Iterator for Peeking<I, A> where
    A: Clone,
    I: Iterator<Item=A>
{
    type Item = (A, Option<A>);

    #[inline]
    fn next(&mut self) -> Option<(A, Option<A>)> {
        match self.iter.next() {
            Some(item) => Some((item,
                match self.iter.peek() {
                    Some(ref item) => Some((*item).clone()),
                    None => None,
                }
            )),
            None => None,
        }
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.iter.size_hint()
    }
}

pub trait PeekingExt {
    fn peeking(self) -> Peeking<Self, Self::Item>
        where Self: Sized+Iterator,
              Self::Item: Clone;
}

impl<A: Clone, I: Sized+Iterator<Item=A>> PeekingExt for I {
    fn peeking(self) -> Peeking<Self, A>
        where Self: Sized+Iterator
    {
         Peeking { iter: self.peekable() }
    }
}

/// An iterator to maintain state while iterating another iterator and allows filtering
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
#[derive(Clone)]
pub struct FilteringScan<I, St, F> {
    iter: I,
    f: F,
    state: St,
}

impl<B, I, St, F> Iterator for FilteringScan<I, St, F> where
    I: Iterator,
    F: FnMut(&mut St, I::Item) -> Option<B>,
{
    type Item = B;

    #[inline]
    fn next(&mut self) -> Option<B> {
        loop {
            match self.iter.next() {
                Some(next) => match (self.f)(&mut self.state, next) {
                    Some(result) => return Some(result),
                    None => continue,
                },
                None => return None,
            }
        }
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        let (_, upper) = self.iter.size_hint();
        (0, upper) // can't know a lower bound, due to the scan function
    }
}

pub trait FilteringScanExt {
    fn scan_filter<St, B, F>(self, initial_state: St, f: F) -> FilteringScan<Self, St, F>
        where Self: Sized+Iterator, F: FnMut(&mut St, Self::Item) -> Option<B>;
}

impl<I: Sized+Iterator> FilteringScanExt for I {
    fn scan_filter<St, B, F>(self, initial_state: St, f: F) -> FilteringScan<Self, St, F>
        where Self: Sized+Iterator, F: FnMut(&mut St, I::Item) -> Option<B>,
    {
        FilteringScan{iter: self, f: f, state: initial_state}
    }
}

#[derive(Clone)]
pub enum ParseResult {
    Continue,
    Halt,
    End,
}

/// An iterator to maintain state while iterating another iterator and allows more complex parsing
// (allowing holding the current iterator value, continue if its goes empty and filter the output)
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
#[derive(Clone)]
pub struct Parser<I, A, St, F> where
    A: Clone,
    I: Iterator<Item=A>,
{
    iter: I,
    f: F,
    state: St,

    iter_state: ParseResult,
    current_elem: Option<A>,
}

impl<B, I, St, F, A> Iterator for Parser<I, A, St, F> where
    A: Clone,
    I: Iterator<Item=A>,
    F: FnMut(&mut St, Option<I::Item>) -> (ParseResult, Option<B>),
{
    type Item = B;

    #[inline]
    fn next(&mut self) -> Option<B> {
        loop {
            match self.iter_state {
                ParseResult::Continue => self.current_elem = self.iter.next(),
                ParseResult::Halt => {},
                ParseResult::End => return None,
            };

            let (new_state, result) = (self.f) (&mut self.state, self.current_elem.clone());

            self.iter_state = new_state;

            match result {
                Some(result) => return Some(result),
                None => continue,
            }
        }
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        (0, None) // can't know a lower or upper bound, due to the parse function
    }
}

pub trait ParserExt {
    fn parsing<St, B, F>(self, initial_state: St, f: F) -> Parser<Self, Self::Item, St, F>
        where Self: Sized+Iterator, Self::Item: Clone, F: Fn(&mut St, Option<Self::Item>) -> (ParseResult, Option<B>);
}

impl<A:Clone, I: Sized+Iterator<Item=A>>ParserExt for I {
    fn parsing<St, B, F>(self, initial_state: St, f: F) -> Parser<Self, A, St, F>
        where F: FnMut(&mut St, Option<A>) -> (ParseResult, Option<B>),
    {
        Parser{iter: self, f: f, state: initial_state, iter_state: ParseResult::Continue, current_elem: None}
    }
}

#[test]
fn peeking_test() {
    let test = vec!["first", "second", "third"];
    let mut index = 0;
    for (elem, peek) in test.into_iter().peeking() {
        match elem {
            "first" => {
                assert_eq!(peek, Some("second"));
                index += 1;
            },
            "second" => {
                assert_eq!(peek, Some("third"));
                index += 1;
            },
            "third" => {
                assert_eq!(peek, None);
                index += 1;
            },
            _ => assert!(false),
        }
    }
    assert_eq!(index, 3);
}

#[test]
fn filtering_scan_test() {
    let test = vec![1, 3, 1, 3];
    let index = 0;
    let mut sum = 0;
    for elem in test.into_iter().scan_filter(index, |index, x| {
        *index += 1;
        if (*index % 2) == 1 {
            Some(x)
        } else {
            None
        }
    }) {
        sum += elem;
    }
    assert_eq!(sum, 2);
}

#[test]
fn parsing_test() {

    struct ParseState {
        one_halt_test: u32,
        two_counter: u32,
        one_more_test: bool,
    }

    let mut result = vec![];

    let test = vec![1, 2, 2, 3];
    for elem in test.into_iter().parsing(
        ParseState {
            one_halt_test: 3,
            two_counter: 1,
            one_more_test: true,
        },
        |state, elem| {
            match elem {
                Some(1) => {
                    state.one_halt_test -= 1;
                    if state.one_halt_test == 0 {
                        (ParseResult::Continue, Some(1))
                    } else {
                        (ParseResult::Halt, None)
                    }
                },
                Some(2) => {
                    state.two_counter += 1;
                    (ParseResult::Continue, Some(state.two_counter))
                },
                Some(3) => {
                    (ParseResult::Continue, Some(4))
                },
                Some(_) => { panic!("found a bug in rust") },
                None => {
                    if state.one_more_test {
                        state.one_more_test = false;
                        (ParseResult::Continue, Some(5))
                    } else {
                        (ParseResult::End, None)
                    }
                }
            }
        }
    ) {
        result.push(elem);
    }

    assert_eq!(result, vec![1, 2, 3, 4, 5]);
}
<commit_msg>First iteration on Waiting Iterator Extension<commit_after>use std::iter::Peekable;

/// An iterator that performs a lookahead of 1, utilizing the existing Peakable Iterator
/// Can be stacked to perform an even greater lookahead
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
#[derive(Clone)]
pub struct Peeking<I, A> where
    A: Clone,
    I: Iterator<Item=A>,
{
    iter: Peekable<I>,
}

impl<I, A> Iterator for Peeking<I, A> where
    A: Clone,
    I: Iterator<Item=A>
{
    type Item = (A, Option<A>);

    #[inline]
    fn next(&mut self) -> Option<(A, Option<A>)> {
        match self.iter.next() {
            Some(item) => Some((item,
                match self.iter.peek() {
                    Some(ref item) => Some((*item).clone()),
                    None => None,
                }
            )),
            None => None,
        }
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        self.iter.size_hint()
    }
}

pub trait PeekingExt {
    fn peeking(self) -> Peeking<Self, Self::Item>
        where Self: Sized+Iterator,
              Self::Item: Clone;
}

impl<A: Clone, I: Sized+Iterator<Item=A>> PeekingExt for I {
    fn peeking(self) -> Peeking<Self, A>
        where Self: Sized+Iterator
    {
         Peeking { iter: self.peekable() }
    }
}

/// An iterator to maintain state while iterating another iterator and allows filtering
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
#[derive(Clone)]
pub struct FilteringScan<I, St, F> {
    iter: I,
    f: F,
    state: St,
}

impl<B, I, St, F> Iterator for FilteringScan<I, St, F> where
    I: Iterator,
    F: FnMut(&mut St, I::Item) -> Option<B>,
{
    type Item = B;

    #[inline]
    fn next(&mut self) -> Option<B> {
        loop {
            match self.iter.next() {
                Some(next) => match (self.f)(&mut self.state, next) {
                    Some(result) => return Some(result),
                    None => continue,
                },
                None => return None,
            }
        }
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        let (_, upper) = self.iter.size_hint();
        (0, upper) // can't know a lower bound, due to the scan function
    }
}

pub trait FilteringScanExt {
    fn scan_filter<St, B, F>(self, initial_state: St, f: F) -> FilteringScan<Self, St, F>
        where Self: Sized+Iterator, F: FnMut(&mut St, Self::Item) -> Option<B>;
}

impl<I: Sized+Iterator> FilteringScanExt for I {
    fn scan_filter<St, B, F>(self, initial_state: St, f: F) -> FilteringScan<Self, St, F>
        where Self: Sized+Iterator, F: FnMut(&mut St, I::Item) -> Option<B>,
    {
        FilteringScan{iter: self, f: f, state: initial_state}
    }
}

#[derive(Clone)]
pub enum ParseResult {
    Continue,
    Halt,
    End,
}

/// An iterator to maintain state while iterating another iterator and allows more complex parsing
// (allowing holding the current iterator value, continue if its goes empty and filter the output)
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
#[derive(Clone)]
pub struct Parser<I, A, St, F> where
    A: Clone,
    I: Iterator<Item=A>,
{
    iter: I,
    f: F,
    state: St,

    iter_state: ParseResult,
    current_elem: Option<A>,
}

impl<B, I, St, F, A> Iterator for Parser<I, A, St, F> where
    A: Clone,
    I: Iterator<Item=A>,
    F: FnMut(&mut St, Option<I::Item>) -> (ParseResult, Option<B>),
{
    type Item = B;

    #[inline]
    fn next(&mut self) -> Option<B> {
        loop {
            match self.iter_state {
                ParseResult::Continue => self.current_elem = self.iter.next(),
                ParseResult::Halt => {},
                ParseResult::End => return None,
            };

            let (new_state, result) = (self.f) (&mut self.state, self.current_elem.clone());

            self.iter_state = new_state;

            match result {
                Some(result) => return Some(result),
                None => continue,
            }
        }
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        (0, None) // can't know a lower or upper bound, due to the parse function
    }
}

pub trait ParserExt {
    fn parsing<St, B, F>(self, initial_state: St, f: F) -> Parser<Self, Self::Item, St, F>
        where Self: Sized+Iterator, Self::Item: Clone, F: Fn(&mut St, Option<Self::Item>) -> (ParseResult, Option<B>);
}

impl<A:Clone, I: Sized+Iterator<Item=A>>ParserExt for I {
    fn parsing<St, B, F>(self, initial_state: St, f: F) -> Parser<Self, A, St, F>
        where F: FnMut(&mut St, Option<A>) -> (ParseResult, Option<B>),
    {
        Parser{iter: self, f: f, state: initial_state, iter_state: ParseResult::Continue, current_elem: None}
    }
}

#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
#[derive(Clone)]
pub struct Wait<B, I, E, F>
    where
    I: Iterator,
    F: FnMut(&mut E, I::Item) -> Option<B> {
        iter: I,
        f: F,
        current_elem: Option<B>,
}

impl<B, I, E, F> Iterator for Wait<B, I, E, F> where
    I: Iterator,
    F: FnMut(&mut E, I::Item) -> Option<B>,
{
    type Item = B;

    #[inline]
    fn next(&mut self) -> Option<B> {
        while let Some(elem) =  self.iter.next() {
            match self.f(self.current_elem, elem) {
                Some(elem) => {
                    let return_val = self.current_elem;
                    self.current_elem = elem;
                    match return_val {
                        Some(value) => return Some(value),
                        None => continue,
                    }
                },
                None => continue,
            }
        }
        None
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        let (_, upper) = self.iter.size_hint();
        (0, upper) // can't know a lower bound, due to the waiting function
    }
}

#[test]
fn peeking_test() {
    let test = vec!["first", "second", "third"];
    let mut index = 0;
    for (elem, peek) in test.into_iter().peeking() {
        match elem {
            "first" => {
                assert_eq!(peek, Some("second"));
                index += 1;
            },
            "second" => {
                assert_eq!(peek, Some("third"));
                index += 1;
            },
            "third" => {
                assert_eq!(peek, None);
                index += 1;
            },
            _ => assert!(false),
        }
    }
    assert_eq!(index, 3);
}

#[test]
fn filtering_scan_test() {
    let test = vec![1, 3, 1, 3];
    let index = 0;
    let mut sum = 0;
    for elem in test.into_iter().scan_filter(index, |index, x| {
        *index += 1;
        if (*index % 2) == 1 {
            Some(x)
        } else {
            None
        }
    }) {
        sum += elem;
    }
    assert_eq!(sum, 2);
}

#[test]
fn parsing_test() {

    struct ParseState {
        one_halt_test: u32,
        two_counter: u32,
        one_more_test: bool,
    }

    let mut result = vec![];

    let test = vec![1, 2, 2, 3];
    for elem in test.into_iter().parsing(
        ParseState {
            one_halt_test: 3,
            two_counter: 1,
            one_more_test: true,
        },
        |state, elem| {
            match elem {
                Some(1) => {
                    state.one_halt_test -= 1;
                    if state.one_halt_test == 0 {
                        (ParseResult::Continue, Some(1))
                    } else {
                        (ParseResult::Halt, None)
                    }
                },
                Some(2) => {
                    state.two_counter += 1;
                    (ParseResult::Continue, Some(state.two_counter))
                },
                Some(3) => {
                    (ParseResult::Continue, Some(4))
                },
                Some(_) => { panic!("found a bug in rust") },
                None => {
                    if state.one_more_test {
                        state.one_more_test = false;
                        (ParseResult::Continue, Some(5))
                    } else {
                        (ParseResult::End, None)
                    }
                }
            }
        }
    ) {
        result.push(elem);
    }

    assert_eq!(result, vec![1, 2, 3, 4, 5]);
}
<|endoftext|>


<commit_before><commit_msg>Add simple test for deposits<commit_after><|endoftext|>


<commit_before>use piston_window;

pub enum InputResult {
    PushScreen(Box<Screen>),
    PopScreen,
}

pub trait Screen {
    fn on_input(&mut self, input: &piston_window::Input) -> InputResult;
    fn on_draw(&mut self, args: &piston_window::RenderArgs, window: &piston_window::PistonWindow);
    fn on_update(&mut self, args: &piston_window::UpdateArgs);
}
<commit_msg>InputResults vector<commit_after>use piston_window;

pub enum InputResult {
    PushScreen(Box<Screen>),
    PopScreen,
}

pub type InputResults = Vec<InputResult>;

pub trait Screen {
    fn on_input(&mut self, input: &piston_window::Input) -> InputResult;
    fn on_draw(&mut self, args: &piston_window::RenderArgs, window: &piston_window::PistonWindow);
    fn on_update(&mut self, args: &piston_window::UpdateArgs);
}
<|endoftext|>


<commit_before>use crate::spec::{FramePointer, LinkerFlavor, SanitizerSet, Target, TargetOptions};

pub fn target() -> Target {
    let mut base = super::apple_base::opts("macos");
    base.cpu = "apple-a12".to_string();
    base.max_atomic_width = Some(128);

    // FIXME: The leak sanitizer currently fails the tests, see #88132.
    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::THREAD;

    base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-arch".to_string(), "arm64".to_string()]);
    base.link_env_remove.extend(super::apple_base::macos_link_env_remove());

    // Clang automatically chooses a more specific target based on
    // MACOSX_DEPLOYMENT_TARGET.  To enable cross-language LTO to work
    // correctly, we do too.
    let arch = "aarch64";
    let llvm_target = super::apple_base::macos_llvm_target(&arch);

    Target {
        llvm_target,
        pointer_width: 64,
        data_layout: "e-m:o-i64:64-i128:128-n32:64-S128".to_string(),
        arch: arch.to_string(),
        options: TargetOptions {
            mcount: "\u{1}mcount".to_string(),
            frame_pointer: FramePointer::NonLeaf,
            ..base
        },
    }
}
<commit_msg>Auto merge of #90478 - rusticstuff:apple-a14, r=wesleywiser<commit_after>use crate::spec::{FramePointer, LinkerFlavor, SanitizerSet, Target, TargetOptions};

pub fn target() -> Target {
    let mut base = super::apple_base::opts("macos");
    base.cpu = "apple-a14".to_string();
    base.max_atomic_width = Some(128);

    // FIXME: The leak sanitizer currently fails the tests, see #88132.
    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::THREAD;

    base.pre_link_args.insert(LinkerFlavor::Gcc, vec!["-arch".to_string(), "arm64".to_string()]);
    base.link_env_remove.extend(super::apple_base::macos_link_env_remove());

    // Clang automatically chooses a more specific target based on
    // MACOSX_DEPLOYMENT_TARGET.  To enable cross-language LTO to work
    // correctly, we do too.
    let arch = "aarch64";
    let llvm_target = super::apple_base::macos_llvm_target(&arch);

    Target {
        llvm_target,
        pointer_width: 64,
        data_layout: "e-m:o-i64:64-i128:128-n32:64-S128".to_string(),
        arch: arch.to_string(),
        options: TargetOptions {
            mcount: "\u{1}mcount".to_string(),
            frame_pointer: FramePointer::NonLeaf,
            ..base
        },
    }
}
<|endoftext|>