<commit_before>// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.

use std::sync::{mpsc, Arc};
use std::thread;
use std::time::Duration;

use grpcio::{ChannelBuilder, Environment};
use kvproto::kvrpcpb::*;
use kvproto::metapb::{Peer, Region};
use kvproto::tikvpb::TikvClient;

use test_raftstore::*;
use tikv_util::HandyRwLock;

fn acquire_pessimistic_lock(
    client: &TikvClient,
    ctx: Context,
    key: Vec<u8>,
    ts: u64,
) -> PessimisticLockResponse {
    let mut req = PessimisticLockRequest::default();
    req.set_context(ctx);
    let mut mutation = Mutation::default();
    mutation.set_op(Op::PessimisticLock);
    mutation.set_key(key.clone());
    mutation.set_value(key.clone());
    req.set_mutations(vec![mutation].into_iter().collect());
    req.primary_lock = key;
    req.start_version = ts;
    req.for_update_ts = ts;
    req.lock_ttl = 20;
    req.is_first_lock = false;
    client.kv_pessimistic_lock(&req).unwrap()
}

fn must_acquire_pessimistic_lock(client: &TikvClient, ctx: Context, key: Vec<u8>, ts: u64) {
    let resp = acquire_pessimistic_lock(client, ctx, key, ts);
    assert!(!resp.has_region_error(), "{:?}", resp.get_region_error());
    assert!(resp.errors.is_empty(), "{:?}", resp.get_errors());
}

fn must_deadlock(client: &TikvClient, ctx: Context, key1: &[u8], ts: u64) {
    let key1 = key1.to_vec();
    let mut key2 = key1.clone();
    key2.push(1);
    must_acquire_pessimistic_lock(client, ctx.clone(), key1.clone(), ts);
    must_acquire_pessimistic_lock(client, ctx.clone(), key2.clone(), ts + 1);

    let client1 = client.clone();
    let ctx1 = ctx.clone();
    let (tx, rx) = mpsc::sync_channel(1);
    thread::spawn(move || {
        let _ = acquire_pessimistic_lock(&client1, ctx1, key1, ts + 1);
        tx.send(1).unwrap();
    });
    // Sleep to make sure txn(ts+1) is waiting for txn(ts)
    thread::sleep(Duration::from_millis(500));
    let resp = acquire_pessimistic_lock(client, ctx, key2, ts);
    assert_eq!(resp.errors.len(), 1);
    assert!(resp.errors[0].has_deadlock());
    rx.recv().unwrap();
}

fn build_leader_client(cluster: &mut Cluster<ServerCluster>, key: &[u8]) -> (TikvClient, Context) {
    let region_id = cluster.get_region_id(key);
    let leader = cluster.leader_of_region(region_id).unwrap();
    let epoch = cluster.get_region_epoch(region_id);

    let env = Arc::new(Environment::new(1));
    let channel =
        ChannelBuilder::new(env).connect(cluster.sim.rl().get_addr(leader.get_store_id()));
    let client = TikvClient::new(channel);

    let mut ctx = Context::default();
    ctx.set_region_id(region_id);
    ctx.set_peer(leader);
    ctx.set_region_epoch(epoch);

    (client, ctx)
}

fn deadlock_detector_leader_must_be(cluster: &mut Cluster<ServerCluster>, store_id: u64) {
    let leader_region = cluster.get_region(b"");
    assert_eq!(
        cluster
            .leader_of_region(leader_region.get_id())
            .unwrap()
            .get_store_id(),
        store_id
    );
    let leader_peer = find_peer_of_store(&leader_region, store_id);
    cluster
        .pd_client
        .region_leader_must_be(leader_region.get_id(), leader_peer);
}

fn must_transfer_leader(cluster: &mut Cluster<ServerCluster>, region_key: &[u8], store_id: u64) {
    let region = cluster.get_region(region_key);
    let target_peer = find_peer_of_store(&region, store_id);
    cluster.must_transfer_leader(region.get_id(), target_peer.clone());
    cluster
        .pd_client
        .region_leader_must_be(region.get_id(), target_peer);
}

fn find_peer_of_store(region: &Region, store_id: u64) -> Peer {
    region
        .get_peers()
        .iter()
        .find(|p| p.get_store_id() == store_id)
        .unwrap()
        .clone()
}

#[test]
fn test_detect_deadlock_when_shuffle_region() {
    let mut cluster = new_server_cluster(0, 4);
    let pd_client = Arc::clone(&cluster.pd_client);
    // Disable default max peer count check.
    pd_client.disable_default_operator();

    // Region 1 has 3 peers. And peer(1, 1) is the leader of region 1.
    let region_id = cluster.run_conf_change();
    pd_client.must_add_peer(region_id, new_peer(2, 2));
    pd_client.must_add_peer(region_id, new_peer(3, 3));

    // The leader of deadlock detector is the leader of region 1.
    deadlock_detector_leader_must_be(&mut cluster, 1);
    let (client, ctx) = build_leader_client(&mut cluster, b"k1");
    must_deadlock(&client, ctx, b"k1", 10);

    // The leader of region 1 has transfered. The leader of deadlock detector should also transfer.
    must_transfer_leader(&mut cluster, b"", 2);
    deadlock_detector_leader_must_be(&mut cluster, 2);
    let (client, ctx) = build_leader_client(&mut cluster, b"k2");
    must_deadlock(&client, ctx, b"k2", 20);

    // Split region and transfer the leader of new region to store(3).
    let region = cluster.get_region(b"");
    cluster.must_split(&region, b"k10");
    must_transfer_leader(&mut cluster, b"k10", 3);
    // Deadlock occurs on store(3) and the leader of deadlock detector is store(2).
    deadlock_detector_leader_must_be(&mut cluster, 2);
    let (client, ctx) = build_leader_client(&mut cluster, b"k10");
    must_deadlock(&client, ctx, b"k10", 30);

    // Transfer the new region from store(1, 2, 3) to store(2, 3, 4).
    let new_region = cluster.get_region(b"k10");
    pd_client.must_add_peer(new_region.get_id(), new_peer(4, 4));
    must_transfer_leader(&mut cluster, b"k10", 4);
    let peer = find_peer_of_store(&new_region, 1);
    pd_client.must_remove_peer(region_id, peer);

    // Transfer the leader of deadlock detector to store(1) and
    // deadlock occours on store(4) again.
    must_transfer_leader(&mut cluster, b"", 1);
    deadlock_detector_leader_must_be(&mut cluster, 1);
    let (client, ctx) = build_leader_client(&mut cluster, b"k11");
    must_deadlock(&client, ctx, b"k11", 30);

    // Add store(1) back again which will send a role change message with empty region key range to
    // the deadlock detector. It misleads the leader of deadlock detector stepping down.
    pd_client.must_add_peer(new_region.get_id(), new_peer(1, 5));
    deadlock_detector_leader_must_be(&mut cluster, 1);
    let (client, ctx) = build_leader_client(&mut cluster, b"k3");
    must_deadlock(&client, ctx, b"k3", 10);
}
<commit_msg>test: stabilize test_detect_deadlock_when_shuffle_region (#6155)<commit_after>// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.

use std::sync::{mpsc, Arc};
use std::thread;
use std::time::Duration;

use grpcio::{ChannelBuilder, Environment};
use kvproto::kvrpcpb::*;
use kvproto::metapb::{Peer, Region};
use kvproto::tikvpb::TikvClient;

use test_raftstore::*;
use tikv_util::HandyRwLock;

fn acquire_pessimistic_lock(
    client: &TikvClient,
    ctx: Context,
    key: Vec<u8>,
    ts: u64,
) -> PessimisticLockResponse {
    let mut req = PessimisticLockRequest::default();
    req.set_context(ctx);
    let mut mutation = Mutation::default();
    mutation.set_op(Op::PessimisticLock);
    mutation.set_key(key.clone());
    mutation.set_value(key.clone());
    req.set_mutations(vec![mutation].into_iter().collect());
    req.primary_lock = key;
    req.start_version = ts;
    req.for_update_ts = ts;
    req.lock_ttl = 20;
    req.is_first_lock = false;
    client.kv_pessimistic_lock(&req).unwrap()
}

fn must_acquire_pessimistic_lock(client: &TikvClient, ctx: Context, key: Vec<u8>, ts: u64) {
    let resp = acquire_pessimistic_lock(client, ctx, key, ts);
    assert!(!resp.has_region_error(), "{:?}", resp.get_region_error());
    assert!(resp.errors.is_empty(), "{:?}", resp.get_errors());
}

fn must_deadlock(client: &TikvClient, ctx: Context, key1: &[u8], ts: u64) {
    let key1 = key1.to_vec();
    let mut key2 = key1.clone();
    key2.push(1);
    must_acquire_pessimistic_lock(client, ctx.clone(), key1.clone(), ts);
    must_acquire_pessimistic_lock(client, ctx.clone(), key2.clone(), ts + 1);

    let client1 = client.clone();
    let ctx1 = ctx.clone();
    let (tx, rx) = mpsc::sync_channel(1);
    thread::spawn(move || {
        let _ = acquire_pessimistic_lock(&client1, ctx1, key1, ts + 1);
        tx.send(1).unwrap();
    });
    // Sleep to make sure txn(ts+1) is waiting for txn(ts)
    thread::sleep(Duration::from_millis(500));
    let resp = acquire_pessimistic_lock(client, ctx, key2, ts);
    assert_eq!(resp.errors.len(), 1);
    assert!(resp.errors[0].has_deadlock());
    rx.recv().unwrap();
}

fn build_leader_client(cluster: &mut Cluster<ServerCluster>, key: &[u8]) -> (TikvClient, Context) {
    let region_id = cluster.get_region_id(key);
    let leader = cluster.leader_of_region(region_id).unwrap();
    let epoch = cluster.get_region_epoch(region_id);

    let env = Arc::new(Environment::new(1));
    let channel =
        ChannelBuilder::new(env).connect(cluster.sim.rl().get_addr(leader.get_store_id()));
    let client = TikvClient::new(channel);

    let mut ctx = Context::default();
    ctx.set_region_id(region_id);
    ctx.set_peer(leader);
    ctx.set_region_epoch(epoch);

    (client, ctx)
}

fn deadlock_detector_leader_must_be(cluster: &mut Cluster<ServerCluster>, store_id: u64) {
    let leader_region = cluster.get_region(b"");
    assert_eq!(
        cluster
            .leader_of_region(leader_region.get_id())
            .unwrap()
            .get_store_id(),
        store_id
    );
    let leader_peer = find_peer_of_store(&leader_region, store_id);
    cluster
        .pd_client
        .region_leader_must_be(leader_region.get_id(), leader_peer);
}

fn must_transfer_leader(cluster: &mut Cluster<ServerCluster>, region_key: &[u8], store_id: u64) {
    let region = cluster.get_region(region_key);
    let target_peer = find_peer_of_store(&region, store_id);
    cluster.must_transfer_leader(region.get_id(), target_peer.clone());
    cluster
        .pd_client
        .region_leader_must_be(region.get_id(), target_peer);
    // Make sure the new leader can get snapshot locally.
    cluster.must_get(region_key);
}

fn find_peer_of_store(region: &Region, store_id: u64) -> Peer {
    region
        .get_peers()
        .iter()
        .find(|p| p.get_store_id() == store_id)
        .unwrap()
        .clone()
}

#[test]
fn test_detect_deadlock_when_shuffle_region() {
    let mut cluster = new_server_cluster(0, 4);
    let pd_client = Arc::clone(&cluster.pd_client);
    // Disable default max peer count check.
    pd_client.disable_default_operator();

    // Region 1 has 3 peers. And peer(1, 1) is the leader of region 1.
    let region_id = cluster.run_conf_change();
    pd_client.must_add_peer(region_id, new_peer(2, 2));
    pd_client.must_add_peer(region_id, new_peer(3, 3));

    // The leader of deadlock detector is the leader of region 1.
    deadlock_detector_leader_must_be(&mut cluster, 1);
    let (client, ctx) = build_leader_client(&mut cluster, b"k1");
    must_deadlock(&client, ctx, b"k1", 10);

    // The leader of region 1 has transfered. The leader of deadlock detector should also transfer.
    must_transfer_leader(&mut cluster, b"", 2);
    deadlock_detector_leader_must_be(&mut cluster, 2);
    let (client, ctx) = build_leader_client(&mut cluster, b"k2");
    must_deadlock(&client, ctx, b"k2", 20);

    // Split region and transfer the leader of new region to store(3).
    let region = cluster.get_region(b"");
    cluster.must_split(&region, b"k10");
    must_transfer_leader(&mut cluster, b"k10", 3);
    // Deadlock occurs on store(3) and the leader of deadlock detector is store(2).
    deadlock_detector_leader_must_be(&mut cluster, 2);
    let (client, ctx) = build_leader_client(&mut cluster, b"k10");
    must_deadlock(&client, ctx, b"k10", 30);

    // Transfer the new region from store(1, 2, 3) to store(2, 3, 4).
    let new_region = cluster.get_region(b"k10");
    pd_client.must_add_peer(new_region.get_id(), new_peer(4, 4));
    must_transfer_leader(&mut cluster, b"k10", 4);
    let peer = find_peer_of_store(&new_region, 1);
    pd_client.must_remove_peer(region_id, peer);

    // Transfer the leader of deadlock detector to store(1) and
    // deadlock occours on store(4) again.
    must_transfer_leader(&mut cluster, b"", 1);
    deadlock_detector_leader_must_be(&mut cluster, 1);
    let (client, ctx) = build_leader_client(&mut cluster, b"k11");
    must_deadlock(&client, ctx, b"k11", 30);

    // Add store(1) back again which will send a role change message with empty region key range to
    // the deadlock detector. It misleads the leader of deadlock detector stepping down.
    pd_client.must_add_peer(new_region.get_id(), new_peer(1, 5));
    deadlock_detector_leader_must_be(&mut cluster, 1);
    let (client, ctx) = build_leader_client(&mut cluster, b"k3");
    must_deadlock(&client, ctx, b"k3", 10);
}
<|endoftext|>


<commit_before>use std::{
    env,
    fs::File,
    io,
    path::{Path, PathBuf},
};

use anyhow::Result;
use flate2::{write::GzEncoder, Compression};
use xshell::{cmd, cp, mkdir_p, pushd, read_file, rm_rf, write_file};

use crate::{date_iso, project_root};

pub(crate) struct DistCmd {
    pub(crate) nightly: bool,
    pub(crate) client_version: Option<String>,
}

impl DistCmd {
    pub(crate) fn run(self) -> Result<()> {
        let dist = project_root().join("dist");
        rm_rf(&dist)?;
        mkdir_p(&dist)?;

        if let Some(version) = self.client_version {
            let release_tag = if self.nightly { "nightly".to_string() } else { date_iso()? };
            dist_client(&version, &release_tag)?;
        }
        dist_server()?;
        Ok(())
    }
}

fn dist_client(version: &str, release_tag: &str) -> Result<()> {
    let _d = pushd("./editors/code")?;
    let nightly = release_tag == "nightly";

    let mut patch = Patch::new("./package.json")?;

    patch
        .replace(r#""version": "0.4.0-dev""#, &format!(r#""version": "{}""#, version))
        .replace(r#""releaseTag": null"#, &format!(r#""releaseTag": "{}""#, release_tag))
        .replace(r#""$generated-start": false,"#, "")
        .replace(",\n        \"$generated-end\": false", "");

    if nightly {
        patch.replace(
            r#""displayName": "rust-analyzer""#,
            r#""displayName": "rust-analyzer (nightly)""#,
        );
    }
    if !nightly {
        patch.replace(r#""enableProposedApi": true,"#, r#""#);
    }
    patch.commit()?;

    cmd!("npm ci").run()?;
    cmd!("npx vsce package -o ../../dist/rust-analyzer.vsix").run()?;
    Ok(())
}

fn dist_server() -> Result<()> {
    let target = get_target();
    if target.contains("-linux-gnu") || target.contains("-linux-musl") {
        env::set_var("CC", "clang");
    }

    cmd!("cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --target {target} --release").run()?;

    let suffix = exe_suffix(&target);
    let src =
        Path::new("target").join(&target).join("release").join(format!("rust-analyzer{}", suffix));
    let dst = Path::new("dist").join(format!("rust-analyzer-{}{}", target, suffix));
    gzip(&src, &dst.with_extension("gz"))?;

    // FIXME: the old names are temporarily kept for client compatibility, but they should be removed
    // Remove this block after a couple of releases
    match target.as_ref() {
        "x86_64-unknown-linux-gnu" => {
            cp(&src, "dist/rust-analyzer-linux")?;
            gzip(&src, Path::new("dist/rust-analyzer-linux.gz"))?;
        }
        "x86_64-pc-windows-msvc" => {
            cp(&src, "dist/rust-analyzer-windows.exe")?;
            gzip(&src, Path::new("dist/rust-analyzer-windows.gz"))?;
        }
        "x86_64-apple-darwin" => {
            cp(&src, "dist/rust-analyzer-mac")?;
            gzip(&src, Path::new("dist/rust-analyzer-mac.gz"))?;
        }
        _ => {}
    }

    Ok(())
}

fn get_target() -> String {
    match env::var("RA_TARGET") {
        Ok(target) => target,
        _ => {
            if cfg!(target_os = "linux") {
                "x86_64-unknown-linux-gnu".to_string()
            } else if cfg!(target_os = "windows") {
                "x86_64-pc-windows-msvc".to_string()
            } else if cfg!(target_os = "macos") {
                "x86_64-apple-darwin".to_string()
            } else {
                panic!("Unsupported OS, maybe try setting RA_TARGET")
            }
        }
    }
}

fn exe_suffix(target: &str) -> String {
    if target.contains("-windows-") {
        ".exe".into()
    } else {
        "".into()
    }
}

fn gzip(src_path: &Path, dest_path: &Path) -> Result<()> {
    let mut encoder = GzEncoder::new(File::create(dest_path)?, Compression::best());
    let mut input = io::BufReader::new(File::open(src_path)?);
    io::copy(&mut input, &mut encoder)?;
    encoder.finish()?;
    Ok(())
}

struct Patch {
    path: PathBuf,
    original_contents: String,
    contents: String,
}

impl Patch {
    fn new(path: impl Into<PathBuf>) -> Result<Patch> {
        let path = path.into();
        let contents = read_file(&path)?;
        Ok(Patch { path, original_contents: contents.clone(), contents })
    }

    fn replace(&mut self, from: &str, to: &str) -> &mut Patch {
        assert!(self.contents.contains(from));
        self.contents = self.contents.replace(from, to);
        self
    }

    fn commit(&self) -> Result<()> {
        write_file(&self.path, &self.contents)?;
        Ok(())
    }
}

impl Drop for Patch {
    fn drop(&mut self) {
        write_file(&self.path, &self.original_contents).unwrap();
    }
}
<commit_msg>minor<commit_after>use std::{
    env,
    fs::File,
    io,
    path::{Path, PathBuf},
};

use anyhow::Result;
use flate2::{write::GzEncoder, Compression};
use xshell::{cmd, cp, mkdir_p, pushd, read_file, rm_rf, write_file};

use crate::{date_iso, project_root};

pub(crate) struct DistCmd {
    pub(crate) nightly: bool,
    pub(crate) client_version: Option<String>,
}

impl DistCmd {
    pub(crate) fn run(self) -> Result<()> {
        let dist = project_root().join("dist");
        rm_rf(&dist)?;
        mkdir_p(&dist)?;

        if let Some(version) = self.client_version {
            let release_tag = if self.nightly { "nightly".to_string() } else { date_iso()? };
            dist_client(&version, &release_tag)?;
        }
        dist_server()?;
        Ok(())
    }
}

fn dist_client(version: &str, release_tag: &str) -> Result<()> {
    let _d = pushd("./editors/code")?;
    let nightly = release_tag == "nightly";

    let mut patch = Patch::new("./package.json")?;

    patch
        .replace(r#""version": "0.4.0-dev""#, &format!(r#""version": "{}""#, version))
        .replace(r#""releaseTag": null"#, &format!(r#""releaseTag": "{}""#, release_tag))
        .replace(r#""$generated-start": false,"#, "")
        .replace(",\n                \"$generated-end\": false", "");

    if nightly {
        patch.replace(
            r#""displayName": "rust-analyzer""#,
            r#""displayName": "rust-analyzer (nightly)""#,
        );
    }
    if !nightly {
        patch.replace(r#""enableProposedApi": true,"#, r#""#);
    }
    patch.commit()?;

    cmd!("npm ci").run()?;
    cmd!("npx vsce package -o ../../dist/rust-analyzer.vsix").run()?;
    Ok(())
}

fn dist_server() -> Result<()> {
    let target = get_target();
    if target.contains("-linux-gnu") || target.contains("-linux-musl") {
        env::set_var("CC", "clang");
    }

    cmd!("cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --target {target} --release").run()?;

    let suffix = exe_suffix(&target);
    let src =
        Path::new("target").join(&target).join("release").join(format!("rust-analyzer{}", suffix));
    let dst = Path::new("dist").join(format!("rust-analyzer-{}{}", target, suffix));
    gzip(&src, &dst.with_extension("gz"))?;

    // FIXME: the old names are temporarily kept for client compatibility, but they should be removed
    // Remove this block after a couple of releases
    match target.as_ref() {
        "x86_64-unknown-linux-gnu" => {
            cp(&src, "dist/rust-analyzer-linux")?;
            gzip(&src, Path::new("dist/rust-analyzer-linux.gz"))?;
        }
        "x86_64-pc-windows-msvc" => {
            cp(&src, "dist/rust-analyzer-windows.exe")?;
            gzip(&src, Path::new("dist/rust-analyzer-windows.gz"))?;
        }
        "x86_64-apple-darwin" => {
            cp(&src, "dist/rust-analyzer-mac")?;
            gzip(&src, Path::new("dist/rust-analyzer-mac.gz"))?;
        }
        _ => {}
    }

    Ok(())
}

fn get_target() -> String {
    match env::var("RA_TARGET") {
        Ok(target) => target,
        _ => {
            if cfg!(target_os = "linux") {
                "x86_64-unknown-linux-gnu".to_string()
            } else if cfg!(target_os = "windows") {
                "x86_64-pc-windows-msvc".to_string()
            } else if cfg!(target_os = "macos") {
                "x86_64-apple-darwin".to_string()
            } else {
                panic!("Unsupported OS, maybe try setting RA_TARGET")
            }
        }
    }
}

fn exe_suffix(target: &str) -> String {
    if target.contains("-windows-") {
        ".exe".into()
    } else {
        "".into()
    }
}

fn gzip(src_path: &Path, dest_path: &Path) -> Result<()> {
    let mut encoder = GzEncoder::new(File::create(dest_path)?, Compression::best());
    let mut input = io::BufReader::new(File::open(src_path)?);
    io::copy(&mut input, &mut encoder)?;
    encoder.finish()?;
    Ok(())
}

struct Patch {
    path: PathBuf,
    original_contents: String,
    contents: String,
}

impl Patch {
    fn new(path: impl Into<PathBuf>) -> Result<Patch> {
        let path = path.into();
        let contents = read_file(&path)?;
        Ok(Patch { path, original_contents: contents.clone(), contents })
    }

    fn replace(&mut self, from: &str, to: &str) -> &mut Patch {
        assert!(self.contents.contains(from));
        self.contents = self.contents.replace(from, to);
        self
    }

    fn commit(&self) -> Result<()> {
        write_file(&self.path, &self.contents)?;
        Ok(())
    }
}

impl Drop for Patch {
    fn drop(&mut self) {
        write_file(&self.path, &self.original_contents).unwrap();
    }
}
<|endoftext|>


<commit_before>//! This module handles auto-magic editing actions applied together with users
//! edits. For example, if the user typed
//!
//! ```text
//!     foo
//!         .bar()
//!         .baz()
//!     |   // <- cursor is here
//! ```
//!
//! and types `.` next, we want to indent the dot.
//!
//! Language server executes such typing assists synchronously. That is, they
//! block user's typing and should be pretty fast for this reason!

mod on_enter;

use ide_db::{
    base_db::{FilePosition, SourceDatabase},
    RootDatabase,
};
use syntax::{
    algo::find_node_at_offset,
    ast::{self, edit::IndentLevel, AstToken},
    AstNode, SourceFile,
    SyntaxKind::{FIELD_EXPR, METHOD_CALL_EXPR},
    TextRange, TextSize,
};

use text_edit::TextEdit;

use crate::SourceChange;

pub(crate) use on_enter::on_enter;

// Don't forget to add new trigger characters to `server_capabilities` in `caps.rs`.
pub(crate) const TRIGGER_CHARS: &str = ".=>{";

// Feature: On Typing Assists
//
// Some features trigger on typing certain characters:
//
// - typing `let =` tries to smartly add `;` if `=` is followed by an existing expression
// - typing `.` in a chain method call auto-indents
//
// VS Code::
//
// Add the following to `settings.json`:
// [source,json]
// ----
// "editor.formatOnType": true,
// ----
//
// image::https://user-images.githubusercontent.com/48062697/113166163-69758500-923a-11eb-81ee-eb33ec380399.gif[]
// image::https://user-images.githubusercontent.com/48062697/113171066-105c2000-923f-11eb-87ab-f4a263346567.gif[]
pub(crate) fn on_char_typed(
    db: &RootDatabase,
    position: FilePosition,
    char_typed: char,
) -> Option<SourceChange> {
    assert!(TRIGGER_CHARS.contains(char_typed));
    let file = &db.parse(position.file_id).tree();
    assert_eq!(file.syntax().text().char_at(position.offset), Some(char_typed));
    let edit = on_char_typed_inner(file, position.offset, char_typed)?;
    Some(SourceChange::from_text_edit(position.file_id, edit))
}

fn on_char_typed_inner(file: &SourceFile, offset: TextSize, char_typed: char) -> Option<TextEdit> {
    assert!(TRIGGER_CHARS.contains(char_typed));
    match char_typed {
        '.' => on_dot_typed(file, offset),
        '=' => on_eq_typed(file, offset),
        '>' => on_arrow_typed(file, offset),
        '{' => on_opening_brace_typed(file, offset),
        _ => unreachable!(),
    }
}

/// Inserts a closing `}` when the user types an opening `{`, wrapping an existing expression in a
/// block.
fn on_opening_brace_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    stdx::always!(file.syntax().text().char_at(offset) == Some('{'));
    let brace_token = file.syntax().token_at_offset(offset).right_biased()?;
    let block = ast::BlockExpr::cast(brace_token.parent()?)?;

    // We expect a block expression enclosing exactly 1 preexisting expression. It can be parsed as
    // either the trailing expr or an ExprStmt.
    let offset = {
        match block.statements().next() {
            Some(ast::Stmt::ExprStmt(it)) => {
                // Use the expression span to place `}` before the `;`
                it.expr()?.syntax().text_range().end()
            }
            None => block.tail_expr()?.syntax().text_range().end(),
            _ => return None,
        }
    };

    Some(TextEdit::insert(offset, "}".to_string()))
}

/// Returns an edit which should be applied after `=` was typed. Primarily,
/// this works when adding `let =`.
// FIXME: use a snippet completion instead of this hack here.
fn on_eq_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    stdx::always!(file.syntax().text().char_at(offset) == Some('='));
    let let_stmt: ast::LetStmt = find_node_at_offset(file.syntax(), offset)?;
    if let_stmt.semicolon_token().is_some() {
        return None;
    }
    if let Some(expr) = let_stmt.initializer() {
        let expr_range = expr.syntax().text_range();
        if expr_range.contains(offset) && offset != expr_range.start() {
            return None;
        }
        if file.syntax().text().slice(offset..expr_range.start()).contains_char('\n') {
            return None;
        }
    } else {
        return None;
    }
    let offset = let_stmt.syntax().text_range().end();
    Some(TextEdit::insert(offset, ";".to_string()))
}

/// Returns an edit which should be applied when a dot ('.') is typed on a blank line, indenting the line appropriately.
fn on_dot_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    stdx::always!(file.syntax().text().char_at(offset) == Some('.'));
    let whitespace =
        file.syntax().token_at_offset(offset).left_biased().and_then(ast::Whitespace::cast)?;

    let current_indent = {
        let text = whitespace.text();
        let newline = text.rfind('\n')?;
        &text[newline + 1..]
    };
    let current_indent_len = TextSize::of(current_indent);

    let parent = whitespace.syntax().parent()?;
    // Make sure dot is a part of call chain
    if !matches!(parent.kind(), FIELD_EXPR | METHOD_CALL_EXPR) {
        return None;
    }
    let prev_indent = IndentLevel::from_node(&parent);
    let target_indent = format!("    {}", prev_indent);
    let target_indent_len = TextSize::of(&target_indent);
    if current_indent_len == target_indent_len {
        return None;
    }

    Some(TextEdit::replace(TextRange::new(offset - current_indent_len, offset), target_indent))
}

/// Adds a space after an arrow when `fn foo() { ... }` is turned into `fn foo() -> { ... }`
fn on_arrow_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    let file_text = file.syntax().text();
    stdx::always!(file_text.char_at(offset) == Some('>'));
    let after_arrow = offset + TextSize::of('>');
    if file_text.char_at(after_arrow) != Some('{') {
        return None;
    }
    if find_node_at_offset::<ast::RetType>(file.syntax(), offset).is_none() {
        return None;
    }

    Some(TextEdit::insert(after_arrow, " ".to_string()))
}

#[cfg(test)]
mod tests {
    use test_utils::{assert_eq_text, extract_offset};

    use super::*;

    fn do_type_char(char_typed: char, before: &str) -> Option<String> {
        let (offset, mut before) = extract_offset(before);
        let edit = TextEdit::insert(offset, char_typed.to_string());
        edit.apply(&mut before);
        let parse = SourceFile::parse(&before);
        on_char_typed_inner(&parse.tree(), offset, char_typed).map(|it| {
            it.apply(&mut before);
            before.to_string()
        })
    }

    fn type_char(char_typed: char, ra_fixture_before: &str, ra_fixture_after: &str) {
        let actual = do_type_char(char_typed, ra_fixture_before)
            .unwrap_or_else(|| panic!("typing `{}` did nothing", char_typed));

        assert_eq_text!(ra_fixture_after, &actual);
    }

    fn type_char_noop(char_typed: char, before: &str) {
        let file_change = do_type_char(char_typed, before);
        assert!(file_change.is_none())
    }

    #[test]
    fn test_on_eq_typed() {
        //     do_check(r"
        // fn foo() {
        //     let foo =$0
        // }
        // ", r"
        // fn foo() {
        //     let foo =;
        // }
        // ");
        type_char(
            '=',
            r"
fn foo() {
    let foo $0 1 + 1
}
",
            r"
fn foo() {
    let foo = 1 + 1;
}
",
        );
        //     do_check(r"
        // fn foo() {
        //     let foo =$0
        //     let bar = 1;
        // }
        // ", r"
        // fn foo() {
        //     let foo =;
        //     let bar = 1;
        // }
        // ");
    }

    #[test]
    fn indents_new_chain_call() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                $0
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    $0
            }
            ",
        )
    }

    #[test]
    fn indents_new_chain_call_with_semi() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                $0;
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .;
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    $0;
            }
            ",
        )
    }

    #[test]
    fn indents_new_chain_call_with_let() {
        type_char(
            '.',
            r#"
fn main() {
    let _ = foo
    $0
    bar()
}
"#,
            r#"
fn main() {
    let _ = foo
        .
    bar()
}
"#,
        );
    }

    #[test]
    fn indents_continued_chain_call() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                    .first()
                $0
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .first()
                    .
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    .first()
                    $0
            }
            ",
        );
    }

    #[test]
    fn indents_middle_of_chain_call() {
        type_char(
            '.',
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                $0
                    .nth(92)
                    .unwrap();
            }
            ",
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                    .
                    .nth(92)
                    .unwrap();
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                    $0
                    .nth(92)
                    .unwrap();
            }
            ",
        );
    }

    #[test]
    fn dont_indent_freestanding_dot() {
        type_char_noop(
            '.',
            r"
            fn main() {
                $0
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
            $0
            }
            ",
        );
    }

    #[test]
    fn adds_space_after_return_type() {
        type_char('>', "fn foo() -$0{ 92 }", "fn foo() -> { 92 }")
    }

    #[test]
    fn adds_closing_brace() {
        type_char('{', r"fn f() { match () { _ => $0() } }", r"fn f() { match () { _ => {()} } }");
        type_char('{', r"fn f() { $0(); }", r"fn f() { {()}; }");
        type_char('{', r"fn f() { let x = $0(); }", r"fn f() { let x = {()}; }");
        type_char(
            '{',
            r"
            const S: () = $0();
            fn f() {}
            ",
            r"
            const S: () = {()};
            fn f() {}
            ",
        );
        type_char(
            '{',
            r"
            fn f() {
                match x {
                    0 => $0(),
                    1 => (),
                }
            }",
            r"
            fn f() {
                match x {
                    0 => {()},
                    1 => (),
                }
            }",
        );
    }
}
<commit_msg>simplify<commit_after>//! This module handles auto-magic editing actions applied together with users
//! edits. For example, if the user typed
//!
//! ```text
//!     foo
//!         .bar()
//!         .baz()
//!     |   // <- cursor is here
//! ```
//!
//! and types `.` next, we want to indent the dot.
//!
//! Language server executes such typing assists synchronously. That is, they
//! block user's typing and should be pretty fast for this reason!

mod on_enter;

use ide_db::{
    base_db::{FilePosition, SourceDatabase},
    RootDatabase,
};
use syntax::{
    algo::find_node_at_offset,
    ast::{self, edit::IndentLevel, AstToken},
    AstNode, SourceFile,
    SyntaxKind::{FIELD_EXPR, METHOD_CALL_EXPR},
    TextRange, TextSize,
};

use text_edit::TextEdit;

use crate::SourceChange;

pub(crate) use on_enter::on_enter;

// Don't forget to add new trigger characters to `server_capabilities` in `caps.rs`.
pub(crate) const TRIGGER_CHARS: &str = ".=>{";

// Feature: On Typing Assists
//
// Some features trigger on typing certain characters:
//
// - typing `let =` tries to smartly add `;` if `=` is followed by an existing expression
// - typing `.` in a chain method call auto-indents
//
// VS Code::
//
// Add the following to `settings.json`:
// [source,json]
// ----
// "editor.formatOnType": true,
// ----
//
// image::https://user-images.githubusercontent.com/48062697/113166163-69758500-923a-11eb-81ee-eb33ec380399.gif[]
// image::https://user-images.githubusercontent.com/48062697/113171066-105c2000-923f-11eb-87ab-f4a263346567.gif[]
pub(crate) fn on_char_typed(
    db: &RootDatabase,
    position: FilePosition,
    char_typed: char,
) -> Option<SourceChange> {
    assert!(TRIGGER_CHARS.contains(char_typed));
    let file = &db.parse(position.file_id).tree();
    assert_eq!(file.syntax().text().char_at(position.offset), Some(char_typed));
    let edit = on_char_typed_inner(file, position.offset, char_typed)?;
    Some(SourceChange::from_text_edit(position.file_id, edit))
}

fn on_char_typed_inner(file: &SourceFile, offset: TextSize, char_typed: char) -> Option<TextEdit> {
    assert!(TRIGGER_CHARS.contains(char_typed));
    match char_typed {
        '.' => on_dot_typed(file, offset),
        '=' => on_eq_typed(file, offset),
        '>' => on_arrow_typed(file, offset),
        '{' => on_opening_brace_typed(file, offset),
        _ => unreachable!(),
    }
}

/// Inserts a closing `}` when the user types an opening `{`, wrapping an existing expression in a
/// block.
fn on_opening_brace_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    stdx::always!(file.syntax().text().char_at(offset) == Some('{'));
    let brace_token = file.syntax().token_at_offset(offset).right_biased()?;
    let block = ast::BlockExpr::cast(brace_token.parent()?)?;

    // We expect a block expression enclosing exactly 1 preexisting expression. It can be parsed as
    // either the trailing expr or an ExprStmt.
    let offset = match block.statements().next() {
        Some(ast::Stmt::ExprStmt(it)) => {
            // Use the expression span to place `}` before the `;`
            it.expr()?.syntax().text_range().end()
        }
        None => block.tail_expr()?.syntax().text_range().end(),
        _ => return None,
    };

    Some(TextEdit::insert(offset, "}".to_string()))
}

/// Returns an edit which should be applied after `=` was typed. Primarily,
/// this works when adding `let =`.
// FIXME: use a snippet completion instead of this hack here.
fn on_eq_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    stdx::always!(file.syntax().text().char_at(offset) == Some('='));
    let let_stmt: ast::LetStmt = find_node_at_offset(file.syntax(), offset)?;
    if let_stmt.semicolon_token().is_some() {
        return None;
    }
    if let Some(expr) = let_stmt.initializer() {
        let expr_range = expr.syntax().text_range();
        if expr_range.contains(offset) && offset != expr_range.start() {
            return None;
        }
        if file.syntax().text().slice(offset..expr_range.start()).contains_char('\n') {
            return None;
        }
    } else {
        return None;
    }
    let offset = let_stmt.syntax().text_range().end();
    Some(TextEdit::insert(offset, ";".to_string()))
}

/// Returns an edit which should be applied when a dot ('.') is typed on a blank line, indenting the line appropriately.
fn on_dot_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    stdx::always!(file.syntax().text().char_at(offset) == Some('.'));
    let whitespace =
        file.syntax().token_at_offset(offset).left_biased().and_then(ast::Whitespace::cast)?;

    let current_indent = {
        let text = whitespace.text();
        let newline = text.rfind('\n')?;
        &text[newline + 1..]
    };
    let current_indent_len = TextSize::of(current_indent);

    let parent = whitespace.syntax().parent()?;
    // Make sure dot is a part of call chain
    if !matches!(parent.kind(), FIELD_EXPR | METHOD_CALL_EXPR) {
        return None;
    }
    let prev_indent = IndentLevel::from_node(&parent);
    let target_indent = format!("    {}", prev_indent);
    let target_indent_len = TextSize::of(&target_indent);
    if current_indent_len == target_indent_len {
        return None;
    }

    Some(TextEdit::replace(TextRange::new(offset - current_indent_len, offset), target_indent))
}

/// Adds a space after an arrow when `fn foo() { ... }` is turned into `fn foo() -> { ... }`
fn on_arrow_typed(file: &SourceFile, offset: TextSize) -> Option<TextEdit> {
    let file_text = file.syntax().text();
    stdx::always!(file_text.char_at(offset) == Some('>'));
    let after_arrow = offset + TextSize::of('>');
    if file_text.char_at(after_arrow) != Some('{') {
        return None;
    }
    if find_node_at_offset::<ast::RetType>(file.syntax(), offset).is_none() {
        return None;
    }

    Some(TextEdit::insert(after_arrow, " ".to_string()))
}

#[cfg(test)]
mod tests {
    use test_utils::{assert_eq_text, extract_offset};

    use super::*;

    fn do_type_char(char_typed: char, before: &str) -> Option<String> {
        let (offset, mut before) = extract_offset(before);
        let edit = TextEdit::insert(offset, char_typed.to_string());
        edit.apply(&mut before);
        let parse = SourceFile::parse(&before);
        on_char_typed_inner(&parse.tree(), offset, char_typed).map(|it| {
            it.apply(&mut before);
            before.to_string()
        })
    }

    fn type_char(char_typed: char, ra_fixture_before: &str, ra_fixture_after: &str) {
        let actual = do_type_char(char_typed, ra_fixture_before)
            .unwrap_or_else(|| panic!("typing `{}` did nothing", char_typed));

        assert_eq_text!(ra_fixture_after, &actual);
    }

    fn type_char_noop(char_typed: char, before: &str) {
        let file_change = do_type_char(char_typed, before);
        assert!(file_change.is_none())
    }

    #[test]
    fn test_on_eq_typed() {
        //     do_check(r"
        // fn foo() {
        //     let foo =$0
        // }
        // ", r"
        // fn foo() {
        //     let foo =;
        // }
        // ");
        type_char(
            '=',
            r"
fn foo() {
    let foo $0 1 + 1
}
",
            r"
fn foo() {
    let foo = 1 + 1;
}
",
        );
        //     do_check(r"
        // fn foo() {
        //     let foo =$0
        //     let bar = 1;
        // }
        // ", r"
        // fn foo() {
        //     let foo =;
        //     let bar = 1;
        // }
        // ");
    }

    #[test]
    fn indents_new_chain_call() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                $0
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    $0
            }
            ",
        )
    }

    #[test]
    fn indents_new_chain_call_with_semi() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                $0;
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .;
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    $0;
            }
            ",
        )
    }

    #[test]
    fn indents_new_chain_call_with_let() {
        type_char(
            '.',
            r#"
fn main() {
    let _ = foo
    $0
    bar()
}
"#,
            r#"
fn main() {
    let _ = foo
        .
    bar()
}
"#,
        );
    }

    #[test]
    fn indents_continued_chain_call() {
        type_char(
            '.',
            r"
            fn main() {
                xs.foo()
                    .first()
                $0
            }
            ",
            r"
            fn main() {
                xs.foo()
                    .first()
                    .
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
                xs.foo()
                    .first()
                    $0
            }
            ",
        );
    }

    #[test]
    fn indents_middle_of_chain_call() {
        type_char(
            '.',
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                $0
                    .nth(92)
                    .unwrap();
            }
            ",
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                    .
                    .nth(92)
                    .unwrap();
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn source_impl() {
                let var = enum_defvariant_list().unwrap()
                    $0
                    .nth(92)
                    .unwrap();
            }
            ",
        );
    }

    #[test]
    fn dont_indent_freestanding_dot() {
        type_char_noop(
            '.',
            r"
            fn main() {
                $0
            }
            ",
        );
        type_char_noop(
            '.',
            r"
            fn main() {
            $0
            }
            ",
        );
    }

    #[test]
    fn adds_space_after_return_type() {
        type_char('>', "fn foo() -$0{ 92 }", "fn foo() -> { 92 }")
    }

    #[test]
    fn adds_closing_brace() {
        type_char('{', r"fn f() { match () { _ => $0() } }", r"fn f() { match () { _ => {()} } }");
        type_char('{', r"fn f() { $0(); }", r"fn f() { {()}; }");
        type_char('{', r"fn f() { let x = $0(); }", r"fn f() { let x = {()}; }");
        type_char(
            '{',
            r"
            const S: () = $0();
            fn f() {}
            ",
            r"
            const S: () = {()};
            fn f() {}
            ",
        );
        type_char(
            '{',
            r"
            fn f() {
                match x {
                    0 => $0(),
                    1 => (),
                }
            }",
            r"
            fn f() {
                match x {
                    0 => {()},
                    1 => (),
                }
            }",
        );
    }
}
<|endoftext|>


<commit_before>#[lang="stack_exhausted"]
extern "C" fn stack_exhausted() {}

#[lang="eh_personality"]
extern "C" fn eh_personality() {}

#[no_mangle]
#[allow(non_snake_case)]
pub extern "C" fn _Unwind_Resume(_ex_obj: *mut ()) { }

/// Memcpy
///
/// Copy N bytes of memory from one location to another.
#[no_mangle]
pub unsafe extern fn memcpy(dest: *mut u8, src: *const u8,
                            n: usize) -> *mut u8 {
    let mut i = 0;
    while i < n {
        *dest.offset(i as isize) = *src.offset(i as isize);
        i += 1;
    }

    dest
}

/// Memmove
///
/// Copy N bytes of memory from src to dest. The memory areas may overlap.
#[no_mangle]
pub unsafe extern fn memmove(dest: *mut u8, src: *const u8,
                             n: usize) -> *mut u8 {
    if src < dest as *const u8 {
        let mut i = n;
        while i != 0 {
            i -= 1;
            *dest.offset(i as isize) = *src.offset(i as isize);
        }
    } else {
        let mut i = 0;
        while i < n {
            *dest.offset(i as isize) = *src.offset(i as isize);
            i += 1;
        }
    }

    dest
}

/// Memset
///
/// Fill a block of memory with a specified value.
#[no_mangle]
pub unsafe extern fn memset(s: *mut u8, c: i32, n: usize) -> *mut u8 {
    let mut i = 0;
    while i < n {
        *s.offset(i as isize) = c as u8;
        i += 1;
    }

    s
}

/// Memcmp
///
/// Compare two blocks of memory.
#[no_mangle]
pub unsafe extern fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
    let mut i = 0;

    while i < n {
        let a = *s1.offset(i as isize);
        let b = *s2.offset(i as isize);
        if a != b {
            return a as i32 - b as i32
        }
        i += 1;
    }

    0
}

/// 64 bit remainder on 32 bit arch
#[no_mangle]
#[cfg(target_arch = "x86")]
pub extern "C" fn __umoddi3(mut a: u64, mut b: u64) -> u64 {
    let mut hig = a >> 32; // The first 32 bits of a
    let mut d = 1;

    if hig >= b {
        hig /= b;
        a -= (hig * b) << 32;
    }

    while b > 0 && b < a {
        b *= 2;
        d *= 2;
    }

    loop {
        if a >= b {
            a -= b;
        }
        b >>= 1;
        d >>= 1;

        if d == 0 {
            break;
        }
    }

    a
}

/// 64 bit division on 32 bit arch
#[no_mangle]
#[cfg(target_arch = "x86")]
pub extern "C" fn __udivdi3(mut a: u64, mut b: u64) -> u64 {
    let mut res = 0;
    let mut hig = a >> 32; // The first 32 bits of a
    let mut d = 1;

    if hig >= b {
        hig /= b;
        res = hig << 32;
        a -= (hig * b) << 32;
    }

    while b > 0 && b < a {
        b *= 2;
        d *= 2;
    }

    loop {
        if a >= b {
            a -= b;
            res += d;
        }
        b >>= 1;
        d >>= 1;

        if d == 0 {
            break;
        }
    }

    res
}

#[no_mangle]
#[cfg(target_arch = "x86")]
/// 64 bit division and rem on 32 bit arch
pub extern "C" fn __udivremi3(mut a: u64, mut b: u64) -> (u64, u64) {
    let mut res = 0;
    let mut hig = a >> 32; // The first 32 bits of a
    let mut d = 1;

    if hig >= b {
        hig /= b;
        res = hig << 32;
        a -= (hig * b) << 32;
    }

    while b > 0 && b < a {
        b *= 2;
        d *= 2;
    }

    loop {
        if a >= b {
            a -= b;
            res += d;
        }
        b >>= 1;
        d >>= 1;

        if d == 0 {
            break;
        }
    }

    (res, a)
}
<commit_msg>Remove stack_exhausted<commit_after>#[lang="eh_personality"]
extern "C" fn eh_personality() {}

#[no_mangle]
#[allow(non_snake_case)]
pub extern "C" fn _Unwind_Resume(_ex_obj: *mut ()) { }

/// Memcpy
///
/// Copy N bytes of memory from one location to another.
#[no_mangle]
pub unsafe extern fn memcpy(dest: *mut u8, src: *const u8,
                            n: usize) -> *mut u8 {
    let mut i = 0;
    while i < n {
        *dest.offset(i as isize) = *src.offset(i as isize);
        i += 1;
    }

    dest
}

/// Memmove
///
/// Copy N bytes of memory from src to dest. The memory areas may overlap.
#[no_mangle]
pub unsafe extern fn memmove(dest: *mut u8, src: *const u8,
                             n: usize) -> *mut u8 {
    if src < dest as *const u8 {
        let mut i = n;
        while i != 0 {
            i -= 1;
            *dest.offset(i as isize) = *src.offset(i as isize);
        }
    } else {
        let mut i = 0;
        while i < n {
            *dest.offset(i as isize) = *src.offset(i as isize);
            i += 1;
        }
    }

    dest
}

/// Memset
///
/// Fill a block of memory with a specified value.
#[no_mangle]
pub unsafe extern fn memset(s: *mut u8, c: i32, n: usize) -> *mut u8 {
    let mut i = 0;
    while i < n {
        *s.offset(i as isize) = c as u8;
        i += 1;
    }

    s
}

/// Memcmp
///
/// Compare two blocks of memory.
#[no_mangle]
pub unsafe extern fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 {
    let mut i = 0;

    while i < n {
        let a = *s1.offset(i as isize);
        let b = *s2.offset(i as isize);
        if a != b {
            return a as i32 - b as i32
        }
        i += 1;
    }

    0
}

/// 64 bit remainder on 32 bit arch
#[no_mangle]
#[cfg(target_arch = "x86")]
pub extern "C" fn __umoddi3(mut a: u64, mut b: u64) -> u64 {
    let mut hig = a >> 32; // The first 32 bits of a
    let mut d = 1;

    if hig >= b {
        hig /= b;
        a -= (hig * b) << 32;
    }

    while b > 0 && b < a {
        b *= 2;
        d *= 2;
    }

    loop {
        if a >= b {
            a -= b;
        }
        b >>= 1;
        d >>= 1;

        if d == 0 {
            break;
        }
    }

    a
}

/// 64 bit division on 32 bit arch
#[no_mangle]
#[cfg(target_arch = "x86")]
pub extern "C" fn __udivdi3(mut a: u64, mut b: u64) -> u64 {
    let mut res = 0;
    let mut hig = a >> 32; // The first 32 bits of a
    let mut d = 1;

    if hig >= b {
        hig /= b;
        res = hig << 32;
        a -= (hig * b) << 32;
    }

    while b > 0 && b < a {
        b *= 2;
        d *= 2;
    }

    loop {
        if a >= b {
            a -= b;
            res += d;
        }
        b >>= 1;
        d >>= 1;

        if d == 0 {
            break;
        }
    }

    res
}

#[no_mangle]
#[cfg(target_arch = "x86")]
/// 64 bit division and rem on 32 bit arch
pub extern "C" fn __udivremi3(mut a: u64, mut b: u64) -> (u64, u64) {
    let mut res = 0;
    let mut hig = a >> 32; // The first 32 bits of a
    let mut d = 1;

    if hig >= b {
        hig /= b;
        res = hig << 32;
        a -= (hig * b) << 32;
    }

    while b > 0 && b < a {
        b *= 2;
        d *= 2;
    }

    loop {
        if a >= b {
            a -= b;
            res += d;
        }
        b >>= 1;
        d >>= 1;

        if d == 0 {
            break;
        }
    }

    (res, a)
}
<|endoftext|>


<commit_before>//
//  0 1 0  Mnemosyne: a functional systems programming language.
//  0 0 1  (c) 2015 Hawk Weisman
//  1 1 1  hi@hawkweisman.me
//
//  Mnemosyne is released under the MIT License. Please refer to
//  the LICENSE file at the top-level directory of this distribution
//  or at https://github.com/hawkw/mnemosyne/.
//
//! Mnemosyne semantic analysis
use std::iter;
use std::fmt::Write;
use std::rc::Rc;

use ::forktable::ForkTable;

use ast::*;
use self::annotations::*;
use self::types::Type;

/// A symbol table is a `ForkTable` mapping `String`s to `Type` annotations.
///
/// As of `84f1de8`, symbol annotations can be type annotations for values
/// or annotations for type definitions. This is provisional; type defs
/// may be moved to a separate table later.
///
/// This table should be forked upon entering a new scope.
pub type SymbolTable<'a> = ForkTable<'a, String, SymbolAnnotation<'a>>;

#[macro_use]
macro_rules! indent {
    ($to:expr) => ( iter::repeat('\t')
                        .take($to)
                        .collect::<String>() )
}

pub mod ast;
pub mod types;
pub mod annotations;

impl<'a> AnnotateTypes<'a> for Unscoped<'a, Form<'a, UnscopedState>> {
    #[allow(unused_variables)]
    fn annotate_types(self, scope: SymbolTable) -> Scoped<'a, Self> {
        unimplemented!() //TODO: implement
    }
}

impl<'a> AnnotateTypes<'a> for Unscoped<'a, DefForm<'a, UnscopedState>> {
    #[allow(unused_variables)]
    fn annotate_types(self, scope: SymbolTable) -> Scoped<'a, Self>{
        unimplemented!() //TODO: implement
    }
}

#[derive(Clone,Debug,PartialEq)]
pub enum SymbolAnnotation<'a> {
    TypeDef(Type)
  , Value { /// The type of the symbol
            ty: Type,
            /// An optional proven value for the symbol.
            ///
            /// This should be defined iff the symbol signifies a constant value
            /// or a constant expression, or if we were able to prove that the value
            /// remains constant within the current scope.
            proven_value: Option<Rc<Expr<'a, ScopedState>>>
          }
}

impl<'a> SymbolAnnotation<'a> {
    // TODO: possibly this should recurse into references? IDK
    pub fn is_fn_type(&self) -> bool {
        match *self {
            SymbolAnnotation::TypeDef(ref ty) =>
                match *ty { Type::Function(_) => true
                          , _                => false
                          }
          , SymbolAnnotation::Value { ref ty, .. } =>
                match *ty { Type::Function(_) => true
                          , _                => false
                          }
        }
    }

    pub fn is_prim_type(&self) -> bool {
        match *self {
            SymbolAnnotation::TypeDef(ref ty) =>
                match *ty { Type::Prim(_) => true
                          , _             => false
                          }
          , _ => false
        }
    }
}
<commit_msg>Remove some unused imports<commit_after>//
//  0 1 0  Mnemosyne: a functional systems programming language.
//  0 0 1  (c) 2015 Hawk Weisman
//  1 1 1  hi@hawkweisman.me
//
//  Mnemosyne is released under the MIT License. Please refer to
//  the LICENSE file at the top-level directory of this distribution
//  or at https://github.com/hawkw/mnemosyne/.
//
//! Mnemosyne semantic analysis
use std::rc::Rc;

use ::forktable::ForkTable;

use ast::*;
use self::annotations::*;
use self::types::Type;

/// A symbol table is a `ForkTable` mapping `String`s to `Type` annotations.
///
/// As of `84f1de8`, symbol annotations can be type annotations for values
/// or annotations for type definitions. This is provisional; type defs
/// may be moved to a separate table later.
///
/// This table should be forked upon entering a new scope.
pub type SymbolTable<'a> = ForkTable<'a, String, SymbolAnnotation<'a>>;

#[macro_use]
macro_rules! indent {
    ($to:expr) => ( iter::repeat('\t')
                        .take($to)
                        .collect::<String>() )
}

pub mod ast;
pub mod types;
pub mod annotations;

impl<'a> AnnotateTypes<'a> for Unscoped<'a, Form<'a, UnscopedState>> {
    #[allow(unused_variables)]
    fn annotate_types(self, scope: SymbolTable) -> Scoped<'a, Self> {
        unimplemented!() //TODO: implement
    }
}

impl<'a> AnnotateTypes<'a> for Unscoped<'a, DefForm<'a, UnscopedState>> {
    #[allow(unused_variables)]
    fn annotate_types(self, scope: SymbolTable) -> Scoped<'a, Self>{
        unimplemented!() //TODO: implement
    }
}

#[derive(Clone,Debug,PartialEq)]
pub enum SymbolAnnotation<'a> {
    TypeDef(Type)
  , Value { /// The type of the symbol
            ty: Type,
            /// An optional proven value for the symbol.
            ///
            /// This should be defined iff the symbol signifies a constant value
            /// or a constant expression, or if we were able to prove that the value
            /// remains constant within the current scope.
            proven_value: Option<Rc<Expr<'a, ScopedState>>>
          }
}

impl<'a> SymbolAnnotation<'a> {
    // TODO: possibly this should recurse into references? IDK
    pub fn is_fn_type(&self) -> bool {
        match *self {
            SymbolAnnotation::TypeDef(ref ty) =>
                match *ty { Type::Function(_) => true
                          , _                => false
                          }
          , SymbolAnnotation::Value { ref ty, .. } =>
                match *ty { Type::Function(_) => true
                          , _                => false
                          }
        }
    }

    pub fn is_prim_type(&self) -> bool {
        match *self {
            SymbolAnnotation::TypeDef(ref ty) =>
                match *ty { Type::Prim(_) => true
                          , _             => false
                          }
          , _ => false
        }
    }
}
<|endoftext|>


<commit_before>use alloc::boxed::Box;

use intrusive::queue::Queue;

use hal::local_irq_disable;

use sync::spin::{InterruptSpinLock, InterruptSpinGuard};

use super::{Thread, ThreadImpl, Scheduler};

pub type InternalQueue = Queue<Box<ThreadImpl>, ThreadImpl>;

#[macro_export]
/// Wait for an event to occur
///
/// This macro allows a thread to wait for a condition to become true.
/// If the condition is false, the thread will block on the [`queue`][queue]
/// given as parameter. This macro *MUST* be called with local irqs enabled.
///
/// [queue]: thread/struct.WaitQueue.html
///
/// Note that this macro does not signify exclusivity. In fact multiple
/// concurrent threads might go through. In that case external atomic (or
/// locked) test on the condition might be necessary.
macro_rules! wait_event {
    ($queue:expr, $cond:expr) => (
        loop {
            $crate::hal::local_irq_disable();

            let locked_queue = $queue.lock();

            if $cond {
                $crate::hal::local_irq_enable();
                break;
            }

            Scheduler::block(locked_queue);
        }
    )
}

pub struct WaitQueue {
    queue: InterruptSpinLock<InternalQueue>,
}

impl WaitQueue {
    /// Create a new WaitQueue
    pub fn new() -> Self {
        WaitQueue {
            queue: InterruptSpinLock::new(Queue::new()),
        }
    }

    #[doc(hidden)]
    pub fn lock(&self) -> InterruptSpinGuard<InternalQueue> {
        self.queue.lock()
    }

    #[inline]
    /// Block the current thread
    pub fn block(&self) {
        local_irq_disable();
        Scheduler::block(self.queue.lock());
    }

    #[inline]
    fn unblock_thread(imp: Box<ThreadImpl>) {
        let thread = Thread {
            t_impl: imp,
        };

        Scheduler::ready(thread);
    }

    /// Unblock the first thread in the queue.
    /// Returns true if a thread was unblocked, false otherwise
    pub fn unblock(&self) {
        if let Some(t) = self.queue.lock().dequeue() {
            WaitQueue::unblock_thread(t);
        }
    }

    /// Unblock all threads in the queue
    pub fn unblock_all(&self) {
        let mut queue = self.queue.lock();

        while let Some(t) = queue.dequeue() {
            WaitQueue::unblock_thread(t);
        }
    }
}

unsafe impl Send for WaitQueue {}
unsafe impl Sync for WaitQueue {}
<commit_msg>thread: Correct build error on wait_event!()<commit_after>use alloc::boxed::Box;

use intrusive::queue::Queue;

use hal::local_irq_disable;

use sync::spin::{InterruptSpinLock, InterruptSpinGuard};

use super::{Thread, ThreadImpl, Scheduler};

pub type InternalQueue = Queue<Box<ThreadImpl>, ThreadImpl>;

#[macro_export]
/// Wait for an event to occur
///
/// This macro allows a thread to wait for a condition to become true.
/// If the condition is false, the thread will block on the [`queue`][queue]
/// given as parameter. This macro *MUST* be called with local irqs enabled.
///
/// [queue]: thread/struct.WaitQueue.html
///
/// Note that this macro does not signify exclusivity. In fact multiple
/// concurrent threads might go through. In that case external atomic (or
/// locked) test on the condition might be necessary.
macro_rules! wait_event {
    ($queue:expr, $cond:expr) => (
        loop {
            $crate::hal::local_irq_disable();

            let locked_queue = $queue.lock();

            if $cond {
                $crate::hal::local_irq_enable();
                break;
            }

            $crate::thread::Scheduler::block(locked_queue);
        }
    )
}

pub struct WaitQueue {
    queue: InterruptSpinLock<InternalQueue>,
}

impl WaitQueue {
    /// Create a new WaitQueue
    pub fn new() -> Self {
        WaitQueue {
            queue: InterruptSpinLock::new(Queue::new()),
        }
    }

    #[doc(hidden)]
    pub fn lock(&self) -> InterruptSpinGuard<InternalQueue> {
        self.queue.lock()
    }

    #[inline]
    /// Block the current thread
    pub fn block(&self) {
        local_irq_disable();
        Scheduler::block(self.queue.lock());
    }

    #[inline]
    fn unblock_thread(imp: Box<ThreadImpl>) {
        let thread = Thread {
            t_impl: imp,
        };

        Scheduler::ready(thread);
    }

    /// Unblock the first thread in the queue.
    /// Returns true if a thread was unblocked, false otherwise
    pub fn unblock(&self) {
        if let Some(t) = self.queue.lock().dequeue() {
            WaitQueue::unblock_thread(t);
        }
    }

    /// Unblock all threads in the queue
    pub fn unblock_all(&self) {
        let mut queue = self.queue.lock();

        while let Some(t) = queue.dequeue() {
            WaitQueue::unblock_thread(t);
        }
    }
}

unsafe impl Send for WaitQueue {}
unsafe impl Sync for WaitQueue {}
<|endoftext|>


<commit_before>// Copyright 2014 Strahinja Val Markovic
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![feature(collections)]
#![feature(unicode)]
#![feature(slice_patterns)]
#![allow(non_snake_case)]
#![allow(unused_attributes)]
#![cfg_attr(test, allow(dead_code))]
#![deny(deprecated)]

extern crate getopts;
extern crate tempdir;
extern crate inlined_parser;

use tempdir::TempDir;
use getopts::Options;
use std::env;
use std::io;
use std::io::Read;
use std::io::Write;
use std::fs::File;
use std::process::Command;
use std::path::Path;
use self::prelude::PRELUDE;
use self::printer::PRINTER_MAIN;
use inlined_parser::{parse, Node};
use std::str;
use std::iter::repeat;

mod generator;
mod prelude;
mod printer;

static TOP_LEVEL_RULE : &'static str = "NGTOP_LEVEL_RULE";

#[derive(Debug)]
enum CliError {
  Io( io::Error ),
  Misc( String ),
}

impl From<io::Error> for CliError {
  fn from( error: io::Error ) -> CliError {
    CliError::Io( error )
  }
}

impl From<String> for CliError {
  fn from( error: String ) -> CliError {
    CliError::Misc( error )
  }
}

fn inputFromFile( input_file: &str ) -> Result<Vec<u8>, CliError> {
  File::open( &Path::new( input_file ) ).and_then( |mut file| {
    let mut data: Vec<u8> = vec!();
    file.read_to_end( &mut data ).map( |_| data )
  }).map_err( |_| CliError::Misc( format!( "Couldn't read: {}", input_file ) ) )
}

fn indentLines( input: &str, num_spaces: usize ) -> String {
  let indent: String = repeat( " " ).take( num_spaces ).collect();
  input.split( '\n' ).map(
    |x| indent.clone() + x + "\n" ).collect::<Vec<String>>().concat()
}

fn printUsage( opts: &getopts::Options ) {
  let program_path = env::args().next().unwrap();
  let program = Path::new( &program_path );
  let short = opts.short_usage(
    program.file_name().unwrap().to_str().unwrap() );
  let usage = opts.usage( &short );
  println!( "{}", usage );
}

fn nameOfFirstRule<'a>( root: &'a Node<'a> ) -> String {
  str::from_utf8(
    &root.preOrder().find( |x| x.name == "Identifier" ).unwrap()
        .matchedData() ).unwrap().trim_matches(' ').to_string()
}

fn codeForGrammar( input: &[u8] ) -> Result<String, CliError> {
  parse( input ).map( |node| {
    let parse_rules = indentLines( &generator::codeForNode( &node ), 2 );
    let prepared_prelude = PRELUDE[ .. PRELUDE.len() -1 ].replace(
      TOP_LEVEL_RULE,
      &nameOfFirstRule( &node ) );

    prepared_prelude + "\n" + &parse_rules + "}"
  } ).ok_or( CliError::Misc( "Failed to parse PEG grammar".to_string() ) )
}

// Returns exit code
fn printParseTree( grammar_code: &str, input_path: &str )
     -> Result<i32, CliError> {
  let final_code = grammar_code.to_owned() + PRINTER_MAIN;
  let temp_dir = TempDir::new( "temp" ).unwrap();
  let code_file = temp_dir.path().join( "printer.rs" );
  let printer = temp_dir.path().join( "printer" );

  try!( File::create( &code_file ).and_then( |mut file| {
    file.write_all( final_code.as_bytes() )
  } ) );

  let status = try!( Command::new( "rustc" ).arg( "-o" )
                     .arg( printer.to_str().unwrap() )
                     .arg( code_file.to_str().unwrap() )
                     .status() );

  if !status.success() {
    return Err( CliError::Misc( "Failed to write code file".to_string() ) )
  }

  let printer = temp_dir.path().join( "printer" );
  let output = try!( Command::new( printer.to_str().unwrap() )
                     .arg( input_path ).output() );

  println!( "{}", String::from_utf8_lossy( &output.stdout ) );
  output.status.code().ok_or(
    CliError::Misc( "No status code for process.".to_string() ) )
}

#[cfg(not(test))]
fn main() {
  let mut opts = Options::new();
  opts.optflag( "h", "help", "Print this help menu." );
  opts.optopt( "i", "input",
               "Path to input file to parse with grammar given to -g option",
               "FILE" );
  opts.optopt( "g", "grammar",
               "Path to PEG grammar. Prints code for grammar if -i not given.",
               "FILE" );

  let args: Vec<_> = env::args().collect();
  let matches = opts.parse( args.tail() ).unwrap();
  if matches.opt_present( "h" ) || args.len() < 2 {
    printUsage( &opts );
    return;
  }

  let exit_code = matches.opt_str( "g" )
    .ok_or( CliError::Misc( "Missing -g option".to_string() ) )
    .and_then( |file| inputFromFile( &file ) )
    .and_then( |input| codeForGrammar( &input ) )
    .and_then( |grammar_code| {
      if let Some( input_path ) = matches.opt_str( "i" ) {
        printParseTree( &grammar_code, &input_path )
      } else {
        println!( "{}", grammar_code );
        Ok( 0 )
      }
    } );

  process::exit( exit_code.unwrap_or_else( |error| {
    println!( "{:?}", error );
    1
  } ) );
}

<commit_msg>Fixing compilation<commit_after>// Copyright 2014 Strahinja Val Markovic
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![feature(collections)]
#![feature(unicode)]
#![feature(slice_patterns)]
#![allow(non_snake_case)]
#![allow(unused_attributes)]
#![cfg_attr(test, allow(dead_code))]
#![deny(deprecated)]

extern crate getopts;
extern crate tempdir;
extern crate inlined_parser;

use tempdir::TempDir;
use getopts::Options;
use std::env;
use std::io;
use std::io::Read;
use std::io::Write;
use std::fs::File;
use std::process::Command;
use std::path::Path;
use self::prelude::PRELUDE;
use self::printer::PRINTER_MAIN;
use inlined_parser::{parse, Node};
use std::str;
use std::iter::repeat;

mod generator;
mod prelude;
mod printer;

static TOP_LEVEL_RULE : &'static str = "NGTOP_LEVEL_RULE";

#[derive(Debug)]
enum CliError {
  Io( io::Error ),
  Misc( String ),
}

impl From<io::Error> for CliError {
  fn from( error: io::Error ) -> CliError {
    CliError::Io( error )
  }
}

impl From<String> for CliError {
  fn from( error: String ) -> CliError {
    CliError::Misc( error )
  }
}

fn inputFromFile( input_file: &str ) -> Result<Vec<u8>, CliError> {
  File::open( &Path::new( input_file ) ).and_then( |mut file| {
    let mut data: Vec<u8> = vec!();
    file.read_to_end( &mut data ).map( |_| data )
  }).map_err( |_| CliError::Misc( format!( "Couldn't read: {}", input_file ) ) )
}

fn indentLines( input: &str, num_spaces: usize ) -> String {
  let indent: String = repeat( " " ).take( num_spaces ).collect();
  input.split( '\n' ).map(
    |x| indent.clone() + x + "\n" ).collect::<Vec<String>>().concat()
}

fn printUsage( opts: &getopts::Options ) {
  let program_path = env::args().next().unwrap();
  let program = Path::new( &program_path );
  let short = opts.short_usage(
    program.file_name().unwrap().to_str().unwrap() );
  let usage = opts.usage( &short );
  println!( "{}", usage );
}

fn nameOfFirstRule<'a>( root: &'a Node<'a> ) -> String {
  str::from_utf8(
    &root.preOrder().find( |x| x.name == "Identifier" ).unwrap()
        .matchedData() ).unwrap().trim_matches(' ').to_string()
}

fn codeForGrammar( input: &[u8] ) -> Result<String, CliError> {
  parse( input ).map( |node| {
    let parse_rules = indentLines( &generator::codeForNode( &node ), 2 );
    let prepared_prelude = PRELUDE[ .. PRELUDE.len() -1 ].replace(
      TOP_LEVEL_RULE,
      &nameOfFirstRule( &node ) );

    prepared_prelude + "\n" + &parse_rules + "}"
  } ).ok_or( CliError::Misc( "Failed to parse PEG grammar".to_string() ) )
}

// Returns exit code
fn printParseTree( grammar_code: &str, input_path: &str )
     -> Result<i32, CliError> {
  let final_code = grammar_code.to_owned() + PRINTER_MAIN;
  let temp_dir = TempDir::new( "temp" ).unwrap();
  let code_file = temp_dir.path().join( "printer.rs" );
  let printer = temp_dir.path().join( "printer" );

  try!( File::create( &code_file ).and_then( |mut file| {
    file.write_all( final_code.as_bytes() )
  } ) );

  let status = try!( Command::new( "rustc" ).arg( "-o" )
                     .arg( printer.to_str().unwrap() )
                     .arg( code_file.to_str().unwrap() )
                     .status() );

  if !status.success() {
    return Err( CliError::Misc( "Failed to write code file".to_string() ) )
  }

  let printer = temp_dir.path().join( "printer" );
  let output = try!( Command::new( printer.to_str().unwrap() )
                     .arg( input_path ).output() );

  println!( "{}", String::from_utf8_lossy( &output.stdout ) );
  output.status.code().ok_or(
    CliError::Misc( "No status code for process.".to_string() ) )
}

#[cfg(not(test))]
fn main() {
  let mut opts = Options::new();
  opts.optflag( "h", "help", "Print this help menu." );
  opts.optopt( "i", "input",
               "Path to input file to parse with grammar given to -g option",
               "FILE" );
  opts.optopt( "g", "grammar",
               "Path to PEG grammar. Prints code for grammar if -i not given.",
               "FILE" );

  let args: Vec<_> = env::args().collect();
  let matches = opts.parse( args.tail() ).unwrap();
  if matches.opt_present( "h" ) || args.len() < 2 {
    printUsage( &opts );
    return;
  }

  let exit_code = matches.opt_str( "g" )
    .ok_or( CliError::Misc( "Missing -g option".to_string() ) )
    .and_then( |file| inputFromFile( &file ) )
    .and_then( |input| codeForGrammar( &input ) )
    .and_then( |grammar_code| {
      if let Some( input_path ) = matches.opt_str( "i" ) {
        printParseTree( &grammar_code, &input_path )
      } else {
        println!( "{}", grammar_code );
        Ok( 0 )
      }
    } );

  std::process::exit( exit_code.unwrap_or_else( |error| {
    println!( "{:?}", error );
    1
  } ) );
}

<|endoftext|>


<commit_before>//! `mbe` (short for Macro By Example) crate contains code for handling
//! `macro_rules` macros. It uses `TokenTree` (from `ra_tt` package) as the
//! interface, although it contains some code to bridge `SyntaxNode`s and
//! `TokenTree`s as well!

mod parser;
mod mbe_expander;
mod syntax_bridge;
mod tt_iter;
mod subtree_source;

pub use tt::{Delimiter, Punct};

use crate::{
    parser::{parse_pattern, Op},
    tt_iter::TtIter,
};

#[derive(Debug, PartialEq, Eq)]
pub enum ParseError {
    Expected(String),
}

#[derive(Debug, PartialEq, Eq)]
pub enum ExpandError {
    NoMatchingRule,
    UnexpectedToken,
    BindingError(String),
    ConversionError,
    InvalidRepeat,
}

pub use crate::syntax_bridge::{
    ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_expr, token_tree_to_items,
    token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty,
};

/// This struct contains AST for a single `macro_rules` definition. What might
/// be very confusing is that AST has almost exactly the same shape as
/// `tt::TokenTree`, but there's a crucial difference: in macro rules, `$ident`
/// and `$()*` have special meaning (see `Var` and `Repeat` data structures)
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MacroRules {
    pub(crate) rules: Vec<Rule>,
    /// Highest id of the token we have in TokenMap
    pub(crate) shift: u32,
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Rule {
    pub(crate) lhs: tt::Subtree,
    pub(crate) rhs: tt::Subtree,
}

// Find the max token id inside a subtree
fn max_id(subtree: &tt::Subtree) -> Option<u32> {
    subtree
        .token_trees
        .iter()
        .filter_map(|tt| match tt {
            tt::TokenTree::Subtree(subtree) => max_id(subtree),
            tt::TokenTree::Leaf(tt::Leaf::Ident(ident))
                if ident.id != tt::TokenId::unspecified() =>
            {
                Some(ident.id.0)
            }
            _ => None,
        })
        .max()
}

/// Shift given TokenTree token id
fn shift_subtree(tt: &mut tt::Subtree, shift: u32) {
    for t in tt.token_trees.iter_mut() {
        match t {
            tt::TokenTree::Leaf(leaf) => match leaf {
                tt::Leaf::Ident(ident) if ident.id != tt::TokenId::unspecified() => {
                    ident.id.0 += shift;
                }
                _ => (),
            },
            tt::TokenTree::Subtree(tt) => shift_subtree(tt, shift),
        }
    }
}

impl MacroRules {
    pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> {
        // Note: this parsing can be implemented using mbe machinery itself, by
        // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing
        // manually seems easier.
        let mut src = TtIter::new(tt);
        let mut rules = Vec::new();
        while src.len() > 0 {
            let rule = Rule::parse(&mut src)?;
            rules.push(rule);
            if let Err(()) = src.expect_char(';') {
                if src.len() > 0 {
                    return Err(ParseError::Expected("expected `:`".to_string()));
                }
                break;
            }
        }

        for rule in rules.iter() {
            validate(&rule.lhs)?;
        }

        // Note that TokenId is started from zero,
        // We have to add 1 to prevent duplication.
        let shift = max_id(tt).unwrap_or(0) + 1;
        Ok(MacroRules { rules, shift })
    }

    pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
        // apply shift
        let mut tt = tt.clone();
        shift_subtree(&mut tt, self.shift);
        mbe_expander::expand(self, &tt)
    }
}

impl Rule {
    fn parse(src: &mut TtIter) -> Result<Rule, ParseError> {
        let mut lhs = src
            .expect_subtree()
            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
            .clone();
        lhs.delimiter = tt::Delimiter::None;
        src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
        src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
        let mut rhs = src
            .expect_subtree()
            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
            .clone();
        rhs.delimiter = tt::Delimiter::None;
        Ok(crate::Rule { lhs, rhs })
    }
}

fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> {
    for op in parse_pattern(pattern) {
        let op = match op {
            Ok(it) => it,
            Err(e) => {
                let msg = match e {
                    ExpandError::InvalidRepeat => "invalid repeat".to_string(),
                    _ => "invalid macro definition".to_string(),
                };
                return Err(ParseError::Expected(msg));
            }
        };
        match op {
            Op::TokenTree(tt::TokenTree::Subtree(subtree)) | Op::Repeat { subtree, .. } => {
                validate(subtree)?
            }
            _ => (),
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests;
<commit_msg>Change to  add 1 if non zero shift<commit_after>//! `mbe` (short for Macro By Example) crate contains code for handling
//! `macro_rules` macros. It uses `TokenTree` (from `ra_tt` package) as the
//! interface, although it contains some code to bridge `SyntaxNode`s and
//! `TokenTree`s as well!

mod parser;
mod mbe_expander;
mod syntax_bridge;
mod tt_iter;
mod subtree_source;

pub use tt::{Delimiter, Punct};

use crate::{
    parser::{parse_pattern, Op},
    tt_iter::TtIter,
};

#[derive(Debug, PartialEq, Eq)]
pub enum ParseError {
    Expected(String),
}

#[derive(Debug, PartialEq, Eq)]
pub enum ExpandError {
    NoMatchingRule,
    UnexpectedToken,
    BindingError(String),
    ConversionError,
    InvalidRepeat,
}

pub use crate::syntax_bridge::{
    ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_expr, token_tree_to_items,
    token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty,
};

/// This struct contains AST for a single `macro_rules` definition. What might
/// be very confusing is that AST has almost exactly the same shape as
/// `tt::TokenTree`, but there's a crucial difference: in macro rules, `$ident`
/// and `$()*` have special meaning (see `Var` and `Repeat` data structures)
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MacroRules {
    pub(crate) rules: Vec<Rule>,
    /// Highest id of the token we have in TokenMap
    pub(crate) shift: u32,
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct Rule {
    pub(crate) lhs: tt::Subtree,
    pub(crate) rhs: tt::Subtree,
}

// Find the max token id inside a subtree
fn max_id(subtree: &tt::Subtree) -> Option<u32> {
    subtree
        .token_trees
        .iter()
        .filter_map(|tt| match tt {
            tt::TokenTree::Subtree(subtree) => max_id(subtree),
            tt::TokenTree::Leaf(tt::Leaf::Ident(ident))
                if ident.id != tt::TokenId::unspecified() =>
            {
                Some(ident.id.0)
            }
            _ => None,
        })
        .max()
}

/// Shift given TokenTree token id
fn shift_subtree(tt: &mut tt::Subtree, shift: u32) {
    for t in tt.token_trees.iter_mut() {
        match t {
            tt::TokenTree::Leaf(leaf) => match leaf {
                tt::Leaf::Ident(ident) if ident.id != tt::TokenId::unspecified() => {
                    ident.id.0 += shift;
                }
                _ => (),
            },
            tt::TokenTree::Subtree(tt) => shift_subtree(tt, shift),
        }
    }
}

impl MacroRules {
    pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> {
        // Note: this parsing can be implemented using mbe machinery itself, by
        // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing
        // manually seems easier.
        let mut src = TtIter::new(tt);
        let mut rules = Vec::new();
        while src.len() > 0 {
            let rule = Rule::parse(&mut src)?;
            rules.push(rule);
            if let Err(()) = src.expect_char(';') {
                if src.len() > 0 {
                    return Err(ParseError::Expected("expected `:`".to_string()));
                }
                break;
            }
        }

        for rule in rules.iter() {
            validate(&rule.lhs)?;
        }

        // Note that TokenId is started from zero,
        // We have to add 1 to prevent duplication.
        let shift = max_id(tt).map_or(0, |it| it + 1);
        Ok(MacroRules { rules, shift })
    }

    pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
        // apply shift
        let mut tt = tt.clone();
        shift_subtree(&mut tt, self.shift);
        mbe_expander::expand(self, &tt)
    }
}

impl Rule {
    fn parse(src: &mut TtIter) -> Result<Rule, ParseError> {
        let mut lhs = src
            .expect_subtree()
            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
            .clone();
        lhs.delimiter = tt::Delimiter::None;
        src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
        src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
        let mut rhs = src
            .expect_subtree()
            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
            .clone();
        rhs.delimiter = tt::Delimiter::None;
        Ok(crate::Rule { lhs, rhs })
    }
}

fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> {
    for op in parse_pattern(pattern) {
        let op = match op {
            Ok(it) => it,
            Err(e) => {
                let msg = match e {
                    ExpandError::InvalidRepeat => "invalid repeat".to_string(),
                    _ => "invalid macro definition".to_string(),
                };
                return Err(ParseError::Expected(msg));
            }
        };
        match op {
            Op::TokenTree(tt::TokenTree::Subtree(subtree)) | Op::Repeat { subtree, .. } => {
                validate(subtree)?
            }
            _ => (),
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests;
<|endoftext|>


<commit_before>//! Updates crates.io index and builds new packages

use super::{DocBuilder, RustwideBuilder};
use crate::db::connect_db;
use crate::error::Result;
use crate::utils::{add_crate_to_queue, get_crate_priority};
use crates_index_diff::{ChangeKind, Index};
use log::{debug, error};

impl DocBuilder {
    /// Updates crates.io-index repository and adds new crates into build queue.
    /// Returns size of queue
    pub fn get_new_crates(&mut self) -> Result<usize> {
        let conn = connect_db()?;
        let index = Index::from_path_or_cloned(&self.options.crates_io_index_path)?;
        let mut changes = index.fetch_changes()?;
        let mut add_count: usize = 0;

        // I belive this will fix ordering of queue if we get more than one crate from changes
        changes.reverse();

        for krate in &changes {
            match krate.kind {
                ChangeKind::Yanked => {
                    // FIXME: remove built doc files? set build as failed?
                    conn.execute(
                        "UPDATE releases SET yanked = TRUE FROM crates WHERE \
                                  crates.id = releases.crate_id AND name = $1 AND version = $2",
                        &[&krate.name, &krate.version],
                    )
                    .ok();
                    debug!("{}-{} yanked", krate.name, krate.version);
                }
                ChangeKind::Added => {
                    let priority = get_crate_priority(&conn, &krate.name)?;
                    add_crate_to_queue(&conn, &krate.name, &krate.version, priority).ok();

                    debug!("{}-{} added into build queue", krate.name, krate.version);
                    add_count += 1;
                }
            }
        }

        Ok(add_count)
    }

    pub fn get_queue_count(&self) -> Result<i64> {
        let conn = connect_db()?;

        Ok(conn
            .query("SELECT COUNT(*) FROM queue WHERE attempt < 5", &[])?
            .get(0)
            .get(0))
    }

    /// Builds the top package from the queue. Returns whether the queue was empty.
    pub(crate) fn build_next_queue_package(
        &mut self,
        builder: &mut RustwideBuilder,
    ) -> Result<bool> {
        let conn = connect_db()?;

        let query = conn.query(
            "SELECT id, name, version
                                     FROM queue
                                     WHERE attempt < 5
                                     ORDER BY priority ASC, attempt ASC, id ASC
                                     LIMIT 1",
            &[],
        )?;

        if query.is_empty() {
            // nothing in the queue; bail
            return Ok(false);
        }

        let id: i32 = query.get(0).get(0);
        let name: String = query.get(0).get(1);
        let version: String = query.get(0).get(2);

        match builder.build_package(self, &name, &version, None) {
            Ok(_) => {
                let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]);
                crate::web::metrics::TOTAL_BUILDS.inc();
            }
            Err(e) => {
                // Increase attempt count
                let rows = conn.query(
                    "UPDATE queue SET attempt = attempt + 1 WHERE id = $1 RETURNING attempt",
                    &[&id],
                )?;
                let attempt: i32 = rows.get(0).get(0);
                if attempt >= 5 {
                    crate::web::metrics::FAILED_BUILDS.inc();
                    crate::web::metrics::TOTAL_BUILDS.inc();
                }
                error!(
                    "Failed to build package {}-{} from queue: {}",
                    name, version, e
                )
            }
        }

        Ok(true)
    }
}

#[cfg(test)]
mod test {
    use crate::{DocBuilder, DocBuilderOptions};
    use log::error;
    use std::path::PathBuf;

    #[test]
    #[ignore]
    fn test_get_new_crates() {
        let _ = env_logger::try_init();
        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
        let mut docbuilder = DocBuilder::new(options);
        let res = docbuilder.get_new_crates();
        if res.is_err() {
            error!("{:?}", res);
        }
        assert!(res.is_ok());
    }
}
<commit_msg>Log errors while marking crates yanked<commit_after>//! Updates crates.io index and builds new packages

use super::{DocBuilder, RustwideBuilder};
use crate::db::connect_db;
use crate::error::Result;
use crate::utils::{add_crate_to_queue, get_crate_priority};
use crates_index_diff::{ChangeKind, Index};
use log::{debug, error};

impl DocBuilder {
    /// Updates crates.io-index repository and adds new crates into build queue.
    /// Returns size of queue
    pub fn get_new_crates(&mut self) -> Result<usize> {
        let conn = connect_db()?;
        let index = Index::from_path_or_cloned(&self.options.crates_io_index_path)?;
        let mut changes = index.fetch_changes()?;
        let mut add_count: usize = 0;

        // I belive this will fix ordering of queue if we get more than one crate from changes
        changes.reverse();

        for krate in &changes {
            match krate.kind {
                ChangeKind::Yanked => {
                    let res = conn.execute(
                        "
                        UPDATE releases
                            SET yanked = TRUE
                        FROM crates
                        WHERE crates.id = releases.crate_id
                            AND name = $1
                            AND version = $2
                        ",
                        &[&krate.name, &krate.version],
                    );
                    match res {
                        Ok(_) => debug!("{}-{} yanked", krate.name, krate.version),
                        Err(err) => error!(
                            "error while setting {}-{} to yanked: {}",
                            krate.name, krate.version, err
                        ),
                    }
                }
                ChangeKind::Added => {
                    let priority = get_crate_priority(&conn, &krate.name)?;
                    add_crate_to_queue(&conn, &krate.name, &krate.version, priority).ok();

                    debug!("{}-{} added into build queue", krate.name, krate.version);
                    add_count += 1;
                }
            }
        }

        Ok(add_count)
    }

    pub fn get_queue_count(&self) -> Result<i64> {
        let conn = connect_db()?;

        Ok(conn
            .query("SELECT COUNT(*) FROM queue WHERE attempt < 5", &[])?
            .get(0)
            .get(0))
    }

    /// Builds the top package from the queue. Returns whether the queue was empty.
    pub(crate) fn build_next_queue_package(
        &mut self,
        builder: &mut RustwideBuilder,
    ) -> Result<bool> {
        let conn = connect_db()?;

        let query = conn.query(
            "SELECT id, name, version
                                     FROM queue
                                     WHERE attempt < 5
                                     ORDER BY priority ASC, attempt ASC, id ASC
                                     LIMIT 1",
            &[],
        )?;

        if query.is_empty() {
            // nothing in the queue; bail
            return Ok(false);
        }

        let id: i32 = query.get(0).get(0);
        let name: String = query.get(0).get(1);
        let version: String = query.get(0).get(2);

        match builder.build_package(self, &name, &version, None) {
            Ok(_) => {
                let _ = conn.execute("DELETE FROM queue WHERE id = $1", &[&id]);
                crate::web::metrics::TOTAL_BUILDS.inc();
            }
            Err(e) => {
                // Increase attempt count
                let rows = conn.query(
                    "UPDATE queue SET attempt = attempt + 1 WHERE id = $1 RETURNING attempt",
                    &[&id],
                )?;
                let attempt: i32 = rows.get(0).get(0);
                if attempt >= 5 {
                    crate::web::metrics::FAILED_BUILDS.inc();
                    crate::web::metrics::TOTAL_BUILDS.inc();
                }
                error!(
                    "Failed to build package {}-{} from queue: {}",
                    name, version, e
                )
            }
        }

        Ok(true)
    }
}

#[cfg(test)]
mod test {
    use crate::{DocBuilder, DocBuilderOptions};
    use log::error;
    use std::path::PathBuf;

    #[test]
    #[ignore]
    fn test_get_new_crates() {
        let _ = env_logger::try_init();
        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
        let mut docbuilder = DocBuilder::new(options);
        let res = docbuilder.get_new_crates();
        if res.is_err() {
            error!("{:?}", res);
        }
        assert!(res.is_ok());
    }
}
<|endoftext|>