<commit_before>//! See https://github.com/matklad/cargo-xtask/.
//!
//! This binary defines various auxiliary build commands, which are not
//! expressible with just `cargo`. Notably, it provides `cargo xtask codegen`
//! for code genetaiont and `cargo xtask install` for installation of
//! rust-analyzer server and client.
//!
//! This binary is integrated into the `cargo` command line by using an alias in
//! `.cargo/config`.
mod help;

use core::fmt::Write;
use core::str;
use pico_args::Arguments;
use std::{env, path::PathBuf};
use xtask::{
    codegen::{self, Mode},
    install_format_hook, run, run_clippy, run_fuzzer, run_rustfmt, run_with_output, Cmd, Result,
};

// Latest stable, feel free to send a PR if this lags behind.
const REQUIRED_RUST_VERSION: u32 = 38;

struct InstallOpt {
    client: Option<ClientOpt>,
    server: Option<ServerOpt>,
}

enum ClientOpt {
    VsCode,
}

struct ServerOpt {
    jemalloc: bool,
}

fn main() -> Result<()> {
    let subcommand = match std::env::args_os().nth(1) {
        None => {
            eprintln!("{}", help::GLOBAL_HELP);
            return Ok(());
        }
        Some(s) => s,
    };
    let mut matches = Arguments::from_vec(std::env::args_os().skip(2).collect());
    let subcommand = &*subcommand.to_string_lossy();
    match subcommand {
        "install" => {
            if matches.contains(["-h", "--help"]) {
                eprintln!("{}", help::INSTALL_HELP);
                return Ok(());
            }
            let server = matches.contains("--server");
            let client_code = matches.contains("--client-code");
            if server && client_code {
                eprintln!("{}", help::INSTALL_RA_CONFLICT);
                return Ok(());
            }
            let jemalloc = matches.contains("--jemalloc");
            matches.finish().or_else(handle_extra_flags)?;
            let opts = InstallOpt {
                client: if server { None } else { Some(ClientOpt::VsCode) },
                server: if client_code { None } else { Some(ServerOpt { jemalloc }) },
            };
            install(opts)?
        }
        "codegen" => {
            if matches.contains(["-h", "--help"]) {
                help::print_no_param_subcommand_help(&subcommand);
                return Ok(());
            }
            codegen::generate_syntax(Mode::Overwrite)?;
            codegen::generate_parser_tests(Mode::Overwrite)?;
            codegen::generate_assists_docs(Mode::Overwrite)?;
        }
        "format" => {
            if matches.contains(["-h", "--help"]) {
                help::print_no_param_subcommand_help(&subcommand);
                return Ok(());
            }
            run_rustfmt(Mode::Overwrite)?
        }
        "format-hook" => {
            if matches.contains(["-h", "--help"]) {
                help::print_no_param_subcommand_help(&subcommand);
                return Ok(());
            }
            install_format_hook()?
        }
        "lint" => {
            if matches.contains(["-h", "--help"]) {
                help::print_no_param_subcommand_help(&subcommand);
                return Ok(());
            }
            run_clippy()?
        }
        "fuzz-tests" => {
            if matches.contains(["-h", "--help"]) {
                help::print_no_param_subcommand_help(&subcommand);
                return Ok(());
            }
            run_fuzzer()?
        }
        _ => eprintln!("{}", help::GLOBAL_HELP),
    }
    Ok(())
}

fn handle_extra_flags(e: pico_args::Error) -> Result<()> {
    if let pico_args::Error::UnusedArgsLeft(flags) = e {
        let mut invalid_flags = String::new();
        for flag in flags {
            write!(&mut invalid_flags, "{}, ", flag)?;
        }
        let (invalid_flags, _) = invalid_flags.split_at(invalid_flags.len() - 2);
        Err(format!("Invalid flags: {}", invalid_flags).into())
    } else {
        Err(e.to_string().into())
    }
}

fn install(opts: InstallOpt) -> Result<()> {
    if cfg!(target_os = "macos") {
        fix_path_for_mac()?
    }
    if let Some(server) = opts.server {
        install_server(server)?;
    }
    if let Some(client) = opts.client {
        install_client(client)?;
    }
    Ok(())
}

fn fix_path_for_mac() -> Result<()> {
    let mut vscode_path: Vec<PathBuf> = {
        const COMMON_APP_PATH: &str =
            r"/Applications/Visual Studio Code.app/Contents/Resources/app/bin";
        const ROOT_DIR: &str = "";
        let home_dir = match env::var("HOME") {
            Ok(home) => home,
            Err(e) => Err(format!("Failed getting HOME from environment with error: {}.", e))?,
        };

        [ROOT_DIR, &home_dir]
            .iter()
            .map(|dir| String::from(*dir) + COMMON_APP_PATH)
            .map(PathBuf::from)
            .filter(|path| path.exists())
            .collect()
    };

    if !vscode_path.is_empty() {
        let vars = match env::var_os("PATH") {
            Some(path) => path,
            None => Err("Could not get PATH variable from env.")?,
        };

        let mut paths = env::split_paths(&vars).collect::<Vec<_>>();
        paths.append(&mut vscode_path);
        let new_paths = env::join_paths(paths)?;
        env::set_var("PATH", &new_paths);
    }

    Ok(())
}

fn install_client(ClientOpt::VsCode: ClientOpt) -> Result<()> {
    let npm_version = Cmd {
        unix: r"npm --version",
        windows: r"cmd.exe /c npm.cmd --version",
        work_dir: "./editors/code",
    }
    .run();

    if npm_version.is_err() {
        eprintln!("\nERROR: `npm --version` failed, `npm` is required to build the VS Code plugin")
    }

    Cmd { unix: r"npm ci", windows: r"cmd.exe /c npm.cmd ci", work_dir: "./editors/code" }.run()?;
    Cmd {
        unix: r"npm run package",
        windows: r"cmd.exe /c npm.cmd run package",
        work_dir: "./editors/code",
    }
    .run()?;

    let code_binary = ["code", "code-insiders", "codium"].iter().find(|bin| {
        Cmd {
            unix: &format!("{} --version", bin),
            windows: &format!("cmd.exe /c {}.cmd --version", bin),
            work_dir: "./editors/code",
        }
        .run()
        .is_ok()
    });

    let code_binary = match code_binary {
        Some(it) => it,
        None => Err("Can't execute `code --version`. Perhaps it is not in $PATH?")?,
    };

    Cmd {
        unix: &format!(r"{} --install-extension ./ra-lsp-0.0.1.vsix --force", code_binary),
        windows: &format!(
            r"cmd.exe /c {}.cmd --install-extension ./ra-lsp-0.0.1.vsix --force",
            code_binary
        ),
        work_dir: "./editors/code",
    }
    .run()?;

    let output = Cmd {
        unix: &format!(r"{} --list-extensions", code_binary),
        windows: &format!(r"cmd.exe /c {}.cmd --list-extensions", code_binary),
        work_dir: ".",
    }
    .run_with_output()?;

    if !str::from_utf8(&output.stdout)?.contains("ra-lsp") {
        Err("Could not install the Visual Studio Code extension. \
             Please make sure you have at least NodeJS 10.x installed and try again.")?;
    }

    Ok(())
}

fn install_server(opts: ServerOpt) -> Result<()> {
    let mut old_rust = false;
    if let Ok(output) = run_with_output("cargo --version", ".") {
        if let Ok(stdout) = String::from_utf8(output.stdout) {
            println!("{}", stdout);
            if !check_version(&stdout, REQUIRED_RUST_VERSION) {
                old_rust = true;
            }
        }
    }

    if old_rust {
        eprintln!(
            "\nWARNING: at least rust 1.{}.0 is required to compile rust-analyzer\n",
            REQUIRED_RUST_VERSION
        )
    }

    let res = if opts.jemalloc {
        run("cargo install --path crates/ra_lsp_server --locked --force --features jemalloc", ".")
    } else {
        run("cargo install --path crates/ra_lsp_server --locked --force", ".")
    };

    if res.is_err() && old_rust {
        eprintln!(
            "\nWARNING: at least rust 1.{}.0 is required to compile rust-analyzer\n",
            REQUIRED_RUST_VERSION
        )
    }

    res
}

fn check_version(version_output: &str, min_minor_version: u32) -> bool {
    // Parse second the number out of
    //      cargo 1.39.0-beta (1c6ec66d5 2019-09-30)
    let minor: Option<u32> = version_output.split('.').nth(1).and_then(|it| it.parse().ok());
    match minor {
        None => true,
        Some(minor) => minor >= min_minor_version,
    }
}
<commit_msg>Fix typo in xtask/src/main.rs.<commit_after>//! See https://github.com/matklad/cargo-xtask/.
//!
//! This binary defines various auxiliary build commands, which are not
//! expressible with just `cargo`. Notably, it provides `cargo xtask codegen`
//! for code generation and `cargo xtask install` for installation of
//! rust-analyzer server and client.
//!
//! This binary is integrated into the `cargo` command line by using an alias in
//! `.cargo/config`.
mod help;

use core::fmt::Write;
use core::str;
use pico_args::Arguments;
use std::{env, path::PathBuf};
use xtask::{
    codegen::{self, Mode},
    install_format_hook, run, run_clippy, run_fuzzer, run_rustfmt, run_with_output, Cmd, Result,
};

// Latest stable, feel free to send a PR if this lags behind.
const REQUIRED_RUST_VERSION: u32 = 38;

struct InstallOpt {
    client: Option<ClientOpt>,
    server: Option<ServerOpt>,
}

enum ClientOpt {
    VsCode,
}

struct ServerOpt {
    jemalloc: bool,
}

fn main() -> Result<()> {
    let subcommand = match std::env::args_os().nth(1) {
        None => {
            eprintln!("{}", help::GLOBAL_HELP);
            return Ok(());
        }
        Some(s) => s,
    };
    let mut matches = Arguments::from_vec(std::env::args_os().skip(2).collect());
    let subcommand = &*subcommand.to_string_lossy();
    match subcommand {
        "install" => {
            if matches.contains(["-h", "--help"]) {
                eprintln!("{}", help::INSTALL_HELP);
                return Ok(());
            }
            let server = matches.contains("--server");
            let client_code = matches.contains("--client-code");
            if server && client_code {
                eprintln!("{}", help::INSTALL_RA_CONFLICT);
                return Ok(());
            }
            let jemalloc = matches.contains("--jemalloc");
            matches.finish().or_else(handle_extra_flags)?;
            let opts = InstallOpt {
                client: if server { None } else { Some(ClientOpt::VsCode) },
                server: if client_code { None } else { Some(ServerOpt { jemalloc }) },
            };
            install(opts)?
        }
        "codegen" => {
            if matches.contains(["-h", "--help"]) {
                help::print_no_param_subcommand_help(&subcommand);
                return Ok(());
            }
            codegen::generate_syntax(Mode::Overwrite)?;
            codegen::generate_parser_tests(Mode::Overwrite)?;
            codegen::generate_assists_docs(Mode::Overwrite)?;
        }
        "format" => {
            if matches.contains(["-h", "--help"]) {
                help::print_no_param_subcommand_help(&subcommand);
                return Ok(());
            }
            run_rustfmt(Mode::Overwrite)?
        }
        "format-hook" => {
            if matches.contains(["-h", "--help"]) {
                help::print_no_param_subcommand_help(&subcommand);
                return Ok(());
            }
            install_format_hook()?
        }
        "lint" => {
            if matches.contains(["-h", "--help"]) {
                help::print_no_param_subcommand_help(&subcommand);
                return Ok(());
            }
            run_clippy()?
        }
        "fuzz-tests" => {
            if matches.contains(["-h", "--help"]) {
                help::print_no_param_subcommand_help(&subcommand);
                return Ok(());
            }
            run_fuzzer()?
        }
        _ => eprintln!("{}", help::GLOBAL_HELP),
    }
    Ok(())
}

fn handle_extra_flags(e: pico_args::Error) -> Result<()> {
    if let pico_args::Error::UnusedArgsLeft(flags) = e {
        let mut invalid_flags = String::new();
        for flag in flags {
            write!(&mut invalid_flags, "{}, ", flag)?;
        }
        let (invalid_flags, _) = invalid_flags.split_at(invalid_flags.len() - 2);
        Err(format!("Invalid flags: {}", invalid_flags).into())
    } else {
        Err(e.to_string().into())
    }
}

fn install(opts: InstallOpt) -> Result<()> {
    if cfg!(target_os = "macos") {
        fix_path_for_mac()?
    }
    if let Some(server) = opts.server {
        install_server(server)?;
    }
    if let Some(client) = opts.client {
        install_client(client)?;
    }
    Ok(())
}

fn fix_path_for_mac() -> Result<()> {
    let mut vscode_path: Vec<PathBuf> = {
        const COMMON_APP_PATH: &str =
            r"/Applications/Visual Studio Code.app/Contents/Resources/app/bin";
        const ROOT_DIR: &str = "";
        let home_dir = match env::var("HOME") {
            Ok(home) => home,
            Err(e) => Err(format!("Failed getting HOME from environment with error: {}.", e))?,
        };

        [ROOT_DIR, &home_dir]
            .iter()
            .map(|dir| String::from(*dir) + COMMON_APP_PATH)
            .map(PathBuf::from)
            .filter(|path| path.exists())
            .collect()
    };

    if !vscode_path.is_empty() {
        let vars = match env::var_os("PATH") {
            Some(path) => path,
            None => Err("Could not get PATH variable from env.")?,
        };

        let mut paths = env::split_paths(&vars).collect::<Vec<_>>();
        paths.append(&mut vscode_path);
        let new_paths = env::join_paths(paths)?;
        env::set_var("PATH", &new_paths);
    }

    Ok(())
}

fn install_client(ClientOpt::VsCode: ClientOpt) -> Result<()> {
    let npm_version = Cmd {
        unix: r"npm --version",
        windows: r"cmd.exe /c npm.cmd --version",
        work_dir: "./editors/code",
    }
    .run();

    if npm_version.is_err() {
        eprintln!("\nERROR: `npm --version` failed, `npm` is required to build the VS Code plugin")
    }

    Cmd { unix: r"npm ci", windows: r"cmd.exe /c npm.cmd ci", work_dir: "./editors/code" }.run()?;
    Cmd {
        unix: r"npm run package",
        windows: r"cmd.exe /c npm.cmd run package",
        work_dir: "./editors/code",
    }
    .run()?;

    let code_binary = ["code", "code-insiders", "codium"].iter().find(|bin| {
        Cmd {
            unix: &format!("{} --version", bin),
            windows: &format!("cmd.exe /c {}.cmd --version", bin),
            work_dir: "./editors/code",
        }
        .run()
        .is_ok()
    });

    let code_binary = match code_binary {
        Some(it) => it,
        None => Err("Can't execute `code --version`. Perhaps it is not in $PATH?")?,
    };

    Cmd {
        unix: &format!(r"{} --install-extension ./ra-lsp-0.0.1.vsix --force", code_binary),
        windows: &format!(
            r"cmd.exe /c {}.cmd --install-extension ./ra-lsp-0.0.1.vsix --force",
            code_binary
        ),
        work_dir: "./editors/code",
    }
    .run()?;

    let output = Cmd {
        unix: &format!(r"{} --list-extensions", code_binary),
        windows: &format!(r"cmd.exe /c {}.cmd --list-extensions", code_binary),
        work_dir: ".",
    }
    .run_with_output()?;

    if !str::from_utf8(&output.stdout)?.contains("ra-lsp") {
        Err("Could not install the Visual Studio Code extension. \
             Please make sure you have at least NodeJS 10.x installed and try again.")?;
    }

    Ok(())
}

fn install_server(opts: ServerOpt) -> Result<()> {
    let mut old_rust = false;
    if let Ok(output) = run_with_output("cargo --version", ".") {
        if let Ok(stdout) = String::from_utf8(output.stdout) {
            println!("{}", stdout);
            if !check_version(&stdout, REQUIRED_RUST_VERSION) {
                old_rust = true;
            }
        }
    }

    if old_rust {
        eprintln!(
            "\nWARNING: at least rust 1.{}.0 is required to compile rust-analyzer\n",
            REQUIRED_RUST_VERSION
        )
    }

    let res = if opts.jemalloc {
        run("cargo install --path crates/ra_lsp_server --locked --force --features jemalloc", ".")
    } else {
        run("cargo install --path crates/ra_lsp_server --locked --force", ".")
    };

    if res.is_err() && old_rust {
        eprintln!(
            "\nWARNING: at least rust 1.{}.0 is required to compile rust-analyzer\n",
            REQUIRED_RUST_VERSION
        )
    }

    res
}

fn check_version(version_output: &str, min_minor_version: u32) -> bool {
    // Parse second the number out of
    //      cargo 1.39.0-beta (1c6ec66d5 2019-09-30)
    let minor: Option<u32> = version_output.split('.').nth(1).and_then(|it| it.parse().ok());
    match minor {
        None => true,
        Some(minor) => minor >= min_minor_version,
    }
}
<|endoftext|>


<commit_before>#[macro_use] extern crate log;
#[macro_use] extern crate nom;
extern crate env_logger;
extern crate mio;
extern crate rustc_serialize;
extern crate yxorp;
extern crate toml;

mod config;
mod command;
mod state;

use std::net::{UdpSocket,ToSocketAddrs};
use std::sync::mpsc::{channel};
use std::collections::HashMap;
use std::thread::JoinHandle;
use yxorp::network;
use yxorp::network::metrics::{METRICS,ProxyMetrics};

use command::{Listener,ListenerType};

fn main() {
  env_logger::init().unwrap();
  info!("starting up");

  // FIXME: should load configuration from a CLI argument
  if let Ok(config) = config::Config::load_from_path("./config.toml") {
    let metrics_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
    let metrics_host   = (&config.metrics.address[..], config.metrics.port).to_socket_addrs().unwrap().next().unwrap();
    METRICS.lock().unwrap().set_up_remote(metrics_socket, metrics_host);
    let metrics_guard = ProxyMetrics::run();

    let mut listeners = HashMap::new();
    let mut jh_opt: Option<JoinHandle<()>> = None;

    for (ref tag, ref ls) in config.listeners {
      let (sender, receiver) = channel::<network::ServerMessage>();
      let mut address = ls.address.clone();
      address.push(':');
      address.push_str(&ls.port.to_string());

      let (tx, jg) = match ls.listener_type {
        ListenerType::HTTP => {
          network::http::start_listener(address.parse().unwrap(), ls.max_connections, sender)
        },
        ListenerType::HTTPS => {
          network::tls::start_listener(address.parse().unwrap(), ls.max_connections, None, sender)
        },
        _ => unimplemented!()
      };
      let l =  Listener::new(tag.clone(), ls.listener_type, ls.address.clone(), ls.port, tx, receiver);
      listeners.insert(tag.clone(), l);
      jh_opt = Some(jg);
    };

    command::start(config.command_socket, listeners);
    if let Some(jh) = jh_opt {
      let _ = jh.join();
      info!("good bye");
    }
  } else {
    println!("could not load configuration, stopping");
  }
}

<commit_msg>Reorder imports to fix macro conflict<commit_after>#[macro_use] extern crate nom;
#[macro_use] extern crate log;
extern crate env_logger;
extern crate mio;
extern crate rustc_serialize;
extern crate yxorp;
extern crate toml;

mod config;
mod command;
mod state;

use std::net::{UdpSocket,ToSocketAddrs};
use std::sync::mpsc::{channel};
use std::collections::HashMap;
use std::thread::JoinHandle;
use yxorp::network;
use yxorp::network::metrics::{METRICS,ProxyMetrics};

use command::{Listener,ListenerType};

fn main() {
  env_logger::init().unwrap();
  info!("starting up");

  // FIXME: should load configuration from a CLI argument
  if let Ok(config) = config::Config::load_from_path("./config.toml") {
    let metrics_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
    let metrics_host   = (&config.metrics.address[..], config.metrics.port).to_socket_addrs().unwrap().next().unwrap();
    METRICS.lock().unwrap().set_up_remote(metrics_socket, metrics_host);
    let metrics_guard = ProxyMetrics::run();

    let mut listeners = HashMap::new();
    let mut jh_opt: Option<JoinHandle<()>> = None;

    for (ref tag, ref ls) in config.listeners {
      let (sender, receiver) = channel::<network::ServerMessage>();
      let mut address = ls.address.clone();
      address.push(':');
      address.push_str(&ls.port.to_string());

      let (tx, jg) = match ls.listener_type {
        ListenerType::HTTP => {
          network::http::start_listener(address.parse().unwrap(), ls.max_connections, sender)
        },
        ListenerType::HTTPS => {
          network::tls::start_listener(address.parse().unwrap(), ls.max_connections, None, sender)
        },
        _ => unimplemented!()
      };
      let l =  Listener::new(tag.clone(), ls.listener_type, ls.address.clone(), ls.port, tx, receiver);
      listeners.insert(tag.clone(), l);
      jh_opt = Some(jg);
    };

    command::start(config.command_socket, listeners);
    if let Some(jh) = jh_opt {
      let _ = jh.join();
      info!("good bye");
    }
  } else {
    println!("could not load configuration, stopping");
  }
}

<|endoftext|>


<commit_before>use sozu_command::config::LoadBalancingAlgorithms;

#[derive(StructOpt, PartialEq, Debug)]
pub struct App {
  #[structopt(short="c", long = "config", help = "Sets a custom config file")]
  pub config: String,
  #[structopt(short="t", long = "timeout", help = "Sets a custom timeout for commands (in milliseconds). 0 disables the timeout")]
  pub timeout: Option<u64>,
  #[structopt(subcommand)]
  pub cmd: SubCmd,
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum SubCmd {
  #[structopt(name = "shutdown", about = "shuts down the proxy without waiting for connections to finish")]
  Shutdown {
    #[structopt(short = "h", long = "hard")]
    hard: bool
  },
  #[structopt(name = "upgrade", about = "upgrade the proxy")]
  Upgrade {
    #[structopt(short = "w", long = "worker", help = "Upgrade the worker with this id")]
    worker: Option<u32>,
  },
  #[structopt(name = "status", about = "gets information on the running workers")]
  Status {
    #[structopt(short = "j", long = "json", help = "Print the command result in JSON format")]
    json: bool
  },
  #[structopt(name = "metrics", about = "gets statistics on the master and its workers")]
  Metrics {
    #[structopt(short = "j", long = "json", help = "Print the command result in JSON format")]
    json: bool
  },
  #[structopt(name = "logging", about = "change logging level")]
  Logging {
    #[structopt(short = "l", long = "level", help = "change logging level")]
    level: String
  },
  #[structopt(name = "state", about = "state management")]
  State {
    #[structopt(subcommand)]
    cmd: StateCmd,
  },
  #[structopt(name = "application", about = "application management")]
  Application {
    #[structopt(subcommand)]
    cmd: ApplicationCmd,
  },
  #[structopt(name = "backend", about = "backend management")]
  Backend {
    #[structopt(subcommand)]
    cmd: BackendCmd,
  },
  #[structopt(name = "frontend", about = "frontend management")]
  Frontend {
    #[structopt(subcommand)]
    cmd: FrontendCmd,
  },
  #[structopt(name = "certificate", about = "certificate management")]
  Certificate {
    #[structopt(subcommand)]
    cmd: CertificateCmd,
  },
  #[structopt(name = "query", about = "configuration state verification")]
  Query {
    #[structopt(short = "j", long = "json", help = "Print the command result in JSON format")]
    json: bool,
    #[structopt(subcommand)]
    cmd: QueryCmd,
  }
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum StateCmd {
  #[structopt(name = "save", about = "Save state to that file")]
  Save {
    #[structopt(short = "f", long = "file")]
    file: String,
  },
  #[structopt(name = "load", about = "Load state from that file")]
  Load {
    #[structopt(short = "f", long = "file")]
    file: String,
  },
  #[structopt(name = "dump")]
  Dump {
    #[structopt(short = "j", long = "json", help = "Print the command result in JSON format")]
    json: bool
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum ApplicationCmd {
  #[structopt(name = "remove")]
  Remove {
    #[structopt(short = "i", long = "id")]
    id: String,
  },
  #[structopt(name = "add")]
  Add{
    #[structopt(short = "i", long = "id")]
    id: String,
    #[structopt(short = "s", long = "sticky-session")]
    sticky_session: bool,
    #[structopt(short = "h", long = "https-redirect")]
    https_redirect: bool,
    #[structopt(long = "send-proxy", help = "Enforces use of the PROXY protocol version 2 over any connection established to this server.")]
    send_proxy: bool,
    #[structopt(long = "expect-proxy", help = "Configures the client-facing connection to receive a PROXY protocol header version 2")]
    expect_proxy: bool,
    #[structopt(long = "load-balancing-policy", help = "Configures the load balancing policy")]
    load_balancing_policy: LoadBalancingAlgorithms,
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum BackendCmd {
  #[structopt(name = "remove")]
  Remove {
    #[structopt(short = "i", long = "id")]
    id: String,
    #[structopt(long = "backend-id")]
    backend_id: String,
    #[structopt(long = "ip")]
    ip: String,
    #[structopt(short = "p", long = "port")]
    port: u16,
  },
  #[structopt(name = "add")]
  Add {
    #[structopt(short = "i", long = "id")]
    id: String,
    #[structopt(long = "backend-id")]
    backend_id: String,
    #[structopt(long = "ip")]
    ip: String,
    #[structopt(short = "p", long = "port")]
    port: u16,
    #[structopt(short = "s", long = "sticky-id", help = "value for the sticky session cookie")]
    sticky_id: Option<String>,
    #[structopt(short = "b", long = "backup", help = "set backend as a backup backend")]
    backup: Option<bool>,
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum FrontendCmd {
  #[structopt(name = "http", about = "HTTP/HTTPS frontend management")]
  Http {
    #[structopt(subcommand)]
    cmd: HttpFrontendCmd,
  },
  #[structopt(name = "tcp", about = "TCP frontend management")]
  Tcp {
    #[structopt(subcommand)]
    cmd: TcpFrontendCmd,
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum HttpFrontendCmd {
  #[structopt(name = "add")]
  Add {
    #[structopt(long = "listener-ip")]
    listener_ip: String,
    #[structopt(long = "listener-port")]
    listener_port: u16,
    #[structopt(short = "i", long = "id", help = "app id of the frontend")]
    id: String,
    #[structopt(short = "host", long = "hostname")]
    hostname: String,
    #[structopt(short = "p", long = "path", help="URL prefix of the frontend")]
    path_begin: Option<String>,
    #[structopt(long = "certificate", help="path to a certificate file")]
    path_to_certificate: Option<String>,
  },
  #[structopt(name = "remove")]
  Remove {
    #[structopt(long = "listener-ip")]
    listener_ip: String,
    #[structopt(long = "listener-port")]
    listener_port: u16,
    #[structopt(short = "i", long = "id", help = "app id of the frontend")]
    id: String,
    #[structopt(short = "host", long = "hostname")]
    hostname: String,
    #[structopt(short = "p", long = "path", help="URL prefix of the frontend")]
    path_begin: Option<String>,
    #[structopt(long = "certificate", help="path to a certificate file")]
    path_to_certificate: Option<String>,
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum TcpFrontendCmd {
  #[structopt(name = "add")]
  Add {
    #[structopt(short = "i", long = "id", help = "app id of the frontend")]
    id: String,
    #[structopt(long = "ip", help = "IP address of the frontend")]
    ip_address: String,
    #[structopt(short = "p", long = "port", help="TCP port of the frontend")]
    port: u16,
  },
  #[structopt(name = "remove")]
  Remove {
    #[structopt(short = "i", long = "id", help = "app id of the frontend")]
    id: String,
    #[structopt(long = "ip", help = "IP address of the frontend")]
    ip_address: String,
    #[structopt(short = "p", long = "port", help="TCP port of the frontend")]
    port: u16,
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum CertificateCmd {
  #[structopt(name = "add")]
  Add {
    #[structopt(short = "i", long = "listener-ip")]
    listener_ip: String,
    #[structopt(short = "p", long = "listener-port")]
    listener_port: u16,
    #[structopt(long = "certificate", help = "path to the certificate")]
    certificate: String,
    #[structopt(long = "certificate-chain", help = "path to the certificate chain")]
    chain: String,
    #[structopt(long = "key", help = "path to the key")]
    key: Option<String>,
  },
  #[structopt(name = "remove")]
  Remove {
    #[structopt(short = "i", long = "listener-ip")]
    listener_ip: String,
    #[structopt(short = "p", long = "listener-port")]
    listener_port: u16,
    #[structopt(short = "cert", long = "certificate", help = "path to the certificate")]
    certificate: String,
  },
  #[structopt(name = "replace")]
  Replace {
    #[structopt(short = "i", long = "listener-ip")]
    listener_ip: String,
    #[structopt(short = "p", long = "listener-port")]
    listener_port: u16,
    #[structopt(long = "new-certificate", help = "path to the new certificate")]
    certificate: String,
    #[structopt(long = "new-certificate-chain", help = "path to the new certificate chain")]
    chain: String,
    #[structopt(long = "new-key", help = "path to the new key")]
    key: Option<String>,
    #[structopt(short = "old-cert", long = "old-certificate", help = "path to the old certificate")]
    old_certificate: String,
  }
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum QueryCmd {
  #[structopt(name = "applications")]
  Applications {
    #[structopt(short = "i", long="id", help="application identifier")]
    id: Option<String>,
    #[structopt(short = "d", long="domain", help="application domain name")]
    domain: Option<String>
  }
}
<commit_msg>conflicting short option<commit_after>use sozu_command::config::LoadBalancingAlgorithms;

#[derive(StructOpt, PartialEq, Debug)]
pub struct App {
  #[structopt(short="c", long = "config", help = "Sets a custom config file")]
  pub config: String,
  #[structopt(short="t", long = "timeout", help = "Sets a custom timeout for commands (in milliseconds). 0 disables the timeout")]
  pub timeout: Option<u64>,
  #[structopt(subcommand)]
  pub cmd: SubCmd,
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum SubCmd {
  #[structopt(name = "shutdown", about = "shuts down the proxy without waiting for connections to finish")]
  Shutdown {
    #[structopt(long = "hard")]
    hard: bool
  },
  #[structopt(name = "upgrade", about = "upgrade the proxy")]
  Upgrade {
    #[structopt(short = "w", long = "worker", help = "Upgrade the worker with this id")]
    worker: Option<u32>,
  },
  #[structopt(name = "status", about = "gets information on the running workers")]
  Status {
    #[structopt(short = "j", long = "json", help = "Print the command result in JSON format")]
    json: bool
  },
  #[structopt(name = "metrics", about = "gets statistics on the master and its workers")]
  Metrics {
    #[structopt(short = "j", long = "json", help = "Print the command result in JSON format")]
    json: bool
  },
  #[structopt(name = "logging", about = "change logging level")]
  Logging {
    #[structopt(short = "l", long = "level", help = "change logging level")]
    level: String
  },
  #[structopt(name = "state", about = "state management")]
  State {
    #[structopt(subcommand)]
    cmd: StateCmd,
  },
  #[structopt(name = "application", about = "application management")]
  Application {
    #[structopt(subcommand)]
    cmd: ApplicationCmd,
  },
  #[structopt(name = "backend", about = "backend management")]
  Backend {
    #[structopt(subcommand)]
    cmd: BackendCmd,
  },
  #[structopt(name = "frontend", about = "frontend management")]
  Frontend {
    #[structopt(subcommand)]
    cmd: FrontendCmd,
  },
  #[structopt(name = "certificate", about = "certificate management")]
  Certificate {
    #[structopt(subcommand)]
    cmd: CertificateCmd,
  },
  #[structopt(name = "query", about = "configuration state verification")]
  Query {
    #[structopt(short = "j", long = "json", help = "Print the command result in JSON format")]
    json: bool,
    #[structopt(subcommand)]
    cmd: QueryCmd,
  }
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum StateCmd {
  #[structopt(name = "save", about = "Save state to that file")]
  Save {
    #[structopt(short = "f", long = "file")]
    file: String,
  },
  #[structopt(name = "load", about = "Load state from that file")]
  Load {
    #[structopt(short = "f", long = "file")]
    file: String,
  },
  #[structopt(name = "dump")]
  Dump {
    #[structopt(short = "j", long = "json", help = "Print the command result in JSON format")]
    json: bool
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum ApplicationCmd {
  #[structopt(name = "remove")]
  Remove {
    #[structopt(short = "i", long = "id")]
    id: String,
  },
  #[structopt(name = "add")]
  Add{
    #[structopt(short = "i", long = "id")]
    id: String,
    #[structopt(short = "s", long = "sticky-session")]
    sticky_session: bool,
    #[structopt(short = "h", long = "https-redirect")]
    https_redirect: bool,
    #[structopt(long = "send-proxy", help = "Enforces use of the PROXY protocol version 2 over any connection established to this server.")]
    send_proxy: bool,
    #[structopt(long = "expect-proxy", help = "Configures the client-facing connection to receive a PROXY protocol header version 2")]
    expect_proxy: bool,
    #[structopt(long = "load-balancing-policy", help = "Configures the load balancing policy")]
    load_balancing_policy: LoadBalancingAlgorithms,
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum BackendCmd {
  #[structopt(name = "remove")]
  Remove {
    #[structopt(short = "i", long = "id")]
    id: String,
    #[structopt(long = "backend-id")]
    backend_id: String,
    #[structopt(long = "ip")]
    ip: String,
    #[structopt(short = "p", long = "port")]
    port: u16,
  },
  #[structopt(name = "add")]
  Add {
    #[structopt(short = "i", long = "id")]
    id: String,
    #[structopt(long = "backend-id")]
    backend_id: String,
    #[structopt(long = "ip")]
    ip: String,
    #[structopt(short = "p", long = "port")]
    port: u16,
    #[structopt(short = "s", long = "sticky-id", help = "value for the sticky session cookie")]
    sticky_id: Option<String>,
    #[structopt(short = "b", long = "backup", help = "set backend as a backup backend")]
    backup: Option<bool>,
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum FrontendCmd {
  #[structopt(name = "http", about = "HTTP/HTTPS frontend management")]
  Http {
    #[structopt(subcommand)]
    cmd: HttpFrontendCmd,
  },
  #[structopt(name = "tcp", about = "TCP frontend management")]
  Tcp {
    #[structopt(subcommand)]
    cmd: TcpFrontendCmd,
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum HttpFrontendCmd {
  #[structopt(name = "add")]
  Add {
    #[structopt(long = "listener-ip")]
    listener_ip: String,
    #[structopt(long = "listener-port")]
    listener_port: u16,
    #[structopt(short = "i", long = "id", help = "app id of the frontend")]
    id: String,
    #[structopt(short = "host", long = "hostname")]
    hostname: String,
    #[structopt(short = "p", long = "path", help="URL prefix of the frontend")]
    path_begin: Option<String>,
    #[structopt(long = "certificate", help="path to a certificate file")]
    path_to_certificate: Option<String>,
  },
  #[structopt(name = "remove")]
  Remove {
    #[structopt(long = "listener-ip")]
    listener_ip: String,
    #[structopt(long = "listener-port")]
    listener_port: u16,
    #[structopt(short = "i", long = "id", help = "app id of the frontend")]
    id: String,
    #[structopt(short = "host", long = "hostname")]
    hostname: String,
    #[structopt(short = "p", long = "path", help="URL prefix of the frontend")]
    path_begin: Option<String>,
    #[structopt(long = "certificate", help="path to a certificate file")]
    path_to_certificate: Option<String>,
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum TcpFrontendCmd {
  #[structopt(name = "add")]
  Add {
    #[structopt(short = "i", long = "id", help = "app id of the frontend")]
    id: String,
    #[structopt(long = "ip", help = "IP address of the frontend")]
    ip_address: String,
    #[structopt(short = "p", long = "port", help="TCP port of the frontend")]
    port: u16,
  },
  #[structopt(name = "remove")]
  Remove {
    #[structopt(short = "i", long = "id", help = "app id of the frontend")]
    id: String,
    #[structopt(long = "ip", help = "IP address of the frontend")]
    ip_address: String,
    #[structopt(short = "p", long = "port", help="TCP port of the frontend")]
    port: u16,
  },
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum CertificateCmd {
  #[structopt(name = "add")]
  Add {
    #[structopt(short = "i", long = "listener-ip")]
    listener_ip: String,
    #[structopt(short = "p", long = "listener-port")]
    listener_port: u16,
    #[structopt(long = "certificate", help = "path to the certificate")]
    certificate: String,
    #[structopt(long = "certificate-chain", help = "path to the certificate chain")]
    chain: String,
    #[structopt(long = "key", help = "path to the key")]
    key: Option<String>,
  },
  #[structopt(name = "remove")]
  Remove {
    #[structopt(short = "i", long = "listener-ip")]
    listener_ip: String,
    #[structopt(short = "p", long = "listener-port")]
    listener_port: u16,
    #[structopt(short = "cert", long = "certificate", help = "path to the certificate")]
    certificate: String,
  },
  #[structopt(name = "replace")]
  Replace {
    #[structopt(short = "i", long = "listener-ip")]
    listener_ip: String,
    #[structopt(short = "p", long = "listener-port")]
    listener_port: u16,
    #[structopt(long = "new-certificate", help = "path to the new certificate")]
    certificate: String,
    #[structopt(long = "new-certificate-chain", help = "path to the new certificate chain")]
    chain: String,
    #[structopt(long = "new-key", help = "path to the new key")]
    key: Option<String>,
    #[structopt(short = "old-cert", long = "old-certificate", help = "path to the old certificate")]
    old_certificate: String,
  }
}

#[derive(StructOpt, PartialEq, Debug)]
pub enum QueryCmd {
  #[structopt(name = "applications")]
  Applications {
    #[structopt(short = "i", long="id", help="application identifier")]
    id: Option<String>,
    #[structopt(short = "d", long="domain", help="application domain name")]
    domain: Option<String>
  }
}
<|endoftext|>


<commit_before>// Solution for http://rosettacode.org/wiki/Execute_a_Markov_algorithm

// Individual markov rule
struct MarkovRule {
    pattern: String,
    replacement: String,
    stop: bool
}

impl MarkovRule {
    fn new(pattern: String, replacement: String, stop: bool) -> MarkovRule {
        MarkovRule {pattern: pattern, replacement: replacement, stop: stop}
    }
}

// The complete markov algorithm
struct MarkovAlgorithm {
    rules: Vec<MarkovRule>
}

impl MarkovAlgorithm {
    // Parse an algorithm description to build a markov algorithm
    pub fn from_str(s: &str) -> Result<MarkovAlgorithm, String> {
        let mut rules: Vec<MarkovRule> = vec!();
        for line in s.lines()
            .map(|l| l.trim()) // Ignore whitespace before and after
            .filter(|l| l.char_len() > 0 && l.char_at(0) != '#') { // Ignore comments

            // check for -> (must be preceded by whitespace)
            // invalid ruleset if absent
            // whitespace rules mean there's 2 possible variations: " ->" and "\t->"
            let arrow_pos = line.find_str(" ->").or_else(|| line.find_str("\t->"));
            match arrow_pos {
                None => {
                    // Ruleset is invalid
                    return Err(format_strbuf!("Invalid rule \"{}\"", line));
                }
                Some(arrow) => {
                    // extract pattern (trim trailing whitespace)
                    let pattern = line.slice_to(arrow).trim_right();

                    // get the string after the arrow
                    // this adds 3 to skip the arrow itself
                    let line_end = line.slice_from(arrow + 3).trim_left();

                    // check for . (stop)
                    let stop = (line_end.char_len() > 0) && (line_end.char_at(0) == '.');

                    // extract replacement
                    let replacement = if stop {line_end.slice_from(1)} else {line_end};

                    // add to rules
                    let new_rule = MarkovRule::new(pattern.to_str(), replacement.to_str(), stop);
                    rules.push(new_rule);
                }
            }
        }
        let rule_set = MarkovAlgorithm{rules: rules};
        Ok(rule_set)
    }

    // Transform a text string by applying the markov algorithm
    pub fn apply(&self, input: &str) -> String {

        // get a writable version of the input to work with
        let mut state = input.to_str();

        // Don't allow input to be used after this
        drop(input);

        // loop while operations are possible
        loop {
            // find the first rule that is applicable
            // (pattern string is in state)
            let possible_rule = self.rules.iter().find(|rule|{
                state.as_slice().find_str(rule.pattern.as_slice()).is_some()
            });

            match possible_rule {
                // stop if no rule found
                None => { break; }
                Some(rule) => {
                    // replace the first instance (only) of the pattern
                    // Note: cannot use str::replace as that replaces all instances

                    // unwrap is safe here as the code for finding a rule
                    // already established that the pattern is present
                    let pos = state.as_slice().find_str(rule.pattern.as_slice()).unwrap();
                    let width = rule.pattern.len();

                    // string parts
                    let left = state.as_slice().slice_to(pos).to_owned();
                    let right = state.as_slice().slice_from(pos + width).to_owned();

                    // construct new string
                    state = format_strbuf!("{}{}{}", left, rule.replacement, right);

                    // stop if required
                    if rule.stop { break; }
                }
            }
        }

        state
    }
}

// A Rosetta Code sample
struct RCSample<'a> {
    ruleset: &'a str,
    input: &'a str,
    expected_result: &'a str
}

// Sample markow algorithms from rosetta code
// The extra whitespaces are trimmed when MarkovAlgorithm::from_str is called
fn get_samples<'a>() -> [RCSample<'a>, ..5] {
    [
        RCSample {
            ruleset:
                "# This rules file is extracted from Wikipedia:
                # http://en.wikipedia.org/wiki/Markov_Algorithm
                A -> apple
                B -> bag
                S -> shop
                T -> the
                the shop -> my brother
                a never used -> .terminating rule",
            input: "I bought a B of As from T S.",
            expected_result: "I bought a bag of apples from my brother.",
        },
        RCSample{
            ruleset:
                "# Slightly modified from the rules on Wikipedia
                A -> apple
                B -> bag
                S -> .shop
                T -> the
                the shop -> my brother
                a never used -> .terminating rule",
            input: "I bought a B of As from T S.",
            expected_result: "I bought a bag of apples from T shop.",
        },
        RCSample{
            ruleset:
                "# BNF Syntax testing rules
                A -> apple
                WWWW -> with
                Bgage -> ->.*
                B -> bag
                ->.* -> money
                W -> WW
                S -> .shop
                T -> the
                the shop -> my brother
                a never used -> .terminating rule",
            input: "I bought a B of As W my Bgage from T S.",
            expected_result: "I bought a bag of apples with my money from T shop.",
        },
        RCSample{
            ruleset:
                "### Unary Multiplication Engine, for testing Markov Algorithm implementations
                ### By Donal Fellows.
                # Unary addition engine
                _+1 -> _1+
                1+1 -> 11+
                # Pass for converting from the splitting of multiplication into ordinary
                # addition
                1! -> !1
                ,! -> !+
                _! -> _
                # Unary multiplication by duplicating left side, right side times
                1*1 -> x,@y
                1x -> xX
                X, -> 1,1
                X1 -> 1X
                _x -> _X
                ,x -> ,X
                y1 -> 1y
                y_ -> _
                # Next phase of applying
                1@1 -> x,@y
                1@_ -> @_
                ,@_ -> !_
                ++ -> +
                # Termination cleanup for addition
                _1 -> 1
                1+_ -> 1
                _+_ -> ",
            input: "_1111*11111_",
            expected_result: "11111111111111111111",
        },
        RCSample{
            ruleset:
                "# Turing machine: three-state busy beaver
                #
                # state A, symbol 0 => write 1, move right, new state B
                A0 -> 1B
                # state A, symbol 1 => write 1, move left, new state C
                0A1 -> C01
                1A1 -> C11
                # state B, symbol 0 => write 1, move left, new state A
                0B0 -> A01
                1B0 -> A11
                # state B, symbol 1 => write 1, move right, new state B
                B1 -> 1B
                # state C, symbol 0 => write 1, move left, new state B
                0C0 -> B01
                1C0 -> B11
                # state C, symbol 1 => write 1, move left, halt
                0C1 -> H01
                1C1 -> H11",
            input: "000000A000000",
            expected_result: "00011H1111000",
        },
    ]
}

#[cfg(not(test))]
fn main() {
    for (index, sample) in get_samples().iter().enumerate() {
        match MarkovAlgorithm::from_str(sample.ruleset) {
            Ok(algorithm) => {
                println!("Sample {}", (index + 1));
                println!("Output: {}", algorithm.apply(sample.input));
            }
            Err(message) => println!("{}", message)
        }
    }
}

#[test]
fn test_samples() {
    for sample in get_samples().iter() {
        match MarkovAlgorithm::from_str(sample.ruleset) {
            Ok(algorithm) => assert_eq!(sample.expected_result,
                                        algorithm.apply(sample.input).as_slice()),
            Err(message) => fail!("{}", message)
        }
    }
}
<commit_msg>markov_algorithm: fix build<commit_after>// Solution for http://rosettacode.org/wiki/Execute_a_Markov_algorithm

// Individual markov rule
struct MarkovRule {
    pattern: String,
    replacement: String,
    stop: bool
}

impl MarkovRule {
    fn new(pattern: String, replacement: String, stop: bool) -> MarkovRule {
        MarkovRule {pattern: pattern, replacement: replacement, stop: stop}
    }
}

// The complete markov algorithm
struct MarkovAlgorithm {
    rules: Vec<MarkovRule>
}

impl MarkovAlgorithm {
    // Parse an algorithm description to build a markov algorithm
    pub fn from_str(s: &str) -> Result<MarkovAlgorithm, String> {
        let mut rules: Vec<MarkovRule> = vec!();
        for line in s.lines()
            .map(|l| l.trim()) // Ignore whitespace before and after
            .filter(|l| l.char_len() > 0 && l.char_at(0) != '#') { // Ignore comments

            // check for -> (must be preceded by whitespace)
            // invalid ruleset if absent
            // whitespace rules mean there's 2 possible variations: " ->" and "\t->"
            let arrow_pos = line.find_str(" ->").or_else(|| line.find_str("\t->"));
            match arrow_pos {
                None => {
                    // Ruleset is invalid
                    return Err(format!("Invalid rule \"{}\"", line));
                }
                Some(arrow) => {
                    // extract pattern (trim trailing whitespace)
                    let pattern = line.slice_to(arrow).trim_right();

                    // get the string after the arrow
                    // this adds 3 to skip the arrow itself
                    let line_end = line.slice_from(arrow + 3).trim_left();

                    // check for . (stop)
                    let stop = (line_end.char_len() > 0) && (line_end.char_at(0) == '.');

                    // extract replacement
                    let replacement = if stop {line_end.slice_from(1)} else {line_end};

                    // add to rules
                    let new_rule = MarkovRule::new(pattern.to_str(), replacement.to_str(), stop);
                    rules.push(new_rule);
                }
            }
        }
        let rule_set = MarkovAlgorithm{rules: rules};
        Ok(rule_set)
    }

    // Transform a text string by applying the markov algorithm
    pub fn apply(&self, input: &str) -> String {

        // get a writable version of the input to work with
        let mut state = input.to_str();

        // Don't allow input to be used after this
        drop(input);

        // loop while operations are possible
        loop {
            // find the first rule that is applicable
            // (pattern string is in state)
            let possible_rule = self.rules.iter().find(|rule|{
                state.as_slice().find_str(rule.pattern.as_slice()).is_some()
            });

            match possible_rule {
                // stop if no rule found
                None => { break; }
                Some(rule) => {
                    // replace the first instance (only) of the pattern
                    // Note: cannot use str::replace as that replaces all instances

                    // unwrap is safe here as the code for finding a rule
                    // already established that the pattern is present
                    let pos = state.as_slice().find_str(rule.pattern.as_slice()).unwrap();
                    let width = rule.pattern.len();

                    // string parts
                    let left = state.as_slice().slice_to(pos).to_owned();
                    let right = state.as_slice().slice_from(pos + width).to_owned();

                    // construct new string
                    state = format!("{}{}{}", left, rule.replacement, right);

                    // stop if required
                    if rule.stop { break; }
                }
            }
        }

        state
    }
}

// A Rosetta Code sample
struct RCSample<'a> {
    ruleset: &'a str,
    input: &'a str,
    expected_result: &'a str
}

// Sample markow algorithms from rosetta code
// The extra whitespaces are trimmed when MarkovAlgorithm::from_str is called
fn get_samples<'a>() -> [RCSample<'a>, ..5] {
    [
        RCSample {
            ruleset:
                "# This rules file is extracted from Wikipedia:
                # http://en.wikipedia.org/wiki/Markov_Algorithm
                A -> apple
                B -> bag
                S -> shop
                T -> the
                the shop -> my brother
                a never used -> .terminating rule",
            input: "I bought a B of As from T S.",
            expected_result: "I bought a bag of apples from my brother.",
        },
        RCSample{
            ruleset:
                "# Slightly modified from the rules on Wikipedia
                A -> apple
                B -> bag
                S -> .shop
                T -> the
                the shop -> my brother
                a never used -> .terminating rule",
            input: "I bought a B of As from T S.",
            expected_result: "I bought a bag of apples from T shop.",
        },
        RCSample{
            ruleset:
                "# BNF Syntax testing rules
                A -> apple
                WWWW -> with
                Bgage -> ->.*
                B -> bag
                ->.* -> money
                W -> WW
                S -> .shop
                T -> the
                the shop -> my brother
                a never used -> .terminating rule",
            input: "I bought a B of As W my Bgage from T S.",
            expected_result: "I bought a bag of apples with my money from T shop.",
        },
        RCSample{
            ruleset:
                "### Unary Multiplication Engine, for testing Markov Algorithm implementations
                ### By Donal Fellows.
                # Unary addition engine
                _+1 -> _1+
                1+1 -> 11+
                # Pass for converting from the splitting of multiplication into ordinary
                # addition
                1! -> !1
                ,! -> !+
                _! -> _
                # Unary multiplication by duplicating left side, right side times
                1*1 -> x,@y
                1x -> xX
                X, -> 1,1
                X1 -> 1X
                _x -> _X
                ,x -> ,X
                y1 -> 1y
                y_ -> _
                # Next phase of applying
                1@1 -> x,@y
                1@_ -> @_
                ,@_ -> !_
                ++ -> +
                # Termination cleanup for addition
                _1 -> 1
                1+_ -> 1
                _+_ -> ",
            input: "_1111*11111_",
            expected_result: "11111111111111111111",
        },
        RCSample{
            ruleset:
                "# Turing machine: three-state busy beaver
                #
                # state A, symbol 0 => write 1, move right, new state B
                A0 -> 1B
                # state A, symbol 1 => write 1, move left, new state C
                0A1 -> C01
                1A1 -> C11
                # state B, symbol 0 => write 1, move left, new state A
                0B0 -> A01
                1B0 -> A11
                # state B, symbol 1 => write 1, move right, new state B
                B1 -> 1B
                # state C, symbol 0 => write 1, move left, new state B
                0C0 -> B01
                1C0 -> B11
                # state C, symbol 1 => write 1, move left, halt
                0C1 -> H01
                1C1 -> H11",
            input: "000000A000000",
            expected_result: "00011H1111000",
        },
    ]
}

#[cfg(not(test))]
fn main() {
    for (index, sample) in get_samples().iter().enumerate() {
        match MarkovAlgorithm::from_str(sample.ruleset) {
            Ok(algorithm) => {
                println!("Sample {}", (index + 1));
                println!("Output: {}", algorithm.apply(sample.input));
            }
            Err(message) => println!("{}", message)
        }
    }
}

#[test]
fn test_samples() {
    for sample in get_samples().iter() {
        match MarkovAlgorithm::from_str(sample.ruleset) {
            Ok(algorithm) => assert_eq!(sample.expected_result,
                                        algorithm.apply(sample.input).as_slice()),
            Err(message) => fail!("{}", message)
        }
    }
}
<|endoftext|>


<commit_before>

use ::backend;
use ::event::{Event, Key};
use std::io::Write;
use termion;
use termion::input::TermRead;
use termion::raw::IntoRawMode;
use ::theme::{BaseColor, Color, ColorStyle, Effect};

pub struct TermionBackend {
    terminal: termion::raw::RawTerminal<::std::io::Stdout>,
}

impl backend::Backend for TermionBackend {
    fn init() -> Self {
        print!("{}", termion::cursor::Hide);
        Self::clear();

        TermionBackend {
            terminal: ::std::io::stdout().into_raw_mode().unwrap(),
        }
    }

    fn finish(&mut self) {
        // Maybe we should clear everything?
        print!("{}{}", termion::cursor::Show, termion::cursor::Goto(1, 1));
        Self::clear();
    }

    fn init_color_style(&mut self, style: ColorStyle, foreground: &Color,
                        background: &Color) {
        // Do we _need_ to save the color?
    }

    fn with_color<F: FnOnce()>(&self, color: ColorStyle, f: F) {
        // TODO: actually use colors
        // TODO: careful! need to remember the previous state
        //                and apply it back
        f();
    }

    fn with_effect<F: FnOnce()>(&self, effect: Effect, f: F) {
        // TODO: actually use effects
        // TODO: careful! need to remember the previous state
        //                and apply it back
        f();
    }

    fn has_colors(&self) -> bool {
        // TODO: color support detection?
        true
    }

    fn screen_size(&self) -> (usize, usize) {
        let (x, y) = termion::terminal_size().unwrap_or((1, 1));
        (x as usize, y as usize)
    }

    fn clear() {
        print!("{}", termion::clear::All);
    }

    fn refresh(&mut self) {
        // Not sure termion needs a refresh phase
        self.terminal.flush().unwrap();
    }

    fn print_at(&self, (x, y): (usize, usize), text: &str) {
        // TODO: terminals are 1-based. Should we add 1 here?
        print!("{}{}", termion::cursor::Goto(x as u16, y as u16), text);
    }

    fn set_refresh_rate(&mut self, fps: u32) {
        // TODO: handle async refresh, when no input is entered.
        // Could be done with a timeout on the event polling,
        // if it was supportedd.
    }

    fn poll_event(&self) -> Event {
        ::std::io::stdin().keys().next();
        Event::Key(Key::Enter)
    }
}
<commit_msg>Take `&self` in TermionBackend::clear()<commit_after>use ::backend;
use ::event::{Event, Key};
use std::io::Write;
use termion;
use termion::input::TermRead;
use termion::raw::IntoRawMode;
use ::theme::{BaseColor, Color, ColorStyle, Effect};

pub struct TermionBackend {
    terminal: termion::raw::RawTerminal<::std::io::Stdout>,
}

impl backend::Backend for TermionBackend {
    fn init() -> Self {
        print!("{}", termion::cursor::Hide);

        let backend = TermionBackend {
            terminal: ::std::io::stdout().into_raw_mode().unwrap(),
        };

        backend.clear();
        backend
    }

    fn finish(&mut self) {
        // Maybe we should clear everything?
        print!("{}{}", termion::cursor::Show, termion::cursor::Goto(1, 1));
        self.clear();
    }

    fn init_color_style(&mut self, style: ColorStyle, foreground: &Color,
                        background: &Color) {
        // Do we _need_ to save the color?
    }

    fn with_color<F: FnOnce()>(&self, color: ColorStyle, f: F) {
        // TODO: actually use colors
        // TODO: careful! need to remember the previous state
        //                and apply it back
        f();
    }

    fn with_effect<F: FnOnce()>(&self, effect: Effect, f: F) {
        // TODO: actually use effects
        // TODO: careful! need to remember the previous state
        //                and apply it back
        f();
    }

    fn has_colors(&self) -> bool {
        // TODO: color support detection?
        true
    }

    fn screen_size(&self) -> (usize, usize) {
        let (x, y) = termion::terminal_size().unwrap_or((1, 1));
        (x as usize, y as usize)
    }

    fn clear(&self) {
        print!("{}", termion::clear::All);
    }

    fn refresh(&mut self) {
        // Not sure termion needs a refresh phase
        self.terminal.flush().unwrap();
    }

    fn print_at(&self, (x, y): (usize, usize), text: &str) {
        // TODO: terminals are 1-based. Should we add 1 here?
        print!("{}{}", termion::cursor::Goto(x as u16, y as u16), text);
    }

    fn set_refresh_rate(&mut self, fps: u32) {
        // TODO: handle async refresh, when no input is entered.
        // Could be done with a timeout on the event polling,
        // if it was supportedd.
    }

    fn poll_event(&self) -> Event {
        ::std::io::stdin().keys().next();
        Event::Key(Key::Enter)
    }
}
<|endoftext|>


<commit_before>use std::str::from_utf8;

use rustc_serialize::*;

use websocket::Message;
use websocket::Sender as WebSocketSender;
use websocket::Receiver as WebSocketReceiver;
use websocket::message::Type;
use websocket::Client;
use websocket::client::request::Url;
use websocket::result::WebSocketError;

use payment_detection::{PaymentDetection, PaymentError};

// For complete blockchain.info API see: https://blockchain.info/de/api/api_websocket

#[derive(RustcDecodable)]
pub struct AddressEvent {
	pub x: Transaction,
}

#[derive(RustcDecodable)]
pub struct Transaction {
	pub inputs: Vec<TransactionInput>,
	pub out: Vec<TransactionOutput>
}

#[derive(RustcDecodable)]
pub struct TransactionOutput {
	pub value: u64,
	pub addr: String,
}

#[derive(RustcDecodable)]
pub struct TransactionInput {
	pub prev_out: TransactionOutput,
}

#[derive(RustcEncodable)]
pub struct AddressSubscription<'a> {
	op: &'a str,
	addr: &'a str
}

const ADDRESS_SUBCRIPTION_OP: &'static str = "addr_sub";

impl<'a> AddressSubscription<'a> {
	pub fn new(address: &str) -> AddressSubscription {
		AddressSubscription {
			op: ADDRESS_SUBCRIPTION_OP,
			addr: address
		}
	}
}

pub struct BlockchainInfo<'a> {
	websocket_url: String,
	websocket_msg: String,
	address: &'a str,
	amount: u64
}

const WEBSOCKET_URL: &'static str = "wss://ws.blockchain.info/inv";

impl<'a> PaymentDetection<'a> for BlockchainInfo<'a> {
	fn new(address: &'a str, amount: u64) -> Self {
		let address_subscription = AddressSubscription::new(address);
		let json_request = json::encode(&address_subscription).unwrap();

		BlockchainInfo {
			websocket_url: WEBSOCKET_URL.to_string(),
			websocket_msg: json_request,
			address: &address,
			amount: amount
		}
	}

	fn wait(&self) -> Result<(), PaymentError> {
		let websocket_url = Url::parse(&self.websocket_url).unwrap();
		println!("Connecting to {}", websocket_url);

		let request = Client::connect(websocket_url).unwrap();

		let response = request.send().unwrap(); // Send the request and retrieve a response

		let mut client = response.begin(); // Get a Client

		println!("Successfully connected");

		client.send_message(&Message::text(self.websocket_msg.to_string())).unwrap(); // Send message

		let mut amount_sum = 0;
		for message in client.incoming_messages() {
			match handle_msg(&self, &message) {
				Ok(incoming_amount) => {
					amount_sum = amount_sum + incoming_amount;
					if amount_sum >= self.amount {
						println!("PAYMENT COMPLETE");
						break;
					}
				}

				Err(e) => match e {
					PaymentError::Timeout => {
						if amount_sum > 0 {
							return Err(PaymentError::InsufficientAmount);
						} else {
						    return Err(PaymentError::Timeout);
						}
					}
					_ => return Err(e)
				}
			}
		}

		Ok(())
	}
}

fn handle_msg(blockchain_info: &BlockchainInfo, message: &Result<Message, WebSocketError>) -> Result<u64, PaymentError> {
    let message: &Message = match *message {
    	Ok(ref m) => m,
    	Err(ref e) => match e {
    		&WebSocketError::NoDataAvailable => return Err(PaymentError::Timeout),
    		_ => return Err(PaymentError::BackendError)
    	}
    };

	match message.opcode {
		Type::Text => {
			match incoming_amount(&*message.payload, &blockchain_info.address.to_string()) {
				Ok(amount) => Ok(amount),
				Err(e) => Err(e)
			}
		}

		Type::Close => {
			Err(PaymentError::Timeout)
		}

		_ => {
			println!("Backend error (unhandled websocket message): {:?}", message);
			Err(PaymentError::BackendError)
		}
	}
}

fn incoming_amount(data: &[u8], address: &str) -> Result<u64, PaymentError> {
	let data = match from_utf8(data) {
        Ok(v) => v,
        Err(e) => {
        	println!("Backend error (UTF-8 decoding): {:?}", e);
        	return Err(PaymentError::BackendError);
        }
    };

	let address_event: AddressEvent = match json::decode(&data) {
		Ok(ae) => ae,
		Err(e) => {
			println!("Backend error (JSON decoding): {:?}", e);
			return Err(PaymentError::BackendError);
		}
	};

	let transaction: Transaction = address_event.x;

	let mut amount_payed = 0;
	for output in &transaction.out {
		if output.addr == address {
			println!("received {} satoshis from {}", output.value, &transaction.inputs[0].prev_out.addr);
			amount_payed = amount_payed + output.value;
		}
	}

	Ok(amount_payed)
}

<commit_msg>final refactoring of blockchain_info client code<commit_after>use std::str::from_utf8;

use rustc_serialize::*;

use websocket::Message;
use websocket::Sender as WebSocketSender;
use websocket::Receiver as WebSocketReceiver;
use websocket::message::Type;
use websocket::Client;
use websocket::client::request::Url;
use websocket::result::WebSocketError;

use payment_detection::{PaymentDetection, PaymentError};

// For complete blockchain.info API see: https://blockchain.info/de/api/api_websocket

#[derive(RustcDecodable)]
pub struct AddressEvent {
	pub x: Transaction,
}

#[derive(RustcDecodable)]
pub struct Transaction {
	pub inputs: Vec<TransactionInput>,
	pub out: Vec<TransactionOutput>
}

#[derive(RustcDecodable)]
pub struct TransactionOutput {
	pub value: u64,
	pub addr: String,
}

#[derive(RustcDecodable)]
pub struct TransactionInput {
	pub prev_out: TransactionOutput,
}

#[derive(RustcEncodable)]
pub struct AddressSubscription<'a> {
	op: &'a str,
	addr: &'a str
}

const ADDRESS_SUBCRIPTION_OP: &'static str = "addr_sub";

impl<'a> AddressSubscription<'a> {
	pub fn new(address: &str) -> AddressSubscription {
		AddressSubscription {
			op: ADDRESS_SUBCRIPTION_OP,
			addr: address
		}
	}
}

pub struct BlockchainInfo<'a> {
	websocket_url: String,
	websocket_msg: String,
	address: &'a str,
	amount: u64
}

const WEBSOCKET_URL: &'static str = "wss://ws.blockchain.info/inv";

impl<'a> PaymentDetection<'a> for BlockchainInfo<'a> {
	fn new(address: &'a str, amount: u64) -> Self {
		let address_subscription = AddressSubscription::new(address);
		let json_request = json::encode(&address_subscription).unwrap();

		BlockchainInfo {
			websocket_url: WEBSOCKET_URL.to_string(),
			websocket_msg: json_request,
			address: &address,
			amount: amount
		}
	}

	fn wait(&self) -> Result<(), PaymentError> {
		let websocket_url = Url::parse(&self.websocket_url).unwrap();
		println!("Connecting to {}", websocket_url);

		let request = Client::connect(websocket_url).unwrap();

		let response = request.send().unwrap(); // Send the request and retrieve a response

		let mut client = response.begin(); // Get a Client

		println!("Successfully connected");

		client.send_message(&Message::text(self.websocket_msg.to_string())).unwrap(); // Send message

		let mut amount_sum = 0;
		for message in client.incoming_messages() {
			match handle_msg(&self, &message) {
				Ok(incoming_amount) => {
					amount_sum = amount_sum + incoming_amount;
					if amount_sum >= self.amount {
						println!("PAYMENT COMPLETE");
						break;
					}
				}

				Err(e) => match e {
					PaymentError::Timeout => {
						if amount_sum > 0 {
							return Err(PaymentError::InsufficientAmount);
						} else {
						    return Err(PaymentError::Timeout);
						}
					}
					_ => return Err(e)
				}
			}
		}

		Ok(())
	}
}

fn handle_msg(blockchain_info: &BlockchainInfo, message: &Result<Message, WebSocketError>) -> Result<u64, PaymentError> {
    let message: &Message = match *message {
    	Ok(ref m) => m,
    	Err(ref e) => match e {
    		&WebSocketError::NoDataAvailable => return Err(PaymentError::Timeout),
    		_ => return Err(PaymentError::BackendError)
    	}
    };

	match message.opcode {
		Type::Text => {
			let data = match from_utf8(&message.payload) {
		        Ok(v) => v,
		        Err(e) => {
		        	println!("Backend error (UTF-8 decoding): {:?}", e);
		        	return Err(PaymentError::BackendError);
		        }
		    };

			let address_event: AddressEvent = match json::decode(&data) {
				Ok(ae) => ae,
				Err(e) => {
					println!("Backend error (JSON decoding): {:?}", e);
					return Err(PaymentError::BackendError);
				}
			};

			let transaction: Transaction = address_event.x;

			let mut amount_payed = 0;
			for output in &transaction.out {
				if output.addr == blockchain_info.address.to_string() {
					println!("received {} satoshis from {}", output.value, &transaction.inputs[0].prev_out.addr);
					amount_payed = amount_payed + output.value;
				}
			}

			Ok(amount_payed)
		}

		Type::Close => {
			Err(PaymentError::Timeout)
		}

		_ => {
			println!("Backend error (unhandled websocket message): {:?}", message);
			Err(PaymentError::BackendError)
		}
	}
}
<|endoftext|>


<commit_before>mod build_lex_table;
mod build_parse_table;
mod coincident_tokens;
mod item;
mod item_set_builder;
mod minimize_parse_table;
mod token_conflicts;

use self::build_lex_table::build_lex_table;
use self::build_parse_table::build_parse_table;
use self::coincident_tokens::CoincidentTokenIndex;
use self::item::TokenSet;
use self::minimize_parse_table::minimize_parse_table;
use self::token_conflicts::TokenConflictMap;
use crate::error::Result;
use crate::generate::grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar};
use crate::generate::nfa::{CharacterSet, NfaCursor};
use crate::generate::rules::{AliasMap, Symbol, SymbolType};
use crate::generate::tables::{LexTable, ParseAction, ParseTable, ParseTableEntry};

pub(crate) fn build_tables(
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
    simple_aliases: &AliasMap,
    inlines: &InlinedProductionMap,
    minimize: bool,
    state_ids_to_log: Vec<usize>,
) -> Result<(ParseTable, LexTable, LexTable, Option<Symbol>)> {
    let (mut parse_table, following_tokens) =
        build_parse_table(syntax_grammar, lexical_grammar, inlines, state_ids_to_log)?;
    let token_conflict_map = TokenConflictMap::new(lexical_grammar, following_tokens);
    let coincident_token_index = CoincidentTokenIndex::new(&parse_table, lexical_grammar);
    let keywords = identify_keywords(
        lexical_grammar,
        &parse_table,
        syntax_grammar.word_token,
        &token_conflict_map,
        &coincident_token_index,
    );
    populate_error_state(
        &mut parse_table,
        syntax_grammar,
        lexical_grammar,
        &coincident_token_index,
        &token_conflict_map,
        &keywords,
    );
    populate_used_symbols(&mut parse_table, syntax_grammar, lexical_grammar);
    mark_fragile_tokens(&mut parse_table, lexical_grammar, &token_conflict_map);
    if minimize {
        minimize_parse_table(
            &mut parse_table,
            syntax_grammar,
            lexical_grammar,
            simple_aliases,
            &token_conflict_map,
            &keywords,
        );
    }
    let (main_lex_table, keyword_lex_table) = build_lex_table(
        &mut parse_table,
        syntax_grammar,
        lexical_grammar,
        &keywords,
        &coincident_token_index,
        &token_conflict_map,
        minimize,
    );
    Ok((
        parse_table,
        main_lex_table,
        keyword_lex_table,
        syntax_grammar.word_token,
    ))
}

fn populate_error_state(
    parse_table: &mut ParseTable,
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
    coincident_token_index: &CoincidentTokenIndex,
    token_conflict_map: &TokenConflictMap,
    keywords: &TokenSet,
) {
    let state = &mut parse_table.states[0];
    let n = lexical_grammar.variables.len();

    // First identify the *conflict-free tokens*: tokens that do not overlap with
    // any other token in any way, besides matching exactly the same string.
    let conflict_free_tokens: TokenSet = (0..n)
        .into_iter()
        .filter_map(|i| {
            let conflicts_with_other_tokens = (0..n).into_iter().any(|j| {
                j != i
                    && !coincident_token_index.contains(Symbol::terminal(i), Symbol::terminal(j))
                    && token_conflict_map.does_match_shorter_or_longer(i, j)
            });
            if conflicts_with_other_tokens {
                None
            } else {
                info!(
                    "error recovery - token {} has no conflicts",
                    lexical_grammar.variables[i].name
                );
                Some(Symbol::terminal(i))
            }
        })
        .collect();

    let recover_entry = ParseTableEntry {
        reusable: false,
        actions: vec![ParseAction::Recover],
    };

    // Exclude from the error-recovery state any token that conflicts with one of
    // the *conflict-free tokens* identified above.
    for i in 0..n {
        let symbol = Symbol::terminal(i);
        if !conflict_free_tokens.contains(&symbol) && !keywords.contains(&symbol) {
            if syntax_grammar.word_token != Some(symbol) {
                if let Some(t) = conflict_free_tokens.iter().find(|t| {
                    !coincident_token_index.contains(symbol, *t)
                        && token_conflict_map.does_conflict(symbol.index, t.index)
                }) {
                    info!(
                        "error recovery - exclude token {} because of conflict with {}",
                        lexical_grammar.variables[i].name, lexical_grammar.variables[t.index].name
                    );
                    continue;
                }
            }
        }
        info!(
            "error recovery - include token {}",
            lexical_grammar.variables[i].name
        );
        state
            .terminal_entries
            .entry(symbol)
            .or_insert_with(|| recover_entry.clone());
    }

    for (i, external_token) in syntax_grammar.external_tokens.iter().enumerate() {
        if external_token.corresponding_internal_token.is_none() {
            state
                .terminal_entries
                .entry(Symbol::external(i))
                .or_insert_with(|| recover_entry.clone());
        }
    }

    state.terminal_entries.insert(Symbol::end(), recover_entry);
}

fn populate_used_symbols(
    parse_table: &mut ParseTable,
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
) {
    let mut terminal_usages = vec![false; lexical_grammar.variables.len()];
    let mut non_terminal_usages = vec![false; syntax_grammar.variables.len()];
    let mut external_usages = vec![false; syntax_grammar.external_tokens.len()];
    for state in &parse_table.states {
        for symbol in state.terminal_entries.keys() {
            match symbol.kind {
                SymbolType::Terminal => terminal_usages[symbol.index] = true,
                SymbolType::External => external_usages[symbol.index] = true,
                _ => {}
            }
        }
        for symbol in state.nonterminal_entries.keys() {
            non_terminal_usages[symbol.index] = true;
        }
    }
    parse_table.symbols.push(Symbol::end());
    for (i, value) in terminal_usages.into_iter().enumerate() {
        if value {
            // Assign the grammar's word token a low numerical index. This ensures that
            // it can be stored in a subtree with no heap allocations, even for grammars with
            // very large numbers of tokens. This is an optimization, but it's also important to
            // ensure that a subtree's symbol can be successfully reassigned to the word token
            // without having to move the subtree to the heap.
            // See https://github.com/tree-sitter/tree-sitter/issues/258
            if syntax_grammar.word_token.map_or(false, |t| t.index == i) {
                parse_table.symbols.insert(1, Symbol::terminal(i));
            } else {
                parse_table.symbols.push(Symbol::terminal(i));
            }
        }
    }
    for (i, value) in external_usages.into_iter().enumerate() {
        if value {
            parse_table.symbols.push(Symbol::external(i));
        }
    }
    for (i, value) in non_terminal_usages.into_iter().enumerate() {
        if value {
            parse_table.symbols.push(Symbol::non_terminal(i));
        }
    }
}

fn identify_keywords(
    lexical_grammar: &LexicalGrammar,
    parse_table: &ParseTable,
    word_token: Option<Symbol>,
    token_conflict_map: &TokenConflictMap,
    coincident_token_index: &CoincidentTokenIndex,
) -> TokenSet {
    if word_token.is_none() {
        return TokenSet::new();
    }

    let word_token = word_token.unwrap();
    let mut cursor = NfaCursor::new(&lexical_grammar.nfa, Vec::new());

    // First find all of the candidate keyword tokens: tokens that start with
    // letters or underscore and can match the same string as a word token.
    let keywords: TokenSet = lexical_grammar
        .variables
        .iter()
        .enumerate()
        .filter_map(|(i, variable)| {
            cursor.reset(vec![variable.start_state]);
            if all_chars_are_alphabetical(&cursor)
                && token_conflict_map.does_match_same_string(i, word_token.index)
            {
                info!(
                    "Keywords - add candidate {}",
                    lexical_grammar.variables[i].name
                );
                Some(Symbol::terminal(i))
            } else {
                None
            }
        })
        .collect();

    // Exclude keyword candidates that shadow another keyword candidate.
    let keywords: TokenSet = keywords
        .iter()
        .filter(|token| {
            for other_token in keywords.iter() {
                if other_token != *token
                    && token_conflict_map.does_match_same_string(other_token.index, token.index)
                {
                    info!(
                        "Keywords - exclude {} because it matches the same string as {}",
                        lexical_grammar.variables[token.index].name,
                        lexical_grammar.variables[other_token.index].name
                    );
                    return false;
                }
            }
            true
        })
        .collect();

    // Exclude keyword candidates for which substituting the keyword capture
    // token would introduce new lexical conflicts with other tokens.
    let keywords = keywords
        .iter()
        .filter(|token| {
            for other_index in 0..lexical_grammar.variables.len() {
                if keywords.contains(&Symbol::terminal(other_index)) {
                    continue;
                }

                // If the word token was already valid in every state containing
                // this keyword candidate, then substituting the word token won't
                // introduce any new lexical conflicts.
                if coincident_token_index
                    .states_with(*token, Symbol::terminal(other_index))
                    .iter()
                    .all(|state_id| {
                        parse_table.states[*state_id]
                            .terminal_entries
                            .contains_key(&word_token)
                    })
                {
                    continue;
                }

                if !token_conflict_map.has_same_conflict_status(
                    token.index,
                    word_token.index,
                    other_index,
                ) {
                    info!(
                        "Keywords - exclude {} because of conflict with {}",
                        lexical_grammar.variables[token.index].name,
                        lexical_grammar.variables[other_index].name
                    );
                    return false;
                }
            }

            info!(
                "Keywords - include {}",
                lexical_grammar.variables[token.index].name,
            );
            true
        })
        .collect();

    keywords
}

fn mark_fragile_tokens(
    parse_table: &mut ParseTable,
    lexical_grammar: &LexicalGrammar,
    token_conflict_map: &TokenConflictMap,
) {
    let n = lexical_grammar.variables.len();
    let mut valid_tokens_mask = Vec::with_capacity(n);
    for state in parse_table.states.iter_mut() {
        valid_tokens_mask.clear();
        valid_tokens_mask.resize(n, false);
        for token in state.terminal_entries.keys() {
            if token.is_terminal() {
                valid_tokens_mask[token.index] = true;
            }
        }
        for (token, entry) in state.terminal_entries.iter_mut() {
            if token.is_terminal() {
                for i in 0..n {
                    if token_conflict_map.does_overlap(i, token.index) {
                        if valid_tokens_mask[i] {
                            entry.reusable = false;
                            break;
                        }
                    }
                }
            }
        }
    }
}

fn all_chars_are_alphabetical(cursor: &NfaCursor) -> bool {
    cursor.transition_chars().all(|(chars, is_sep)| {
        if is_sep {
            true
        } else if let CharacterSet::Include(chars) = chars {
            chars.iter().all(|c| c.is_alphabetic() || *c == '_')
        } else {
            false
        }
    })
}
<commit_msg>Loosen keyword identification criteria slightly<commit_after>mod build_lex_table;
mod build_parse_table;
mod coincident_tokens;
mod item;
mod item_set_builder;
mod minimize_parse_table;
mod token_conflicts;

use self::build_lex_table::build_lex_table;
use self::build_parse_table::build_parse_table;
use self::coincident_tokens::CoincidentTokenIndex;
use self::item::TokenSet;
use self::minimize_parse_table::minimize_parse_table;
use self::token_conflicts::TokenConflictMap;
use crate::error::Result;
use crate::generate::grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar};
use crate::generate::nfa::{CharacterSet, NfaCursor};
use crate::generate::rules::{AliasMap, Symbol, SymbolType};
use crate::generate::tables::{LexTable, ParseAction, ParseTable, ParseTableEntry};

pub(crate) fn build_tables(
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
    simple_aliases: &AliasMap,
    inlines: &InlinedProductionMap,
    minimize: bool,
    state_ids_to_log: Vec<usize>,
) -> Result<(ParseTable, LexTable, LexTable, Option<Symbol>)> {
    let (mut parse_table, following_tokens) =
        build_parse_table(syntax_grammar, lexical_grammar, inlines, state_ids_to_log)?;
    let token_conflict_map = TokenConflictMap::new(lexical_grammar, following_tokens);
    let coincident_token_index = CoincidentTokenIndex::new(&parse_table, lexical_grammar);
    let keywords = identify_keywords(
        lexical_grammar,
        &parse_table,
        syntax_grammar.word_token,
        &token_conflict_map,
        &coincident_token_index,
    );
    populate_error_state(
        &mut parse_table,
        syntax_grammar,
        lexical_grammar,
        &coincident_token_index,
        &token_conflict_map,
        &keywords,
    );
    populate_used_symbols(&mut parse_table, syntax_grammar, lexical_grammar);
    mark_fragile_tokens(&mut parse_table, lexical_grammar, &token_conflict_map);
    if minimize {
        minimize_parse_table(
            &mut parse_table,
            syntax_grammar,
            lexical_grammar,
            simple_aliases,
            &token_conflict_map,
            &keywords,
        );
    }
    let (main_lex_table, keyword_lex_table) = build_lex_table(
        &mut parse_table,
        syntax_grammar,
        lexical_grammar,
        &keywords,
        &coincident_token_index,
        &token_conflict_map,
        minimize,
    );
    Ok((
        parse_table,
        main_lex_table,
        keyword_lex_table,
        syntax_grammar.word_token,
    ))
}

fn populate_error_state(
    parse_table: &mut ParseTable,
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
    coincident_token_index: &CoincidentTokenIndex,
    token_conflict_map: &TokenConflictMap,
    keywords: &TokenSet,
) {
    let state = &mut parse_table.states[0];
    let n = lexical_grammar.variables.len();

    // First identify the *conflict-free tokens*: tokens that do not overlap with
    // any other token in any way, besides matching exactly the same string.
    let conflict_free_tokens: TokenSet = (0..n)
        .into_iter()
        .filter_map(|i| {
            let conflicts_with_other_tokens = (0..n).into_iter().any(|j| {
                j != i
                    && !coincident_token_index.contains(Symbol::terminal(i), Symbol::terminal(j))
                    && token_conflict_map.does_match_shorter_or_longer(i, j)
            });
            if conflicts_with_other_tokens {
                None
            } else {
                info!(
                    "error recovery - token {} has no conflicts",
                    lexical_grammar.variables[i].name
                );
                Some(Symbol::terminal(i))
            }
        })
        .collect();

    let recover_entry = ParseTableEntry {
        reusable: false,
        actions: vec![ParseAction::Recover],
    };

    // Exclude from the error-recovery state any token that conflicts with one of
    // the *conflict-free tokens* identified above.
    for i in 0..n {
        let symbol = Symbol::terminal(i);
        if !conflict_free_tokens.contains(&symbol) && !keywords.contains(&symbol) {
            if syntax_grammar.word_token != Some(symbol) {
                if let Some(t) = conflict_free_tokens.iter().find(|t| {
                    !coincident_token_index.contains(symbol, *t)
                        && token_conflict_map.does_conflict(symbol.index, t.index)
                }) {
                    info!(
                        "error recovery - exclude token {} because of conflict with {}",
                        lexical_grammar.variables[i].name, lexical_grammar.variables[t.index].name
                    );
                    continue;
                }
            }
        }
        info!(
            "error recovery - include token {}",
            lexical_grammar.variables[i].name
        );
        state
            .terminal_entries
            .entry(symbol)
            .or_insert_with(|| recover_entry.clone());
    }

    for (i, external_token) in syntax_grammar.external_tokens.iter().enumerate() {
        if external_token.corresponding_internal_token.is_none() {
            state
                .terminal_entries
                .entry(Symbol::external(i))
                .or_insert_with(|| recover_entry.clone());
        }
    }

    state.terminal_entries.insert(Symbol::end(), recover_entry);
}

fn populate_used_symbols(
    parse_table: &mut ParseTable,
    syntax_grammar: &SyntaxGrammar,
    lexical_grammar: &LexicalGrammar,
) {
    let mut terminal_usages = vec![false; lexical_grammar.variables.len()];
    let mut non_terminal_usages = vec![false; syntax_grammar.variables.len()];
    let mut external_usages = vec![false; syntax_grammar.external_tokens.len()];
    for state in &parse_table.states {
        for symbol in state.terminal_entries.keys() {
            match symbol.kind {
                SymbolType::Terminal => terminal_usages[symbol.index] = true,
                SymbolType::External => external_usages[symbol.index] = true,
                _ => {}
            }
        }
        for symbol in state.nonterminal_entries.keys() {
            non_terminal_usages[symbol.index] = true;
        }
    }
    parse_table.symbols.push(Symbol::end());
    for (i, value) in terminal_usages.into_iter().enumerate() {
        if value {
            // Assign the grammar's word token a low numerical index. This ensures that
            // it can be stored in a subtree with no heap allocations, even for grammars with
            // very large numbers of tokens. This is an optimization, but it's also important to
            // ensure that a subtree's symbol can be successfully reassigned to the word token
            // without having to move the subtree to the heap.
            // See https://github.com/tree-sitter/tree-sitter/issues/258
            if syntax_grammar.word_token.map_or(false, |t| t.index == i) {
                parse_table.symbols.insert(1, Symbol::terminal(i));
            } else {
                parse_table.symbols.push(Symbol::terminal(i));
            }
        }
    }
    for (i, value) in external_usages.into_iter().enumerate() {
        if value {
            parse_table.symbols.push(Symbol::external(i));
        }
    }
    for (i, value) in non_terminal_usages.into_iter().enumerate() {
        if value {
            parse_table.symbols.push(Symbol::non_terminal(i));
        }
    }
}

fn identify_keywords(
    lexical_grammar: &LexicalGrammar,
    parse_table: &ParseTable,
    word_token: Option<Symbol>,
    token_conflict_map: &TokenConflictMap,
    coincident_token_index: &CoincidentTokenIndex,
) -> TokenSet {
    if word_token.is_none() {
        return TokenSet::new();
    }

    let word_token = word_token.unwrap();
    let mut cursor = NfaCursor::new(&lexical_grammar.nfa, Vec::new());

    // First find all of the candidate keyword tokens: tokens that start with
    // letters or underscore and can match the same string as a word token.
    let keyword_candidates: TokenSet = lexical_grammar
        .variables
        .iter()
        .enumerate()
        .filter_map(|(i, variable)| {
            cursor.reset(vec![variable.start_state]);
            if all_chars_are_alphabetical(&cursor)
                && token_conflict_map.does_match_same_string(i, word_token.index)
            {
                info!(
                    "Keywords - add candidate {}",
                    lexical_grammar.variables[i].name
                );
                Some(Symbol::terminal(i))
            } else {
                None
            }
        })
        .collect();

    // Exclude keyword candidates that shadow another keyword candidate.
    let keywords: TokenSet = keyword_candidates
        .iter()
        .filter(|token| {
            for other_token in keyword_candidates.iter() {
                if other_token != *token
                    && token_conflict_map.does_match_same_string(other_token.index, token.index)
                {
                    info!(
                        "Keywords - exclude {} because it matches the same string as {}",
                        lexical_grammar.variables[token.index].name,
                        lexical_grammar.variables[other_token.index].name
                    );
                    return false;
                }
            }
            true
        })
        .collect();

    // Exclude keyword candidates for which substituting the keyword capture
    // token would introduce new lexical conflicts with other tokens.
    let keywords = keywords
        .iter()
        .filter(|token| {
            for other_index in 0..lexical_grammar.variables.len() {
                if keyword_candidates.contains(&Symbol::terminal(other_index)) {
                    continue;
                }

                // If the word token was already valid in every state containing
                // this keyword candidate, then substituting the word token won't
                // introduce any new lexical conflicts.
                if coincident_token_index
                    .states_with(*token, Symbol::terminal(other_index))
                    .iter()
                    .all(|state_id| {
                        parse_table.states[*state_id]
                            .terminal_entries
                            .contains_key(&word_token)
                    })
                {
                    continue;
                }

                if !token_conflict_map.has_same_conflict_status(
                    token.index,
                    word_token.index,
                    other_index,
                ) {
                    info!(
                        "Keywords - exclude {} because of conflict with {}",
                        lexical_grammar.variables[token.index].name,
                        lexical_grammar.variables[other_index].name
                    );
                    return false;
                }
            }

            info!(
                "Keywords - include {}",
                lexical_grammar.variables[token.index].name,
            );
            true
        })
        .collect();

    keywords
}

fn mark_fragile_tokens(
    parse_table: &mut ParseTable,
    lexical_grammar: &LexicalGrammar,
    token_conflict_map: &TokenConflictMap,
) {
    let n = lexical_grammar.variables.len();
    let mut valid_tokens_mask = Vec::with_capacity(n);
    for state in parse_table.states.iter_mut() {
        valid_tokens_mask.clear();
        valid_tokens_mask.resize(n, false);
        for token in state.terminal_entries.keys() {
            if token.is_terminal() {
                valid_tokens_mask[token.index] = true;
            }
        }
        for (token, entry) in state.terminal_entries.iter_mut() {
            if token.is_terminal() {
                for i in 0..n {
                    if token_conflict_map.does_overlap(i, token.index) {
                        if valid_tokens_mask[i] {
                            entry.reusable = false;
                            break;
                        }
                    }
                }
            }
        }
    }
}

fn all_chars_are_alphabetical(cursor: &NfaCursor) -> bool {
    cursor.transition_chars().all(|(chars, is_sep)| {
        if is_sep {
            true
        } else if let CharacterSet::Include(chars) = chars {
            chars.iter().all(|c| c.is_alphabetic() || *c == '_')
        } else {
            false
        }
    })
}
<|endoftext|>