<commit_before>mod attestation_producer;
mod block_producer;
mod config;
mod duties;
pub mod error;
mod service;
mod signer;

use crate::config::{
    Config as ClientConfig, KeySource, DEFAULT_SERVER, DEFAULT_SERVER_GRPC_PORT,
    DEFAULT_SERVER_HTTP_PORT,
};
use crate::service::Service as ValidatorService;
use clap::{App, Arg, ArgMatches, SubCommand};
use eth2_config::Eth2Config;
use lighthouse_bootstrap::Bootstrapper;
use protos::services_grpc::ValidatorServiceClient;
use slog::{crit, error, info, o, Drain, Level, Logger};
use types::{InteropEthSpec, Keypair, MainnetEthSpec, MinimalEthSpec};

pub const DEFAULT_SPEC: &str = "minimal";
pub const DEFAULT_DATA_DIR: &str = ".lighthouse-validator";
pub const CLIENT_CONFIG_FILENAME: &str = "validator-client.toml";
pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml";

type Result<T> = core::result::Result<T, String>;

fn main() {
    // Logging
    let decorator = slog_term::TermDecorator::new().build();
    let decorator = logging::AlignedTermDecorator::new(decorator, logging::MAX_MESSAGE_WIDTH);
    let drain = slog_term::FullFormat::new(decorator).build().fuse();
    let drain = slog_async::Async::new(drain).build().fuse();

    // CLI
    let matches = App::new("Lighthouse Validator Client")
        .version("0.0.1")
        .author("Sigma Prime <contact@sigmaprime.io>")
        .about("Eth 2.0 Validator Client")
        .arg(
            Arg::with_name("datadir")
                .long("datadir")
                .short("d")
                .value_name("DIR")
                .help("Data directory for keys and databases.")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("logfile")
                .long("logfile")
                .value_name("logfile")
                .help("File path where output will be written.")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("spec")
                .short("s")
                .long("spec")
                .value_name("TITLE")
                .help("Specifies the default eth2 spec type.")
                .takes_value(true)
                .possible_values(&["mainnet", "minimal", "interop"])
                .conflicts_with("eth2-config")
                .global(true)
        )
        .arg(
            Arg::with_name("eth2-config")
                .long("eth2-config")
                .short("e")
                .value_name("TOML_FILE")
                .help("Path to Ethereum 2.0 config and specification file (e.g., eth2_spec.toml).")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("server")
                .long("server")
                .value_name("NETWORK_ADDRESS")
                .help("Address to connect to BeaconNode.")
                .default_value(DEFAULT_SERVER)
                .takes_value(true),
        )
        .arg(
            Arg::with_name("server-grpc-port")
                .long("g")
                .value_name("PORT")
                .help("Port to use for gRPC API connection to the server.")
                .default_value(DEFAULT_SERVER_GRPC_PORT)
                .takes_value(true),
        )
        .arg(
            Arg::with_name("server-http-port")
                .long("h")
                .value_name("PORT")
                .help("Port to use for HTTP API connection to the server.")
                .default_value(DEFAULT_SERVER_HTTP_PORT)
                .takes_value(true),
        )
        .arg(
            Arg::with_name("debug-level")
                .long("debug-level")
                .value_name("LEVEL")
                .short("s")
                .help("The title of the spec constants for chain config.")
                .takes_value(true)
                .possible_values(&["info", "debug", "trace", "warn", "error", "crit"])
                .default_value("info"),
        )
        /*
         * The "testnet" sub-command.
         *
         * Used for starting testnet validator clients.
         */
        .subcommand(SubCommand::with_name("testnet")
            .about("Starts a testnet validator using INSECURE, predicatable private keys, based off the canonical \
                   validator index. ONLY USE FOR TESTING PURPOSES!")
            .arg(
                Arg::with_name("bootstrap")
                    .short("b")
                    .long("bootstrap")
                    .help("Connect to the RPC server to download the eth2_config via the HTTP API.")
            )
            .subcommand(SubCommand::with_name("range")
                .about("Uses the standard, predicatable `interop` keygen method to produce a range \
                        of predicatable private keys and starts performing their validator duties.")
                .arg(Arg::with_name("first_validator")
                    .value_name("VALIDATOR_INDEX")
                    .required(true)
                    .help("The first validator public key to be generated for this client."))
                .arg(Arg::with_name("validator_count")
                    .value_name("COUNT")
                    .required(true)
                    .help("The number of validators."))
            )
        )
        .get_matches();

    let drain = match matches.value_of("debug-level") {
        Some("info") => drain.filter_level(Level::Info),
        Some("debug") => drain.filter_level(Level::Debug),
        Some("trace") => drain.filter_level(Level::Trace),
        Some("warn") => drain.filter_level(Level::Warning),
        Some("error") => drain.filter_level(Level::Error),
        Some("crit") => drain.filter_level(Level::Critical),
        _ => unreachable!("guarded by clap"),
    };
    let log = slog::Logger::root(drain.fuse(), o!());
    let (client_config, eth2_config) = match get_configs(&matches, &log) {
        Ok(tuple) => tuple,
        Err(e) => {
            crit!(
                log,
                "Unable to initialize configuration";
                "error" => e
            );
            return;
        }
    };

    info!(
        log,
        "Starting validator client";
        "datadir" => client_config.full_data_dir().expect("Unable to find datadir").to_str(),
    );

    let result = match eth2_config.spec_constants.as_str() {
        "mainnet" => ValidatorService::<ValidatorServiceClient, Keypair, MainnetEthSpec>::start(
            client_config,
            eth2_config,
            log.clone(),
        ),
        "minimal" => ValidatorService::<ValidatorServiceClient, Keypair, MinimalEthSpec>::start(
            client_config,
            eth2_config,
            log.clone(),
        ),
        "interop" => ValidatorService::<ValidatorServiceClient, Keypair, InteropEthSpec>::start(
            client_config,
            eth2_config,
            log.clone(),
        ),
        other => {
            crit!(log, "Unknown spec constants"; "title" => other);
            return;
        }
    };

    // start the validator service.
    // this specifies the GRPC and signer type to use as the duty manager beacon node.
    match result {
        Ok(_) => info!(log, "Validator client shutdown successfully."),
        Err(e) => crit!(log, "Validator client exited with error"; "error" => e.to_string()),
    }
}

/// Parses the CLI arguments and attempts to load the client and eth2 configuration.
///
/// This is not a pure function, it reads from disk and may contact network servers.
pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<(ClientConfig, Eth2Config)> {
    let mut client_config = ClientConfig::default();

    if let Some(server) = cli_args.value_of("server") {
        client_config.server = server.to_string();
    }

    if let Some(port) = cli_args.value_of("server-http-port") {
        client_config.server_http_port = port
            .parse::<u16>()
            .map_err(|e| format!("Unable to parse HTTP port: {:?}", e))?;
    }

    if let Some(port) = cli_args.value_of("server-grpc-port") {
        client_config.server_grpc_port = port
            .parse::<u16>()
            .map_err(|e| format!("Unable to parse gRPC port: {:?}", e))?;
    }

    info!(
        log,
        "Beacon node connection info";
        "grpc_port" => client_config.server_grpc_port,
        "http_port" => client_config.server_http_port,
        "server" => &client_config.server,
    );

    match cli_args.subcommand() {
        ("testnet", Some(sub_cli_args)) => {
            if cli_args.is_present("eth2-config") && sub_cli_args.is_present("bootstrap") {
                return Err(
                    "Cannot specify --eth2-config and --bootstrap as it may result \
                     in ambiguity."
                        .into(),
                );
            }
            process_testnet_subcommand(sub_cli_args, client_config, log)
        }
        _ => return Err("You must use the testnet command. See '--help'.".into()),
    }
}

/// Parses the `testnet` CLI subcommand.
///
/// This is not a pure function, it reads from disk and may contact network servers.
fn process_testnet_subcommand(
    cli_args: &ArgMatches,
    mut client_config: ClientConfig,
    log: &Logger,
) -> Result<(ClientConfig, Eth2Config)> {
    let eth2_config = if cli_args.is_present("bootstrap") {
        let bootstrapper = Bootstrapper::from_server_string(format!(
            "http://{}:{}",
            client_config.server, client_config.server_http_port
        ))?;

        let eth2_config = bootstrapper.eth2_config()?;

        info!(
            log,
            "Bootstrapped eth2 config via HTTP";
            "slot_time_millis" => eth2_config.spec.milliseconds_per_slot,
            "spec" => &eth2_config.spec_constants,
        );

        eth2_config
    } else {
        match cli_args.value_of("spec") {
            Some("mainnet") => Eth2Config::mainnet(),
            Some("minimal") => Eth2Config::minimal(),
            Some("interop") => Eth2Config::interop(),
            _ => return Err("No --spec flag provided. See '--help'.".into()),
        }
    };

    client_config.key_source = match cli_args.subcommand() {
        ("range", Some(sub_cli_args)) => {
            let first = sub_cli_args
                .value_of("first_validator")
                .ok_or_else(|| "No first validator supplied")?
                .parse::<usize>()
                .map_err(|e| format!("Unable to parse first validator: {:?}", e))?;
            let count = sub_cli_args
                .value_of("validator_count")
                .ok_or_else(|| "No validator count supplied")?
                .parse::<usize>()
                .map_err(|e| format!("Unable to parse validator count: {:?}", e))?;

            info!(
                log,
                "Generating unsafe testing keys";
                "first_validator" => first,
                "count" => count
            );

            KeySource::TestingKeypairRange(first..first + count)
        }
        _ => KeySource::Disk,
    };

    Ok((client_config, eth2_config))
}
<commit_msg>Rename key gen subcommand in val client<commit_after>mod attestation_producer;
mod block_producer;
mod config;
mod duties;
pub mod error;
mod service;
mod signer;

use crate::config::{
    Config as ClientConfig, KeySource, DEFAULT_SERVER, DEFAULT_SERVER_GRPC_PORT,
    DEFAULT_SERVER_HTTP_PORT,
};
use crate::service::Service as ValidatorService;
use clap::{App, Arg, ArgMatches, SubCommand};
use eth2_config::Eth2Config;
use lighthouse_bootstrap::Bootstrapper;
use protos::services_grpc::ValidatorServiceClient;
use slog::{crit, error, info, o, Drain, Level, Logger};
use types::{InteropEthSpec, Keypair, MainnetEthSpec, MinimalEthSpec};

pub const DEFAULT_SPEC: &str = "minimal";
pub const DEFAULT_DATA_DIR: &str = ".lighthouse-validator";
pub const CLIENT_CONFIG_FILENAME: &str = "validator-client.toml";
pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml";

type Result<T> = core::result::Result<T, String>;

fn main() {
    // Logging
    let decorator = slog_term::TermDecorator::new().build();
    let decorator = logging::AlignedTermDecorator::new(decorator, logging::MAX_MESSAGE_WIDTH);
    let drain = slog_term::FullFormat::new(decorator).build().fuse();
    let drain = slog_async::Async::new(drain).build().fuse();

    // CLI
    let matches = App::new("Lighthouse Validator Client")
        .version("0.0.1")
        .author("Sigma Prime <contact@sigmaprime.io>")
        .about("Eth 2.0 Validator Client")
        .arg(
            Arg::with_name("datadir")
                .long("datadir")
                .short("d")
                .value_name("DIR")
                .help("Data directory for keys and databases.")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("logfile")
                .long("logfile")
                .value_name("logfile")
                .help("File path where output will be written.")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("spec")
                .short("s")
                .long("spec")
                .value_name("TITLE")
                .help("Specifies the default eth2 spec type.")
                .takes_value(true)
                .possible_values(&["mainnet", "minimal", "interop"])
                .conflicts_with("eth2-config")
                .global(true)
        )
        .arg(
            Arg::with_name("eth2-config")
                .long("eth2-config")
                .short("e")
                .value_name("TOML_FILE")
                .help("Path to Ethereum 2.0 config and specification file (e.g., eth2_spec.toml).")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("server")
                .long("server")
                .value_name("NETWORK_ADDRESS")
                .help("Address to connect to BeaconNode.")
                .default_value(DEFAULT_SERVER)
                .takes_value(true),
        )
        .arg(
            Arg::with_name("server-grpc-port")
                .long("g")
                .value_name("PORT")
                .help("Port to use for gRPC API connection to the server.")
                .default_value(DEFAULT_SERVER_GRPC_PORT)
                .takes_value(true),
        )
        .arg(
            Arg::with_name("server-http-port")
                .long("h")
                .value_name("PORT")
                .help("Port to use for HTTP API connection to the server.")
                .default_value(DEFAULT_SERVER_HTTP_PORT)
                .takes_value(true),
        )
        .arg(
            Arg::with_name("debug-level")
                .long("debug-level")
                .value_name("LEVEL")
                .short("s")
                .help("The title of the spec constants for chain config.")
                .takes_value(true)
                .possible_values(&["info", "debug", "trace", "warn", "error", "crit"])
                .default_value("info"),
        )
        /*
         * The "testnet" sub-command.
         *
         * Used for starting testnet validator clients.
         */
        .subcommand(SubCommand::with_name("testnet")
            .about("Starts a testnet validator using INSECURE, predicatable private keys, based off the canonical \
                   validator index. ONLY USE FOR TESTING PURPOSES!")
            .arg(
                Arg::with_name("bootstrap")
                    .short("b")
                    .long("bootstrap")
                    .help("Connect to the RPC server to download the eth2_config via the HTTP API.")
            )
            .subcommand(SubCommand::with_name("insecure")
                .about("Uses the standard, predicatable `interop` keygen method to produce a range \
                        of predicatable private keys and starts performing their validator duties.")
                .arg(Arg::with_name("first_validator")
                    .value_name("VALIDATOR_INDEX")
                    .required(true)
                    .help("The first validator public key to be generated for this client."))
                .arg(Arg::with_name("validator_count")
                    .value_name("COUNT")
                    .required(true)
                    .help("The number of validators."))
            )
        )
        .get_matches();

    let drain = match matches.value_of("debug-level") {
        Some("info") => drain.filter_level(Level::Info),
        Some("debug") => drain.filter_level(Level::Debug),
        Some("trace") => drain.filter_level(Level::Trace),
        Some("warn") => drain.filter_level(Level::Warning),
        Some("error") => drain.filter_level(Level::Error),
        Some("crit") => drain.filter_level(Level::Critical),
        _ => unreachable!("guarded by clap"),
    };
    let log = slog::Logger::root(drain.fuse(), o!());
    let (client_config, eth2_config) = match get_configs(&matches, &log) {
        Ok(tuple) => tuple,
        Err(e) => {
            crit!(
                log,
                "Unable to initialize configuration";
                "error" => e
            );
            return;
        }
    };

    info!(
        log,
        "Starting validator client";
        "datadir" => client_config.full_data_dir().expect("Unable to find datadir").to_str(),
    );

    let result = match eth2_config.spec_constants.as_str() {
        "mainnet" => ValidatorService::<ValidatorServiceClient, Keypair, MainnetEthSpec>::start(
            client_config,
            eth2_config,
            log.clone(),
        ),
        "minimal" => ValidatorService::<ValidatorServiceClient, Keypair, MinimalEthSpec>::start(
            client_config,
            eth2_config,
            log.clone(),
        ),
        "interop" => ValidatorService::<ValidatorServiceClient, Keypair, InteropEthSpec>::start(
            client_config,
            eth2_config,
            log.clone(),
        ),
        other => {
            crit!(log, "Unknown spec constants"; "title" => other);
            return;
        }
    };

    // start the validator service.
    // this specifies the GRPC and signer type to use as the duty manager beacon node.
    match result {
        Ok(_) => info!(log, "Validator client shutdown successfully."),
        Err(e) => crit!(log, "Validator client exited with error"; "error" => e.to_string()),
    }
}

/// Parses the CLI arguments and attempts to load the client and eth2 configuration.
///
/// This is not a pure function, it reads from disk and may contact network servers.
pub fn get_configs(cli_args: &ArgMatches, log: &Logger) -> Result<(ClientConfig, Eth2Config)> {
    let mut client_config = ClientConfig::default();

    if let Some(server) = cli_args.value_of("server") {
        client_config.server = server.to_string();
    }

    if let Some(port) = cli_args.value_of("server-http-port") {
        client_config.server_http_port = port
            .parse::<u16>()
            .map_err(|e| format!("Unable to parse HTTP port: {:?}", e))?;
    }

    if let Some(port) = cli_args.value_of("server-grpc-port") {
        client_config.server_grpc_port = port
            .parse::<u16>()
            .map_err(|e| format!("Unable to parse gRPC port: {:?}", e))?;
    }

    info!(
        log,
        "Beacon node connection info";
        "grpc_port" => client_config.server_grpc_port,
        "http_port" => client_config.server_http_port,
        "server" => &client_config.server,
    );

    match cli_args.subcommand() {
        ("testnet", Some(sub_cli_args)) => {
            if cli_args.is_present("eth2-config") && sub_cli_args.is_present("bootstrap") {
                return Err(
                    "Cannot specify --eth2-config and --bootstrap as it may result \
                     in ambiguity."
                        .into(),
                );
            }
            process_testnet_subcommand(sub_cli_args, client_config, log)
        }
        _ => return Err("You must use the testnet command. See '--help'.".into()),
    }
}

/// Parses the `testnet` CLI subcommand.
///
/// This is not a pure function, it reads from disk and may contact network servers.
fn process_testnet_subcommand(
    cli_args: &ArgMatches,
    mut client_config: ClientConfig,
    log: &Logger,
) -> Result<(ClientConfig, Eth2Config)> {
    let eth2_config = if cli_args.is_present("bootstrap") {
        let bootstrapper = Bootstrapper::from_server_string(format!(
            "http://{}:{}",
            client_config.server, client_config.server_http_port
        ))?;

        let eth2_config = bootstrapper.eth2_config()?;

        info!(
            log,
            "Bootstrapped eth2 config via HTTP";
            "slot_time_millis" => eth2_config.spec.milliseconds_per_slot,
            "spec" => &eth2_config.spec_constants,
        );

        eth2_config
    } else {
        match cli_args.value_of("spec") {
            Some("mainnet") => Eth2Config::mainnet(),
            Some("minimal") => Eth2Config::minimal(),
            Some("interop") => Eth2Config::interop(),
            _ => return Err("No --spec flag provided. See '--help'.".into()),
        }
    };

    client_config.key_source = match cli_args.subcommand() {
        ("insecure", Some(sub_cli_args)) => {
            let first = sub_cli_args
                .value_of("first_validator")
                .ok_or_else(|| "No first validator supplied")?
                .parse::<usize>()
                .map_err(|e| format!("Unable to parse first validator: {:?}", e))?;
            let count = sub_cli_args
                .value_of("validator_count")
                .ok_or_else(|| "No validator count supplied")?
                .parse::<usize>()
                .map_err(|e| format!("Unable to parse validator count: {:?}", e))?;

            info!(
                log,
                "Generating unsafe testing keys";
                "first_validator" => first,
                "count" => count
            );

            KeySource::TestingKeypairRange(first..first + count)
        }
        _ => KeySource::Disk,
    };

    Ok((client_config, eth2_config))
}
<|endoftext|>


<commit_before>//
// zhtta.rs
//
// Running on Rust 0.8
//
// Starting code for PS3
//
// Note: it would be very unwise to run this server on a machine that is
// on the Internet and contains any sensitive files!
//
// University of Virginia - cs4414 Fall 2013
// Weilin Xu and David Evans
// Version 0.3

extern mod extra;

use std::rt::io::*;
use std::rt::io::net::ip::{SocketAddr, Ipv4Addr};
use std::io::println;
use std::cell::Cell;
use std::{os, str, io, run, task, vec};
use extra::arc;
use extra::priority_queue::PriorityQueue;
use std::comm::*;
use std::cast;
use extra::comm::DuplexStream;
use std::hashmap::HashSet;
use std::path::Path;

static PORT:  int = 4414;
static IP: &'static str = "127.0.0.1";
static mut visitor_count: uint = 0;

#[deriving(Clone)]
struct access_t {
    filepath: ~std::path::PosixPath,
    size: i64,
    data: ~[u8]
}

struct sched_msg {
    stream: Option<std::rt::io::net::tcp::TcpStream>,
    filepath: ~std::path::PosixPath,
    priority: uint
}

impl std::cmp::Ord for sched_msg {
    fn lt(&self, other: &sched_msg) -> bool {
        return self.priority > other.priority;
    }
}

impl std::cmp::Ord for access_t {
    fn lt(&self, other: &access_t)  ->  bool {
        self.size > other.size
    }
}

fn main() {
    let req_heap: PriorityQueue<sched_msg> = PriorityQueue::new();
    let shared_req_heap = arc::RWArc::new(req_heap);
    let add_vec = shared_req_heap.clone();
    let take_vec = shared_req_heap.clone();

    let cache: PriorityQueue<access_t> = PriorityQueue::new();
    let shared_cache = arc::RWArc::new(cache);
    let rem_cache = shared_cache.clone();
    let check_cache = shared_cache.clone();

    let (port, chan) = stream();
    let chan = SharedChan::new(chan);

    /*do spawn {
        loop {
            timer::sleep(7000);
            do rem_cache.write |rc| {
                (*rc).clear();
            }
        }
    }*/

    // dequeue file requests, and send responses.
    // FIFO
    do spawn {
        let (sm_port, sm_chan) = stream();

        // a task for sending responses.
        do spawn {
            loop {
                let mut tf: sched_msg = sm_port.recv(); // wait for the dequeued request to handle
                let fpath = tf.filepath.clone();
                println(fmt!("begin serving file [%?]", tf.filepath.to_str()));
                // A web server should always reply a HTTP header for any legal HTTP request.
                let extension = fpath.components().last();
                match extension.rfind('.') {
                    Some(x) =>  {
                        match extension.slice_from(x+1) {
                            "html"|"xhtml"|"txt"|"xml"  => { tf.stream.write("HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=UTF-8\r\n\r\n".as_bytes()); }
                            _                           => { tf.stream.write("HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream; charset=UTF-8\r\n\r\n".as_bytes()); }
                        }
                    }
                    _       =>  { tf.stream.write("HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream; charset=UTF-8\r\n\r\n".as_bytes()); }
                }

                do check_cache.write |cc| {
                    let mut cached = false;
                    for elem in (*cc).iter() {
                        if elem.filepath == fpath {
                            tf.stream.write(elem.data);
                            cached = true;
                            break;
                        }
                    }
                    if !cached {
                        if tf.filepath.get_mode().unwrap() % 2 == 1 {
                            execFile(&mut tf);
                            // No cache for dynamically-generated files.
                        }
                        else {
                            let data = writeFile(&mut tf);
                            let size = fpath.get_size().unwrap();
                            if (*cc).len() < 10 {
                                (*cc).push(access_t { filepath: fpath.clone(), size: size, data: data });
                            }
                            else if (*cc).top().size < size { (*cc).push_pop(access_t { filepath: fpath.clone(), size: size, data: data }); }
                        }
                    }
                }

                println(fmt!("finish file [%?]", fpath.to_str()));
            }
        }

        loop {
            port.recv(); // wait for arrving notification
            do take_vec.write |vec| {
                if ((*vec).len() > 0) {
                    // LIFO didn't make sense in service scheduling, so we modify it as FIFO by using shift_opt() rather than pop().
                    let tf: sched_msg = (*vec).pop();
                    println(fmt!("shift from queue, size: %ud", (*vec).len()));
                    sm_chan.send(tf); // send the request to send-response-task to serve.
                }
            }
        }
    }

    // IP addresses to give higher priority
    let mut ip_vals: HashSet<u32> = HashSet::with_capacity(9000);
    ip_vals.insert((192 as u32 << 24) + (168 as u32 << 16));
    ip_vals.insert((127 as u32 << 24) + (143 as u32 << 16));
    ip_vals.insert((137 as u32 << 24) + (54 as u32 << 16));
    ip_vals.insert(0);
    let shared_ip_map = arc::RWArc::new(ip_vals);

    let shared_count = arc::RWArc::new(0);

    let ip = match FromStr::from_str(IP) { Some(ip) => ip,
                                           None => { println(fmt!("Error: Invalid IP address <%s>", IP));
                                                     return;},
                                         };

    let socket = net::tcp::TcpListener::bind(SocketAddr {ip: ip, port: PORT as u16});

    println(fmt!("Listening on %s:%d ...", ip.to_str(), PORT));
    let mut acceptor = socket.listen().unwrap();

    for stream in acceptor.incoming() {
        let stream = Cell::new(stream);

        let incr_count = shared_count.clone();

        // Start a new task to handle the each connection
        let child_chan = chan.clone();
        let shared_ip_map = shared_ip_map.clone();
        let child_add_vec = add_vec.clone();

        do spawn {
            do incr_count.write |count| {
                *count += 1;
            }

            let mut stream = stream.take();
            let mut buf = [0, ..500];
            stream.read(buf);
            let request_str = str::from_utf8(buf);

            let req_group : ~[&str]= request_str.splitn_iter(' ', 3).collect();
            if req_group.len() > 2 {
                let path = req_group[1];
                println(fmt!("Request for path: \n%?", path));
                // More better path security!
                let unclean_path = os::getcwd().push(Path(path).to_str()).to_str();
                let mut file_path = ~os::getcwd();
                // paths are always normalized so a/b/../c becomes a/c
                if unclean_path.starts_with(file_path.to_str()) {
                    file_path = ~file_path.push(path);
                }
                if !os::path_exists(file_path) || os::path_is_dir(file_path) {
                    println(fmt!("Request received:\n%s", request_str));
                    let response: ~str = fmt!(
                        "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=UTF-8\r\n\r\n
                         <doctype !html><title>Hello, Rust!</title>

Greetings, Krusty!

Visitor count: %u

                         \r\n", incr_count.read(|c| { *c }));

                    stream.write(response.as_bytes());
                }
                else {
                    // Requests scheduling

                    let mut priority = file_path.stat().unwrap().st_size as uint;
                    unsafe {
                        match stream {
                            Some(ref s) => {
                                    let stream = cast::transmute_mut(s);
                                    let pn = stream.peer_name().unwrap();
                                    println(fmt!("Peer is: %?", pn));
                                    match pn.ip {
                                        Ipv4Addr(a, b, c, d) => {
                                                                // Since we are sharing the ip_map it must be read
                                                                do shared_ip_map.read |map| {
                                                                    if check_ip(a,b,c,d, map) {
                                                                        priority = 1;
                                                                        println("local request!");
                                                                    }
                                                                }
                                                            },
                                        _                    =>  fail!()
                                    }
                            },
                            _    => fail!()
                        };
                    }
                    let msg: sched_msg = sched_msg{stream: stream, filepath: file_path.clone(), priority: priority};
                    let (sm_port, sm_chan) = std::comm::stream();
                    sm_chan.send(msg);

                    do child_add_vec.write |vec| {
                        let msg = sm_port.recv();
                        (*vec).push(msg); // enqueue new request.
                        println("add to queue");
                    }
                    child_chan.send(""); //notify the new arriving request.
                    println(fmt!("get file request: %?", file_path));
                }
            }
            println!("connection terminates")
        }
    }
}

// Looks up an ip prefix in the hashset by trying each octet
fn check_ip(a: u8, b: u8, c: u8, d: u8, map: &HashSet<u32>) -> bool {
    let mut mut_ip = a as u32 << 24;
    if map.contains(&mut_ip) {
        return true;
    }
    mut_ip += (b as u32 << 16);
    if map.contains(&mut_ip) {
        return true;
    }
    mut_ip += (c as u32 << 8);
    if map.contains(&mut_ip) {
        return true;
    }
    mut_ip += (d as u32);
    if map.contains(&mut_ip) {
        return true;
    }
    false
}

fn execFile(file_data: &mut sched_msg) {
    let (port, chan) = DuplexStream();
    do task::spawn_supervised {
        do_gash(&chan)
    }
    let mut file_data = file_data;
    match io::file_reader(file_data.filepath) {
        Ok(rd)      =>  {   let closer = ~['\"',' ', '-','-','>'];
                            while !rd.eof() {
                            let rd_byte = rd.read_byte() as u8;

                            match rd_byte {
                                0x3c =>  { let mut open: ~[u8] = ~[0,0,0,0,0,0,0,0,0,0,0,0,0,0];
                                                          rd.read(open, 14);
                                                          if str::from_utf8(open) == ~"!--#exec cmd=\"" {
                                                                let mut cmd: ~[u8] = ~[];
                                                                let mut cmd_byte: u8;
                                                                let mut i = 0;
                                                                loop {
                                                                    cmd_byte = rd.read_byte() as u8;
                                                                    if cmd_byte == closer[i] as u8 {
                                                                        if i == closer.len() - 1 {
                                                                            break;
                                                                        }

                                                                        i += 1;
                                                                    }
                                                                    else {
                                                                        i = 0;
                                                                    }

                                                                    if cmd_byte != 0xFF {
                                                                        cmd.push(cmd_byte);
                                                                    }
                                                                }

                                                          port.send(str::from_utf8(cmd.slice_to(cmd.len() - 4)));
                                                          let result = port.recv();
                                                          file_data.stream.write(result.as_bytes());
                                                          }
                                                        },
                                _                   =>  { if rd_byte != 0xFF {
                                                                file_data.stream.write(&[rd_byte]);
                                                          }
                                                        }
                            }
                          }
                          port.send(~"end");
                        }
        Err(err)    =>  { println(err); }
    }

}

fn do_gash(chan: &DuplexStream<~str, ~str>) {
    let mut gash = run::Process::new("./gash", &[], run::ProcessOptions::new());
    let mut cmd: ~str;
    let mut result: ~str;
    let gin = gash.input();
    let gout = gash.output();
    let mut res: ~[u8];
    let mut prompt: ~[u8] = ~[];
    let mut res_byte: u8;
    for _ in range(0,8) {
        prompt.push(0);
    }

    gout.read(prompt, 8);
    // Insecure. We should use some other means of communicating the need to die.
    loop {
        cmd = chan.recv();
        if cmd == ~"end" {
            break;
        }

        cmd.push_str("\n");
        res = ~[];
        gin.write(cmd.as_bytes());
        loop {
            res_byte = gout.read_byte() as u8;
            if res_byte == '\0' as u8{
                break;
            }

            res.push(res_byte);
        }

        gout.read(prompt, 7);
        result = str::from_utf8(res);
        chan.send(result.clone());
    }

    gash.destroy();
}

fn writeFile(tf: &mut sched_msg) ->  ~[u8] {
    let mut file: ~[u8] = ~[];
    let mut writes = 0;
    match io::file_reader(tf.filepath) {
        Ok(rd)      =>  { while !rd.eof() {
                            let mut buffer: ~[u8] = vec::with_capacity(20971520u);
                            unsafe { vec::raw::set_len(&mut buffer, 20971520u); }
                            let read = rd.read(buffer, 20971520u);
                            unsafe { vec::raw::set_len(&mut buffer, read); }
                            file.push_all(buffer);
                            tf.stream.write(buffer);
                            writes += 1;
                          }
                          println(fmt!("%d", writes));
                        }
        Err(err)    =>  { println(err); }
    }
    file
}

<commit_msg>Cache clearing seems to work<commit_after>//
// zhtta.rs
//
// Running on Rust 0.8
//
// Starting code for PS3
//
// Note: it would be very unwise to run this server on a machine that is
// on the Internet and contains any sensitive files!
//
// University of Virginia - cs4414 Fall 2013
// Weilin Xu and David Evans
// Version 0.3

extern mod extra;

use std::rt::io::*;
use std::rt::io::net::ip::{SocketAddr, Ipv4Addr};
use std::io::println;
use std::cell::Cell;
use std::{os, str, io, run, task, vec};
use extra::arc;
use extra::priority_queue::PriorityQueue;
use std::comm::*;
use std::cast;
use extra::comm::DuplexStream;
use std::hashmap::HashSet;
use std::path::Path;

static PORT:  int = 4414;
static IP: &'static str = "127.0.0.1";
static mut visitor_count: uint = 0;

#[deriving(Clone)]
struct access_t {
    filepath: ~std::path::PosixPath,
    size: i64,
    data: ~[u8]
}

struct sched_msg {
    stream: Option<std::rt::io::net::tcp::TcpStream>,
    filepath: ~std::path::PosixPath,
    priority: uint
}

impl std::cmp::Ord for sched_msg {
    fn lt(&self, other: &sched_msg) -> bool {
        return self.priority > other.priority;
    }
}

impl std::cmp::Ord for access_t {
    fn lt(&self, other: &access_t)  ->  bool {
        self.size > other.size
    }
}

fn main() {
    let req_heap: PriorityQueue<sched_msg> = PriorityQueue::new();
    let shared_req_heap = arc::RWArc::new(req_heap);
    let add_vec = shared_req_heap.clone();
    let take_vec = shared_req_heap.clone();

    let cache: PriorityQueue<access_t> = PriorityQueue::new();
    let shared_cache = arc::RWArc::new(cache);
    let rem_cache = shared_cache.clone();
    let check_cache = shared_cache.clone();

    let (port, chan) = stream();
    let chan = SharedChan::new(chan);

    do spawn {
        loop {
            timer::sleep(9000);
            do rem_cache.write |rc| {
                (*rc).clear();
            }
        }
    }

    // dequeue file requests, and send responses.
    // FIFO
    do spawn {
        let (sm_port, sm_chan) = stream();

        // a task for sending responses.
        do spawn {
            loop {
                let mut tf: sched_msg = sm_port.recv(); // wait for the dequeued request to handle
                let fpath = tf.filepath.clone();
                println(fmt!("begin serving file [%?]", tf.filepath.to_str()));
                // A web server should always reply a HTTP header for any legal HTTP request.
                let extension = fpath.components().last();
                match extension.rfind('.') {
                    Some(x) =>  {
                        match extension.slice_from(x+1) {
                            "html"|"xhtml"|"txt"|"xml"  => { tf.stream.write("HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=UTF-8\r\n\r\n".as_bytes()); }
                            _                           => { tf.stream.write("HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream; charset=UTF-8\r\n\r\n".as_bytes()); }
                        }
                    }
                    _       =>  { tf.stream.write("HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream; charset=UTF-8\r\n\r\n".as_bytes()); }
                }

                do check_cache.write |cc| {
                    let mut cached = false;
                    for elem in (*cc).iter() {
                        if elem.filepath == fpath {
                            tf.stream.write(elem.data);
                            cached = true;
                            break;
                        }
                    }
                    if !cached {
                        if tf.filepath.get_mode().unwrap() % 2 == 1 {
                            execFile(&mut tf);
                            // No cache for dynamically-generated files.
                        }
                        else {
                            let data = writeFile(&mut tf);
                            let size = fpath.get_size().unwrap();
                            if (*cc).len() < 10 {
                                (*cc).push(access_t { filepath: fpath.clone(), size: size, data: data });
                            }
                            else if (*cc).top().size < size { (*cc).replace(access_t { filepath: fpath.clone(), size: size, data: data }); }
                        }
                    }
                }

                println(fmt!("finish file [%?]", fpath.to_str()));
            }
        }

        loop {
            port.recv(); // wait for arrving notification
            do take_vec.write |vec| {
                if ((*vec).len() > 0) {
                    // LIFO didn't make sense in service scheduling, so we modify it as FIFO by using shift_opt() rather than pop().
                    let tf: sched_msg = (*vec).pop();
                    println(fmt!("shift from queue, size: %ud", (*vec).len()));
                    sm_chan.send(tf); // send the request to send-response-task to serve.
                }
            }
        }
    }

    // IP addresses to give higher priority
    let mut ip_vals: HashSet<u32> = HashSet::with_capacity(9000);
    ip_vals.insert((192 as u32 << 24) + (168 as u32 << 16));
    ip_vals.insert((127 as u32 << 24) + (143 as u32 << 16));
    ip_vals.insert((137 as u32 << 24) + (54 as u32 << 16));
    ip_vals.insert(0);
    let shared_ip_map = arc::RWArc::new(ip_vals);

    let shared_count = arc::RWArc::new(0);

    let ip = match FromStr::from_str(IP) { Some(ip) => ip,
                                           None => { println(fmt!("Error: Invalid IP address <%s>", IP));
                                                     return;},
                                         };

    let socket = net::tcp::TcpListener::bind(SocketAddr {ip: ip, port: PORT as u16});

    println(fmt!("Listening on %s:%d ...", ip.to_str(), PORT));
    let mut acceptor = socket.listen().unwrap();

    for stream in acceptor.incoming() {
        let stream = Cell::new(stream);

        let incr_count = shared_count.clone();

        // Start a new task to handle the each connection
        let child_chan = chan.clone();
        let shared_ip_map = shared_ip_map.clone();
        let child_add_vec = add_vec.clone();

        do spawn {
            do incr_count.write |count| {
                *count += 1;
            }

            let mut stream = stream.take();
            let mut buf = [0, ..500];
            stream.read(buf);
            let request_str = str::from_utf8(buf);

            let req_group : ~[&str]= request_str.splitn_iter(' ', 3).collect();
            if req_group.len() > 2 {
                let path = req_group[1];
                println(fmt!("Request for path: \n%?", path));
                // More better path security!
                let unclean_path = os::getcwd().push(Path(path).to_str()).to_str();
                let mut file_path = ~os::getcwd();
                // paths are always normalized so a/b/../c becomes a/c
                if unclean_path.starts_with(file_path.to_str()) {
                    file_path = ~file_path.push(path);
                }
                if !os::path_exists(file_path) || os::path_is_dir(file_path) {
                    println(fmt!("Request received:\n%s", request_str));
                    let response: ~str = fmt!(
                        "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=UTF-8\r\n\r\n
                         <doctype !html><title>Hello, Rust!</title>

Greetings, Krusty!

Visitor count: %u

                         \r\n", incr_count.read(|c| { *c }));

                    stream.write(response.as_bytes());
                }
                else {
                    // Requests scheduling

                    let mut priority = file_path.stat().unwrap().st_size as uint;
                    unsafe {
                        match stream {
                            Some(ref s) => {
                                    let stream = cast::transmute_mut(s);
                                    let pn = stream.peer_name().unwrap();
                                    println(fmt!("Peer is: %?", pn));
                                    match pn.ip {
                                        Ipv4Addr(a, b, c, d) => {
                                                                // Since we are sharing the ip_map it must be read
                                                                do shared_ip_map.read |map| {
                                                                    if check_ip(a,b,c,d, map) {
                                                                        priority = 1;
                                                                        println("local request!");
                                                                    }
                                                                }
                                                            },
                                        _                    =>  fail!()
                                    }
                            },
                            _    => fail!()
                        };
                    }
                    let msg: sched_msg = sched_msg{stream: stream, filepath: file_path.clone(), priority: priority};
                    let (sm_port, sm_chan) = std::comm::stream();
                    sm_chan.send(msg);

                    do child_add_vec.write |vec| {
                        let msg = sm_port.recv();
                        (*vec).push(msg); // enqueue new request.
                        println("add to queue");
                    }
                    child_chan.send(""); //notify the new arriving request.
                    println(fmt!("get file request: %?", file_path));
                }
            }
            println!("connection terminates")
        }
    }
}

// Looks up an ip prefix in the hashset by trying each octet
fn check_ip(a: u8, b: u8, c: u8, d: u8, map: &HashSet<u32>) -> bool {
    let mut mut_ip = a as u32 << 24;
    if map.contains(&mut_ip) {
        return true;
    }
    mut_ip += (b as u32 << 16);
    if map.contains(&mut_ip) {
        return true;
    }
    mut_ip += (c as u32 << 8);
    if map.contains(&mut_ip) {
        return true;
    }
    mut_ip += (d as u32);
    if map.contains(&mut_ip) {
        return true;
    }
    false
}

fn execFile(file_data: &mut sched_msg) {
    let (port, chan) = DuplexStream();
    do task::spawn_supervised {
        do_gash(&chan)
    }
    match io::file_reader(file_data.filepath) {
        Ok(rd)      =>  {   let closer = ~['\"',' ', '-','-','>'];
                            while !rd.eof() {
                            let rd_byte = rd.read_byte() as u8;

                            match rd_byte {
                                0x3c =>  { let mut open: ~[u8] = ~[0,0,0,0,0,0,0,0,0,0,0,0,0,0];
                                                          rd.read(open, 14);
                                                          if str::from_utf8(open) == ~"!--#exec cmd=\"" {
                                                                let mut cmd: ~[u8] = ~[];
                                                                let mut cmd_byte: u8;
                                                                let mut i = 0;
                                                                loop {
                                                                    cmd_byte = rd.read_byte() as u8;
                                                                    if cmd_byte == closer[i] as u8 {
                                                                        if i == closer.len() - 1 {
                                                                            break;
                                                                        }

                                                                        i += 1;
                                                                    }
                                                                    else {
                                                                        i = 0;
                                                                    }

                                                                    if cmd_byte != 0xFF {
                                                                        cmd.push(cmd_byte);
                                                                    }
                                                                }

                                                          port.send(str::from_utf8(cmd.slice_to(cmd.len() - 4)));
                                                          let result = port.recv();
                                                          file_data.stream.write(result.as_bytes());
                                                          }
                                                        },
                                _                   =>  { if rd_byte != 0xFF {
                                                                file_data.stream.write(&[rd_byte]);
                                                          }
                                                        }
                            }
                          }
                          port.send(~"end");
                        }
        Err(err)    =>  { println(err); }
    }

}

fn do_gash(chan: &DuplexStream<~str, ~str>) {
    let mut gash = run::Process::new("./gash", &[], run::ProcessOptions::new());
    let mut cmd: ~str;
    let mut result: ~str;
    let gin = gash.input();
    let gout = gash.output();
    let mut res: ~[u8];
    let mut prompt: ~[u8] = ~[];
    let mut res_byte: u8;
    for _ in range(0,8) {
        prompt.push(0);
    }

    gout.read(prompt, 8);
    // Insecure. We should use some other means of communicating the need to die.
    loop {
        cmd = chan.recv();
        if cmd == ~"end" {
            break;
        }

        cmd.push_str("\n");
        res = ~[];
        gin.write(cmd.as_bytes());
        loop {
            res_byte = gout.read_byte() as u8;
            if res_byte == '\0' as u8{
                break;
            }

            res.push(res_byte);
        }

        gout.read(prompt, 7);
        result = str::from_utf8(res);
        chan.send(result.clone());
    }

    gash.destroy();
}

fn writeFile(tf: &mut sched_msg) ->  ~[u8] {
    let mut file: ~[u8] = ~[];
    let mut writes = 0;
    match io::file_reader(tf.filepath) {
        Ok(rd)      =>  { while !rd.eof() {
                            let mut buffer: ~[u8] = vec::with_capacity(20971520u);
                            unsafe { vec::raw::set_len(&mut buffer, 20971520u); }
                            let read = rd.read(buffer, 20971520u);
                            unsafe { vec::raw::set_len(&mut buffer, read); }
                            file.push_all(buffer);
                            tf.stream.write(buffer);
                            writes += 1;
                          }
                          println(fmt!("%d", writes));
                        }
        Err(err)    =>  { println(err); }
    }
    file
}

<|endoftext|>


<commit_before>// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::fmt;

#[repr(C)]
enum CEnum {
    Hello = 30,
    World = 60
}

fn test1(c: CEnum) -> i32 {
  let c2 = CEnum::Hello;
  match (c, c2) {
    (CEnum::Hello, CEnum::Hello) => 42,
    (CEnum::World, CEnum::Hello) => 0,
    _ => 1
  }
}

#[repr(packed)]
struct Pakd {
    a: u64,
    b: u32,
    c: u16,
    d: u8,
    e: ()
}

// It is unsafe to use #[derive(Debug)] on a packed struct because the code generated by the derive
// macro takes references to the fields instead of accessing them directly.
impl fmt::Debug for Pakd {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        // It's important that we load the fields into locals by-value here. This will do safe
        // unaligned loads into the locals, then pass references to the properly-aligned locals to
        // the formatting code.
        let Pakd { a, b, c, d, e } = *self;
        f.debug_struct("Pakd")
            .field("a", &a)
            .field("b", &b)
            .field("c", &c)
            .field("d", &d)
            .field("e", &e)
            .finish()
    }
}

// It is unsafe to use #[derive(PartialEq)] on a packed struct because the code generated by the
// derive macro takes references to the fields instead of accessing them directly.
impl PartialEq for Pakd {
    fn eq(&self, other: &Pakd) -> bool {
        self.a == other.a &&
            self.b == other.b &&
            self.c == other.c &&
            self.d == other.d &&
            self.e == other.e
    }
}

impl Drop for Pakd {
    fn drop(&mut self) {}
}

fn test2() -> Pakd {
    Pakd { a: 42, b: 42, c: 42, d: 42, e: () }
}

#[derive(PartialEq, Debug)]
struct TupleLike(u64, u32);

fn test3() -> TupleLike {
    TupleLike(42, 42)
}

fn test4(x: fn(u64, u32) -> TupleLike) -> (TupleLike, TupleLike) {
    let y = TupleLike;
    (x(42, 84), y(42, 84))
}

fn test5(x: fn(u32) -> Option<u32>) -> (Option<u32>, Option<u32>) {
    let y = Some;
    (x(42), y(42))
}

fn main() {
  assert_eq!(test1(CEnum::Hello), 42);
  assert_eq!(test1(CEnum::World), 0);
  assert_eq!(test2(), Pakd { a: 42, b: 42, c: 42, d: 42, e: () });
  assert_eq!(test3(), TupleLike(42, 42));
  let t4 = test4(TupleLike);
  assert_eq!(t4.0, t4.1);
  let t5 = test5(Some);
  assert_eq!(t5.0, t5.1);
}
<commit_msg>Fix indentation in test.<commit_after>// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::fmt;

#[repr(C)]
enum CEnum {
    Hello = 30,
    World = 60
}

fn test1(c: CEnum) -> i32 {
    let c2 = CEnum::Hello;
    match (c, c2) {
        (CEnum::Hello, CEnum::Hello) => 42,
        (CEnum::World, CEnum::Hello) => 0,
        _ => 1
    }
}

#[repr(packed)]
struct Pakd {
    a: u64,
    b: u32,
    c: u16,
    d: u8,
    e: ()
}

// It is unsafe to use #[derive(Debug)] on a packed struct because the code generated by the derive
// macro takes references to the fields instead of accessing them directly.
impl fmt::Debug for Pakd {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        // It's important that we load the fields into locals by-value here. This will do safe
        // unaligned loads into the locals, then pass references to the properly-aligned locals to
        // the formatting code.
        let Pakd { a, b, c, d, e } = *self;
        f.debug_struct("Pakd")
            .field("a", &a)
            .field("b", &b)
            .field("c", &c)
            .field("d", &d)
            .field("e", &e)
            .finish()
    }
}

// It is unsafe to use #[derive(PartialEq)] on a packed struct because the code generated by the
// derive macro takes references to the fields instead of accessing them directly.
impl PartialEq for Pakd {
    fn eq(&self, other: &Pakd) -> bool {
        self.a == other.a &&
            self.b == other.b &&
            self.c == other.c &&
            self.d == other.d &&
            self.e == other.e
    }
}

impl Drop for Pakd {
    fn drop(&mut self) {}
}

fn test2() -> Pakd {
    Pakd { a: 42, b: 42, c: 42, d: 42, e: () }
}

#[derive(PartialEq, Debug)]
struct TupleLike(u64, u32);

fn test3() -> TupleLike {
    TupleLike(42, 42)
}

fn test4(x: fn(u64, u32) -> TupleLike) -> (TupleLike, TupleLike) {
    let y = TupleLike;
    (x(42, 84), y(42, 84))
}

fn test5(x: fn(u32) -> Option<u32>) -> (Option<u32>, Option<u32>) {
    let y = Some;
    (x(42), y(42))
}

fn main() {
    assert_eq!(test1(CEnum::Hello), 42);
    assert_eq!(test1(CEnum::World), 0);
    assert_eq!(test2(), Pakd { a: 42, b: 42, c: 42, d: 42, e: () });
    assert_eq!(test3(), TupleLike(42, 42));
    let t4 = test4(TupleLike);
    assert_eq!(t4.0, t4.1);
    let t5 = test5(Some);
    assert_eq!(t5.0, t5.1);
}
<|endoftext|>


<commit_before>use std;
import std::str;

fn test(actual: str, expected: str) {
    log actual;
    log expected;
    assert (str::eq(actual, expected));
}

fn main() {
    test(#fmt["hello %d friends and %s things", 10, "formatted"],
         "hello 10 friends and formatted things");

    test(#fmt["test"], "test");

    // a quadratic optimization in LLVM (jump-threading) makes this test a
    // bit slow to compile unless we break it up
    part1();
    part2();
    part3();
    part4();
    part5();
    part6();
}

fn part1() {
    // Simple tests for types

    test(#fmt["%d", 1], "1");
    test(#fmt["%i", 2], "2");
    test(#fmt["%i", -1], "-1");
    test(#fmt["%u", 10u], "10");
    test(#fmt["%s", "test"], "test");
    test(#fmt["%b", true], "true");
    test(#fmt["%b", false], "false");
    test(#fmt["%c", 'A'], "A");
    test(#fmt["%x", 0xff_u], "ff");
    test(#fmt["%X", 0x12ab_u], "12AB");
    test(#fmt["%o", 10u], "12");
    test(#fmt["%t", 0b11010101_u], "11010101");
    test(#fmt["%f", 5.82], "5.82");
    // 32-bit limits

    test(#fmt["%i", -2147483648], "-2147483648");
    test(#fmt["%i", 2147483647], "2147483647");
    test(#fmt["%u", 4294967295u], "4294967295");
    test(#fmt["%x", 0xffffffff_u], "ffffffff");
    test(#fmt["%o", 0xffffffff_u], "37777777777");
    test(#fmt["%t", 0xffffffff_u], "11111111111111111111111111111111");
}
fn part2() {
    // Widths

    test(#fmt["%1d", 500], "500");
    test(#fmt["%10d", 500], "       500");
    test(#fmt["%10d", -500], "      -500");
    test(#fmt["%10u", 500u], "       500");
    test(#fmt["%10s", "test"], "      test");
    test(#fmt["%10b", true], "      true");
    test(#fmt["%10x", 0xff_u], "        ff");
    test(#fmt["%10X", 0xff_u], "        FF");
    test(#fmt["%10o", 10u], "        12");
    test(#fmt["%10t", 0xff_u], "  11111111");
    test(#fmt["%10c", 'A'], "         A");
    test(#fmt["%10f", 5.82], "      5.82");
    // Left justify

    test(#fmt["%-10d", 500], "500       ");
    test(#fmt["%-10d", -500], "-500      ");
    test(#fmt["%-10u", 500u], "500       ");
    test(#fmt["%-10s", "test"], "test      ");
    test(#fmt["%-10b", true], "true      ");
    test(#fmt["%-10x", 0xff_u], "ff        ");
    test(#fmt["%-10X", 0xff_u], "FF        ");
    test(#fmt["%-10o", 10u], "12        ");
    test(#fmt["%-10t", 0xff_u], "11111111  ");
    test(#fmt["%-10c", 'A'], "A         ");
    test(#fmt["%-10f", 5.82], "5.82      ");
}

fn part3() {
    // Precision

    test(#fmt["%.d", 0], "");
    test(#fmt["%.u", 0u], "");
    test(#fmt["%.x", 0u], "");
    test(#fmt["%.t", 0u], "");
    test(#fmt["%.d", 10], "10");
    test(#fmt["%.d", -10], "-10");
    test(#fmt["%.u", 10u], "10");
    test(#fmt["%.s", "test"], "");
    test(#fmt["%.x", 127u], "7f");
    test(#fmt["%.o", 10u], "12");
    test(#fmt["%.t", 3u], "11");
    test(#fmt["%.c", 'A'], "A");
    test(#fmt["%.f", 5.82], "5");
    test(#fmt["%.0d", 0], "");
    test(#fmt["%.0u", 0u], "");
    test(#fmt["%.0x", 0u], "");
    test(#fmt["%.0t", 0u], "");
    test(#fmt["%.0d", 10], "10");
    test(#fmt["%.0d", -10], "-10");
    test(#fmt["%.0u", 10u], "10");
    test(#fmt["%.0s", "test"], "");
    test(#fmt["%.0x", 127u], "7f");
    test(#fmt["%.0o", 10u], "12");
    test(#fmt["%.0t", 3u], "11");
    test(#fmt["%.0c", 'A'], "A");
    test(#fmt["%.0f", 5.892], "5");
    test(#fmt["%.1d", 0], "0");
    test(#fmt["%.1u", 0u], "0");
    test(#fmt["%.1x", 0u], "0");
    test(#fmt["%.1t", 0u], "0");
    test(#fmt["%.1d", 10], "10");
    test(#fmt["%.1d", -10], "-10");
    test(#fmt["%.1u", 10u], "10");
    test(#fmt["%.1s", "test"], "t");
    test(#fmt["%.1x", 127u], "7f");
    test(#fmt["%.1o", 10u], "12");
    test(#fmt["%.1t", 3u], "11");
    test(#fmt["%.1c", 'A'], "A");
    test(#fmt["%.1f", 5.82], "5.8");
}
fn part4() {
    test(#fmt["%.5d", 0], "00000");
    test(#fmt["%.5u", 0u], "00000");
    test(#fmt["%.5x", 0u], "00000");
    test(#fmt["%.5t", 0u], "00000");
    test(#fmt["%.5d", 10], "00010");
    test(#fmt["%.5d", -10], "-00010");
    test(#fmt["%.5u", 10u], "00010");
    test(#fmt["%.5s", "test"], "test");
    test(#fmt["%.5x", 127u], "0007f");
    test(#fmt["%.5o", 10u], "00012");
    test(#fmt["%.5t", 3u], "00011");
    test(#fmt["%.5c", 'A'], "A");
    test(#fmt["%.5f", 5.82], "5.82000");
    // Bool precision. I'm not sure if it's good or bad to have bool
    // conversions support precision - it's not standard printf so we
    // can do whatever. For now I'm making it behave the same as string
    // conversions.

    test(#fmt["%.b", true], "");
    test(#fmt["%.0b", true], "");
    test(#fmt["%.1b", true], "t");
}

fn part5() {
    // Explicit + sign. Only for signed conversions

    test(#fmt["%+d", 0], "+0");
    test(#fmt["%+d", 1], "+1");
    test(#fmt["%+d", -1], "-1");
    test(#fmt["%+f", 0.0], "+0");
    // Leave space for sign

    test(#fmt["% d", 0], " 0");
    test(#fmt["% d", 1], " 1");
    test(#fmt["% d", -1], "-1");
    test(#fmt["% f", 0.0], " 0");
    // Plus overrides space

    test(#fmt["% +d", 0], "+0");
    test(#fmt["%+ d", 0], "+0");
    test(#fmt["% +f", 0.0], "+0");
    test(#fmt["%+ f", 0.0], "+0");
    // 0-padding

    test(#fmt["%05d", 0], "00000");
    test(#fmt["%05d", 1], "00001");
    test(#fmt["%05d", -1], "-0001");
    test(#fmt["%05u", 1u], "00001");
    test(#fmt["%05x", 127u], "0007f");
    test(#fmt["%05X", 127u], "0007F");
    test(#fmt["%05o", 10u], "00012");
    test(#fmt["%05t", 3u], "00011");
    test(#fmt["%05f", 5.82], "05.82");
    // 0-padding a string is undefined but glibc does this:

    test(#fmt["%05s", "test"], " test");
    test(#fmt["%05c", 'A'], "    A");
    test(#fmt["%05b", true], " true");
    // Left-justify overrides 0-padding

    test(#fmt["%-05d", 0], "0    ");
    test(#fmt["%-05d", 1], "1    ");
    test(#fmt["%-05d", -1], "-1   ");
    test(#fmt["%-05u", 1u], "1    ");
    test(#fmt["%-05x", 127u], "7f   ");
    test(#fmt["%-05X", 127u], "7F   ");
    test(#fmt["%-05o", 10u], "12   ");
    test(#fmt["%-05t", 3u], "11   ");
    test(#fmt["%-05s", "test"], "test ");
    test(#fmt["%-05c", 'A'], "A    ");
    test(#fmt["%-05b", true], "true ");
    test(#fmt["%-05f", 5.82], "5.82 ");
}
fn part6() {
    // Precision overrides 0-padding

    test(#fmt["%06.5d", 0], " 00000");
    test(#fmt["%06.5u", 0u], " 00000");
    test(#fmt["%06.5x", 0u], " 00000");
    test(#fmt["%06.5d", 10], " 00010");
    test(#fmt["%06.5d", -10], "-00010");
    test(#fmt["%06.5u", 10u], " 00010");
    test(#fmt["%06.5s", "test"], "  test");
    test(#fmt["%06.5c", 'A'], "     A");
    test(#fmt["%06.5x", 127u], " 0007f");
    test(#fmt["%06.5X", 127u], " 0007F");
    test(#fmt["%06.5o", 10u], " 00012");
    test(#fmt["%08.5f", 5.82], " 5.82000");
    // Signed combinations

    test(#fmt["% 5d", 1], "    1");
    test(#fmt["% 5d", -1], "   -1");
    test(#fmt["%+5d", 1], "   +1");
    test(#fmt["%+5d", -1], "   -1");
    test(#fmt["% 05d", 1], " 0001");
    test(#fmt["% 05d", -1], "-0001");
    test(#fmt["%+05d", 1], "+0001");
    test(#fmt["%+05d", -1], "-0001");
    test(#fmt["%- 5d", 1], " 1   ");
    test(#fmt["%- 5d", -1], "-1   ");
    test(#fmt["%-+5d", 1], "+1   ");
    test(#fmt["%-+5d", -1], "-1   ");
    test(#fmt["%- 05d", 1], " 1   ");
    test(#fmt["%- 05d", -1], "-1   ");
    test(#fmt["%-+05d", 1], "+1   ");
    test(#fmt["%-+05d", -1], "-1   ");
}
<commit_msg>Disable a test that's failing on 32-bit due to imprecise literal handling<commit_after>use std;
import std::str;

fn test(actual: str, expected: str) {
    log actual;
    log expected;
    assert (str::eq(actual, expected));
}

fn main() {
    test(#fmt["hello %d friends and %s things", 10, "formatted"],
         "hello 10 friends and formatted things");

    test(#fmt["test"], "test");

    // a quadratic optimization in LLVM (jump-threading) makes this test a
    // bit slow to compile unless we break it up
    part1();
    part2();
    part3();
    part4();
    part5();
    part6();
}

fn part1() {
    // Simple tests for types

    test(#fmt["%d", 1], "1");
    test(#fmt["%i", 2], "2");
    test(#fmt["%i", -1], "-1");
    test(#fmt["%u", 10u], "10");
    test(#fmt["%s", "test"], "test");
    test(#fmt["%b", true], "true");
    test(#fmt["%b", false], "false");
    test(#fmt["%c", 'A'], "A");
    test(#fmt["%x", 0xff_u], "ff");
    test(#fmt["%X", 0x12ab_u], "12AB");
    test(#fmt["%o", 10u], "12");
    test(#fmt["%t", 0b11010101_u], "11010101");
    test(#fmt["%f", 5.82], "5.82");
    // 32-bit limits

    // FIXME Disabled until issue 1252 is resolved.
    // test(#fmt["%i", -2147483648], "-2147483648");
    test(#fmt["%i", 2147483647], "2147483647");
    test(#fmt["%u", 4294967295u], "4294967295");
    test(#fmt["%x", 0xffffffff_u], "ffffffff");
    test(#fmt["%o", 0xffffffff_u], "37777777777");
    test(#fmt["%t", 0xffffffff_u], "11111111111111111111111111111111");
}
fn part2() {
    // Widths

    test(#fmt["%1d", 500], "500");
    test(#fmt["%10d", 500], "       500");
    test(#fmt["%10d", -500], "      -500");
    test(#fmt["%10u", 500u], "       500");
    test(#fmt["%10s", "test"], "      test");
    test(#fmt["%10b", true], "      true");
    test(#fmt["%10x", 0xff_u], "        ff");
    test(#fmt["%10X", 0xff_u], "        FF");
    test(#fmt["%10o", 10u], "        12");
    test(#fmt["%10t", 0xff_u], "  11111111");
    test(#fmt["%10c", 'A'], "         A");
    test(#fmt["%10f", 5.82], "      5.82");
    // Left justify

    test(#fmt["%-10d", 500], "500       ");
    test(#fmt["%-10d", -500], "-500      ");
    test(#fmt["%-10u", 500u], "500       ");
    test(#fmt["%-10s", "test"], "test      ");
    test(#fmt["%-10b", true], "true      ");
    test(#fmt["%-10x", 0xff_u], "ff        ");
    test(#fmt["%-10X", 0xff_u], "FF        ");
    test(#fmt["%-10o", 10u], "12        ");
    test(#fmt["%-10t", 0xff_u], "11111111  ");
    test(#fmt["%-10c", 'A'], "A         ");
    test(#fmt["%-10f", 5.82], "5.82      ");
}

fn part3() {
    // Precision

    test(#fmt["%.d", 0], "");
    test(#fmt["%.u", 0u], "");
    test(#fmt["%.x", 0u], "");
    test(#fmt["%.t", 0u], "");
    test(#fmt["%.d", 10], "10");
    test(#fmt["%.d", -10], "-10");
    test(#fmt["%.u", 10u], "10");
    test(#fmt["%.s", "test"], "");
    test(#fmt["%.x", 127u], "7f");
    test(#fmt["%.o", 10u], "12");
    test(#fmt["%.t", 3u], "11");
    test(#fmt["%.c", 'A'], "A");
    test(#fmt["%.f", 5.82], "5");
    test(#fmt["%.0d", 0], "");
    test(#fmt["%.0u", 0u], "");
    test(#fmt["%.0x", 0u], "");
    test(#fmt["%.0t", 0u], "");
    test(#fmt["%.0d", 10], "10");
    test(#fmt["%.0d", -10], "-10");
    test(#fmt["%.0u", 10u], "10");
    test(#fmt["%.0s", "test"], "");
    test(#fmt["%.0x", 127u], "7f");
    test(#fmt["%.0o", 10u], "12");
    test(#fmt["%.0t", 3u], "11");
    test(#fmt["%.0c", 'A'], "A");
    test(#fmt["%.0f", 5.892], "5");
    test(#fmt["%.1d", 0], "0");
    test(#fmt["%.1u", 0u], "0");
    test(#fmt["%.1x", 0u], "0");
    test(#fmt["%.1t", 0u], "0");
    test(#fmt["%.1d", 10], "10");
    test(#fmt["%.1d", -10], "-10");
    test(#fmt["%.1u", 10u], "10");
    test(#fmt["%.1s", "test"], "t");
    test(#fmt["%.1x", 127u], "7f");
    test(#fmt["%.1o", 10u], "12");
    test(#fmt["%.1t", 3u], "11");
    test(#fmt["%.1c", 'A'], "A");
    test(#fmt["%.1f", 5.82], "5.8");
}
fn part4() {
    test(#fmt["%.5d", 0], "00000");
    test(#fmt["%.5u", 0u], "00000");
    test(#fmt["%.5x", 0u], "00000");
    test(#fmt["%.5t", 0u], "00000");
    test(#fmt["%.5d", 10], "00010");
    test(#fmt["%.5d", -10], "-00010");
    test(#fmt["%.5u", 10u], "00010");
    test(#fmt["%.5s", "test"], "test");
    test(#fmt["%.5x", 127u], "0007f");
    test(#fmt["%.5o", 10u], "00012");
    test(#fmt["%.5t", 3u], "00011");
    test(#fmt["%.5c", 'A'], "A");
    test(#fmt["%.5f", 5.82], "5.82000");
    // Bool precision. I'm not sure if it's good or bad to have bool
    // conversions support precision - it's not standard printf so we
    // can do whatever. For now I'm making it behave the same as string
    // conversions.

    test(#fmt["%.b", true], "");
    test(#fmt["%.0b", true], "");
    test(#fmt["%.1b", true], "t");
}

fn part5() {
    // Explicit + sign. Only for signed conversions

    test(#fmt["%+d", 0], "+0");
    test(#fmt["%+d", 1], "+1");
    test(#fmt["%+d", -1], "-1");
    test(#fmt["%+f", 0.0], "+0");
    // Leave space for sign

    test(#fmt["% d", 0], " 0");
    test(#fmt["% d", 1], " 1");
    test(#fmt["% d", -1], "-1");
    test(#fmt["% f", 0.0], " 0");
    // Plus overrides space

    test(#fmt["% +d", 0], "+0");
    test(#fmt["%+ d", 0], "+0");
    test(#fmt["% +f", 0.0], "+0");
    test(#fmt["%+ f", 0.0], "+0");
    // 0-padding

    test(#fmt["%05d", 0], "00000");
    test(#fmt["%05d", 1], "00001");
    test(#fmt["%05d", -1], "-0001");
    test(#fmt["%05u", 1u], "00001");
    test(#fmt["%05x", 127u], "0007f");
    test(#fmt["%05X", 127u], "0007F");
    test(#fmt["%05o", 10u], "00012");
    test(#fmt["%05t", 3u], "00011");
    test(#fmt["%05f", 5.82], "05.82");
    // 0-padding a string is undefined but glibc does this:

    test(#fmt["%05s", "test"], " test");
    test(#fmt["%05c", 'A'], "    A");
    test(#fmt["%05b", true], " true");
    // Left-justify overrides 0-padding

    test(#fmt["%-05d", 0], "0    ");
    test(#fmt["%-05d", 1], "1    ");
    test(#fmt["%-05d", -1], "-1   ");
    test(#fmt["%-05u", 1u], "1    ");
    test(#fmt["%-05x", 127u], "7f   ");
    test(#fmt["%-05X", 127u], "7F   ");
    test(#fmt["%-05o", 10u], "12   ");
    test(#fmt["%-05t", 3u], "11   ");
    test(#fmt["%-05s", "test"], "test ");
    test(#fmt["%-05c", 'A'], "A    ");
    test(#fmt["%-05b", true], "true ");
    test(#fmt["%-05f", 5.82], "5.82 ");
}
fn part6() {
    // Precision overrides 0-padding

    test(#fmt["%06.5d", 0], " 00000");
    test(#fmt["%06.5u", 0u], " 00000");
    test(#fmt["%06.5x", 0u], " 00000");
    test(#fmt["%06.5d", 10], " 00010");
    test(#fmt["%06.5d", -10], "-00010");
    test(#fmt["%06.5u", 10u], " 00010");
    test(#fmt["%06.5s", "test"], "  test");
    test(#fmt["%06.5c", 'A'], "     A");
    test(#fmt["%06.5x", 127u], " 0007f");
    test(#fmt["%06.5X", 127u], " 0007F");
    test(#fmt["%06.5o", 10u], " 00012");
    test(#fmt["%08.5f", 5.82], " 5.82000");
    // Signed combinations

    test(#fmt["% 5d", 1], "    1");
    test(#fmt["% 5d", -1], "   -1");
    test(#fmt["%+5d", 1], "   +1");
    test(#fmt["%+5d", -1], "   -1");
    test(#fmt["% 05d", 1], " 0001");
    test(#fmt["% 05d", -1], "-0001");
    test(#fmt["%+05d", 1], "+0001");
    test(#fmt["%+05d", -1], "-0001");
    test(#fmt["%- 5d", 1], " 1   ");
    test(#fmt["%- 5d", -1], "-1   ");
    test(#fmt["%-+5d", 1], "+1   ");
    test(#fmt["%-+5d", -1], "-1   ");
    test(#fmt["%- 05d", 1], " 1   ");
    test(#fmt["%- 05d", -1], "-1   ");
    test(#fmt["%-+05d", 1], "+1   ");
    test(#fmt["%-+05d", -1], "-1   ");
}
<|endoftext|>


<commit_before>use std::error::Error;
use std::path::{Path, PathBuf};
use yaml_rust::{Yaml, YamlEmitter, YamlLoader};

/// List of directories containing files to expand. The first tuple element is the source
/// directory, while the second tuple element is the destination directory.
#[rustfmt::skip]
static TO_EXPAND: &[(&str, &str)] = &[
    ("src/ci/github-actions", ".github/workflows"),
];

/// Name of a special key that will be removed from all the maps in expanded configuration files.
/// This key can then be used to contain shared anchors.
static REMOVE_MAP_KEY: &str = "x--expand-yaml-anchors--remove";

/// Message that will be included at the top of all the expanded files. {source} will be replaced
/// with the source filename relative to the base path.
static HEADER_MESSAGE: &str = "\
#############################################################
#   WARNING: automatically generated file, DO NOT CHANGE!   #
#############################################################

# This file was automatically generated by the expand-yaml-anchors tool. The
# source file that generated this one is:
#
#   {source}
#
# Once you make changes to that file you need to run:
#
#   ./x.py run src/tools/expand-yaml-anchors/
#
# The CI build will fail if the tool is not run after changes to this file.

";

enum Mode {
    Check,
    Generate,
}

struct App {
    mode: Mode,
    base: PathBuf,
}

impl App {
    fn from_args() -> Result<Self, Box<dyn Error>> {
        // Parse CLI arguments
        let args = std::env::args().skip(1).collect::<Vec<_>>();
        let (mode, base) = match args.iter().map(|s| s.as_str()).collect::<Vec<_>>().as_slice() {
            &["generate", ref base] => (Mode::Generate, PathBuf::from(base)),
            &["check", ref base] => (Mode::Check, PathBuf::from(base)),
            _ => {
                eprintln!("usage: expand-yaml-anchors <source-dir> <dest-dir>");
                std::process::exit(1);
            }
        };

        Ok(App { mode, base })
    }

    fn run(&self) -> Result<(), Box<dyn Error>> {
        for (source, dest) in TO_EXPAND {
            let source = self.base.join(source);
            let dest = self.base.join(dest);
            for entry in std::fs::read_dir(&source)? {
                let path = entry?.path();
                if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") {
                    continue;
                }

                let dest_path = dest.join(path.file_name().unwrap());
                self.expand(&path, &dest_path).with_context(|| match self.mode {
                    Mode::Generate => format!(
                        "failed to expand {} into {}",
                        self.path(&path),
                        self.path(&dest_path)
                    ),
                    Mode::Check => format!("{} is not up to date", self.path(&dest_path)),
                })?;
            }
        }
        Ok(())
    }

    fn expand(&self, source: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
        let content = std::fs::read_to_string(source)
            .with_context(|| format!("failed to read {}", self.path(source)))?;

        let mut buf = HEADER_MESSAGE.replace("{source}", &self.path(source).to_string());

        let documents = YamlLoader::load_from_str(&content)
            .with_context(|| format!("failed to parse {}", self.path(source)))?;
        for mut document in documents.into_iter() {
            document = yaml_merge_keys::merge_keys(document)
                .with_context(|| format!("failed to expand {}", self.path(source)))?;
            document = filter_document(document);

            YamlEmitter::new(&mut buf).dump(&document).map_err(|err| WithContext {
                context: "failed to serialize the expanded yaml".into(),
                source: Box::new(err),
            })?;
            buf.push('\n');
        }

        match self.mode {
            Mode::Check => {
                let old = std::fs::read_to_string(dest)
                    .with_context(|| format!("failed to read {}", self.path(dest)))?;
                if old != buf {
                    return Err(Box::new(StrError(format!(
                        "{} and {} are different",
                        self.path(source),
                        self.path(dest),
                    ))));
                }
            }
            Mode::Generate => {
                std::fs::write(dest, buf.as_bytes())
                    .with_context(|| format!("failed to write to {}", self.path(dest)))?;
            }
        }
        Ok(())
    }

    fn path<'a>(&self, path: &'a Path) -> impl std::fmt::Display + 'a {
        path.strip_prefix(&self.base).unwrap_or(path).display()
    }
}

fn filter_document(document: Yaml) -> Yaml {
    match document {
        Yaml::Hash(map) => Yaml::Hash(
            map.into_iter()
                .filter(|(key, _)| {
                    if let Yaml::String(string) = &key { string != REMOVE_MAP_KEY } else { true }
                })
                .map(|(key, value)| (filter_document(key), filter_document(value)))
                .collect(),
        ),
        Yaml::Array(vec) => {
            Yaml::Array(vec.into_iter().map(|item| filter_document(item)).collect())
        }
        other => other,
    }
}

fn main() {
    if let Err(err) = App::from_args().and_then(|app| app.run()) {
        eprintln!("error: {}", err);

        let mut source = err.as_ref() as &dyn Error;
        while let Some(err) = source.source() {
            eprintln!("caused by: {}", err);
            source = err;
        }

        std::process::exit(1);
    }
}

#[derive(Debug)]
struct StrError(String);

impl Error for StrError {}

impl std::fmt::Display for StrError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        std::fmt::Display::fmt(&self.0, f)
    }
}

#[derive(Debug)]
struct WithContext {
    context: String,
    source: Box<dyn Error>,
}

impl std::fmt::Display for WithContext {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.context)
    }
}

impl Error for WithContext {
    fn source(&self) -> Option<&(dyn Error + 'static)> {
        Some(self.source.as_ref())
    }
}

pub(crate) trait ResultExt<T> {
    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>>;
}

impl<T, E: Into<Box<dyn Error>>> ResultExt<T> for Result<T, E> {
    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>> {
        match self {
            Ok(ok) => Ok(ok),
            Err(err) => Err(WithContext { source: err.into(), context: f() }.into()),
        }
    }
}
<commit_msg>expand-yaml-anchors: fix clippy::match_ref_pats and clippy::redundant_closure<commit_after>use std::error::Error;
use std::path::{Path, PathBuf};
use yaml_rust::{Yaml, YamlEmitter, YamlLoader};

/// List of directories containing files to expand. The first tuple element is the source
/// directory, while the second tuple element is the destination directory.
#[rustfmt::skip]
static TO_EXPAND: &[(&str, &str)] = &[
    ("src/ci/github-actions", ".github/workflows"),
];

/// Name of a special key that will be removed from all the maps in expanded configuration files.
/// This key can then be used to contain shared anchors.
static REMOVE_MAP_KEY: &str = "x--expand-yaml-anchors--remove";

/// Message that will be included at the top of all the expanded files. {source} will be replaced
/// with the source filename relative to the base path.
static HEADER_MESSAGE: &str = "\
#############################################################
#   WARNING: automatically generated file, DO NOT CHANGE!   #
#############################################################

# This file was automatically generated by the expand-yaml-anchors tool. The
# source file that generated this one is:
#
#   {source}
#
# Once you make changes to that file you need to run:
#
#   ./x.py run src/tools/expand-yaml-anchors/
#
# The CI build will fail if the tool is not run after changes to this file.

";

enum Mode {
    Check,
    Generate,
}

struct App {
    mode: Mode,
    base: PathBuf,
}

impl App {
    fn from_args() -> Result<Self, Box<dyn Error>> {
        // Parse CLI arguments
        let args = std::env::args().skip(1).collect::<Vec<_>>();
        let (mode, base) = match args.iter().map(|s| s.as_str()).collect::<Vec<_>>().as_slice() {
            ["generate", ref base] => (Mode::Generate, PathBuf::from(base)),
            ["check", ref base] => (Mode::Check, PathBuf::from(base)),
            _ => {
                eprintln!("usage: expand-yaml-anchors <source-dir> <dest-dir>");
                std::process::exit(1);
            }
        };

        Ok(App { mode, base })
    }

    fn run(&self) -> Result<(), Box<dyn Error>> {
        for (source, dest) in TO_EXPAND {
            let source = self.base.join(source);
            let dest = self.base.join(dest);
            for entry in std::fs::read_dir(&source)? {
                let path = entry?.path();
                if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") {
                    continue;
                }

                let dest_path = dest.join(path.file_name().unwrap());
                self.expand(&path, &dest_path).with_context(|| match self.mode {
                    Mode::Generate => format!(
                        "failed to expand {} into {}",
                        self.path(&path),
                        self.path(&dest_path)
                    ),
                    Mode::Check => format!("{} is not up to date", self.path(&dest_path)),
                })?;
            }
        }
        Ok(())
    }

    fn expand(&self, source: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
        let content = std::fs::read_to_string(source)
            .with_context(|| format!("failed to read {}", self.path(source)))?;

        let mut buf = HEADER_MESSAGE.replace("{source}", &self.path(source).to_string());

        let documents = YamlLoader::load_from_str(&content)
            .with_context(|| format!("failed to parse {}", self.path(source)))?;
        for mut document in documents.into_iter() {
            document = yaml_merge_keys::merge_keys(document)
                .with_context(|| format!("failed to expand {}", self.path(source)))?;
            document = filter_document(document);

            YamlEmitter::new(&mut buf).dump(&document).map_err(|err| WithContext {
                context: "failed to serialize the expanded yaml".into(),
                source: Box::new(err),
            })?;
            buf.push('\n');
        }

        match self.mode {
            Mode::Check => {
                let old = std::fs::read_to_string(dest)
                    .with_context(|| format!("failed to read {}", self.path(dest)))?;
                if old != buf {
                    return Err(Box::new(StrError(format!(
                        "{} and {} are different",
                        self.path(source),
                        self.path(dest),
                    ))));
                }
            }
            Mode::Generate => {
                std::fs::write(dest, buf.as_bytes())
                    .with_context(|| format!("failed to write to {}", self.path(dest)))?;
            }
        }
        Ok(())
    }

    fn path<'a>(&self, path: &'a Path) -> impl std::fmt::Display + 'a {
        path.strip_prefix(&self.base).unwrap_or(path).display()
    }
}

fn filter_document(document: Yaml) -> Yaml {
    match document {
        Yaml::Hash(map) => Yaml::Hash(
            map.into_iter()
                .filter(|(key, _)| {
                    if let Yaml::String(string) = &key { string != REMOVE_MAP_KEY } else { true }
                })
                .map(|(key, value)| (filter_document(key), filter_document(value)))
                .collect(),
        ),
        Yaml::Array(vec) => Yaml::Array(vec.into_iter().map(filter_document).collect()),
        other => other,
    }
}

fn main() {
    if let Err(err) = App::from_args().and_then(|app| app.run()) {
        eprintln!("error: {}", err);

        let mut source = err.as_ref() as &dyn Error;
        while let Some(err) = source.source() {
            eprintln!("caused by: {}", err);
            source = err;
        }

        std::process::exit(1);
    }
}

#[derive(Debug)]
struct StrError(String);

impl Error for StrError {}

impl std::fmt::Display for StrError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        std::fmt::Display::fmt(&self.0, f)
    }
}

#[derive(Debug)]
struct WithContext {
    context: String,
    source: Box<dyn Error>,
}

impl std::fmt::Display for WithContext {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.context)
    }
}

impl Error for WithContext {
    fn source(&self) -> Option<&(dyn Error + 'static)> {
        Some(self.source.as_ref())
    }
}

pub(crate) trait ResultExt<T> {
    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>>;
}

impl<T, E: Into<Box<dyn Error>>> ResultExt<T> for Result<T, E> {
    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>> {
        match self {
            Ok(ok) => Ok(ok),
            Err(err) => Err(WithContext { source: err.into(), context: f() }.into()),
        }
    }
}
<|endoftext|>


<commit_before>//! Utilities for interacting with tasks implemented in the rust-rosetta repository.

use std::collections::HashSet;
use std::fs;
use std::path::{Path, PathBuf};

use cargo_metadata;
use failure::{self, Error, ResultExt};
use reqwest::Url;
use toml::Value;
use walkdir::WalkDir;

use remote;
use TASK_URL_RE;

/// A local (in repository) implementation of a Rosetta Code task.
#[derive(Debug, Clone)]
pub struct LocalTask {
    /// The name of the package implementing this task.
    pub package_name: String,

    /// The path to the crate manifest for the task.
    pub manifest_path: PathBuf,

    /// A set of filenames containing the Rust source code of the tasks.
    pub source: HashSet<PathBuf>,

    /// The URL of the task on the Rosetta Code wiki.
    ///
    /// This is parsed from the crate metadata, so it may not point to an actual Rosetta Code URL.
    pub url: Url,

    /// The title of the task on Rosetta Code.
    ///
    /// The title is normalized, so it can be used to uniquely identify the task on the wiki.
    pub title: String,
}

/// Given a path to the root `Cargo.toml`, returns a list of tasks implemented in the rust-rosetta
/// repository.
pub fn parse_tasks<P>(manifest_path: P) -> Result<Vec<LocalTask>, Error>
where
    P: AsRef<Path>,
{
    let metadata = cargo_metadata::metadata(Some(&manifest_path.as_ref())).unwrap();
    let packages = &metadata.packages;

    let mut tasks = vec![];

    for member in &metadata.workspace_members {
        // Skip if we encounter known non-task crates.
        if member.name == "rust-rosetta" || member.name == "meta" {
            continue;
        }

        let package = packages.iter().find(|p| p.name == member.name).unwrap();

        // If the package has a proc-macro or dylib target, it's probably just a dependency of
        // another task. Skip it.
        if package.targets.iter().any(|t| {
            t.kind.contains(&String::from("dylib")) || t.kind.contains(&String::from("proc-macro"))
        }) {
            continue;
        }

        let manifest_path = Path::new(&package.manifest_path);

        let rosetta_url = parse_rosetta_url(manifest_path).context(format!(
            "could not parse rosetta code URL from {}",
            manifest_path.display()
        ))?;

        let title = {
            let caps = TASK_URL_RE.captures(rosetta_url.as_str()).ok_or_else(|| {
                failure::err_msg(format!(
                    "task URL does not match rosetta code regex: {}",
                    rosetta_url
                ))
            })?;
            remote::decode_title(&caps[1])
        };

        tasks.push(LocalTask {
            package_name: member.name.clone(),
            manifest_path: manifest_path.to_owned(),
            source: find_sources(manifest_path.parent().unwrap())?,
            url: rosetta_url,
            title,
        });
    }

    Ok(tasks)
}

/// Parses the Rosetta Code URL from the package metadata in a task's crate manifest.
fn parse_rosetta_url<P>(manifest_path: P) -> Result<Url, Error>
where
    P: AsRef<Path>,
{
    let manifest: Value = fs::read_string(manifest_path)?.parse()?;

    let url = manifest
        .get("package")
        .and_then(|p| p.get("metadata"))
        .and_then(|m| m.get("rosettacode"))
        .and_then(|m| m.get("url"))
        .and_then(|u| u.as_str())
        .ok_or_else(|| failure::err_msg("unexpected metadata format"))?;

    Ok(Url::parse(url)?)
}

/// Collect the paths to the source files of a given package.
fn find_sources<P>(directory: P) -> Result<HashSet<PathBuf>, Error>
where
    P: AsRef<Path>,
{
    let mut sources = HashSet::new();

    for entry in WalkDir::new(directory) {
        let entry = entry?;

        if let Some("rs") = entry.path().extension().and_then(|s| s.to_str()) {
            sources.insert(entry.path().to_owned());
        }
    }

    Ok(sources)
}
<commit_msg>use new name for `fs::read_to_string`<commit_after>//! Utilities for interacting with tasks implemented in the rust-rosetta repository.

use std::collections::HashSet;
use std::fs;
use std::path::{Path, PathBuf};

use cargo_metadata;
use failure::{self, Error, ResultExt};
use reqwest::Url;
use toml::Value;
use walkdir::WalkDir;

use remote;
use TASK_URL_RE;

/// A local (in repository) implementation of a Rosetta Code task.
#[derive(Debug, Clone)]
pub struct LocalTask {
    /// The name of the package implementing this task.
    pub package_name: String,

    /// The path to the crate manifest for the task.
    pub manifest_path: PathBuf,

    /// A set of filenames containing the Rust source code of the tasks.
    pub source: HashSet<PathBuf>,

    /// The URL of the task on the Rosetta Code wiki.
    ///
    /// This is parsed from the crate metadata, so it may not point to an actual Rosetta Code URL.
    pub url: Url,

    /// The title of the task on Rosetta Code.
    ///
    /// The title is normalized, so it can be used to uniquely identify the task on the wiki.
    pub title: String,
}

/// Given a path to the root `Cargo.toml`, returns a list of tasks implemented in the rust-rosetta
/// repository.
pub fn parse_tasks<P>(manifest_path: P) -> Result<Vec<LocalTask>, Error>
where
    P: AsRef<Path>,
{
    let metadata = cargo_metadata::metadata(Some(&manifest_path.as_ref())).unwrap();
    let packages = &metadata.packages;

    let mut tasks = vec![];

    for member in &metadata.workspace_members {
        // Skip if we encounter known non-task crates.
        if member.name == "rust-rosetta" || member.name == "meta" {
            continue;
        }

        let package = packages.iter().find(|p| p.name == member.name).unwrap();

        // If the package has a proc-macro or dylib target, it's probably just a dependency of
        // another task. Skip it.
        if package.targets.iter().any(|t| {
            t.kind.contains(&String::from("dylib")) || t.kind.contains(&String::from("proc-macro"))
        }) {
            continue;
        }

        let manifest_path = Path::new(&package.manifest_path);

        let rosetta_url = parse_rosetta_url(manifest_path).context(format!(
            "could not parse rosetta code URL from {}",
            manifest_path.display()
        ))?;

        let title = {
            let caps = TASK_URL_RE.captures(rosetta_url.as_str()).ok_or_else(|| {
                failure::err_msg(format!(
                    "task URL does not match rosetta code regex: {}",
                    rosetta_url
                ))
            })?;
            remote::decode_title(&caps[1])
        };

        tasks.push(LocalTask {
            package_name: member.name.clone(),
            manifest_path: manifest_path.to_owned(),
            source: find_sources(manifest_path.parent().unwrap())?,
            url: rosetta_url,
            title,
        });
    }

    Ok(tasks)
}

/// Parses the Rosetta Code URL from the package metadata in a task's crate manifest.
fn parse_rosetta_url<P>(manifest_path: P) -> Result<Url, Error>
where
    P: AsRef<Path>,
{
    let manifest: Value = fs::read_to_string(manifest_path)?.parse()?;

    let url = manifest
        .get("package")
        .and_then(|p| p.get("metadata"))
        .and_then(|m| m.get("rosettacode"))
        .and_then(|m| m.get("url"))
        .and_then(|u| u.as_str())
        .ok_or_else(|| failure::err_msg("unexpected metadata format"))?;

    Ok(Url::parse(url)?)
}

/// Collect the paths to the source files of a given package.
fn find_sources<P>(directory: P) -> Result<HashSet<PathBuf>, Error>
where
    P: AsRef<Path>,
{
    let mut sources = HashSet::new();

    for entry in WalkDir::new(directory) {
        let entry = entry?;

        if let Some("rs") = entry.path().extension().and_then(|s| s.to_str()) {
            sources.insert(entry.path().to_owned());
        }
    }

    Ok(sources)
}
<|endoftext|>