<commit_before>#![warn(clippy::all, rust_2018_idioms)]

use cargo_registry::{env_optional, metrics::LogEncoder, util::errors::AppResult, App, Env};
use std::{fs::File, process::Command, sync::Arc, time::Duration};

use conduit_hyper::Service;
use futures_util::future::FutureExt;
use prometheus::Encoder;
use reqwest::blocking::Client;
use std::io::{self, Write};
use tokio::io::AsyncWriteExt;
use tokio::signal::unix::{signal, SignalKind};

const CORE_THREADS: usize = 4;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let _sentry = cargo_registry::sentry::init();

    // Initialize logging
    cargo_registry::util::tracing::init();

    let config = cargo_registry::config::Server::default();
    let env = config.env();
    let client = Client::new();
    let app = Arc::new(App::new(config, Some(client)));

    // Start the background thread periodically persisting download counts to the database.
    downloads_counter_thread(app.clone());

    // Start the background thread periodically logging instance metrics.
    log_instance_metrics_thread(app.clone());

    let handler = cargo_registry::build_handler(app.clone());

    let heroku = dotenv::var("HEROKU").is_ok();
    let fastboot = dotenv::var("USE_FASTBOOT").is_ok();
    let dev_docker = dotenv::var("DEV_DOCKER").is_ok();

    let ip = if dev_docker {
        [0, 0, 0, 0]
    } else {
        [127, 0, 0, 1]
    };
    let port = match (heroku, env_optional("PORT")) {
        (false, Some(port)) => port,
        _ => 8888,
    };

    let threads = dotenv::var("SERVER_THREADS")
        .map(|s| s.parse().expect("SERVER_THREADS was not a valid number"))
        .unwrap_or_else(|_| match env {
            Env::Development => 5,
            // A large default because this can be easily changed via env and in production we
            // want the logging middleware to accurately record the start time.
            _ => 500,
        });

    let rt = tokio::runtime::Builder::new_multi_thread()
        .enable_all()
        .worker_threads(CORE_THREADS)
        .max_blocking_threads(threads as usize)
        .build()
        .unwrap();

    let handler = Arc::new(conduit_hyper::BlockingHandler::new(handler));
    let make_service =
        hyper::service::make_service_fn(move |socket: &hyper::server::conn::AddrStream| {
            let addr = socket.remote_addr();
            let handler = handler.clone();
            async move { Service::from_blocking(handler, addr) }
        });

    let (addr, server) = rt.block_on(async {
        let server = hyper::Server::bind(&(ip, port).into()).serve(make_service);

        // When the user configures PORT=0 the operating system will allocate a random unused port.
        // This fetches that random port and uses it to display the the correct url later.
        let addr = server.local_addr();

        let mut sig_int = signal(SignalKind::interrupt())?;
        let mut sig_term = signal(SignalKind::terminate())?;
        let server = server.with_graceful_shutdown(async move {
            // Wait for either signal
            tokio::select! {
                _ = sig_int.recv().fuse() => {},
                _ = sig_term.recv().fuse() => {},
            };
            tokio::io::stdout()
                .write_all(b"Starting graceful shutdown\n")
                .await
                .ok();
        });

        Ok::<_, io::Error>((addr, server))
    })?;

    // Do not change this line! Removing the line or changing its contents in any way will break
    // the test suite :)
    println!("Listening at http://{addr}");

    // Creating this file tells heroku to tell nginx that the application is ready
    // to receive traffic.
    if heroku {
        let path = if fastboot {
            "/tmp/backend-initialized"
        } else {
            "/tmp/app-initialized"
        };
        println!("Writing to {path}");
        File::create(path).unwrap();

        // Launch nginx via the Heroku nginx buildpack
        // `wait()` is never called on the child process, but it should be okay to leave a zombie
        // process around on shutdown when Heroku is tearing down the entire container anyway.
        Command::new("./script/start-web.sh")
            .spawn()
            .expect("Couldn't spawn nginx");
    }

    // Block the main thread until the server has shutdown
    rt.block_on(server)?;

    println!("Persisting remaining downloads counters");
    match app.downloads_counter.persist_all_shards(&app) {
        Ok(stats) => stats.log(),
        Err(err) => println!("downloads_counter error: {err}"),
    }

    println!("Server has gracefully shutdown!");
    Ok(())
}

fn downloads_counter_thread(app: Arc<App>) {
    let interval = Duration::from_millis(
        (app.config.downloads_persist_interval_ms / app.downloads_counter.shards_count()) as u64,
    );

    std::thread::spawn(move || loop {
        std::thread::sleep(interval);

        match app.downloads_counter.persist_next_shard(&app) {
            Ok(stats) => stats.log(),
            Err(err) => println!("downloads_counter error: {err}"),
        }
    });
}

fn log_instance_metrics_thread(app: Arc<App>) {
    // Only run the thread if the configuration is provided
    let interval = if let Some(secs) = app.config.instance_metrics_log_every_seconds {
        Duration::from_secs(secs)
    } else {
        return;
    };

    std::thread::spawn(move || loop {
        if let Err(err) = log_instance_metrics_inner(&app) {
            eprintln!("log_instance_metrics error: {err}");
        }
        std::thread::sleep(interval);
    });
}

fn log_instance_metrics_inner(app: &App) -> AppResult<()> {
    let families = app.instance_metrics.gather(app)?;

    let mut stdout = std::io::stdout();
    LogEncoder::new().encode(&families, &mut stdout)?;
    stdout.flush()?;

    Ok(())
}
<commit_msg>bin/server: Use `tracing` macros instead of `println!()`<commit_after>#![warn(clippy::all, rust_2018_idioms)]

#[macro_use]
extern crate tracing;

use cargo_registry::{env_optional, metrics::LogEncoder, util::errors::AppResult, App, Env};
use std::{fs::File, process::Command, sync::Arc, time::Duration};

use conduit_hyper::Service;
use futures_util::future::FutureExt;
use prometheus::Encoder;
use reqwest::blocking::Client;
use std::io::{self, Write};
use tokio::signal::unix::{signal, SignalKind};

const CORE_THREADS: usize = 4;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let _sentry = cargo_registry::sentry::init();

    // Initialize logging
    cargo_registry::util::tracing::init();

    let config = cargo_registry::config::Server::default();
    let env = config.env();
    let client = Client::new();
    let app = Arc::new(App::new(config, Some(client)));

    // Start the background thread periodically persisting download counts to the database.
    downloads_counter_thread(app.clone());

    // Start the background thread periodically logging instance metrics.
    log_instance_metrics_thread(app.clone());

    let handler = cargo_registry::build_handler(app.clone());

    let heroku = dotenv::var("HEROKU").is_ok();
    let fastboot = dotenv::var("USE_FASTBOOT").is_ok();
    let dev_docker = dotenv::var("DEV_DOCKER").is_ok();

    let ip = if dev_docker {
        [0, 0, 0, 0]
    } else {
        [127, 0, 0, 1]
    };
    let port = match (heroku, env_optional("PORT")) {
        (false, Some(port)) => port,
        _ => 8888,
    };

    let threads = dotenv::var("SERVER_THREADS")
        .map(|s| s.parse().expect("SERVER_THREADS was not a valid number"))
        .unwrap_or_else(|_| match env {
            Env::Development => 5,
            // A large default because this can be easily changed via env and in production we
            // want the logging middleware to accurately record the start time.
            _ => 500,
        });

    let rt = tokio::runtime::Builder::new_multi_thread()
        .enable_all()
        .worker_threads(CORE_THREADS)
        .max_blocking_threads(threads as usize)
        .build()
        .unwrap();

    let handler = Arc::new(conduit_hyper::BlockingHandler::new(handler));
    let make_service =
        hyper::service::make_service_fn(move |socket: &hyper::server::conn::AddrStream| {
            let addr = socket.remote_addr();
            let handler = handler.clone();
            async move { Service::from_blocking(handler, addr) }
        });

    let (addr, server) = rt.block_on(async {
        let server = hyper::Server::bind(&(ip, port).into()).serve(make_service);

        // When the user configures PORT=0 the operating system will allocate a random unused port.
        // This fetches that random port and uses it to display the the correct url later.
        let addr = server.local_addr();

        let mut sig_int = signal(SignalKind::interrupt())?;
        let mut sig_term = signal(SignalKind::terminate())?;
        let server = server.with_graceful_shutdown(async move {
            // Wait for either signal
            tokio::select! {
                _ = sig_int.recv().fuse() => {},
                _ = sig_term.recv().fuse() => {},
            };

            info!("Starting graceful shutdown");
        });

        Ok::<_, io::Error>((addr, server))
    })?;

    // Do not change this line! Removing the line or changing its contents in any way will break
    // the test suite :)
    info!("Listening at http://{addr}");

    // Creating this file tells heroku to tell nginx that the application is ready
    // to receive traffic.
    if heroku {
        let path = if fastboot {
            "/tmp/backend-initialized"
        } else {
            "/tmp/app-initialized"
        };
        info!("Writing to {path}");
        File::create(path).unwrap();

        // Launch nginx via the Heroku nginx buildpack
        // `wait()` is never called on the child process, but it should be okay to leave a zombie
        // process around on shutdown when Heroku is tearing down the entire container anyway.
        Command::new("./script/start-web.sh")
            .spawn()
            .expect("Couldn't spawn nginx");
    }

    // Block the main thread until the server has shutdown
    rt.block_on(server)?;

    info!("Persisting remaining downloads counters");
    match app.downloads_counter.persist_all_shards(&app) {
        Ok(stats) => stats.log(),
        Err(err) => error!(?err, "downloads_counter error"),
    }

    info!("Server has gracefully shutdown!");
    Ok(())
}

fn downloads_counter_thread(app: Arc<App>) {
    let interval = Duration::from_millis(
        (app.config.downloads_persist_interval_ms / app.downloads_counter.shards_count()) as u64,
    );

    std::thread::spawn(move || loop {
        std::thread::sleep(interval);

        match app.downloads_counter.persist_next_shard(&app) {
            Ok(stats) => stats.log(),
            Err(err) => error!(?err, "downloads_counter error"),
        }
    });
}

fn log_instance_metrics_thread(app: Arc<App>) {
    // Only run the thread if the configuration is provided
    let interval = if let Some(secs) = app.config.instance_metrics_log_every_seconds {
        Duration::from_secs(secs)
    } else {
        return;
    };

    std::thread::spawn(move || loop {
        if let Err(err) = log_instance_metrics_inner(&app) {
            error!(?err, "log_instance_metrics error");
        }
        std::thread::sleep(interval);
    });
}

fn log_instance_metrics_inner(app: &App) -> AppResult<()> {
    let families = app.instance_metrics.gather(app)?;

    let mut stdout = std::io::stdout();
    LogEncoder::new().encode(&families, &mut stdout)?;
    stdout.flush()?;

    Ok(())
}
<|endoftext|>


<commit_before>extern crate zwreec;
extern crate getopts;
extern crate libc;
#[macro_use] extern crate log;
extern crate time;
extern crate term;

use std::env;
use std::vec::Vec;
use std::error::Error;
use std::fs::File;
use std::io::{Read,Write};
use std::path::Path;
use std::process::exit;

use zwreec::config;
use zwreec::config::Config;

mod logger;

// found in:
// http://stackoverflow.com/a/27590832
macro_rules! print_stderr(
    ($($arg:tt)*) => (
        match write!(&mut ::std::io::stderr(), $($arg)* ) {
            Ok(_) => {},
            Err(x) => panic!("Unable to write to stderr: {}", x),
        }
    )
);

/// Returns the primary options for zwreec. It is e.g. used to generate the
/// usage information.
fn short_options() -> getopts::Options {
    let mut opts = getopts::Options::new();
    opts.optflagmulti("v", "verbose", "Be more verbose. Can be used multiple times.");
    opts.optflag("q", "quiet", "Be quiet");
    opts.optflag("w", "overwrite", "Overwrite output file if necessary.");
    opts.optflagopt("l", "logfile", "Specify log file (default standard error)", "LOGFILE");
    opts.optopt("o", "", "Name of the output file", "FILE");
    opts.optflag("h", "help", "Display this help and exit");
    opts.optflag("V", "version", "Display version");

    opts
}

/// Prints usage information
/// The verbose flag signals if all options should be shown.
/// NOTE: This is similar to librustc_driver's usage function.
fn usage(verbose: bool) {
    let options = if verbose {
        config::zwreec_options(short_options())
    } else {
        short_options()
    };

    let brief = format!("Usage: zwreec [-hV] [-vqwf] [-l [LOGFILE]] [-o OUTPUT] INPUT");

    println!("{}", config::zwreec_usage(verbose, options, &brief));
}

/// Parse command line arguments
///
/// Parses command line arguments to set up the logger and extract input and
/// output parameters. Will display the usage and exit depending on arguments.
///
/// Returns `getopts::Matches` and a `std::fs::File` for the input and output
/// file. The `getopts::Matches` can be used to
///
/// # Examples
///
/// ```
/// let mut opts = getopts::Options::new();
/// opts.optflag("h", "help", "display this help and exit");
///
/// let (matches, mut input, mut output) = parse_arguments(
///     env::args().collect(),
///     opts
/// );
/// ```
///
/// # Failures
///
/// Depending on encountered arguments or parsing errors this function will
/// print the usage and/or call `exit(1)`.
///
/// NOTE: This is similar to librustc_driver's handle_options function.
fn parse_arguments(args: Vec<String>, opts: getopts::Options) -> (getopts::Matches, config::Config) {
    let mut loggers: Vec<Box<logger::SharedLogger>> = vec![];

    if args.is_empty() {
        // No options provided, print usage and exit
        usage(false);
        exit(1);
    }

    let matches = match opts.parse(&args[1..]) {
        Ok(m)  => m,
        Err(f) => {
            // parsing error
            // display usage and return
            print_stderr!("{}\n", f.to_string());
            exit(1);
        }
    };

    if matches.opt_present("help") {
        usage(matches.opt_present("verbose"));
        exit(0);
    }

    if matches.opt_present("version") {
        println!("zwreec {}", match option_env!("CFG_VERSION") {
            Some(s) => s.to_string(),
            None => format!("{}.{}.{}{}",
                            env!("CARGO_PKG_VERSION_MAJOR"),
                            env!("CARGO_PKG_VERSION_MINOR"),
                            env!("CARGO_PKG_VERSION_PATCH"),
                            option_env!("CARGO_PKG_VERSION_PRE").unwrap_or(""))
        });
        exit(0);
    }

    if matches.opt_present("verbose") {
        // set log level to verbose
        loggers.push(logger::TermLogger::new(
                match matches.opt_count("v") {
                    1 => logger::LogLevelFilter::Info,
                    2 => logger::LogLevelFilter::Debug,
                    _ => logger::LogLevelFilter::Trace,
                }));
    } else if matches.opt_present("quiet") {
        // set log level to error
        loggers.push(logger::TermLogger::new(logger::LogLevelFilter::Error));
    } else {
        // set log level to warn
        loggers.push(logger::TermLogger::new(logger::LogLevelFilter::Warn));
    }

    if matches.opt_present("logfile") {
        // sets a logger to output to logfile
        let name = if let Some(n) = matches.opt_str("logfile") {
            n
        } else {
            "zwreec.log".to_string()
        };
        loggers.push(logger::FileLogger::new(
                        logger::LogLevelFilter::Trace,
                        File::create(name).unwrap())
            );
    }

    // TODO: This might not belong in a function called parse_arguments
    // activate logger
    let _ = logger::CombinedLogger::init(loggers);

    let cfg = Config::from_matches(&matches);
    (matches, cfg)
}

fn parse_input(matches: &getopts::Matches) -> Option<Box<Read>> {
    if matches.free.len() == 1 {
        let path = Path::new(&matches.free[0]);
        match File::open(path) {
            Err(why) => {
                error!("Couldn't open {}: {}",
                    path.display(), Error::description(&why));
                None
            },
            Ok(file) => {
                info!("Opened input: {}", path.display());
                Some(Box::new(file))
            }
        }
    } else if unsafe { libc::isatty(libc::STDIN_FILENO as i32) } == 0 {
        // Not connected to a terminal, assuming safe to read from stdin
        info!("Reading input from stdin");
        Some(Box::new(std::io::stdin()))
    } else {
        None
    }
}

fn parse_output(matches: &getopts::Matches) -> Option<Box<Write>> {
    let name = matches.opt_str("o").unwrap_or("a.z8".to_string());

    if name == "-" {
        // tty requested
        if unsafe { libc::isatty(libc::STDOUT_FILENO as i32)  } == 0 {
            // Not connected to a terminal, assuming safe to write to stdin
            // NOTE: this should be considered unsafe, as the library is *not*
            // guaranteed to only print to stderr
            warn!("Writing to stdout can lead to unusable output!");
            warn!("You should specify an output name using -o 'FILE'");
            info!("Writing output to stdout");
            Some(Box::new(std::io::stdout()))
        } else {
            error!("stdout is connected to a terminal.");
            error!("Zcode is a binary format and should not be printed to a tty.");
            None
        }
    } else {
        // opening file
        let path = Path::new(&name);

        if name == "a.z8" {
            debug!("No output file specified, using {}", path.display());
        }

        // Check if FILE exists and issue warning.
        if File::open(path).is_ok() {
            if !matches.opt_present("w") {
                error!("Output file {} already exists. Use '-o NAME' to use a different name or '-w' to overwrite!",
                       path.display());
                return None;
            } else {
                warn!("Output file {} already exists", path.display());
            }
        }

        match File::create(path) {
            Err(why) => {
                error!("Couldn't open {}: {}",
                       path.display(), Error::description(&why));
                None
            },
            Ok(file) => {
                info!("Opened output: {}", path.display());
                Some(Box::new(file))
            }
        }
    }
}

fn main() {
    // handle command line parameters
    let (matches, cfg) = parse_arguments(
        env::args().collect(),
        config::zwreec_options(short_options())
    );

    let mut input = parse_input(&matches);
    let mut output = parse_output(&matches);

    debug!("Parsed command line options");
    info!("Main started");

    // call library
    if !cfg.test_cases.is_empty() {
        zwreec::test_library(cfg, &mut input, &mut output);
    } else {
        // unwrap input and output
        let mut _input = match input {
            Some(i) => i,
            None => panic!("Missing input file! Compile aborted")
        };
        let mut _output = match output {
            Some(o) => o,
            None => panic!("Missing output file! Compile aborted")
        };
        zwreec::compile(cfg, &mut _input, &mut _output);
    }

    info!("Main finished");
}
<commit_msg>Make logfile option more predictable<commit_after>extern crate zwreec;
extern crate getopts;
extern crate libc;
#[macro_use] extern crate log;
extern crate time;
extern crate term;

use std::env;
use std::vec::Vec;
use std::error::Error;
use std::fs::File;
use std::io::{Read,Write};
use std::path::Path;
use std::process::exit;

use zwreec::config;
use zwreec::config::Config;

mod logger;

// found in:
// http://stackoverflow.com/a/27590832
macro_rules! print_stderr(
    ($($arg:tt)*) => (
        match write!(&mut ::std::io::stderr(), $($arg)* ) {
            Ok(_) => {},
            Err(x) => panic!("Unable to write to stderr: {}", x),
        }
    )
);

/// Returns the primary options for zwreec. It is e.g. used to generate the
/// usage information.
fn short_options() -> getopts::Options {
    let mut opts = getopts::Options::new();
    opts.optflagmulti("v", "verbose", "Be more verbose. Can be used multiple times.");
    opts.optflag("q", "quiet", "Be quiet");
    opts.optflag("w", "overwrite", "Overwrite output file if necessary.");
    opts.optflagopt("l", "logfile", "Specify log file (additionally to logging on stderr)", "LOGFILE");
    opts.optopt("o", "", "Name of the output file", "FILE");
    opts.optflag("h", "help", "Display this help and exit");
    opts.optflag("V", "version", "Display version");

    opts
}

/// Prints usage information
/// The verbose flag signals if all options should be shown.
/// NOTE: This is similar to librustc_driver's usage function.
fn usage(verbose: bool) {
    let options = if verbose {
        config::zwreec_options(short_options())
    } else {
        short_options()
    };

    let brief = format!("Usage: zwreec [-hV] [-vqwf] [-l [LOGFILE]] [-o OUTPUT] INPUT");

    println!("{}", config::zwreec_usage(verbose, options, &brief));
}

/// Parse command line arguments
///
/// Parses command line arguments to set up the logger and extract input and
/// output parameters. Will display the usage and exit depending on arguments.
///
/// Returns `getopts::Matches` and a `std::fs::File` for the input and output
/// file. The `getopts::Matches` can be used to
///
/// # Examples
///
/// ```
/// let mut opts = getopts::Options::new();
/// opts.optflag("h", "help", "display this help and exit");
///
/// let (matches, mut input, mut output) = parse_arguments(
///     env::args().collect(),
///     opts
/// );
/// ```
///
/// # Failures
///
/// Depending on encountered arguments or parsing errors this function will
/// print the usage and/or call `exit(1)`.
///
/// NOTE: This is similar to librustc_driver's handle_options function.
fn parse_arguments(args: Vec<String>, opts: getopts::Options) -> (getopts::Matches, config::Config) {
    let mut loggers: Vec<Box<logger::SharedLogger>> = vec![];

    if args.is_empty() {
        // No options provided, print usage and exit
        usage(false);
        exit(1);
    }

    let matches = match opts.parse(&args[1..]) {
        Ok(m)  => m,
        Err(f) => {
            // parsing error
            // display usage and return
            print_stderr!("{}\n", f.to_string());
            exit(1);
        }
    };

    if matches.opt_present("help") {
        usage(matches.opt_present("verbose"));
        exit(0);
    }

    if matches.opt_present("version") {
        println!("zwreec {}", match option_env!("CFG_VERSION") {
            Some(s) => s.to_string(),
            None => format!("{}.{}.{}{}",
                            env!("CARGO_PKG_VERSION_MAJOR"),
                            env!("CARGO_PKG_VERSION_MINOR"),
                            env!("CARGO_PKG_VERSION_PATCH"),
                            option_env!("CARGO_PKG_VERSION_PRE").unwrap_or(""))
        });
        exit(0);
    }

    if matches.opt_present("verbose") {
        // set log level to verbose
        loggers.push(logger::TermLogger::new(
                match matches.opt_count("v") {
                    1 => logger::LogLevelFilter::Info,
                    2 => logger::LogLevelFilter::Debug,
                    _ => logger::LogLevelFilter::Trace,
                }));
    } else if matches.opt_present("quiet") {
        // set log level to error
        loggers.push(logger::TermLogger::new(logger::LogLevelFilter::Error));
    } else {
        // set log level to warn
        loggers.push(logger::TermLogger::new(logger::LogLevelFilter::Warn));
    }

    if matches.opt_present("logfile") {
        // sets a logger to output to logfile
        let name = if let Some(n) = matches.opt_str("logfile") {
            n
        } else {
            println!("Error: logfile option specified but no logfile name given");
            usage(false);
            exit(1);
        };
        loggers.push(logger::FileLogger::new(
                        logger::LogLevelFilter::Trace,
                        File::create(name).unwrap())
            );
    }

    // TODO: This might not belong in a function called parse_arguments
    // activate logger
    let _ = logger::CombinedLogger::init(loggers);

    let cfg = Config::from_matches(&matches);
    (matches, cfg)
}

fn parse_input(matches: &getopts::Matches) -> Option<Box<Read>> {
    if matches.free.len() == 1 {
        let path = Path::new(&matches.free[0]);
        match File::open(path) {
            Err(why) => {
                error!("Couldn't open {}: {}",
                    path.display(), Error::description(&why));
                None
            },
            Ok(file) => {
                info!("Opened input: {}", path.display());
                Some(Box::new(file))
            }
        }
    } else if unsafe { libc::isatty(libc::STDIN_FILENO as i32) } == 0 {
        // Not connected to a terminal, assuming safe to read from stdin
        info!("Reading input from stdin");
        Some(Box::new(std::io::stdin()))
    } else {
        None
    }
}

fn parse_output(matches: &getopts::Matches) -> Option<Box<Write>> {
    let name = matches.opt_str("o").unwrap_or("a.z8".to_string());

    if name == "-" {
        // tty requested
        if unsafe { libc::isatty(libc::STDOUT_FILENO as i32)  } == 0 {
            // Not connected to a terminal, assuming safe to write to stdin
            // NOTE: this should be considered unsafe, as the library is *not*
            // guaranteed to only print to stderr
            warn!("Writing to stdout can lead to unusable output!");
            warn!("You should specify an output name using -o 'FILE'");
            info!("Writing output to stdout");
            Some(Box::new(std::io::stdout()))
        } else {
            error!("stdout is connected to a terminal.");
            error!("Zcode is a binary format and should not be printed to a tty.");
            None
        }
    } else {
        // opening file
        let path = Path::new(&name);

        if name == "a.z8" {
            debug!("No output file specified, using {}", path.display());
        }

        // Check if FILE exists and issue warning.
        if File::open(path).is_ok() {
            if !matches.opt_present("w") {
                error!("Output file {} already exists. Use '-o NAME' to use a different name or '-w' to overwrite!",
                       path.display());
                return None;
            } else {
                warn!("Output file {} already exists", path.display());
            }
        }

        match File::create(path) {
            Err(why) => {
                error!("Couldn't open {}: {}",
                       path.display(), Error::description(&why));
                None
            },
            Ok(file) => {
                info!("Opened output: {}", path.display());
                Some(Box::new(file))
            }
        }
    }
}

fn main() {
    // handle command line parameters
    let (matches, cfg) = parse_arguments(
        env::args().collect(),
        config::zwreec_options(short_options())
    );

    let mut input = parse_input(&matches);
    let mut output = parse_output(&matches);

    debug!("Parsed command line options");
    info!("Main started");

    // call library
    if !cfg.test_cases.is_empty() {
        zwreec::test_library(cfg, &mut input, &mut output);
    } else {
        // unwrap input and output
        let mut _input = match input {
            Some(i) => i,
            None => panic!("Missing input file! Compile aborted")
        };
        let mut _output = match output {
            Some(o) => o,
            None => panic!("Missing output file! Compile aborted")
        };
        zwreec::compile(cfg, &mut _input, &mut _output);
    }

    info!("Main finished");
}
<|endoftext|>


<commit_before>use bit_vec::BitVec;
use std::slice::Iter;
use lfsr;
use lfsr::LFSR;

#[derive(PartialEq, Debug)]
pub struct GaloisLFSR {
    state: BitVec,
    mask: BitVec,
}

impl GaloisLFSR {
    pub fn from_iter(iter: Iter<usize>) -> Self {
        let &len = iter.clone().max().unwrap();

        let state = BitVec::from_elem(len, true);
        let mut mask = BitVec::from_elem(len, false);

        for &t in iter {
            if t < len {
                // Set the _counterpart_ mask index
                mask.set(len - t, true);
            }
        }
        mask.set(0, true);

        GaloisLFSR { state: state, mask: mask, }
    }

    fn shift(&mut self) {
        lfsr::shift(&mut self.state);
    }

    fn feedback(&mut self) {
        for (i, mb) in self.mask.iter().enumerate() {
            let sb = self.state[i];
            self.state.set(i, mb ^ sb);
        }
    }

    pub fn iter(&mut self) -> lfsr::Iter<GaloisLFSR> {
        lfsr::Iter { lfsr: self }
    }
}

impl LFSR for GaloisLFSR {
    fn output(&self) -> bool {
        let len = self.state.len();
        self.state[len - 1]
    }

    fn step(&mut self) {
        let output = self.output();

        self.shift();

        if output {
            self.feedback();
        }
    }
}
impl IntoIterator for GaloisLFSR {
    type Item = bool;
    type IntoIter = lfsr::IntoIter<GaloisLFSR>;

    fn into_iter(self) -> Self::IntoIter {
        lfsr::IntoIter { lfsr: self }
    }
}

impl<'a> IntoIterator for &'a mut GaloisLFSR {
    type Item = bool;
    type IntoIter = lfsr::Iter<'a, GaloisLFSR>;

    fn into_iter(self) -> Self::IntoIter {
        lfsr::Iter { lfsr: self }
    }
}
<commit_msg>Newline nit<commit_after>use bit_vec::BitVec;
use std::slice::Iter;
use lfsr;
use lfsr::LFSR;

#[derive(PartialEq, Debug)]
pub struct GaloisLFSR {
    state: BitVec,
    mask: BitVec,
}

impl GaloisLFSR {
    pub fn from_iter(iter: Iter<usize>) -> Self {
        let &len = iter.clone().max().unwrap();

        let state = BitVec::from_elem(len, true);
        let mut mask = BitVec::from_elem(len, false);

        for &t in iter {
            if t < len {
                // Set the _counterpart_ mask index
                mask.set(len - t, true);
            }
        }
        mask.set(0, true);

        GaloisLFSR { state: state, mask: mask, }
    }

    fn shift(&mut self) {
        lfsr::shift(&mut self.state);
    }

    fn feedback(&mut self) {
        for (i, mb) in self.mask.iter().enumerate() {
            let sb = self.state[i];
            self.state.set(i, mb ^ sb);
        }
    }

    pub fn iter(&mut self) -> lfsr::Iter<GaloisLFSR> {
        lfsr::Iter { lfsr: self }
    }
}

impl LFSR for GaloisLFSR {
    fn output(&self) -> bool {
        let len = self.state.len();
        self.state[len - 1]
    }

    fn step(&mut self) {
        let output = self.output();

        self.shift();

        if output {
            self.feedback();
        }
    }
}

impl IntoIterator for GaloisLFSR {
    type Item = bool;
    type IntoIter = lfsr::IntoIter<GaloisLFSR>;

    fn into_iter(self) -> Self::IntoIter {
        lfsr::IntoIter { lfsr: self }
    }
}

impl<'a> IntoIterator for &'a mut GaloisLFSR {
    type Item = bool;
    type IntoIter = lfsr::Iter<'a, GaloisLFSR>;

    fn into_iter(self) -> Self::IntoIter {
        lfsr::Iter { lfsr: self }
    }
}
<|endoftext|>


<commit_before>use bit_vec::BitVec;
use std::slice::Iter;
use lfsr;

#[derive(PartialEq, Debug)]
pub struct GaloisLFSR {
    state: BitVec,
    mask: BitVec,
}

impl GaloisLFSR {
    pub fn from_iter(iter: Iter<usize>) -> Self {
        let &len = iter.clone().max().unwrap();

        let state = BitVec::from_elem(len, true);
        let mut mask = BitVec::from_elem(len, false);

        for &t in iter {
            if t < len {
                // Set the _counterpart_ mask index
                mask.set(len - t, true);
            }
        }
        mask.set(0, true);

        GaloisLFSR { state: state, mask: mask, }
    }

    fn shift(&mut self) {
        lfsr::shift(&mut self.state);
    }

    fn feedback(&mut self) {
        for (i, mb) in self.mask.iter().enumerate() {
            let sb = self.state[i];
            self.state.set(i, mb ^ sb);
        }
    }

    pub fn iter(&mut self) -> lfsr::Iter<GaloisLFSR> {
        lfsr::Iter { lfsr: self }
    }
}

impl lfsr::LFSR for GaloisLFSR {
    fn output(&self) -> bool {
        let len = self.state.len();
        self.state[len - 1]
    }

    fn step(&mut self) {
        let output = self.output();

        self.shift();

        if output {
            self.feedback();
        }
    }
}

impl IntoIterator for GaloisLFSR {
    type Item = bool;
    type IntoIter = lfsr::IntoIter<GaloisLFSR>;

    fn into_iter(self) -> Self::IntoIter {
        lfsr::IntoIter { lfsr: self }
    }
}

impl<'a> IntoIterator for &'a mut GaloisLFSR {
    type Item = bool;
    type IntoIter = lfsr::Iter<'a, GaloisLFSR>;

    fn into_iter(self) -> Self::IntoIter {
        lfsr::Iter { lfsr: self }
    }
}
<commit_msg>Add FromIterator impl for GaloisLFSR<commit_after>use bit_vec::BitVec;
use std::slice::Iter;
use std::iter::FromIterator;
use lfsr;

#[derive(PartialEq, Debug)]
pub struct GaloisLFSR {
    state: BitVec,
    mask: BitVec,
}

impl GaloisLFSR {
    fn shift(&mut self) {
        lfsr::shift(&mut self.state);
    }

    fn feedback(&mut self) {
        for (i, mb) in self.mask.iter().enumerate() {
            let sb = self.state[i];
            self.state.set(i, mb ^ sb);
        }
    }

    pub fn iter(&mut self) -> lfsr::Iter<GaloisLFSR> {
        lfsr::Iter { lfsr: self }
    }
}

impl lfsr::LFSR for GaloisLFSR {
    fn output(&self) -> bool {
        let len = self.state.len();
        self.state[len - 1]
    }

    fn step(&mut self) {
        let output = self.output();

        self.shift();

        if output {
            self.feedback();
        }
    }
}

impl IntoIterator for GaloisLFSR {
    type Item = bool;
    type IntoIter = lfsr::IntoIter<GaloisLFSR>;

    fn into_iter(self) -> Self::IntoIter {
        lfsr::IntoIter { lfsr: self }
    }
}

impl<'a> IntoIterator for &'a mut GaloisLFSR {
    type Item = bool;
    type IntoIter = lfsr::Iter<'a, GaloisLFSR>;

    fn into_iter(self) -> Self::IntoIter {
        lfsr::Iter { lfsr: self }
    }
}

impl FromIterator<usize> for GaloisLFSR {
    fn from_iter<T>(source: T) -> Self where T: IntoIterator<Item=usize> {
        let mut taps: Vec<usize> = source.into_iter().map(|t| t).collect();
        taps.sort();
        taps.dedup();

        let &len = taps.iter().max().unwrap();

        let state = BitVec::from_elem(len, true);
        let mut mask = BitVec::from_elem(len, false);

        for &t in &taps {
            if t < len {
                // Set the _counterpart_ mask index
                mask.set(len - t, true);
            }
        }
        mask.set(0, true);

        GaloisLFSR { state: state, mask: mask, }
    }
}
<|endoftext|>


<commit_before>
use std::fmt;
use std::io::Read;  // Trait providing read_to_string()
use std::env;

use url;
use hyper;
use serde;
use serde_json;

// use Groups;

use ::errors::*;

pub const API_VERSION: u16 = 3;

#[derive(Default, Clone, Copy, Debug)]
pub struct Pagination {
    pub page: u16,
    pub per_page: u16,
}

pub struct GitLab {
    url: url::Url,
    private_token: String,
    pagination: Option<Pagination>,
    client: hyper::Client,
}

// Explicitly implement Debug trait for GitLab so we can hide the token.
impl fmt::Debug for GitLab {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,
               "GitLab {{ scheme: {}, domain: {}, port: {}, private_token: XXXXXXXXXXXXXXXXXXXX, \
                pagination: {:?} }}",
               self.url.scheme(),
               self.url.domain().unwrap_or("bad hostname provided"),
               self.url
                   .port()
                   .map(|port_u16| port_u16.to_string())
                   .unwrap_or("no port provided".to_string()),
               self.pagination)
    }
}

fn validate_url(scheme: &str, domain: &str, port: u16) -> Result<url::Url> {

    match domain.find('.') {
        None => {
            // pass
        }
        Some(index) => {
            if index == 0 {
                bail!(format!("invalid domain: '{}' cannot start with a dot", domain));
            }
        }
    };

    if domain.ends_with('.') {
        bail!(format!("invalid domain: '{}' cannot end with a dot", domain));
    }

    let url_string = format!("{}://{}/api/v{}/", scheme, domain, API_VERSION);
    let mut url = url::Url::parse(&url_string)
        .chain_err(|| format!("failure to parse URL '{}'", url_string))?;
    url.set_port(Some(port)).expect("bad port provided");

    {
        let url_host = url.host_str();
        if url_host.is_none() {
            bail!("failure to get URL's hostname");
        }
        if url_host.unwrap() != domain {
            bail!(format!("invalid hostname '{}'", domain));
        }
    }

    Ok(url)
}

impl GitLab {
    pub fn _new(scheme: &str, domain: &str, port: u16, private_token: &str) -> Result<GitLab> {
        if private_token.len() != 20 {
            bail!(format!("private token should be a 20 characters string (not {})",
                          private_token.len()));
        }

        let url: url::Url = validate_url(scheme, domain, port).chain_err(|| "invalid URL")?;

        Ok(GitLab {
            url: url,
            private_token: private_token.to_string(),
            pagination: None,
            client: match env::var("HTTP_PROXY") {
                Ok(proxy) => {
                    let proxy: Vec<&str> = proxy.trim_left_matches("http://").split(':').collect();
                    let hostname = proxy[0].to_string();
                    let port = proxy[1].parse()
                        .chain_err(|| format!("failure to set port to {}", port))?;

                    hyper::Client::with_http_proxy(hostname, port)
                }
                Err(_) => hyper::Client::new(),
            },
        })
    }

    pub fn new_insecure(domain: &str, private_token: &str) -> Result<GitLab> {
        warn!("Using insecure http:// protocol: Token will be sent in clear!");
        GitLab::_new("http", domain, 80, private_token)
    }

    pub fn new(domain: &str, private_token: &str) -> Result<GitLab> {
        GitLab::_new("https", domain, 443, private_token)
    }

    pub fn port(mut self, port: u16) -> Self {
        self.url.set_port(Some(port)).unwrap();
        self
    }

    pub fn scheme(mut self, scheme: &str) -> Self {
        self.url.set_scheme(scheme).unwrap();
        self
    }

    /// Build a URL used to access GitLab instance, including some parameters.
    ///
    /// # Examples
    ///
    /// Example from GitLab: https://docs.gitlab.com/ce/api/#basic-usage
    ///
    /// ```
    /// use gitlab_api::GitLab;
    ///
    /// let expected_url = "https://gitlab.example.com\
    ///                     /api/v3/groups?order_by=path&private_token=XXXXXXXXXXXXXXXXXXXX";
    ///
    /// let gl = GitLab::new("gitlab.example.com", "XXXXXXXXXXXXXXXXXXXX").unwrap();
    ///
    /// assert_eq!(gl.build_url("groups?order_by=path").unwrap(), expected_url);
    /// ```
    pub fn build_url(&self, query: &str) -> Result<String> {
        let mut new_url = self.url
            .clone()
            .join(query)
            .chain_err(|| {
                format!("Failure to join query '{}' to url {}",
                        query,
                        self.url.as_str())
            })?;
        new_url.query_pairs_mut().append_pair("private_token", &self.private_token);
        self.pagination.as_ref().map(|pagination| {
            new_url.query_pairs_mut().append_pair("page", &pagination.page.to_string());
            new_url.query_pairs_mut().append_pair("per_page", &pagination.per_page.to_string());
        });

        Ok(new_url.into_string())
    }

    // pub fn attempt_connection(&self) -> Result<hyper::client::Response, hyper::Error> {
    //     let url = self.build_url("version");
    //     // Close connections after each GET.
    //     self.client.get(&url).header(hyper::header::Connection::close()).send()
    // }

    /// Set pagination information
    ///
    /// # Examples
    ///
    /// ```
    /// use gitlab_api::{GitLab, Pagination};
    ///
    /// let expected_url = "https://gitlab.example.com\
    ///                     /api/v3/groups?order_by=path&\
    ///                     private_token=XXXXXXXXXXXXXXXXXXXX&page=2&per_page=5";
    ///
    /// let mut gl = GitLab::new("gitlab.example.com", "XXXXXXXXXXXXXXXXXXXX").unwrap();
    /// gl.set_pagination(Pagination {page: 2, per_page: 5});
    /// assert_eq!(gl.build_url("groups?order_by=path").unwrap(), expected_url);
    /// ```
    pub fn set_pagination(&mut self, pagination: Pagination) {
        self.pagination = Some(pagination);
    }

    pub fn get<T>(&self, query: &str) -> Result<T>
        where T: serde::Deserialize
    {
        let url = self.build_url(query)
            .chain_err(|| format!("failure to build url for query '{}'", query))?;
        info!("url: {:?}", url);

        // Close connections after each GET.
        let mut res: hyper::client::Response = self.client
            .get(&url)
            .header(hyper::header::Connection::close())
            .send()
            .chain_err(|| format!("cannot send request '{}' to {:?}", query, self))?;
        info!("res.status: {:?}", res.status);
        debug!("res.headers: {:?}", res.headers);
        debug!("res.url: {:?}", res.url);

        let mut body = String::new();
        res.read_to_string(&mut body).chain_err(|| "cannot read response body")?;
        debug!("body:\n{:?}", body);

        // assert_eq!(res.status, hyper::status::StatusCode::Ok);
        if res.status != hyper::status::StatusCode::Ok {
            bail!(format!("status code '{}', not '200 OK'", res.status));
        }

        serde_json::from_str(body.as_str())
            .chain_err(|| format!("cannot build Rust struct from JSON data: {}", body))
    }

    pub fn version(&self) -> Result<::Version> {
        self.get("version").chain_err(|| "cannot query 'version'")
    }

    pub fn groups(&self) -> ::groups::GroupsLister {
        ::groups::GroupsLister::new(self)
    }

    pub fn projects(&self) -> ::projects::ProjectsLister {
        ::projects::ProjectsLister::new(self)
    }

    pub fn issues(&self) -> ::issues::IssuesLister {
        ::issues::IssuesLister::new(self)
    }

    pub fn merge_requests(&self, project_id: i64) -> ::merge_requests::MergeRequestsLister {
        ::merge_requests::MergeRequestsLister::new(self, project_id)
    }

    // pub fn groups(&mut self, listing: ::groups::Listing) -> Result<Groups, serde_json::Error> {
    //     let query = listing.build_query();
    //     // self.get(&query)
    //     unimplemented!();
    // }

    // pub fn owned_groups(&mut self) -> Result<Groups, serde_json::Error> {
    //     let query = ::groups::owned_groups::Listing::new().build_query();
    //     info!("query: {:?}", query);
    //     self.get(&query)
    // }
}

#[cfg(test)]
mod tests {
    use std::fmt;
    use gitlab::*;
    use errors::*;

    fn verify_ok<T>(result: &Result<T>) {
        if let &Err(ref e) = result {
            println!("error: {}", e);

            for e in e.iter().skip(1) {
                println!("caused by: {}", e);
            }

            // The backtrace is not always generated. Try to run this example
            // with `RUST_BACKTRACE=1`.
            if let Some(backtrace) = e.backtrace() {
                println!("backtrace: {:?}", backtrace);
            }
        }
        assert!(result.is_ok());
    }

    fn verify_err<T>(result: &Result<T>)
        where T: fmt::Debug
    {
        match result {
            &Err(_) => {
                // pass
            }
            &Ok(ref t) => {
                panic!(format!("Expected an Err(), got an Ok(t), with t: {:?}", t));
            }
        }
    }

    #[test]
    fn new_valid() {
        let gl = GitLab::new("gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_ok(&gl);

        let gl = GitLab::new_insecure("gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_ok(&gl);

        let gl = GitLab::new("localhost", "XXXXXXXXXXXXXXXXXXXX");
        verify_ok(&gl);

        let gl = GitLab::new_insecure("localhost", "XXXXXXXXXXXXXXXXXXXX");
        verify_ok(&gl);
    }

    #[test]
    fn new_invalid_url_1() {
        let gl = GitLab::new("", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_2() {
        let gl = GitLab::new("gitla/b.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("gitla/b.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_3() {
        let gl = GitLab::new("/gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("/gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_4() {
        let gl = GitLab::new("http:/gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("http:/gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_5() {
        let gl = GitLab::new("http:///gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("http:///gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_6() {
        let gl = GitLab::new(".gitlab", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure(".gitlab", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_7() {
        let gl = GitLab::new(".gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure(".gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_8() {
        let gl = GitLab::new("gitlab.", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("gitlab.", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_10() {
        let gl = GitLab::new("gitlab.com.", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("gitlab.com.", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_token() {
        let gl = GitLab::new("gitlab.com", "");
        assert!(gl.is_err());

        let gl = GitLab::new("gitlab.com", "X");
        assert!(gl.is_err());

        let gl = GitLab::new("gitlab.com", "XXXXXXXXXXXXXXXXXXX");
        assert!(gl.is_err());

        let gl = GitLab::new("gitlab.com", "XXXXXXXXXXXXXXXXXXXXX");
        assert!(gl.is_err());
    }
}
<commit_msg>Add unit test for Debug trait<commit_after>
use std::fmt;
use std::io::Read;  // Trait providing read_to_string()
use std::env;

use url;
use hyper;
use serde;
use serde_json;

// use Groups;

use ::errors::*;

pub const API_VERSION: u16 = 3;

#[derive(Default, Clone, Copy, Debug)]
pub struct Pagination {
    pub page: u16,
    pub per_page: u16,
}

pub struct GitLab {
    url: url::Url,
    private_token: String,
    pagination: Option<Pagination>,
    client: hyper::Client,
}

// Explicitly implement Debug trait for GitLab so we can hide the token.
impl fmt::Debug for GitLab {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f,
               "GitLab {{ scheme: {}, domain: {}, port: {}, private_token: XXXXXXXXXXXXXXXXXXXX, \
                pagination: {:?} }}",
               self.url.scheme(),
               self.url.domain().unwrap_or("bad hostname provided"),
               self.url
                   .port()
                   .map(|port_u16| port_u16.to_string())
                   .unwrap_or("no port provided".to_string()),
               self.pagination)
    }
}

fn validate_url(scheme: &str, domain: &str, port: u16) -> Result<url::Url> {

    match domain.find('.') {
        None => {
            // pass
        }
        Some(index) => {
            if index == 0 {
                bail!(format!("invalid domain: '{}' cannot start with a dot", domain));
            }
        }
    };

    if domain.ends_with('.') {
        bail!(format!("invalid domain: '{}' cannot end with a dot", domain));
    }

    let url_string = format!("{}://{}/api/v{}/", scheme, domain, API_VERSION);
    let mut url = url::Url::parse(&url_string)
        .chain_err(|| format!("failure to parse URL '{}'", url_string))?;
    url.set_port(Some(port)).expect("bad port provided");

    {
        let url_host = url.host_str();
        if url_host.is_none() {
            bail!("failure to get URL's hostname");
        }
        if url_host.unwrap() != domain {
            bail!(format!("invalid hostname '{}'", domain));
        }
    }

    Ok(url)
}

impl GitLab {
    pub fn _new(scheme: &str, domain: &str, port: u16, private_token: &str) -> Result<GitLab> {
        if private_token.len() != 20 {
            bail!(format!("private token should be a 20 characters string (not {})",
                          private_token.len()));
        }

        let url: url::Url = validate_url(scheme, domain, port).chain_err(|| "invalid URL")?;

        Ok(GitLab {
            url: url,
            private_token: private_token.to_string(),
            pagination: None,
            client: match env::var("HTTP_PROXY") {
                Ok(proxy) => {
                    let proxy: Vec<&str> = proxy.trim_left_matches("http://").split(':').collect();
                    let hostname = proxy[0].to_string();
                    let port = proxy[1].parse()
                        .chain_err(|| format!("failure to set port to {}", port))?;

                    hyper::Client::with_http_proxy(hostname, port)
                }
                Err(_) => hyper::Client::new(),
            },
        })
    }

    pub fn new_insecure(domain: &str, private_token: &str) -> Result<GitLab> {
        warn!("Using insecure http:// protocol: Token will be sent in clear!");
        GitLab::_new("http", domain, 80, private_token)
    }

    pub fn new(domain: &str, private_token: &str) -> Result<GitLab> {
        GitLab::_new("https", domain, 443, private_token)
    }

    pub fn port(mut self, port: u16) -> Self {
        self.url.set_port(Some(port)).unwrap();
        self
    }

    pub fn scheme(mut self, scheme: &str) -> Self {
        self.url.set_scheme(scheme).unwrap();
        self
    }

    /// Build a URL used to access GitLab instance, including some parameters.
    ///
    /// # Examples
    ///
    /// Example from GitLab: https://docs.gitlab.com/ce/api/#basic-usage
    ///
    /// ```
    /// use gitlab_api::GitLab;
    ///
    /// let expected_url = "https://gitlab.example.com\
    ///                     /api/v3/groups?order_by=path&private_token=XXXXXXXXXXXXXXXXXXXX";
    ///
    /// let gl = GitLab::new("gitlab.example.com", "XXXXXXXXXXXXXXXXXXXX").unwrap();
    ///
    /// assert_eq!(gl.build_url("groups?order_by=path").unwrap(), expected_url);
    /// ```
    pub fn build_url(&self, query: &str) -> Result<String> {
        let mut new_url = self.url
            .clone()
            .join(query)
            .chain_err(|| {
                format!("Failure to join query '{}' to url {}",
                        query,
                        self.url.as_str())
            })?;
        new_url.query_pairs_mut().append_pair("private_token", &self.private_token);
        self.pagination.as_ref().map(|pagination| {
            new_url.query_pairs_mut().append_pair("page", &pagination.page.to_string());
            new_url.query_pairs_mut().append_pair("per_page", &pagination.per_page.to_string());
        });

        Ok(new_url.into_string())
    }

    // pub fn attempt_connection(&self) -> Result<hyper::client::Response, hyper::Error> {
    //     let url = self.build_url("version");
    //     // Close connections after each GET.
    //     self.client.get(&url).header(hyper::header::Connection::close()).send()
    // }

    /// Set pagination information
    ///
    /// # Examples
    ///
    /// ```
    /// use gitlab_api::{GitLab, Pagination};
    ///
    /// let expected_url = "https://gitlab.example.com\
    ///                     /api/v3/groups?order_by=path&\
    ///                     private_token=XXXXXXXXXXXXXXXXXXXX&page=2&per_page=5";
    ///
    /// let mut gl = GitLab::new("gitlab.example.com", "XXXXXXXXXXXXXXXXXXXX").unwrap();
    /// gl.set_pagination(Pagination {page: 2, per_page: 5});
    /// assert_eq!(gl.build_url("groups?order_by=path").unwrap(), expected_url);
    /// ```
    pub fn set_pagination(&mut self, pagination: Pagination) {
        self.pagination = Some(pagination);
    }

    pub fn get<T>(&self, query: &str) -> Result<T>
        where T: serde::Deserialize
    {
        let url = self.build_url(query)
            .chain_err(|| format!("failure to build url for query '{}'", query))?;
        info!("url: {:?}", url);

        // Close connections after each GET.
        let mut res: hyper::client::Response = self.client
            .get(&url)
            .header(hyper::header::Connection::close())
            .send()
            .chain_err(|| format!("cannot send request '{}' to {:?}", query, self))?;
        info!("res.status: {:?}", res.status);
        debug!("res.headers: {:?}", res.headers);
        debug!("res.url: {:?}", res.url);

        let mut body = String::new();
        res.read_to_string(&mut body).chain_err(|| "cannot read response body")?;
        debug!("body:\n{:?}", body);

        // assert_eq!(res.status, hyper::status::StatusCode::Ok);
        if res.status != hyper::status::StatusCode::Ok {
            bail!(format!("status code '{}', not '200 OK'", res.status));
        }

        serde_json::from_str(body.as_str())
            .chain_err(|| format!("cannot build Rust struct from JSON data: {}", body))
    }

    pub fn version(&self) -> Result<::Version> {
        self.get("version").chain_err(|| "cannot query 'version'")
    }

    pub fn groups(&self) -> ::groups::GroupsLister {
        ::groups::GroupsLister::new(self)
    }

    pub fn projects(&self) -> ::projects::ProjectsLister {
        ::projects::ProjectsLister::new(self)
    }

    pub fn issues(&self) -> ::issues::IssuesLister {
        ::issues::IssuesLister::new(self)
    }

    pub fn merge_requests(&self, project_id: i64) -> ::merge_requests::MergeRequestsLister {
        ::merge_requests::MergeRequestsLister::new(self, project_id)
    }

    // pub fn groups(&mut self, listing: ::groups::Listing) -> Result<Groups, serde_json::Error> {
    //     let query = listing.build_query();
    //     // self.get(&query)
    //     unimplemented!();
    // }

    // pub fn owned_groups(&mut self) -> Result<Groups, serde_json::Error> {
    //     let query = ::groups::owned_groups::Listing::new().build_query();
    //     info!("query: {:?}", query);
    //     self.get(&query)
    // }
}

#[cfg(test)]
mod tests {
    use std::fmt;
    use gitlab::*;
    use errors::*;

    fn verify_ok<T>(result: &Result<T>) {
        if let &Err(ref e) = result {
            println!("error: {}", e);

            for e in e.iter().skip(1) {
                println!("caused by: {}", e);
            }

            // The backtrace is not always generated. Try to run this example
            // with `RUST_BACKTRACE=1`.
            if let Some(backtrace) = e.backtrace() {
                println!("backtrace: {:?}", backtrace);
            }
        }
        assert!(result.is_ok());
    }

    fn verify_err<T>(result: &Result<T>)
        where T: fmt::Debug
    {
        match result {
            &Err(_) => {
                // pass
            }
            &Ok(ref t) => {
                panic!(format!("Expected an Err(), got an Ok(t), with t: {:?}", t));
            }
        }
    }

    #[test]
    fn impl_debug_for_gitlab() {
        let gl = GitLab::new("gitlab.com", "XXXXXXXXXXXXXXXXXXXX").unwrap();

        let debug = format!("{:?}", gl);
        assert_eq!("GitLab { scheme: https, domain: gitlab.com, port: no port provided, private_token: XXXXXXXXXXXXXXXXXXXX, pagination: None }", debug);

        let gl = gl.scheme("http").port(80);
        let debug = format!("{:?}", gl);
        assert_eq!("GitLab { scheme: http, domain: gitlab.com, port: no port provided, private_token: XXXXXXXXXXXXXXXXXXXX, pagination: None }", debug);

        let mut gl = gl.port(81);
        let debug = format!("{:?}", gl);
        assert_eq!("GitLab { scheme: http, domain: gitlab.com, port: 81, private_token: XXXXXXXXXXXXXXXXXXXX, pagination: None }", debug);

        gl.set_pagination(Pagination {page: 2, per_page: 5});
        let debug = format!("{:?}", gl);
        assert_eq!("GitLab { scheme: http, domain: gitlab.com, port: 81, private_token: XXXXXXXXXXXXXXXXXXXX, pagination: Some(Pagination { page: 2, per_page: 5 }) }", debug);
    }

    #[test]
    fn new_valid() {
        let gl = GitLab::new("gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_ok(&gl);

        let gl = GitLab::new_insecure("gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_ok(&gl);

        let gl = GitLab::new("localhost", "XXXXXXXXXXXXXXXXXXXX");
        verify_ok(&gl);

        let gl = GitLab::new_insecure("localhost", "XXXXXXXXXXXXXXXXXXXX");
        verify_ok(&gl);
    }

    #[test]
    fn new_invalid_url_1() {
        let gl = GitLab::new("", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_2() {
        let gl = GitLab::new("gitla/b.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("gitla/b.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_3() {
        let gl = GitLab::new("/gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("/gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_4() {
        let gl = GitLab::new("http:/gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("http:/gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_5() {
        let gl = GitLab::new("http:///gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("http:///gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_6() {
        let gl = GitLab::new(".gitlab", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure(".gitlab", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_7() {
        let gl = GitLab::new(".gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure(".gitlab.com", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_8() {
        let gl = GitLab::new("gitlab.", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("gitlab.", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_url_10() {
        let gl = GitLab::new("gitlab.com.", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);

        let gl = GitLab::new_insecure("gitlab.com.", "XXXXXXXXXXXXXXXXXXXX");
        verify_err(&gl);
    }

    #[test]
    fn new_invalid_token() {
        let gl = GitLab::new("gitlab.com", "");
        assert!(gl.is_err());

        let gl = GitLab::new("gitlab.com", "X");
        assert!(gl.is_err());

        let gl = GitLab::new("gitlab.com", "XXXXXXXXXXXXXXXXXXX");
        assert!(gl.is_err());

        let gl = GitLab::new("gitlab.com", "XXXXXXXXXXXXXXXXXXXXX");
        assert!(gl.is_err());
    }
}
<|endoftext|>


<commit_before>use core::{mem, slice, str};

use raw;

#[cfg(feature = "no-panic")]
use no_panic::no_panic;

/// Safe API for formatting floating point numbers to text.
///
/// ## Example
///
/// ```edition2018
/// let mut buffer = ryu::Buffer::new();
/// let printed = buffer.format(1.234);
/// assert_eq!(printed, "1.234");
/// ```
#[derive(Copy, Clone)]
pub struct Buffer {
    bytes: [u8; 24],
}

impl Buffer {
    /// This is a cheap operation; you don't need to worry about reusing buffers
    /// for efficiency.
    #[inline]
    #[cfg_attr(feature = "no-panic", no_panic)]
    pub fn new() -> Self {
        Buffer {
            bytes: unsafe { mem::uninitialized() },
        }
    }

    /// Print a floating point number into this buffer and return a reference to
    /// its string representation within the buffer.
    ///
    /// # Special cases
    ///
    /// This function **does not** check for NaN or infinity. If the input
    /// number is not a finite float, the printed representation will be some
    /// correctly formatted but unspecified numerical value.
    ///
    /// Please check [`is_finite`] yourself before calling this function, or
    /// check [`is_nan`] and [`is_infinite`] and handle those cases yourself.
    ///
    /// [`is_finite`]: https://doc.rust-lang.org/std/primitive.f64.html#method.is_finite
    /// [`is_nan`]: https://doc.rust-lang.org/std/primitive.f64.html#method.is_nan
    /// [`is_infinite`]: https://doc.rust-lang.org/std/primitive.f64.html#method.is_infinite
    #[inline]
    #[cfg_attr(feature = "no-panic", no_panic)]
    pub fn format<F: Float>(&mut self, f: F) -> &str {
        unsafe {
            let n = f.write_to_ryu_buffer(&mut self.bytes[0]);
            debug_assert!(n <= self.bytes.len());
            let slice = slice::from_raw_parts(&self.bytes[0], n);
            str::from_utf8_unchecked(slice)
        }
    }
}

impl Default for Buffer {
    #[inline]
    #[cfg_attr(feature = "no-panic", no_panic)]
    fn default() -> Self {
        Buffer::new()
    }
}

/// A floating point number, f32 or f64, that can be written into a
/// [`ryu::Buffer`][Buffer].
///
/// This trait is sealed and cannot be implemented for types outside of the
/// `ryu` crate.
pub trait Float: Sealed {
    // Not public API.
    #[doc(hidden)]
    unsafe fn write_to_ryu_buffer(self, result: *mut u8) -> usize;
}

impl Float for f32 {
    #[inline]
    #[cfg_attr(feature = "no-panic", no_panic)]
    unsafe fn write_to_ryu_buffer(self, result: *mut u8) -> usize {
        raw::format32(self, result)
    }
}

impl Float for f64 {
    #[inline]
    #[cfg_attr(feature = "no-panic", no_panic)]
    unsafe fn write_to_ryu_buffer(self, result: *mut u8) -> usize {
        raw::format64(self, result)
    }
}

pub trait Sealed {}
impl Sealed for f32 {}
impl Sealed for f64 {}
<commit_msg>Hide Float methods in trait impl<commit_after>use core::{mem, slice, str};

use raw;

#[cfg(feature = "no-panic")]
use no_panic::no_panic;

/// Safe API for formatting floating point numbers to text.
///
/// ## Example
///
/// ```edition2018
/// let mut buffer = ryu::Buffer::new();
/// let printed = buffer.format(1.234);
/// assert_eq!(printed, "1.234");
/// ```
#[derive(Copy, Clone)]
pub struct Buffer {
    bytes: [u8; 24],
}

impl Buffer {
    /// This is a cheap operation; you don't need to worry about reusing buffers
    /// for efficiency.
    #[inline]
    #[cfg_attr(feature = "no-panic", no_panic)]
    pub fn new() -> Self {
        Buffer {
            bytes: unsafe { mem::uninitialized() },
        }
    }

    /// Print a floating point number into this buffer and return a reference to
    /// its string representation within the buffer.
    ///
    /// # Special cases
    ///
    /// This function **does not** check for NaN or infinity. If the input
    /// number is not a finite float, the printed representation will be some
    /// correctly formatted but unspecified numerical value.
    ///
    /// Please check [`is_finite`] yourself before calling this function, or
    /// check [`is_nan`] and [`is_infinite`] and handle those cases yourself.
    ///
    /// [`is_finite`]: https://doc.rust-lang.org/std/primitive.f64.html#method.is_finite
    /// [`is_nan`]: https://doc.rust-lang.org/std/primitive.f64.html#method.is_nan
    /// [`is_infinite`]: https://doc.rust-lang.org/std/primitive.f64.html#method.is_infinite
    #[inline]
    #[cfg_attr(feature = "no-panic", no_panic)]
    pub fn format<F: Float>(&mut self, f: F) -> &str {
        unsafe {
            let n = f.write_to_ryu_buffer(&mut self.bytes[0]);
            debug_assert!(n <= self.bytes.len());
            let slice = slice::from_raw_parts(&self.bytes[0], n);
            str::from_utf8_unchecked(slice)
        }
    }
}

impl Default for Buffer {
    #[inline]
    #[cfg_attr(feature = "no-panic", no_panic)]
    fn default() -> Self {
        Buffer::new()
    }
}

/// A floating point number, f32 or f64, that can be written into a
/// [`ryu::Buffer`][Buffer].
///
/// This trait is sealed and cannot be implemented for types outside of the
/// `ryu` crate.
pub trait Float: Sealed {
    // Not public API.
    #[doc(hidden)]
    unsafe fn write_to_ryu_buffer(self, result: *mut u8) -> usize;
}

impl Float for f32 {
    #[doc(hidden)]
    #[inline]
    #[cfg_attr(feature = "no-panic", no_panic)]
    unsafe fn write_to_ryu_buffer(self, result: *mut u8) -> usize {
        raw::format32(self, result)
    }
}

impl Float for f64 {
    #[doc(hidden)]
    #[inline]
    #[cfg_attr(feature = "no-panic", no_panic)]
    unsafe fn write_to_ryu_buffer(self, result: *mut u8) -> usize {
        raw::format64(self, result)
    }
}

pub trait Sealed {}
impl Sealed for f32 {}
impl Sealed for f64 {}
<|endoftext|>


<commit_before>#![cfg(feature = "parsing")]

use nom::IResult;

macro_rules! punct {
    ($i:expr, $punct:expr) => {
        $crate::helper::punct($i, $punct)
    };
}

pub fn punct<'a>(input: &'a str, token: &'static str) -> IResult<&'a str, &'a str> {
    let mut chars = input.char_indices();
    while let Some((i, ch)) = chars.next() {
        if !ch.is_whitespace() {
            return if input[i..].starts_with(token) {
                let end = i + token.len();
                IResult::Done(&input[end..], &input[i..end])
            } else {
                IResult::Error
            };
        }
    }
    IResult::Error
}

macro_rules! option {
    ($i:expr, $submac:ident!( $($args:tt)* )) => {
        match $submac!($i, $($args)*) {
            ::nom::IResult::Done(i, o) => ::nom::IResult::Done(i, Some(o)),
            ::nom::IResult::Error => ::nom::IResult::Done($i, None),
        }
    };

    ($i:expr, $f:expr) => {
        option!($i, call!($f));
    };
}

macro_rules! opt_vec {
    ($i:expr, $submac:ident!( $($args:tt)* )) => {
        match $submac!($i, $($args)*) {
            ::nom::IResult::Done(i, o) => ::nom::IResult::Done(i, o),
            ::nom::IResult::Error => ::nom::IResult::Done($i, Vec::new()),
        }
    };
}

macro_rules! epsilon {
    ($i:expr,) => {
        call!($i, {
            fn epsilon<T>(input: T) -> ::nom::IResult<T, ()> {
                ::nom::IResult::Done(input, ())
            }
            epsilon
        })
    };
}
<commit_msg>Simplify epsilon macro<commit_after>#![cfg(feature = "parsing")]

use nom::IResult;

macro_rules! punct {
    ($i:expr, $punct:expr) => {
        $crate::helper::punct($i, $punct)
    };
}

pub fn punct<'a>(input: &'a str, token: &'static str) -> IResult<&'a str, &'a str> {
    let mut chars = input.char_indices();
    while let Some((i, ch)) = chars.next() {
        if !ch.is_whitespace() {
            return if input[i..].starts_with(token) {
                let end = i + token.len();
                IResult::Done(&input[end..], &input[i..end])
            } else {
                IResult::Error
            };
        }
    }
    IResult::Error
}

macro_rules! option {
    ($i:expr, $submac:ident!( $($args:tt)* )) => {
        match $submac!($i, $($args)*) {
            ::nom::IResult::Done(i, o) => ::nom::IResult::Done(i, Some(o)),
            ::nom::IResult::Error => ::nom::IResult::Done($i, None),
        }
    };

    ($i:expr, $f:expr) => {
        option!($i, call!($f));
    };
}

macro_rules! opt_vec {
    ($i:expr, $submac:ident!( $($args:tt)* )) => {
        match $submac!($i, $($args)*) {
            ::nom::IResult::Done(i, o) => ::nom::IResult::Done(i, o),
            ::nom::IResult::Error => ::nom::IResult::Done($i, Vec::new()),
        }
    };
}

macro_rules! epsilon {
    ($i:expr,) => {
        value!($i, ())
    };
}
<|endoftext|>


<commit_before>
use super::DocBuilder;
use super::crates::crates_from_path;
use DocBuilderError;
use utils::{get_package, source_path, copy_dir, copy_doc_dir, update_sources};
use db::{connect_db, add_package_into_database, add_build_into_database, add_path_into_database};
use cargo::core::Package;
use std::process::{Command, Output};
use std::path::PathBuf;
use postgres::Connection;

use regex::Regex;

type CommandResult = Result<String, String>;

#[derive(Debug)]
pub struct ChrootBuilderResult {
    pub output: String,
    pub build_success: bool,
    pub have_doc: bool,
    pub have_examples: bool,
    pub rustc_version: String,
    pub cratesfyi_version: String,
}

impl DocBuilder {
    /// Builds every package documentation in chroot environment
    pub fn build_world(&mut self) -> Result<(), DocBuilderError> {
        try!(update_sources());

        let mut count = 0;

        crates(self.options.crates_io_index_path.clone(), |name, version| {
            match self.build_package(name, version) {
                Ok(status) => {
                    count += 1;
                    if status && count % 10 == 0 {
                        let _ = self.save_cache();
                    }
                }
                Err(err) => warn!("Failed to build package {}-{}: {}", name, version, err),
            }
            self.cache.insert(format!("{}-{}", name, version));
        })
    }

    /// Builds package documentation in chroot environment and adds into cratesfyi database
    pub fn build_package(&mut self, name: &str, version: &str) -> Result<bool, DocBuilderError> {
        info!("Building package {}-{}", name, version);

        // Database connection
        let conn = try!(connect_db());

        // Skip crates according to options
        if (self.options.skip_if_log_exists &&
            self.cache.contains(&format!("{}-{}", name, version)[..])) ||
           (self.options.skip_if_exists &&
            self.db_cache.contains(&format!("{}-{}", name, version)[..])) {
            info!("Skipping");
            return Ok(false);
        }

        // get_package (and cargo) is using semver, add '=' in front of version.
        let pkg = try!(get_package(name, Some(&format!("={}", version)[..])));
        let res = self.build_package_in_chroot(&pkg);

        // copy sources and documentation
        try!(self.add_sources_into_database(&conn, &pkg));
        if res.have_doc {
            try!(self.copy_documentation(&pkg, &res.rustc_version));
            try!(self.add_documentation_into_database(&conn, &pkg));
        }

        let release_id = try!(add_package_into_database(&conn, &pkg, &res));
        try!(add_build_into_database(&conn, &release_id, &res));

        // remove documentation, source and build directory after we are done
        try!(self.clean(&pkg));

        // add package into build cache
        self.cache.insert(format!("{}-{}", name, version));

        Ok(res.build_success)
    }

    /// Builds documentation of a package with cratesfyi in chroot environment
    fn build_package_in_chroot(&self, package: &Package) -> ChrootBuilderResult {
        debug!("Building package in chroot");
        let (rustc_version, cratesfyi_version) = self.get_versions();
        let cmd = format!("cratesfyi doc {} ={}",
                          package.manifest().name(),
                          package.manifest().version());
        match self.chroot_command(cmd) {
            Ok(o) => {
                ChrootBuilderResult {
                    output: o,
                    build_success: true,
                    have_doc: self.have_documentation(&package),
                    have_examples: self.have_examples(&package),
                    rustc_version: rustc_version,
                    cratesfyi_version: cratesfyi_version,
                }
            }
            Err(e) => {
                ChrootBuilderResult {
                    output: e,
                    build_success: false,
                    have_doc: false,
                    have_examples: self.have_examples(&package),
                    rustc_version: rustc_version,
                    cratesfyi_version: cratesfyi_version,
                }
            }
        }
    }

    /// Copies source files of a package into source_path
    #[allow(dead_code)]  // I've been using this function before storing files in database
    fn copy_sources(&self, package: &Package) -> Result<(), DocBuilderError> {
        debug!("Copying sources");
        let destination = PathBuf::from(&self.options.sources_path).join(format!("{}/{}",
                          package.manifest().name(),
                          package.manifest().version()));
        // unwrap is safe here, this function will be always called after get_package
        match copy_dir(source_path(&package).unwrap(), &destination) {
            Ok(_) => Ok(()),
            Err(e) => Err(DocBuilderError::Io(e)),
        }
    }

    /// Copies documentation to destination directory
    fn copy_documentation(&self,
                          package: &Package,
                          rustc_version: &str)
                          -> Result<(), DocBuilderError> {
        debug!("Copying documentation");
        let crate_doc_path = PathBuf::from(&self.options.chroot_path)
                                 .join("home")
                                 .join(&self.options.chroot_user)
                                 .join(canonical_name(&package));
        let destination = PathBuf::from(&self.options.destination).join(format!("{}/{}",
                          package.manifest().name(),
                          package.manifest().version()));
        copy_doc_dir(crate_doc_path,
                     destination,
                     parse_rustc_version(rustc_version).trim())
            .map_err(DocBuilderError::Io)
    }

    /// Removes build directory of a package in chroot
    fn remove_build_dir(&self, package: &Package) -> Result<(), DocBuilderError> {
        let _ = self.chroot_command(format!("rm -rf {}", canonical_name(&package)));
        Ok(())
    }

    /// Remove documentation, build directory and sources directory of a package
    fn clean(&self, package: &Package) -> Result<(), DocBuilderError> {
        debug!("Cleaning package");
        use std::fs::remove_dir_all;
        let documentation_path = PathBuf::from(&self.options.destination).join(format!("{}/{}",
                          package.manifest().name(),
                          package.manifest().version()));
        let source_path = source_path(&package).unwrap();
        // Some crates don't have documentation, so we don't care if removing_dir_all fails
        let _ = self.remove_build_dir(&package);
        let _ = remove_dir_all(documentation_path);
        let _ = remove_dir_all(source_path);
        Ok(())
    }

    /// Runs a command in a chroot environment
    fn chroot_command<T: AsRef<str>>(&self, cmd: T) -> CommandResult {
        command_result(Command::new("sudo")
                           .arg("lxc-attach")
                           .arg("-n")
                           .arg(&self.options.container_name)
                           .arg("--")
                           .arg("su")
                           .arg("-")
                           .arg(&self.options.chroot_user)
                           .arg("-c")
                           .arg(cmd.as_ref())
                           .output()
                           .unwrap())
    }

    /// Checks a package build directory to determine if package have docs
    ///
    /// This function is checking first target in targets to see if documentation exists for a
    /// crate. Package must be successfully built in chroot environment first.
    fn have_documentation(&self, package: &Package) -> bool {
        let crate_doc_path = PathBuf::from(&self.options.chroot_path)
                                 .join("home")
                                 .join(&self.options.chroot_user)
                                 .join(canonical_name(&package))
                                 .join("doc")
                                 .join(package.targets()[0].name().to_string());
        crate_doc_path.exists()
    }

    /// Checks if package have examples
    fn have_examples(&self, package: &Package) -> bool {
        let path = source_path(&package).unwrap().join("examples");
        path.exists() && path.is_dir()
    }

    /// Gets rustc and cratesfyi version from chroot environment
    fn get_versions(&self) -> (String, String) {
        // It is safe to use expect here
        // chroot environment must always have rustc and cratesfyi installed
        (String::from(self.chroot_command("rustc --version")
                          .expect("Failed to get rustc version")
                          .trim()),
         String::from(self.chroot_command("cratesfyi --version")
                          .expect("Failed to get cratesfyi version")
                          .trim()))
    }

    /// Adds sources into database
    fn add_sources_into_database(&self,
                                 conn: &Connection,
                                 package: &Package)
                                 -> Result<(), DocBuilderError> {
        debug!("Adding sources into database");
        let prefix = format!("sources/{}/{}",
                             package.manifest().name(),
                             package.manifest().version());
        add_path_into_database(conn, &prefix, source_path(&package).unwrap())
    }

    /// Adds documentations into database
    fn add_documentation_into_database(&self,
                                       conn: &Connection,
                                       package: &Package)
                                       -> Result<(), DocBuilderError> {
        debug!("Adding documentation into database");
        let prefix = format!("rustdoc/{}/{}",
                             package.manifest().name(),
                             package.manifest().version());
        let crate_doc_path = PathBuf::from(&self.options.destination).join(format!("{}/{}",
                          package.manifest().name(),
                          package.manifest().version()));
        add_path_into_database(conn, &prefix, crate_doc_path)
    }
}

/// Simple function to capture command output
fn command_result(output: Output) -> CommandResult {
    let mut command_out = String::from_utf8_lossy(&output.stdout).into_owned();
    command_out.push_str(&String::from_utf8_lossy(&output.stderr).into_owned()[..]);
    match output.status.success() {
        true => Ok(command_out),
        false => Err(command_out),
    }
}

/// Returns canonical name of a package.
///
/// It's just package-version. All directory structure used in cratesfyi is
/// following this naming scheme.
fn canonical_name(package: &Package) -> String {
    format!("{}-{}",
            package.manifest().name(),
            package.manifest().version())
}

/// Parses rustc commit hash from rustc version string
fn parse_rustc_version<S: AsRef<str>>(version: S) -> String {
    let version_regex = Regex::new(r" ([\w-.]+) \((\w+) (\d+)-(\d+)-(\d+)\)").unwrap();
    let captures = version_regex.captures(version.as_ref()).expect("Failed to parse rustc version");

    format!("{}{}{}-{}-{}",
            captures.at(3).unwrap(),
            captures.at(4).unwrap(),
            captures.at(5).unwrap(),
            captures.at(1).unwrap(),
            captures.at(2).unwrap())
}

/// Runs `func` with the all crates from crates-io.index repository path.
///
/// First argument of func is the name of crate and
/// second argument is the version of crate. Func will be run for every crate.
fn crates<F>(path: PathBuf, mut func: F) -> Result<(), DocBuilderError>
    where F: FnMut(&str, &str) -> ()
{
    crates_from_path(&path, &mut func)
}

#[cfg(test)]
mod test {
    extern crate env_logger;
    use super::parse_rustc_version;
    use std::path::PathBuf;
    use {DocBuilder, DocBuilderOptions};

    #[test]
    #[ignore]
    fn test_build_world() {
        let _ = env_logger::init();
        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
        let mut docbuilder = DocBuilder::new(options);
        // This test is building WHOLE WORLD and may take forever
        assert!(docbuilder.build_world().is_ok());
    }

    #[test]
    #[ignore]
    fn test_build_package() {
        let _ = env_logger::init();
        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
        let mut docbuilder = DocBuilder::new(options);
        let res = docbuilder.build_package("rand", "0.3.14");
        assert!(res.is_ok());
    }

    #[test]
    fn test_parse_rustc_version() {
        assert_eq!(parse_rustc_version("rustc 1.10.0-nightly (57ef01513 2016-05-23)"),
                   "20160523-1.10.0-nightly-57ef01513");
        assert_eq!(parse_rustc_version("cratesfyi 0.2.0 (ba9ae23 2016-05-26)"),
                   "20160526-0.2.0-ba9ae23");
    }
}
<commit_msg>Remove correct documentation directory<commit_after>
use super::DocBuilder;
use super::crates::crates_from_path;
use DocBuilderError;
use utils::{get_package, source_path, copy_dir, copy_doc_dir, update_sources};
use db::{connect_db, add_package_into_database, add_build_into_database, add_path_into_database};
use cargo::core::Package;
use std::process::{Command, Output};
use std::path::PathBuf;
use postgres::Connection;

use regex::Regex;

type CommandResult = Result<String, String>;

#[derive(Debug)]
pub struct ChrootBuilderResult {
    pub output: String,
    pub build_success: bool,
    pub have_doc: bool,
    pub have_examples: bool,
    pub rustc_version: String,
    pub cratesfyi_version: String,
}

impl DocBuilder {
    /// Builds every package documentation in chroot environment
    pub fn build_world(&mut self) -> Result<(), DocBuilderError> {
        try!(update_sources());

        let mut count = 0;

        crates(self.options.crates_io_index_path.clone(), |name, version| {
            match self.build_package(name, version) {
                Ok(status) => {
                    count += 1;
                    if status && count % 10 == 0 {
                        let _ = self.save_cache();
                    }
                }
                Err(err) => warn!("Failed to build package {}-{}: {}", name, version, err),
            }
            self.cache.insert(format!("{}-{}", name, version));
        })
    }

    /// Builds package documentation in chroot environment and adds into cratesfyi database
    pub fn build_package(&mut self, name: &str, version: &str) -> Result<bool, DocBuilderError> {
        info!("Building package {}-{}", name, version);

        // Database connection
        let conn = try!(connect_db());

        // Skip crates according to options
        if (self.options.skip_if_log_exists &&
            self.cache.contains(&format!("{}-{}", name, version)[..])) ||
           (self.options.skip_if_exists &&
            self.db_cache.contains(&format!("{}-{}", name, version)[..])) {
            info!("Skipping");
            return Ok(false);
        }

        // get_package (and cargo) is using semver, add '=' in front of version.
        let pkg = try!(get_package(name, Some(&format!("={}", version)[..])));
        let res = self.build_package_in_chroot(&pkg);

        // copy sources and documentation
        try!(self.add_sources_into_database(&conn, &pkg));
        if res.have_doc {
            try!(self.copy_documentation(&pkg, &res.rustc_version));
            try!(self.add_documentation_into_database(&conn, &pkg));
        }

        let release_id = try!(add_package_into_database(&conn, &pkg, &res));
        try!(add_build_into_database(&conn, &release_id, &res));

        // remove documentation, source and build directory after we are done
        try!(self.clean(&pkg));

        // add package into build cache
        self.cache.insert(format!("{}-{}", name, version));

        Ok(res.build_success)
    }

    /// Builds documentation of a package with cratesfyi in chroot environment
    fn build_package_in_chroot(&self, package: &Package) -> ChrootBuilderResult {
        debug!("Building package in chroot");
        let (rustc_version, cratesfyi_version) = self.get_versions();
        let cmd = format!("cratesfyi doc {} ={}",
                          package.manifest().name(),
                          package.manifest().version());
        match self.chroot_command(cmd) {
            Ok(o) => {
                ChrootBuilderResult {
                    output: o,
                    build_success: true,
                    have_doc: self.have_documentation(&package),
                    have_examples: self.have_examples(&package),
                    rustc_version: rustc_version,
                    cratesfyi_version: cratesfyi_version,
                }
            }
            Err(e) => {
                ChrootBuilderResult {
                    output: e,
                    build_success: false,
                    have_doc: false,
                    have_examples: self.have_examples(&package),
                    rustc_version: rustc_version,
                    cratesfyi_version: cratesfyi_version,
                }
            }
        }
    }

    /// Copies source files of a package into source_path
    #[allow(dead_code)]  // I've been using this function before storing files in database
    fn copy_sources(&self, package: &Package) -> Result<(), DocBuilderError> {
        debug!("Copying sources");
        let destination = PathBuf::from(&self.options.sources_path).join(format!("{}/{}",
                          package.manifest().name(),
                          package.manifest().version()));
        // unwrap is safe here, this function will be always called after get_package
        match copy_dir(source_path(&package).unwrap(), &destination) {
            Ok(_) => Ok(()),
            Err(e) => Err(DocBuilderError::Io(e)),
        }
    }

    /// Copies documentation to destination directory
    fn copy_documentation(&self,
                          package: &Package,
                          rustc_version: &str)
                          -> Result<(), DocBuilderError> {
        debug!("Copying documentation");
        let crate_doc_path = PathBuf::from(&self.options.chroot_path)
                                 .join("home")
                                 .join(&self.options.chroot_user)
                                 .join(canonical_name(&package));
        let destination = PathBuf::from(&self.options.destination).join(format!("{}/{}",
                          package.manifest().name(),
                          package.manifest().version()));
        copy_doc_dir(crate_doc_path,
                     destination,
                     parse_rustc_version(rustc_version).trim())
            .map_err(DocBuilderError::Io)
    }

    /// Removes build directory of a package in chroot
    fn remove_build_dir(&self, package: &Package) -> Result<(), DocBuilderError> {
        let _ = self.chroot_command(format!("rm -rf {}", canonical_name(&package)));
        Ok(())
    }

    /// Remove documentation, build directory and sources directory of a package
    fn clean(&self, package: &Package) -> Result<(), DocBuilderError> {
        debug!("Cleaning package");
        use std::fs::remove_dir_all;
        let documentation_path = PathBuf::from(&self.options.destination)
                                     .join(package.manifest().name());
        let source_path = source_path(&package).unwrap();
        // Some crates don't have documentation, so we don't care if removing_dir_all fails
        let _ = self.remove_build_dir(&package);
        let _ = remove_dir_all(documentation_path);
        let _ = remove_dir_all(source_path);
        Ok(())
    }

    /// Runs a command in a chroot environment
    fn chroot_command<T: AsRef<str>>(&self, cmd: T) -> CommandResult {
        command_result(Command::new("sudo")
                           .arg("lxc-attach")
                           .arg("-n")
                           .arg(&self.options.container_name)
                           .arg("--")
                           .arg("su")
                           .arg("-")
                           .arg(&self.options.chroot_user)
                           .arg("-c")
                           .arg(cmd.as_ref())
                           .output()
                           .unwrap())
    }

    /// Checks a package build directory to determine if package have docs
    ///
    /// This function is checking first target in targets to see if documentation exists for a
    /// crate. Package must be successfully built in chroot environment first.
    fn have_documentation(&self, package: &Package) -> bool {
        let crate_doc_path = PathBuf::from(&self.options.chroot_path)
                                 .join("home")
                                 .join(&self.options.chroot_user)
                                 .join(canonical_name(&package))
                                 .join("doc")
                                 .join(package.targets()[0].name().to_string());
        crate_doc_path.exists()
    }

    /// Checks if package have examples
    fn have_examples(&self, package: &Package) -> bool {
        let path = source_path(&package).unwrap().join("examples");
        path.exists() && path.is_dir()
    }

    /// Gets rustc and cratesfyi version from chroot environment
    fn get_versions(&self) -> (String, String) {
        // It is safe to use expect here
        // chroot environment must always have rustc and cratesfyi installed
        (String::from(self.chroot_command("rustc --version")
                          .expect("Failed to get rustc version")
                          .trim()),
         String::from(self.chroot_command("cratesfyi --version")
                          .expect("Failed to get cratesfyi version")
                          .trim()))
    }

    /// Adds sources into database
    fn add_sources_into_database(&self,
                                 conn: &Connection,
                                 package: &Package)
                                 -> Result<(), DocBuilderError> {
        debug!("Adding sources into database");
        let prefix = format!("sources/{}/{}",
                             package.manifest().name(),
                             package.manifest().version());
        add_path_into_database(conn, &prefix, source_path(&package).unwrap())
    }

    /// Adds documentations into database
    fn add_documentation_into_database(&self,
                                       conn: &Connection,
                                       package: &Package)
                                       -> Result<(), DocBuilderError> {
        debug!("Adding documentation into database");
        let prefix = format!("rustdoc/{}/{}",
                             package.manifest().name(),
                             package.manifest().version());
        let crate_doc_path = PathBuf::from(&self.options.destination).join(format!("{}/{}",
                          package.manifest().name(),
                          package.manifest().version()));
        add_path_into_database(conn, &prefix, crate_doc_path)
    }
}

/// Simple function to capture command output
fn command_result(output: Output) -> CommandResult {
    let mut command_out = String::from_utf8_lossy(&output.stdout).into_owned();
    command_out.push_str(&String::from_utf8_lossy(&output.stderr).into_owned()[..]);
    match output.status.success() {
        true => Ok(command_out),
        false => Err(command_out),
    }
}

/// Returns canonical name of a package.
///
/// It's just package-version. All directory structure used in cratesfyi is
/// following this naming scheme.
fn canonical_name(package: &Package) -> String {
    format!("{}-{}",
            package.manifest().name(),
            package.manifest().version())
}

/// Parses rustc commit hash from rustc version string
fn parse_rustc_version<S: AsRef<str>>(version: S) -> String {
    let version_regex = Regex::new(r" ([\w-.]+) \((\w+) (\d+)-(\d+)-(\d+)\)").unwrap();
    let captures = version_regex.captures(version.as_ref()).expect("Failed to parse rustc version");

    format!("{}{}{}-{}-{}",
            captures.at(3).unwrap(),
            captures.at(4).unwrap(),
            captures.at(5).unwrap(),
            captures.at(1).unwrap(),
            captures.at(2).unwrap())
}

/// Runs `func` with the all crates from crates-io.index repository path.
///
/// First argument of func is the name of crate and
/// second argument is the version of crate. Func will be run for every crate.
fn crates<F>(path: PathBuf, mut func: F) -> Result<(), DocBuilderError>
    where F: FnMut(&str, &str) -> ()
{
    crates_from_path(&path, &mut func)
}

#[cfg(test)]
mod test {
    extern crate env_logger;
    use super::parse_rustc_version;
    use std::path::PathBuf;
    use {DocBuilder, DocBuilderOptions};

    #[test]
    #[ignore]
    fn test_build_world() {
        let _ = env_logger::init();
        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
        let mut docbuilder = DocBuilder::new(options);
        // This test is building WHOLE WORLD and may take forever
        assert!(docbuilder.build_world().is_ok());
    }

    #[test]
    #[ignore]
    fn test_build_package() {
        let _ = env_logger::init();
        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
        let mut docbuilder = DocBuilder::new(options);
        let res = docbuilder.build_package("rand", "0.3.14");
        assert!(res.is_ok());
    }

    #[test]
    fn test_parse_rustc_version() {
        assert_eq!(parse_rustc_version("rustc 1.10.0-nightly (57ef01513 2016-05-23)"),
                   "20160523-1.10.0-nightly-57ef01513");
        assert_eq!(parse_rustc_version("cratesfyi 0.2.0 (ba9ae23 2016-05-26)"),
                   "20160526-0.2.0-ba9ae23");
    }
}
<|endoftext|>


<commit_before>// run-pass

#![feature(unsized_fn_params)]
#![allow(dead_code)]
#[repr(align(256))]
struct A {
    v: u8,
}

impl A {
    fn f(&self) -> *const A {
        self
    }
}

fn f2(v: u8) -> Box<dyn FnOnce() -> *const A> {
    let a = A { v };
    Box::new(move || a.f())
}

fn main() {
    let addr = f2(0)();
    assert_eq!(addr as usize % 256, 0, "addr: {:?}", addr);
}
<commit_msg>This flag is not really needed in the test<commit_after>// run-pass

#![allow(dead_code)]
#[repr(align(256))]
struct A {
    v: u8,
}

impl A {
    fn f(&self) -> *const A {
        self
    }
}

fn f2(v: u8) -> Box<dyn FnOnce() -> *const A> {
    let a = A { v };
    Box::new(move || a.f())
}

fn main() {
    let addr = f2(0)();
    assert_eq!(addr as usize % 256, 0, "addr: {:?}", addr);
}
<|endoftext|>