<commit_before>extern crate "cargo-registry" as cargo_registry;
extern crate migrate;
extern crate postgres;
extern crate r2d2;

use std::os;
use migrate::Migration;
use postgres::{PostgresTransaction, PostgresResult};

use cargo_registry::krate::Crate;

fn main() {
    let db_config = r2d2::Config {
        pool_size: 1,
        helper_tasks: 1,
        test_on_check_out: false,
    };
    let database = cargo_registry::db::pool(env("DATABASE_URL").as_slice(),
                                            db_config);
    let conn = database.get().unwrap();
    let migrations = migrations();

    if os::args().as_slice().get(1).map(|s| s.as_slice()) == Some("rollback") {
        rollback(conn.transaction().unwrap(), migrations).unwrap();
    } else {
        apply(conn.transaction().unwrap(), migrations).unwrap();
    }

    fn env(s: &str) -> String {
        match os::getenv(s) {
            Some(s) => s,
            None => fail!("must have `{}` defined", s),
        }
    }
}

fn apply(tx: PostgresTransaction,
         migrations: Vec<Migration>) -> PostgresResult<()> {
    let mut mgr = try!(migrate::Manager::new(tx));
    for m in migrations.into_iter() {
        try!(mgr.apply(m));
    }
    mgr.set_commit();
    mgr.finish()
}

fn rollback(tx: PostgresTransaction,
            migrations: Vec<Migration>) -> PostgresResult<()> {
    let mut mgr = try!(migrate::Manager::new(tx));
    for m in migrations.into_iter().rev() {
        if mgr.contains(m.version()) {
            try!(mgr.rollback(m));
            break
        }
    }
    mgr.set_commit();
    mgr.finish()
}

fn migrations() -> Vec<Migration> {
    // Generate a new id via `date +"%Y%m%d%H%M%S"`
    vec![
        Migration::add_table(20140924113530, "users", "
            id              SERIAL PRIMARY KEY,
            email           VARCHAR NOT NULL UNIQUE,
            gh_access_token VARCHAR NOT NULL,
            api_token       VARCHAR NOT NULL
        "),
        Migration::add_table(20140924114003, "packages", "
            id              SERIAL PRIMARY KEY,
            name            VARCHAR NOT NULL UNIQUE,
            user_id         INTEGER NOT NULL
        "),
        Migration::add_table(20140924114059, "versions", "
            id              SERIAL PRIMARY KEY,
            package_id      INTEGER NOT NULL,
            num             VARCHAR NOT NULL
        "),
        Migration::run(20140924115329,
                       format!("ALTER TABLE versions ADD CONSTRAINT \
                                unique_num UNIQUE (package_id, num)"),
                       format!("ALTER TABLE versions DROP CONSTRAINT \
                                unique_num")),
        Migration::add_table(20140924120803, "version_dependencies", "
            version_id      INTEGER NOT NULL,
            depends_on_id   INTEGER NOT NULL
        "),
        Migration::add_column(20140925132248, "packages", "updated_at",
                              "TIMESTAMP NOT NULL DEFAULT now()"),
        Migration::add_column(20140925132249, "packages", "created_at",
                              "TIMESTAMP NOT NULL DEFAULT now()"),
        Migration::new(20140925132250, proc(tx) {
            try!(tx.execute("UPDATE packages SET updated_at = now() \
                             WHERE updated_at IS NULL", []));
            try!(tx.execute("UPDATE packages SET created_at = now() \
                             WHERE created_at IS NULL", []));
            Ok(())
        }, proc(_) Ok(())),
        Migration::add_column(20140925132251, "versions", "updated_at",
                              "TIMESTAMP NOT NULL DEFAULT now()"),
        Migration::add_column(20140925132252, "versions", "created_at",
                              "TIMESTAMP NOT NULL DEFAULT now()"),
        Migration::new(20140925132253, proc(tx) {
            try!(tx.execute("UPDATE versions SET updated_at = now() \
                             WHERE updated_at IS NULL", []));
            try!(tx.execute("UPDATE versions SET created_at = now() \
                             WHERE created_at IS NULL", []));
            Ok(())
        }, proc(_) Ok(())),
        Migration::new(20140925132254, proc(tx) {
            try!(tx.execute("ALTER TABLE versions ALTER COLUMN updated_at \
                             DROP DEFAULT", []));
            try!(tx.execute("ALTER TABLE versions ALTER COLUMN created_at \
                             DROP DEFAULT", []));
            try!(tx.execute("ALTER TABLE packages ALTER COLUMN updated_at \
                             DROP DEFAULT", []));
            try!(tx.execute("ALTER TABLE packages ALTER COLUMN created_at \
                             DROP DEFAULT", []));
            Ok(())
        }, proc(_) Ok(())),
        Migration::add_table(20140925153704, "metadata", "
            total_downloads        BIGINT NOT NULL
        "),
        Migration::new(20140925153705, proc(tx) {
            try!(tx.execute("INSERT INTO metadata (total_downloads) \
                             VALUES ($1)", &[&0i64]));
            Ok(())
        }, proc(tx) {
            try!(tx.execute("DELETE FROM metadata", [])); Ok(())
        }),
        Migration::add_column(20140925161623, "packages", "downloads",
                              "INTEGER NOT NULL DEFAULT 0"),
        Migration::add_column(20140925161624, "versions", "downloads",
                              "INTEGER NOT NULL DEFAULT 0"),
        Migration::new(20140925161625, proc(tx) {
            try!(tx.execute("ALTER TABLE versions ALTER COLUMN downloads \
                             DROP DEFAULT", []));
            try!(tx.execute("ALTER TABLE packages ALTER COLUMN downloads \
                             DROP DEFAULT", []));
            Ok(())
        }, proc(_) Ok(())),
        Migration::add_column(20140926130044, "packages", "max_version",
                              "VARCHAR"),
        Migration::new(20140926130045, proc(tx) {
            let stmt = try!(tx.prepare("SELECT * FROM packages"));
            for row in try!(stmt.query(&[])) {
                let pkg = Crate::from_row(&row);
                let versions = pkg.versions(tx).unwrap();
                let v = versions.iter().max_by(|v| &v.num).unwrap();
                let max = v.num.to_string();
                try!(tx.execute("UPDATE packages SET max_version = $1 \
                                 WHERE id = $2",
                                &[&max, &pkg.id]));
            }
            Ok(())
        }, proc(_) Ok(())),
        Migration::new(20140926130046, proc(tx) {
            try!(tx.execute("ALTER TABLE versions ALTER COLUMN downloads \
                             SET NOT NULL", []));
            Ok(())
        }, proc(tx) {
            try!(tx.execute("ALTER TABLE versions ALTER COLUMN downloads \
                             DROP NOT NULL", []));
            Ok(())
        }),
        Migration::new(20140926174020, proc(tx) {
            try!(tx.execute("ALTER TABLE packages RENAME TO crates", []));
            try!(tx.execute("ALTER TABLE versions RENAME COLUMN package_id \
                             TO crate_id", []));
            Ok(())
        }, proc(tx) {
            try!(tx.execute("ALTER TABLE crates RENAME TO packages", []));
            try!(tx.execute("ALTER TABLE versions RENAME COLUMN crate_id \
                             TO package_id", []));
            Ok(())
        }),
        Migration::run(20140929103749,
                       "CREATE INDEX crate_updated_at ON crates (updated_at)",
                       "DROP INDEX crate_updated_at"),
        Migration::run(20140929103750,
                       "CREATE INDEX crate_created_at ON crates (created_at)",
                       "DROP INDEX crate_created_at"),
        Migration::run(20140929103751,
                       "CREATE INDEX crate_downloads ON crates (downloads)",
                       "DROP INDEX crate_downloads"),
        Migration::run(20140929103752,
                       "CREATE INDEX version_crate_id ON versions (crate_id)",
                       "DROP INDEX version_crate_id"),
        Migration::run(20140929103753,
                       "CREATE INDEX version_num ON versions (num)",
                       "DROP INDEX version_num"),
        Migration::run(20140929103754,
                       "CREATE INDEX version_dependencies_version_id \
                        ON version_dependencies (version_id)",
                       "DROP INDEX version_dependencies_version_id"),
        Migration::run(20140929103755,
                       "CREATE INDEX version_dependencies_depends_on_id \
                        ON version_dependencies (depends_on_id)",
                       "DROP INDEX version_dependencies_depends_on_id"),
    ]
}
<commit_msg>Add download date tables<commit_after>extern crate "cargo-registry" as cargo_registry;
extern crate migrate;
extern crate postgres;
extern crate r2d2;

use std::os;
use std::collections::HashSet;
use migrate::Migration;
use postgres::{PostgresTransaction, PostgresResult};

use cargo_registry::krate::Crate;

fn main() {
    let db_config = r2d2::Config {
        pool_size: 1,
        helper_tasks: 1,
        test_on_check_out: false,
    };
    let database = cargo_registry::db::pool(env("DATABASE_URL").as_slice(),
                                            db_config);
    let conn = database.get().unwrap();
    let migrations = migrations();

    if os::args().as_slice().get(1).map(|s| s.as_slice()) == Some("rollback") {
        rollback(conn.transaction().unwrap(), migrations).unwrap();
    } else {
        apply(conn.transaction().unwrap(), migrations).unwrap();
    }

    fn env(s: &str) -> String {
        match os::getenv(s) {
            Some(s) => s,
            None => fail!("must have `{}` defined", s),
        }
    }
}

fn apply(tx: PostgresTransaction,
         migrations: Vec<Migration>) -> PostgresResult<()> {
    let mut mgr = try!(migrate::Manager::new(tx));
    for m in migrations.into_iter() {
        try!(mgr.apply(m));
    }
    mgr.set_commit();
    mgr.finish()
}

fn rollback(tx: PostgresTransaction,
            migrations: Vec<Migration>) -> PostgresResult<()> {
    let mut mgr = try!(migrate::Manager::new(tx));
    for m in migrations.into_iter().rev() {
        if mgr.contains(m.version()) {
            try!(mgr.rollback(m));
            break
        }
    }
    mgr.set_commit();
    mgr.finish()
}

fn migrations() -> Vec<Migration> {
    // Generate a new id via `date +"%Y%m%d%H%M%S"`
    let migrations = vec![
        Migration::add_table(20140924113530, "users", "
            id              SERIAL PRIMARY KEY,
            email           VARCHAR NOT NULL UNIQUE,
            gh_access_token VARCHAR NOT NULL,
            api_token       VARCHAR NOT NULL
        "),
        Migration::add_table(20140924114003, "packages", "
            id              SERIAL PRIMARY KEY,
            name            VARCHAR NOT NULL UNIQUE,
            user_id         INTEGER NOT NULL
        "),
        Migration::add_table(20140924114059, "versions", "
            id              SERIAL PRIMARY KEY,
            package_id      INTEGER NOT NULL,
            num             VARCHAR NOT NULL
        "),
        Migration::run(20140924115329,
                       format!("ALTER TABLE versions ADD CONSTRAINT \
                                unique_num UNIQUE (package_id, num)"),
                       format!("ALTER TABLE versions DROP CONSTRAINT \
                                unique_num")),
        Migration::add_table(20140924120803, "version_dependencies", "
            version_id      INTEGER NOT NULL,
            depends_on_id   INTEGER NOT NULL
        "),
        Migration::add_column(20140925132248, "packages", "updated_at",
                              "TIMESTAMP NOT NULL DEFAULT now()"),
        Migration::add_column(20140925132249, "packages", "created_at",
                              "TIMESTAMP NOT NULL DEFAULT now()"),
        Migration::new(20140925132250, proc(tx) {
            try!(tx.execute("UPDATE packages SET updated_at = now() \
                             WHERE updated_at IS NULL", []));
            try!(tx.execute("UPDATE packages SET created_at = now() \
                             WHERE created_at IS NULL", []));
            Ok(())
        }, proc(_) Ok(())),
        Migration::add_column(20140925132251, "versions", "updated_at",
                              "TIMESTAMP NOT NULL DEFAULT now()"),
        Migration::add_column(20140925132252, "versions", "created_at",
                              "TIMESTAMP NOT NULL DEFAULT now()"),
        Migration::new(20140925132253, proc(tx) {
            try!(tx.execute("UPDATE versions SET updated_at = now() \
                             WHERE updated_at IS NULL", []));
            try!(tx.execute("UPDATE versions SET created_at = now() \
                             WHERE created_at IS NULL", []));
            Ok(())
        }, proc(_) Ok(())),
        Migration::new(20140925132254, proc(tx) {
            try!(tx.execute("ALTER TABLE versions ALTER COLUMN updated_at \
                             DROP DEFAULT", []));
            try!(tx.execute("ALTER TABLE versions ALTER COLUMN created_at \
                             DROP DEFAULT", []));
            try!(tx.execute("ALTER TABLE packages ALTER COLUMN updated_at \
                             DROP DEFAULT", []));
            try!(tx.execute("ALTER TABLE packages ALTER COLUMN created_at \
                             DROP DEFAULT", []));
            Ok(())
        }, proc(_) Ok(())),
        Migration::add_table(20140925153704, "metadata", "
            total_downloads        BIGINT NOT NULL
        "),
        Migration::new(20140925153705, proc(tx) {
            try!(tx.execute("INSERT INTO metadata (total_downloads) \
                             VALUES ($1)", &[&0i64]));
            Ok(())
        }, proc(tx) {
            try!(tx.execute("DELETE FROM metadata", [])); Ok(())
        }),
        Migration::add_column(20140925161623, "packages", "downloads",
                              "INTEGER NOT NULL DEFAULT 0"),
        Migration::add_column(20140925161624, "versions", "downloads",
                              "INTEGER NOT NULL DEFAULT 0"),
        Migration::new(20140925161625, proc(tx) {
            try!(tx.execute("ALTER TABLE versions ALTER COLUMN downloads \
                             DROP DEFAULT", []));
            try!(tx.execute("ALTER TABLE packages ALTER COLUMN downloads \
                             DROP DEFAULT", []));
            Ok(())
        }, proc(_) Ok(())),
        Migration::add_column(20140926130044, "packages", "max_version",
                              "VARCHAR"),
        Migration::new(20140926130045, proc(tx) {
            let stmt = try!(tx.prepare("SELECT * FROM packages"));
            for row in try!(stmt.query(&[])) {
                let pkg = Crate::from_row(&row);
                let versions = pkg.versions(tx).unwrap();
                let v = versions.iter().max_by(|v| &v.num).unwrap();
                let max = v.num.to_string();
                try!(tx.execute("UPDATE packages SET max_version = $1 \
                                 WHERE id = $2",
                                &[&max, &pkg.id]));
            }
            Ok(())
        }, proc(_) Ok(())),
        Migration::new(20140926130046, proc(tx) {
            try!(tx.execute("ALTER TABLE versions ALTER COLUMN downloads \
                             SET NOT NULL", []));
            Ok(())
        }, proc(tx) {
            try!(tx.execute("ALTER TABLE versions ALTER COLUMN downloads \
                             DROP NOT NULL", []));
            Ok(())
        }),
        Migration::new(20140926174020, proc(tx) {
            try!(tx.execute("ALTER TABLE packages RENAME TO crates", []));
            try!(tx.execute("ALTER TABLE versions RENAME COLUMN package_id \
                             TO crate_id", []));
            Ok(())
        }, proc(tx) {
            try!(tx.execute("ALTER TABLE crates RENAME TO packages", []));
            try!(tx.execute("ALTER TABLE versions RENAME COLUMN crate_id \
                             TO package_id", []));
            Ok(())
        }),
        Migration::run(20140929103749,
                       "CREATE INDEX index_crate_updated_at ON crates (updated_at)",
                       "DROP INDEX index_crate_updated_at"),
        Migration::run(20140929103750,
                       "CREATE INDEX index_crate_created_at ON crates (created_at)",
                       "DROP INDEX index_crate_created_at"),
        Migration::run(20140929103751,
                       "CREATE INDEX index_crate_downloads ON crates (downloads)",
                       "DROP INDEX index_crate_downloads"),
        Migration::run(20140929103752,
                       "CREATE INDEX index_version_crate_id ON versions (crate_id)",
                       "DROP INDEX index_version_crate_id"),
        Migration::run(20140929103753,
                       "CREATE INDEX index_version_num ON versions (num)",
                       "DROP INDEX index_version_num"),
        Migration::run(20140929103754,
                       "CREATE INDEX index_version_dependencies_version_id \
                        ON version_dependencies (version_id)",
                       "DROP INDEX index_version_dependencies_version_id"),
        Migration::run(20140929103755,
                       "CREATE INDEX index_version_dependencies_depends_on_id \
                        ON version_dependencies (depends_on_id)",
                       "DROP INDEX index_version_dependencies_depends_on_id"),
        Migration::add_table(20140929103756, "crate_downloads", "
            crate_id        INTEGER NOT NULL,
            downloads       INTEGER NOT NULL,
            date            TIMESTAMP NOT NULL,
            processed       BOOLEAN NOT NULL
        "),
        Migration::run(20140929103757,
                       "CREATE INDEX index_crate_downloads_crate_id \
                        ON crate_downloads (crate_id)",
                       "DROP INDEX index_crate_downloads_crate_id"),
        Migration::run(20140929103758,
                       "CREATE INDEX index_crate_downloads_date \
                        ON crate_downloads (date(date))",
                       "DROP INDEX index_crate_downloads_date"),
        Migration::add_table(20140929103759, "version_downloads", "
            version_id      INTEGER NOT NULL,
            downloads       INTEGER NOT NULL,
            date            TIMESTAMP NOT NULL,
            processed       BOOLEAN NOT NULL
        "),
        Migration::run(20140929103760,
                       "CREATE INDEX index_version_downloads_version_id \
                        ON version_downloads (version_id)",
                       "DROP INDEX index_version_downloads_version_id"),
        Migration::run(20140929103761,
                       "CREATE INDEX index_version_downloads_date \
                        ON version_downloads (date(date))",
                       "DROP INDEX index_version_downloads_date"),
        Migration::run(20140929103762,
                       "CREATE INDEX index_crate_downloads_processed \
                        ON crate_downloads (processed)
                        WHERE processed = FALSE",
                       "DROP INDEX index_crate_downloads_processed"),
        Migration::run(20140929103763,
                       "CREATE INDEX index_version_downloads_processed \
                        ON version_downloads (processed)
                        WHERE processed = FALSE",
                       "DROP INDEX index_version_downloads_processed"),
    ];

    let mut seen = HashSet::new();
    for m in migrations.iter() {
        if !seen.insert(m.version()) {
            fail!("duplicate id: {}", m.version());
        }
    }
    return migrations;
}
<|endoftext|>


<commit_before>// The target triplets have the form of 'arch-vendor-system'.
//
// When building for Linux (e.g. the 'system' part is
// 'linux-something'), replace the vendor with 'unknown'
// so that mapping to rust standard targets happens correctly.
fn convert_custom_linux_target(target: String) -> String {
    let mut parts: Vec<&str> = target.split('-').collect();
    let system = parts[2];
    if system == "linux" {
        parts[1] = "unknown";
    };
    parts.join("-")
}
<commit_msg>Fix panicking when target triplets has less than three elements<commit_after>// The target triplets have the form of 'arch-vendor-system'.
//
// When building for Linux (e.g. the 'system' part is
// 'linux-something'), replace the vendor with 'unknown'
// so that mapping to rust standard targets happens correctly.
fn convert_custom_linux_target(target: String) -> String {
    let mut parts: Vec<&str> = target.split('-').collect();
    let system = parts.get(2);
    if system == Some(&"linux") {
        parts[1] = "unknown";
    };
    parts.join("-")
}
<|endoftext|>


<commit_before>use anyhow::Result;
use clap::Parser;
use regex::Regex;
use std::collections::HashMap;
use std::fs::File;
use std::io::{self, BufRead};
use std::path::PathBuf;
use std::time::Instant;

use polars::frame::DataFrame;
use polars::prelude::*;

#[derive(Parser, Debug)]
#[clap(author, version, about, long_about = None)]
struct Args {
    /// List of input files. Ideally with headers
    #[clap(short, long, multiple_values(true))]
    input: Vec<PathBuf>,

    /// An output file
    #[clap(short, long)]
    output: String,

    /// The column to select. If files have no headers, use "column_N"
    #[clap(short, long, multiple_values(true))]
    select: Vec<String>,

    /// The size of chunks to process at once
    #[clap(short, long, default_value_t = 100)]
    chunksize: usize,
}

fn load_data(path: &PathBuf, delim: u8) -> Result<DataFrame> {
    let mut df: DataFrame = CsvReader::from_path(path)?
        .has_header(true)
        .with_delimiter(delim)
        .with_null_values(Some(NullValues::AllColumns("NA".to_string())))
        .infer_schema(None)
        .finish()
        .unwrap();

    // hstack the experiment name (derived from the filename) into the DataFrame
    let iter_exp =
        vec![path.file_name().unwrap().to_str().unwrap().split('-').collect::<Vec<&str>>()[0..=2]
            .join("-")]
        .into_iter();
    let exp_col: Series = Series::new(
        "experiment",
        iter_exp.flat_map(|n| std::iter::repeat(n).take(df.height())).collect::<Vec<String>>(),
    );
    // exp_col.rename("experiment");
    df.hstack_mut(&[exp_col]).unwrap();

    Ok(df)
}

fn baseline_get_median_gt_zero(str_val: &Series) -> Series {
    let lists = str_val
        .utf8()
        .unwrap()
        .into_iter()
        .map(|x| {
            x.unwrap()
                .split(',')
                .into_iter()
                .map(|y| y.parse::<f64>().unwrap())
                .collect::<Vec<f64>>()
        });

    let medians: Vec<bool> =
        lists.into_iter().map(|x| Series::from_iter(x).median().unwrap() > 0.0).collect();
    Series::from_iter(medians)
}

fn filter_input(input: &mut DataFrame, baseline: bool) -> DataFrame {
    if baseline {
        filter_baseline(input)
    } else {
        filter_differential(input)
    }
}

fn filter_baseline(input: &mut DataFrame) -> DataFrame {
    // This is a baseline experiment.
    // Find columns starting with lower case g, then apply the function to convert to
    // medain and select greater than zero
    let mut meas = Vec::<String>::new();

    for col in input.get_column_names_owned() {
        if col.starts_with('g') {
            input.apply(&col, baseline_get_median_gt_zero).unwrap();
            meas.push(col);
        }
    }
    println!("{:?}", meas);
    // Selection should now have all the gN column names in it
    input
        .clone()
        .lazy()
        .filter(any_exprs(meas.into_iter().map(|x| col(&x)).collect::<Vec<Expr>>()))
        .collect()
        .unwrap()
}

fn filter_differential(input: &DataFrame) -> DataFrame {
    // find the p value and log fold columns
    let pv_regex = Regex::new(r".*p-value.*").unwrap();
    let log_fold_regex = Regex::new(r".*log2.*").unwrap();
    let mut pv: String = "p-value".to_string();
    let mut lf: String = "log2fold".to_string();
    for col in input.get_column_names_owned() {
        if pv_regex.is_match(&col) {
            pv = col;
        } else if log_fold_regex.is_match(&col) {
            lf = col
        }
    }

    input
        .clone()
        .lazy()
        .filter(all_exprs([col(&pv).is_not_null(), col(&lf).neq(lit(0))]))
        .collect()
        .unwrap()
}

fn load_chunk(
    paths: &mut Vec<PathBuf>,
    select: &mut Vec<String>,
) -> Result<DataFrame, anyhow::Error> {
    /*
    Work with the chunks of input to reduce them into a single dataframe of the genes we want from
    that particular set of experiments. Should return a single dataframe
    */
    let mut output: Option<DataFrame> = None;
    while let Some(infile) = paths.pop() {
        // Load the input Csv
        let mut input: DataFrame = if infile.extension().unwrap() == "tsv" {
            load_data(&infile, b'\t')
        } else {
            load_data(&infile, b',')
        }
        .unwrap_or_else(|error| match error.downcast_ref::<PolarsError>() {
            Some(PolarsError::Io(_string)) => panic!("Input file does not exist! {:?}", infile),
            _ => panic!("An error occurred! {:?}", error),
        });

        // Rename columns to remove . in the names. Now also remove spaces
        let mut new_cols = Vec::new();
        for nm in input.get_column_names().iter() {
            new_cols.push(nm.replace('.', ""));//.replace(' ', ""));
        }

        if new_cols != input.get_column_names() {
            input.set_column_names(&new_cols).unwrap_or_else(|error| {
                panic!("Failed to set column names for some reason {:?}", error);
            });
        }

        // Need to detect which type of experiment it is - differential or baseline
        let type_re = Regex::new(r"tpms").unwrap();
        let filtered = filter_input(&mut input, type_re.is_match(infile.to_str().unwrap()));
        // Drop everything from the input except the genes
        let genes: DataFrame = filtered.select(select.iter()).unwrap_or_else(|error| match error {
            PolarsError::NotFound(_string) => {
                panic!(
                    "{:?} was not found in the header, does the file have a header?\n{:?}",
                    select,
                    input.get_column_names()
                );
            },
            _ => panic!("Error selecting column from input: {:?}", error),
        });

        output = match output {
            None => {
                let output_df = genes.clone();
                Some(output_df)
            },
            Some(mut output_df) => {
                // genes =/= output, so we are handling a new file
                output_df.extend(&genes).unwrap_or_else(|error| {
                    panic!("Failed to extend the output dataframe. {:?}", error)
                });
                output_df =
                    output_df.unique(None, UniqueKeepStrategy::First).unwrap_or_else(|error| {
                        panic!("Failed to parse selected column for uniqueness. {:?}", error)
                    });
                Some(output_df)
            },
        }
    }
    Ok(output.unwrap())
}

fn get_taxid(paths: &mut Vec<PathBuf>, lookup_table: &mut HashMap<String, String>) {
    let tax_regex = Regex::new(r".*Taxon_([0-9]{4,})").unwrap();

    while let Some(path) = paths.pop() {
        let exp_name =
            path.file_name().unwrap().to_str().unwrap().split('-').collect::<Vec<&str>>()[0..=2]
                .join("-")
                .replace(".condensed", "");

        let file = File::open(path).unwrap();
        let lines = io::BufReader::new(file).lines();
        for line in lines {
            match line {
                Ok(line) => {
                    let captures = tax_regex.captures(line.as_str());
                    match captures {
                        Some(caps) => {
                            let taxid = caps.get(1).map_or("", |m| m.as_str());
                            lookup_table.insert(exp_name, taxid.to_string());
                            break;
                        },
                        None => {},
                    };
                },
                Err(_idc) => {},
            }
        }
    }
}

fn main() {
    let timer = Instant::now();

    let mut args = Args::parse();

    // Always select the experiment
    args.select.push("experiment".to_string());

    // separate expression data from taxa data
    let mut taxa_files: Vec<PathBuf> = Vec::new();
    let mut expr_files: Vec<PathBuf> = Vec::new();

    let re = Regex::new(r"sdrf.tsv").unwrap();

    for infile in args.input {
        if re.is_match(infile.to_str().unwrap()) {
            // File is experiment setup data, parse for taxa
            taxa_files.push(infile);
        } else {
            // Should only be expression data, parse for genes
            expr_files.push(infile);
        }
    }

    println!("{} taxa files to parse", taxa_files.len());
    println!("{} expression files to parse", expr_files.len());

    let n_files = expr_files.len();

    let mut dataframes: Vec<DataFrame> = Vec::new();

    let chunked_input: Vec<Vec<PathBuf>> =
        expr_files.chunks(args.chunksize).map(|x| x.to_vec()).collect();
    // Read everything into a big vector
    let mut n_chunks = 0;
    for mut files_chunk in chunked_input {
        dataframes.push(
            load_chunk(&mut files_chunk, &mut args.select)
                .unwrap_or_else(|_error| panic!("Something wrong in one of the reads, aborting")),
        );
        n_chunks += 1;
        println!("Done {} files", n_chunks * args.chunksize);
    }

    let mut output: DataFrame = dataframes[0].clone();

    for item in dataframes.iter() {
        output.extend(item).unwrap_or_else(|error| {
            panic!("Unable to extend the DataFrame! Out of memory? {:?}", error)
        });
        output.unique(None, UniqueKeepStrategy::First).unwrap();
    }

    // Now parse the experiment files to get taxa
    let chunked_taxa: Vec<Vec<PathBuf>> =
        taxa_files.chunks(args.chunksize).map(|x| x.to_vec()).collect();

    let mut lookup_table: HashMap<String, String> = HashMap::new();

    for mut tax_chunk in chunked_taxa {
        get_taxid(&mut tax_chunk, &mut lookup_table);
    }

    // Now we have the experiment - taxid lookup_table, we just need to add the column to the output df

    let expt_col = output.select(["experiment"]).unwrap();
    let mut tax_ids: Vec<String> = Vec::with_capacity(expt_col.height());
    for ex in expt_col.iter() {
        let mut warn_flag: bool = true;
        let uhy = ex.utf8().unwrap();
        for (idx, x) in uhy.into_iter().enumerate() {
            if lookup_table.contains_key(x.unwrap()) {
                tax_ids.insert(idx, lookup_table.get(x.unwrap()).unwrap().to_string());
            } else {
                tax_ids.insert(idx, String::new());
                if warn_flag {
                    println!("Experiment {} does not name a taxon", x.unwrap());
                    warn_flag = false;
                }
            }
        }
    }

    output.hstack_mut(&[Series::new("taxid", tax_ids.into_iter())]).unwrap();

    let out_stream: File = File::create(args.output).unwrap();
    CsvWriter::new(out_stream)
        .has_header(true)
        .finish(&mut output)
        .unwrap_or_else(|error| panic!("Something wrong writing file: {:?}", error));

    println!("Processed {} in {} seconds", n_files, timer.elapsed().as_secs());
}
<commit_msg>Changes to allow for multiple baseline measures (e.g. E-GEOD-38430) and to deal with duplicated column names (e.g. GeneID and Gene ID after removing spaces)<commit_after>use anyhow::Result;
use clap::Parser;
use regex::Regex;
use std::collections::HashMap;
use std::fs::File;
use std::io::{self, BufRead};
use std::path::PathBuf;
use std::time::Instant;

use polars::frame::DataFrame;
use polars::prelude::*;

#[derive(Parser, Debug)]
#[clap(author, version, about, long_about = None)]
struct Args {
    /// List of input files. Ideally with headers
    #[clap(short, long, multiple_values(true))]
    input: Vec<PathBuf>,

    /// An output file
    #[clap(short, long)]
    output: String,

    /// The column to select. If files have no headers, use "column_N"
    #[clap(short, long, multiple_values(true))]
    select: Vec<String>,

    /// The size of chunks to process at once
    #[clap(short, long, default_value_t = 100)]
    chunksize: usize,
}

fn load_data(path: &PathBuf, delim: u8) -> Result<DataFrame> {
    let mut df: DataFrame = CsvReader::from_path(path)?
        .has_header(true)
        .with_delimiter(delim)
        .with_null_values(Some(NullValues::AllColumns("NA".to_string())))
        .infer_schema(None)
        .finish()
        .unwrap();

    // hstack the experiment name (derived from the filename) into the DataFrame
    let iter_exp =
        vec![path.file_name().unwrap().to_str().unwrap().split('-').collect::<Vec<&str>>()[0..=2]
            .join("-")]
        .into_iter();
    let exp_col: Series = Series::new(
        "experiment",
        iter_exp.flat_map(|n| std::iter::repeat(n).take(df.height())).collect::<Vec<String>>(),
    );
    // exp_col.rename("experiment");
    df.hstack_mut(&[exp_col]).unwrap();

    Ok(df)
}

fn baseline_get_median_gt_zero(str_val: &Series) -> Series {
    let lists = str_val.utf8().unwrap().into_iter().map(|x| {
        x.unwrap().split(',').into_iter().map(|y| y.parse::<f64>().unwrap()).collect::<Vec<f64>>()
    });

    let medians: Vec<bool> =
        lists.into_iter().map(|x| Series::from_iter(x).median().unwrap() > 0.0).collect();
    Series::from_iter(medians)
}

fn filter_input(input: &mut DataFrame, baseline: bool) -> DataFrame {
    if baseline {
        filter_baseline(input)
    } else {
        filter_differential(input)
    }
}

fn filter_baseline(input: &mut DataFrame) -> DataFrame {
    // This is a baseline experiment.
    // Find columns starting with lower case g, then apply the function to convert to
    // medain and select greater than zero
    let mut meas = Vec::<String>::new();

    for col in input.get_column_names_owned() {
        if col.starts_with('g') {
            input.apply(&col, baseline_get_median_gt_zero).unwrap();
            meas.push(col);
        }
    }
    println!("{:?}", meas);
    // Selection should now have all the gN column names in it
    input
        .clone()
        .lazy()
        .filter(any_exprs(meas.into_iter().map(|x| col(&x)).collect::<Vec<Expr>>()))
        .collect()
        .unwrap()
}

fn filter_differential(input: &DataFrame) -> DataFrame {
    // find the p value and log fold columns
    let pv_regex = Regex::new(r".*p-value.*").unwrap();
    let log_fold_regex = Regex::new(r".*log2.*").unwrap();
    let mut exprs: Vec<Expr> = Vec::new();
    let mut pv: Expr;
    let mut lf: Expr;
    for column in input.get_column_names_owned() {
        if pv_regex.is_match(&column) {
            pv = col(&column).is_not_null();
            exprs.push(pv);
        } else if log_fold_regex.is_match(&column) {
            lf = col(&column).neq(lit(0));
            exprs.push(lf);
        }
    }

    input.clone().lazy().filter(all_exprs(exprs)).collect().unwrap()
}

fn load_chunk(
    paths: &mut Vec<PathBuf>,
    select: &mut Vec<String>,
) -> Result<DataFrame, anyhow::Error> {
    /*
    Work with the chunks of input to reduce them into a single dataframe of the genes we want from
    that particular set of experiments. Should return a single dataframe
    */
    let mut output: Option<DataFrame> = None;
    while let Some(infile) = paths.pop() {
        // Load the input Csv
        let mut input: DataFrame = if infile.extension().unwrap() == "tsv" {
            load_data(&infile, b'\t')
        } else {
            load_data(&infile, b',')
        }
        .unwrap_or_else(|error| match error.downcast_ref::<PolarsError>() {
            Some(PolarsError::Io(_string)) => panic!("Input file does not exist! {:?}", infile),
            _ => panic!("An error occurred! {:?}", error),
        });

        // Rename columns to remove . in the names. Now also remove spaces
        let mut new_cols = Vec::new();
        let mut c: u8 = 0;
        for nm in input.get_column_names().iter() {
            let mut new_name = nm.replace('.', "").replace(' ', "");
            if new_cols.contains(&new_name) {
                c += 1;
                new_name += &c.to_string(); // Hopefully avoid duplicate names
            }
            new_cols.push(new_name);
        }

        if new_cols != input.get_column_names() {
            input.set_column_names(&new_cols).unwrap_or_else(|error| {
                panic!("Failed to set column names for some reason {:?}", error);
            });
        }

        // Need to detect which type of experiment it is - differential or baseline
        let type_re = Regex::new(r"tpms").unwrap();
        let filtered = filter_input(&mut input, type_re.is_match(infile.to_str().unwrap()));
        // Drop everything from the input except the genes
        let genes: DataFrame = filtered.select(select.iter()).unwrap_or_else(|error| match error {
            PolarsError::NotFound(_string) => {
                panic!(
                    "{:?} was not found in the header, does the file have a header?\n{:?}",
                    select,
                    input.get_column_names()
                );
            },
            _ => panic!("Error selecting column from input: {:?}", error),
        });

        output = match output {
            None => {
                let output_df = genes.clone();
                Some(output_df)
            },
            Some(mut output_df) => {
                // genes =/= output, so we are handling a new file
                output_df.extend(&genes).unwrap_or_else(|error| {
                    panic!("Failed to extend the output dataframe. {:?}", error)
                });
                output_df =
                    output_df.unique(None, UniqueKeepStrategy::First).unwrap_or_else(|error| {
                        panic!("Failed to parse selected column for uniqueness. {:?}", error)
                    });
                Some(output_df)
            },
        }
    }
    Ok(output.unwrap())
}

fn get_taxid(paths: &mut Vec<PathBuf>, lookup_table: &mut HashMap<String, String>) {
    let tax_regex = Regex::new(r".*Taxon_([0-9]{4,})").unwrap();

    while let Some(path) = paths.pop() {
        let exp_name =
            path.file_name().unwrap().to_str().unwrap().split('-').collect::<Vec<&str>>()[0..=2]
                .join("-")
                .replace(".condensed", "");

        let file = File::open(path).unwrap();
        let lines = io::BufReader::new(file).lines();
        for line in lines {
            match line {
                Ok(line) => {
                    let captures = tax_regex.captures(line.as_str());
                    match captures {
                        Some(caps) => {
                            let taxid = caps.get(1).map_or("", |m| m.as_str());
                            lookup_table.insert(exp_name, taxid.to_string());
                            break;
                        },
                        None => {},
                    };
                },
                Err(_idc) => {},
            }
        }
    }
}

fn main() {
    let timer = Instant::now();

    let mut args = Args::parse();

    // Always select the experiment
    args.select.push("experiment".to_string());

    // separate expression data from taxa data
    let mut taxa_files: Vec<PathBuf> = Vec::new();
    let mut expr_files: Vec<PathBuf> = Vec::new();

    let re = Regex::new(r"sdrf.tsv").unwrap();

    for infile in args.input {
        if re.is_match(infile.to_str().unwrap()) {
            // File is experiment setup data, parse for taxa
            taxa_files.push(infile);
        } else {
            // Should only be expression data, parse for genes
            expr_files.push(infile);
        }
    }

    println!("{} taxa files to parse", taxa_files.len());
    println!("{} expression files to parse", expr_files.len());

    let n_files = expr_files.len();

    let mut dataframes: Vec<DataFrame> = Vec::new();

    let chunked_input: Vec<Vec<PathBuf>> =
        expr_files.chunks(args.chunksize).map(|x| x.to_vec()).collect();
    // Read everything into a big vector
    let mut n_chunks = 0;
    for mut files_chunk in chunked_input {
        dataframes.push(
            load_chunk(&mut files_chunk, &mut args.select)
                .unwrap_or_else(|_error| panic!("Something wrong in one of the reads, aborting")),
        );
        n_chunks += 1;
        println!("Done {} files", n_chunks * args.chunksize);
    }

    let mut output: DataFrame = dataframes[0].clone();

    for item in dataframes.iter() {
        output.extend(item).unwrap_or_else(|error| {
            panic!("Unable to extend the DataFrame! Out of memory? {:?}", error)
        });
        output.unique(None, UniqueKeepStrategy::First).unwrap();
    }

    // Now parse the experiment files to get taxa
    let chunked_taxa: Vec<Vec<PathBuf>> =
        taxa_files.chunks(args.chunksize).map(|x| x.to_vec()).collect();

    let mut lookup_table: HashMap<String, String> = HashMap::new();

    for mut tax_chunk in chunked_taxa {
        get_taxid(&mut tax_chunk, &mut lookup_table);
    }

    // Now we have the experiment - taxid lookup_table, we just need to add the column to the output df

    let expt_col = output.select(["experiment"]).unwrap();
    let mut tax_ids: Vec<String> = Vec::with_capacity(expt_col.height());
    for ex in expt_col.iter() {
        let mut warn_flag: bool = true;
        let uhy = ex.utf8().unwrap();
        for (idx, x) in uhy.into_iter().enumerate() {
            if lookup_table.contains_key(x.unwrap()) {
                tax_ids.insert(idx, lookup_table.get(x.unwrap()).unwrap().to_string());
            } else {
                tax_ids.insert(idx, String::new());
                if warn_flag {
                    println!("Experiment {} does not name a taxon", x.unwrap());
                    warn_flag = false;
                }
            }
        }
    }

    output.hstack_mut(&[Series::new("taxid", tax_ids.into_iter())]).unwrap();

    let out_stream: File = File::create(args.output).unwrap();
    CsvWriter::new(out_stream)
        .has_header(true)
        .finish(&mut output)
        .unwrap_or_else(|error| panic!("Something wrong writing file: {:?}", error));

    println!("Processed {} in {} seconds", n_files, timer.elapsed().as_secs());
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use std::ffi::CString;

macro_rules! extensions {
    ($sname:ident, $($ext:ident => $s:expr,)*) => (
        /// List of extensions that are enabled or available.
        #[derive(Debug, Copy, Clone, PartialEq, Eq)]
        #[allow(missing_docs)]
        pub struct $sname {
            $(
                pub $ext: bool,
            )*
        }

        impl $sname {
            /// Returns an `Extensions` object with all members set to `false`.
            #[inline]
            pub fn none() -> $sname {
                $sname {
                    $($ext: false,)*
                }
            }

            /// Builds a Vec containing the list of extensions.
            pub fn build_extensions_list(&self) -> Vec<CString> {
                let mut data = Vec::new();
                $(if self.$ext { data.push(CString::new(&$s[..]).unwrap()); })*
                data
            }
        }
    );
}

extensions! {
    InstanceExtensions,
    khr_surface => b"VK_KHR_surface",
    khr_display => b"VK_KHR_display",
    khr_xlib_surface => b"VK_KHR_xlib_surface",
    khr_xcb_surface => b"VK_KHR_xcb_surface",
    khr_wayland_surface => b"VK_KHR_wayland_surface",
    khr_mir_surface => b"VK_KHR_mir_surface",
    khr_android_surface => b"VK_KHR_android_surface",
    khr_win32_surface => b"VK_KHR_win32_surface",
    ext_debug_report => b"VK_EXT_debug_report",
}

extensions! {
    DeviceExtensions,
    khr_swapchain => b"VK_KHR_swapchain",
    khr_display_swapchain => b"VK_KHR_display_swapchain",
}

#[cfg(test)]
mod tests {
    use instance::InstanceExtensions;

    #[test]
    fn empty_extensions() {
        let s = InstanceExtensions::none().build_extensions_list();
        assert!(s.is_empty());
    }
}
<commit_msg>Slightly improve test for empty extensions<commit_after>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use std::ffi::CString;

macro_rules! extensions {
    ($sname:ident, $($ext:ident => $s:expr,)*) => (
        /// List of extensions that are enabled or available.
        #[derive(Debug, Copy, Clone, PartialEq, Eq)]
        #[allow(missing_docs)]
        pub struct $sname {
            $(
                pub $ext: bool,
            )*
        }

        impl $sname {
            /// Returns an `Extensions` object with all members set to `false`.
            #[inline]
            pub fn none() -> $sname {
                $sname {
                    $($ext: false,)*
                }
            }

            /// Builds a Vec containing the list of extensions.
            pub fn build_extensions_list(&self) -> Vec<CString> {
                let mut data = Vec::new();
                $(if self.$ext { data.push(CString::new(&$s[..]).unwrap()); })*
                data
            }
        }
    );
}

extensions! {
    InstanceExtensions,
    khr_surface => b"VK_KHR_surface",
    khr_display => b"VK_KHR_display",
    khr_xlib_surface => b"VK_KHR_xlib_surface",
    khr_xcb_surface => b"VK_KHR_xcb_surface",
    khr_wayland_surface => b"VK_KHR_wayland_surface",
    khr_mir_surface => b"VK_KHR_mir_surface",
    khr_android_surface => b"VK_KHR_android_surface",
    khr_win32_surface => b"VK_KHR_win32_surface",
    ext_debug_report => b"VK_EXT_debug_report",
}

extensions! {
    DeviceExtensions,
    khr_swapchain => b"VK_KHR_swapchain",
    khr_display_swapchain => b"VK_KHR_display_swapchain",
}

#[cfg(test)]
mod tests {
    use instance::InstanceExtensions;
    use instance::DeviceExtensions;

    #[test]
    fn empty_extensions() {
        let i = InstanceExtensions::none().build_extensions_list();
        assert!(i.is_empty());

        let d = DeviceExtensions::none().build_extensions_list();
        assert!(d.is_empty());
    }
}
<|endoftext|>


<commit_before>use if_chain::if_chain;
use rustc_errors::Applicability;
use rustc_hir::*;
use rustc_lint::{LateContext, LateLintPass};
use rustc_session::{declare_lint_pass, declare_tool_lint};

use crate::utils::span_lint_and_sugg;

declare_clippy_lint! {
    /// **What it does:**
    /// Checks for function invocations of the form `primitive::from_str_radix(s, 10)`
    ///
    /// **Why is this bad?**
    /// This specific common use case can be rewritten as `s.parse::<primitive>()`
    /// (and in most cases, the turbofish can be removed), which reduces code length
    /// and complexity.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:**
    ///
    /// ```rust
    /// let input: &str = get_input();
    /// let num = u16::from_str_radix(input, 10)?;
    /// ```
    /// Use instead:
    /// ```rust
    /// let input: &str = get_input();
    /// let num: u16 = input.parse()?;
    /// ```
    pub FROM_STR_RADIX_10,
    style,
    "from_str_radix with radix 10"
}

declare_lint_pass!(FromStrRadix10 => [FROM_STR_RADIX_10]);

impl LateLintPass<'tcx> for FromStrRadix10 {
    fn check_expr(&mut self, cx: &LateContext<'tcx>, exp: &Expr<'tcx>) {
        if_chain! {
            if let ExprKind::Call(maybe_path, arguments) = &exp.kind;
            if let ExprKind::Path(qpath) = &maybe_path.kind;
            if let QPath::TypeRelative(ty, pathseg) = &qpath;

            // check if the first part of the path is some integer primitive
            if let TyKind::Path(ty_qpath) = &ty.kind;
            let ty_res = cx.qpath_res(ty_qpath, ty.hir_id);
            if let def::Res::PrimTy(prim_ty) = ty_res;
            if is_primitive_integer_ty(prim_ty);

            // check if the second part of the path indeed calls the associated
            // function `from_str_radix`
            if pathseg.ident.name.as_str() == "from_str_radix";

            // check if the second argument is a primitive `10`
            if arguments.len() == 2;
            if let ExprKind::Lit(lit) = &arguments[1].kind;
            if let rustc_ast::ast::LitKind::Int(10, _) = lit.node;

            then {
                let orig_string = crate::utils::snippet(cx, arguments[0].span, "string");
                span_lint_and_sugg(
                    cx,
                    FROM_STR_RADIX_10,
                    exp.span,
                    "This call to `from_str_radix` can be shortened to a call to str::parse",
                    "try",
                    format!("({}).parse()", orig_string),
                    Applicability::MaybeIncorrect
                );
            }
        }
    }
}

fn is_primitive_integer_ty(ty: PrimTy) -> bool {
    match ty {
        PrimTy::Int(_) => true,
        PrimTy::Uint(_) => true,
        _ => false,
    }
}
<commit_msg>Fixed doctests that shouldn't have been compiled<commit_after>use if_chain::if_chain;
use rustc_errors::Applicability;
use rustc_hir::*;
use rustc_lint::{LateContext, LateLintPass};
use rustc_session::{declare_lint_pass, declare_tool_lint};

use crate::utils::span_lint_and_sugg;

declare_clippy_lint! {
    /// **What it does:**
    /// Checks for function invocations of the form `primitive::from_str_radix(s, 10)`
    ///
    /// **Why is this bad?**
    /// This specific common use case can be rewritten as `s.parse::<primitive>()`
    /// (and in most cases, the turbofish can be removed), which reduces code length
    /// and complexity.
    ///
    /// **Known problems:** None.
    ///
    /// **Example:**
    ///
    /// ```ignore
    /// let input: &str = get_input();
    /// let num = u16::from_str_radix(input, 10)?;
    /// ```
    /// Use instead:
    /// ```ignore
    /// let input: &str = get_input();
    /// let num: u16 = input.parse()?;
    /// ```
    pub FROM_STR_RADIX_10,
    style,
    "from_str_radix with radix 10"
}

declare_lint_pass!(FromStrRadix10 => [FROM_STR_RADIX_10]);

impl LateLintPass<'tcx> for FromStrRadix10 {
    fn check_expr(&mut self, cx: &LateContext<'tcx>, exp: &Expr<'tcx>) {
        if_chain! {
            if let ExprKind::Call(maybe_path, arguments) = &exp.kind;
            if let ExprKind::Path(qpath) = &maybe_path.kind;
            if let QPath::TypeRelative(ty, pathseg) = &qpath;

            // check if the first part of the path is some integer primitive
            if let TyKind::Path(ty_qpath) = &ty.kind;
            let ty_res = cx.qpath_res(ty_qpath, ty.hir_id);
            if let def::Res::PrimTy(prim_ty) = ty_res;
            if is_primitive_integer_ty(prim_ty);

            // check if the second part of the path indeed calls the associated
            // function `from_str_radix`
            if pathseg.ident.name.as_str() == "from_str_radix";

            // check if the second argument is a primitive `10`
            if arguments.len() == 2;
            if let ExprKind::Lit(lit) = &arguments[1].kind;
            if let rustc_ast::ast::LitKind::Int(10, _) = lit.node;

            then {
                let orig_string = crate::utils::snippet(cx, arguments[0].span, "string");
                span_lint_and_sugg(
                    cx,
                    FROM_STR_RADIX_10,
                    exp.span,
                    "This call to `from_str_radix` can be shortened to a call to str::parse",
                    "try",
                    format!("({}).parse()", orig_string),
                    Applicability::MaybeIncorrect
                );
            }
        }
    }
}

fn is_primitive_integer_ty(ty: PrimTy) -> bool {
    match ty {
        PrimTy::Int(_) => true,
        PrimTy::Uint(_) => true,
        _ => false,
    }
}
<|endoftext|>


<commit_before>use std::path::Path;
use std::io::{Seek, SeekFrom, Write, Read};
use std::fs::{OpenOptions, File};

use utils::counter::Counter;
use page_manager::{UnfullPage, Page, PageId, BlockId, PageStore, PAGESIZE, BLOCKSIZE};

pub struct FsPageManager {
    pages: File,
    count: Counter,
    last_page_last_block: BlockId,
    unpopulated_pages: Vec<PageId>,
}

impl FsPageManager {
    pub fn new(path: &Path) -> Self {
        FsPageManager {
            pages: OpenOptions::new()
                .read(true)
                .write(true)
                .create(true)
                .truncate(true)
                .open(path)
                .unwrap(),
            count: Counter::new(),
            last_page_last_block: BlockId(PAGESIZE as u16),
            unpopulated_pages: Vec::new(),
        }
    }

    fn write_page(&mut self, page: Page, page_id: PageId) {
        let id = page_id.0;
        let mut f = self.pages.try_clone().unwrap();
        f.seek(SeekFrom::Start(id * PAGESIZE as u64 * BLOCKSIZE as u64)).unwrap();
        f.write_all(page.as_slice()).unwrap();
        self.last_page_last_block = BlockId(PAGESIZE as u16);
    }
}

impl PageStore for FsPageManager {
    fn store_full(&mut self, page: Page) -> PageId {
        let id = self.unpopulated_pages.pop().unwrap_or(PageId(self.count.retrieve_and_inc()));
        self.write_page(page, id);
        id
    }

    fn store_unfull(&mut self, page: Page, block_id: BlockId) -> UnfullPage {
        let (mut container_page, page_id) = if self.last_page_last_block.0 + block_id.0 >
                                               PAGESIZE as u16 {
            // New Page
            self.last_page_last_block = BlockId(1);
            (Page::empty(), PageId(self.count.retrieve_and_inc()))
        } else {
            // Fits on same page
            let page_id = PageId(self.count.retrieve() - 1);
            (self.get_page(page_id), page_id)
        };
        let first_block = self.last_page_last_block;
        // First byte of an unfull page acts as reference counter.
        container_page[BlockId::first()].0[0] += 1;
        // Now copy the full blocks over to the unfull page
        for i in 0..block_id.0 {
            container_page[BlockId(first_block.0 + i)] = page[BlockId(i)];
        }
        // Write the new page
        self.write_page(container_page, page_id);
        // And set the last_page_last_block
        self.last_page_last_block = BlockId(first_block.0 + block_id.0);
        UnfullPage::new(page_id, first_block, self.last_page_last_block)
    }

    #[inline]
    fn delete_page(&mut self, page_id: PageId) {
        self.unpopulated_pages.push(page_id);
    }

    ///This method deletes an unfull page.
    ///Actually it just decreases the refcount of that page
    ///In case it becomes zero, the page is added to the unpopulated pages
    ///Otherwise the new refcount is written
    fn delete_unfull(&mut self, page_id: PageId) {
        let mut refcount: [u8; 1] = [0; 1];
        let mut f = self.pages.try_clone().unwrap();
        //Seek start of page
        f.seek(SeekFrom::Start(page_id.0 * PAGESIZE as u64 * BLOCKSIZE as u64)).unwrap();
        //Read one byte...
        f.read_exact(&mut refcount).unwrap();
        //Decrease it
        refcount[0] -= 1;
        //If its zero it means no more relevant data is on that page
        //Throw it into the unpopulated pages
        if refcount[0] == 0 {
            self.unpopulated_pages.push(page_id);
        } else {
            //Otherwise we have to write the refcount back to page... alas
            f.seek(SeekFrom::Start(page_id.0 * PAGESIZE as u64 * BLOCKSIZE as u64)).unwrap();
            f.write(&refcount).unwrap();
        }
    }

    fn get_page(&self, page_id: PageId) -> Page {
        let mut f = self.pages.try_clone().unwrap();
        f.seek(SeekFrom::Start(page_id.0 * PAGESIZE as u64 * BLOCKSIZE as u64)).unwrap();
        Page::from_read(&mut f)
    }
}

#[cfg(test)]
mod tests {
    use test_utils::create_test_dir;

    use super::FsPageManager;
    use page_manager::{UnfullPage, Page, PageStore, PageId, Block, BlockId, BLOCKSIZE, PAGESIZE};

    fn new_pmgr(name: &str) -> FsPageManager {
        let path = &create_test_dir(format!("fs_page_manager/{}", name).as_str());
        FsPageManager::new(&path.join("pages.bin"))
    }

    #[test]
    fn delete_unfull_basic() {
        let mut pmgr = new_pmgr("delete_unfull_basic");
        assert_eq!(pmgr.store_unfull(Page::empty(), BlockId(1)),
                   UnfullPage::new(PageId(0), BlockId(1), BlockId(2)));
        pmgr.delete_unfull(PageId(0));
        assert_eq!(pmgr.unpopulated_pages, vec![PageId(0)]);
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
    }

    #[test]
    fn delete_multitenant_unfull() {
        let mut pmgr = new_pmgr("multitenant_unfull");
        let mut ref_p = Page::empty();
        for i in 0..PAGESIZE - 1 {
            ref_p[BlockId(i as u16 + 1u16)] = Block([(i % 255) as u8; BLOCKSIZE]);
            let mut p = Page::empty();
            p[BlockId::first()] = Block([(i % 255) as u8; BLOCKSIZE]);
            assert_eq!(pmgr.store_unfull(p, BlockId(1)),
                       UnfullPage::new(PageId(0), BlockId(i as u16 + 1), BlockId(i as u16 + 2)));
        }
        ref_p[BlockId::first()].0[0] = (PAGESIZE - 1) as u8;
        assert_eq!(pmgr.get_page(PageId(0)), ref_p);
        pmgr.delete_unfull(PageId(0));
        assert_eq!(pmgr.unpopulated_pages, vec![]);
        assert_eq!(pmgr.store_full(Page::empty()), PageId(1));
        for _ in 0..PAGESIZE -2 {
            assert_eq!(pmgr.unpopulated_pages, vec![]);
            pmgr.delete_unfull(PageId(0));
        }
        assert_eq!(pmgr.unpopulated_pages, vec![PageId(0)]);
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
    }

    #[test]
    fn basic_unfull() {
        let mut pmgr = new_pmgr("basic_unfull");
        assert_eq!(pmgr.store_unfull(Page::empty(), BlockId(1)),
                   UnfullPage::new(PageId(0), BlockId(1), BlockId(2)));
        let mut p = Page::empty();
        p[BlockId::first()].0[0] = 1;
        assert_eq!(pmgr.get_page(PageId(0)), p);
    }

    #[test]
    fn filled_unfull() {
        let mut pmgr = new_pmgr("filled_unfull");
        let mut p = Page::empty();
        let mut ref_p = Page::empty();
        for i in 0..PAGESIZE - 1 {
            p[BlockId(i as u16)] = Block([(i % 255) as u8; BLOCKSIZE]);
            ref_p[BlockId(i as u16 + 1u16)] = Block([(i % 255) as u8; BLOCKSIZE]);
        }
        ref_p[BlockId::first()].0[0] = 1;
        assert_eq!(pmgr.store_unfull(p, BlockId::last()),
                   UnfullPage::new(PageId(0), BlockId(1), BlockId(PAGESIZE as u16)));
        assert_eq!(pmgr.get_page(PageId(0)), ref_p);
    }

    #[test]
    fn multitenant_unfull() {
        let mut pmgr = new_pmgr("multitenant_unfull");
        let mut ref_p = Page::empty();
        for i in 0..PAGESIZE - 1 {
            ref_p[BlockId(i as u16 + 1u16)] = Block([(i % 255) as u8; BLOCKSIZE]);
            let mut p = Page::empty();
            p[BlockId::first()] = Block([(i % 255) as u8; BLOCKSIZE]);
            assert_eq!(pmgr.store_unfull(p, BlockId(1)),
                       UnfullPage::new(PageId(0), BlockId(i as u16 + 1), BlockId(i as u16 + 2)));
        }
        ref_p[BlockId::first()].0[0] = (PAGESIZE - 1) as u8;
        assert_eq!(pmgr.get_page(PageId(0)), ref_p);
    }

    #[test]
    fn overflowing_unfull() {
        let mut pmgr = new_pmgr("overflowing_unfull");
        let mut ref_p = Page::empty();
        for i in 0..PAGESIZE - 1 {
            ref_p[BlockId(i as u16 + 1u16)] = Block([(i % 255) as u8; BLOCKSIZE]);
            let mut p = Page::empty();
            p[BlockId::first()] = Block([(i % 255) as u8; BLOCKSIZE]);
            assert_eq!(pmgr.store_unfull(p, BlockId(1)),
                       UnfullPage::new(PageId(0), BlockId(i as u16 + 1), BlockId(i as u16 + 2)));
        }
        for i in 0..PAGESIZE - 1 {
            let mut p = Page::empty();
            p[BlockId::first()] = Block([(i % 255) as u8; BLOCKSIZE]);
            assert_eq!(pmgr.store_unfull(p, BlockId(1)),
                       UnfullPage::new(PageId(1), BlockId(i as u16 + 1), BlockId(i as u16 + 2)));
        }
        ref_p[BlockId::first()].0[0] = (PAGESIZE - 1) as u8;
        assert_eq!(pmgr.get_page(PageId(0)), ref_p);
        assert_eq!(pmgr.get_page(PageId(1)), ref_p);
    }

    #[test]
    fn unfull_after_full() {
        let mut pmgr = new_pmgr("unfull_after_full");
        let mut p = Page::empty();
        p[BlockId::first()] = Block([1; BLOCKSIZE]);
        assert_eq!(pmgr.store_full(p), PageId(0));
        assert_eq!(pmgr.get_page(PageId(0)), p);
        assert_eq!(pmgr.store_unfull(Page::empty(), BlockId(1)),
                   UnfullPage::new(PageId(1), BlockId(1), BlockId(2)));
        let mut unf_p = Page::empty();
        unf_p[BlockId::first()].0[0] = 1;
        assert_eq!(pmgr.get_page(PageId(1)), unf_p);
        assert_eq!(pmgr.store_full(p), PageId(2));
        assert_eq!(pmgr.get_page(PageId(2)), p);
        assert_eq!(pmgr.store_unfull(Page::empty(), BlockId(1)),
                   UnfullPage::new(PageId(3), BlockId(1), BlockId(2)));
        assert_eq!(pmgr.get_page(PageId(1)), unf_p);
    }

    #[test]
    fn store_page() {
        let path = &create_test_dir("fs_page_manager/store_page");
        let mut pmgr = FsPageManager::new(&path.join("pages.bin"));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
    }

    #[test]
    fn delete_page() {
        let path = &create_test_dir("fs_page_manager/delete_page");
        let mut pmgr = FsPageManager::new(&path.join("pages.bin"));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(1));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(2));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(3));
        pmgr.delete_page(PageId(1));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(1));
    }

    #[test]
    fn get_page() {
        let path = &create_test_dir("fs_page_manager/get_page");
        let mut pmgr = FsPageManager::new(&path.join("pages.bin"));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
        assert_eq!(pmgr.get_page(PageId(0)), Page::empty());
        let mut p = Page::empty();
        p[BlockId::first()] = Block([1; BLOCKSIZE]);
        assert_eq!(pmgr.store_full(p), PageId(1));
        assert_eq!(pmgr.get_page(PageId(1)), p);
    }

    #[test]
    fn combined() {
        let path = &create_test_dir("fs_page_manager/combined");
        let mut pmgr = FsPageManager::new(&path.join("pages.bin"));
        let mut p = Page::empty();
        p[BlockId::first()] = Block([1; BLOCKSIZE]);
        p[BlockId(1)] = Block([2; BLOCKSIZE]);
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
        assert_eq!(pmgr.store_full(p), PageId(1));
        assert_eq!(pmgr.store_full(p), PageId(2));
        p[BlockId(2)] = Block([3; BLOCKSIZE]);
        assert_eq!(pmgr.store_full(p), PageId(3));
        assert_eq!(pmgr.get_page(PageId(3)), p);
        assert!(pmgr.get_page(PageId(1)) != p);
        pmgr.delete_page(PageId(1));
        assert_eq!(pmgr.store_full(p), PageId(1));
        assert_eq!(pmgr.get_page(PageId(1)), p);
    }
}
<commit_msg>fix bug in unittests<commit_after>use std::path::Path;
use std::io::{Seek, SeekFrom, Write, Read};
use std::fs::{OpenOptions, File};

use utils::counter::Counter;
use page_manager::{UnfullPage, Page, PageId, BlockId, PageStore, PAGESIZE, BLOCKSIZE};

pub struct FsPageManager {
    pages: File,
    count: Counter,
    last_page_last_block: BlockId,
    unpopulated_pages: Vec<PageId>,
}

impl FsPageManager {
    pub fn new(path: &Path) -> Self {
        FsPageManager {
            pages: OpenOptions::new()
                .read(true)
                .write(true)
                .create(true)
                .truncate(true)
                .open(path)
                .unwrap(),
            count: Counter::new(),
            last_page_last_block: BlockId(PAGESIZE as u16),
            unpopulated_pages: Vec::new(),
        }
    }

    fn write_page(&mut self, page: Page, page_id: PageId) {
        let id = page_id.0;
        let mut f = self.pages.try_clone().unwrap();
        f.seek(SeekFrom::Start(id * PAGESIZE as u64 * BLOCKSIZE as u64)).unwrap();
        f.write_all(page.as_slice()).unwrap();
        self.last_page_last_block = BlockId(PAGESIZE as u16);
    }
}

impl PageStore for FsPageManager {
    fn store_full(&mut self, page: Page) -> PageId {
        let id = self.unpopulated_pages.pop().unwrap_or(PageId(self.count.retrieve_and_inc()));
        self.write_page(page, id);
        id
    }

    fn store_unfull(&mut self, page: Page, block_id: BlockId) -> UnfullPage {
        let (mut container_page, page_id) = if self.last_page_last_block.0 + block_id.0 >
                                               PAGESIZE as u16 {
            // New Page
            self.last_page_last_block = BlockId(1);
            (Page::empty(), PageId(self.count.retrieve_and_inc()))
        } else {
            // Fits on same page
            let page_id = PageId(self.count.retrieve() - 1);
            (self.get_page(page_id), page_id)
        };
        let first_block = self.last_page_last_block;
        // First byte of an unfull page acts as reference counter.
        container_page[BlockId::first()].0[0] += 1;
        // Now copy the full blocks over to the unfull page
        for i in 0..block_id.0 {
            container_page[BlockId(first_block.0 + i)] = page[BlockId(i)];
        }
        // Write the new page
        self.write_page(container_page, page_id);
        // And set the last_page_last_block
        self.last_page_last_block = BlockId(first_block.0 + block_id.0);
        UnfullPage::new(page_id, first_block, self.last_page_last_block)
    }

    #[inline]
    fn delete_page(&mut self, page_id: PageId) {
        self.unpopulated_pages.push(page_id);
    }

    ///This method deletes an unfull page.
    ///Actually it just decreases the refcount of that page
    ///In case it becomes zero, the page is added to the unpopulated pages
    ///Otherwise the new refcount is written
    fn delete_unfull(&mut self, page_id: PageId) {
        let mut refcount: [u8; 1] = [0; 1];
        let mut f = self.pages.try_clone().unwrap();
        //Seek start of page
        f.seek(SeekFrom::Start(page_id.0 * PAGESIZE as u64 * BLOCKSIZE as u64)).unwrap();
        //Read one byte...
        f.read_exact(&mut refcount).unwrap();
        //Decrease it
        refcount[0] -= 1;
        //If its zero it means no more relevant data is on that page
        //Throw it into the unpopulated pages
        if refcount[0] == 0 {
            self.unpopulated_pages.push(page_id);
        } else {
            //Otherwise we have to write the refcount back to page... alas
            f.seek(SeekFrom::Start(page_id.0 * PAGESIZE as u64 * BLOCKSIZE as u64)).unwrap();
            f.write(&refcount).unwrap();
        }
    }

    fn get_page(&self, page_id: PageId) -> Page {
        let mut f = self.pages.try_clone().unwrap();
        f.seek(SeekFrom::Start(page_id.0 * PAGESIZE as u64 * BLOCKSIZE as u64)).unwrap();
        Page::from_read(&mut f)
    }
}

#[cfg(test)]
mod tests {
    use test_utils::create_test_dir;

    use super::FsPageManager;
    use page_manager::{UnfullPage, Page, PageStore, PageId, Block, BlockId, BLOCKSIZE, PAGESIZE};

    fn new_pmgr(name: &str) -> FsPageManager {
        let path = &create_test_dir(format!("fs_page_manager/{}", name).as_str());
        FsPageManager::new(&path.join("pages.bin"))
    }

    #[test]
    fn delete_unfull_basic() {
        let mut pmgr = new_pmgr("delete_unfull_basic");
        assert_eq!(pmgr.store_unfull(Page::empty(), BlockId(1)),
                   UnfullPage::new(PageId(0), BlockId(1), BlockId(2)));
        pmgr.delete_unfull(PageId(0));
        assert_eq!(pmgr.unpopulated_pages, vec![PageId(0)]);
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
    }

    #[test]
    fn delete_multitenant_unfull() {
        let mut pmgr = new_pmgr("delete_multitenant_unfull");
        let mut ref_p = Page::empty();
        for i in 0..PAGESIZE - 1 {
            ref_p[BlockId(i as u16 + 1u16)] = Block([(i % 255) as u8; BLOCKSIZE]);
            let mut p = Page::empty();
            p[BlockId::first()] = Block([(i % 255) as u8; BLOCKSIZE]);
            assert_eq!(pmgr.store_unfull(p, BlockId(1)),
                       UnfullPage::new(PageId(0), BlockId(i as u16 + 1), BlockId(i as u16 + 2)));
        }
        ref_p[BlockId::first()].0[0] = (PAGESIZE - 1) as u8;
        assert_eq!(pmgr.get_page(PageId(0)), ref_p);
        pmgr.delete_unfull(PageId(0));
        assert_eq!(pmgr.unpopulated_pages, vec![]);
        assert_eq!(pmgr.store_full(Page::empty()), PageId(1));
        for _ in 0..PAGESIZE -2 {
            assert_eq!(pmgr.unpopulated_pages, vec![]);
            pmgr.delete_unfull(PageId(0));
        }
        assert_eq!(pmgr.unpopulated_pages, vec![PageId(0)]);
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
    }

    #[test]
    fn basic_unfull() {
        let mut pmgr = new_pmgr("basic_unfull");
        assert_eq!(pmgr.store_unfull(Page::empty(), BlockId(1)),
                   UnfullPage::new(PageId(0), BlockId(1), BlockId(2)));
        let mut p = Page::empty();
        p[BlockId::first()].0[0] = 1;
        assert_eq!(pmgr.get_page(PageId(0)), p);
    }

    #[test]
    fn filled_unfull() {
        let mut pmgr = new_pmgr("filled_unfull");
        let mut p = Page::empty();
        let mut ref_p = Page::empty();
        for i in 0..PAGESIZE - 1 {
            p[BlockId(i as u16)] = Block([(i % 255) as u8; BLOCKSIZE]);
            ref_p[BlockId(i as u16 + 1u16)] = Block([(i % 255) as u8; BLOCKSIZE]);
        }
        ref_p[BlockId::first()].0[0] = 1;
        assert_eq!(pmgr.store_unfull(p, BlockId::last()),
                   UnfullPage::new(PageId(0), BlockId(1), BlockId(PAGESIZE as u16)));
        assert_eq!(pmgr.get_page(PageId(0)), ref_p);
    }

    #[test]
    fn multitenant_unfull() {
        let mut pmgr = new_pmgr("multitenant_unfull");
        let mut ref_p = Page::empty();
        for i in 0..PAGESIZE - 1 {
            ref_p[BlockId(i as u16 + 1u16)] = Block([(i % 255) as u8; BLOCKSIZE]);
            let mut p = Page::empty();
            p[BlockId::first()] = Block([(i % 255) as u8; BLOCKSIZE]);
            assert_eq!(pmgr.store_unfull(p, BlockId(1)),
                       UnfullPage::new(PageId(0), BlockId(i as u16 + 1), BlockId(i as u16 + 2)));
        }
        ref_p[BlockId::first()].0[0] = (PAGESIZE - 1) as u8;
        assert_eq!(pmgr.get_page(PageId(0)), ref_p);
    }

    #[test]
    fn overflowing_unfull() {
        let mut pmgr = new_pmgr("overflowing_unfull");
        let mut ref_p = Page::empty();
        for i in 0..PAGESIZE - 1 {
            ref_p[BlockId(i as u16 + 1u16)] = Block([(i % 255) as u8; BLOCKSIZE]);
            let mut p = Page::empty();
            p[BlockId::first()] = Block([(i % 255) as u8; BLOCKSIZE]);
            assert_eq!(pmgr.store_unfull(p, BlockId(1)),
                       UnfullPage::new(PageId(0), BlockId(i as u16 + 1), BlockId(i as u16 + 2)));
        }
        for i in 0..PAGESIZE - 1 {
            let mut p = Page::empty();
            p[BlockId::first()] = Block([(i % 255) as u8; BLOCKSIZE]);
            assert_eq!(pmgr.store_unfull(p, BlockId(1)),
                       UnfullPage::new(PageId(1), BlockId(i as u16 + 1), BlockId(i as u16 + 2)));
        }
        ref_p[BlockId::first()].0[0] = (PAGESIZE - 1) as u8;
        assert_eq!(pmgr.get_page(PageId(0)), ref_p);
        assert_eq!(pmgr.get_page(PageId(1)), ref_p);
    }

    #[test]
    fn unfull_after_full() {
        let mut pmgr = new_pmgr("unfull_after_full");
        let mut p = Page::empty();
        p[BlockId::first()] = Block([1; BLOCKSIZE]);
        assert_eq!(pmgr.store_full(p), PageId(0));
        assert_eq!(pmgr.get_page(PageId(0)), p);
        assert_eq!(pmgr.store_unfull(Page::empty(), BlockId(1)),
                   UnfullPage::new(PageId(1), BlockId(1), BlockId(2)));
        let mut unf_p = Page::empty();
        unf_p[BlockId::first()].0[0] = 1;
        assert_eq!(pmgr.get_page(PageId(1)), unf_p);
        assert_eq!(pmgr.store_full(p), PageId(2));
        assert_eq!(pmgr.get_page(PageId(2)), p);
        assert_eq!(pmgr.store_unfull(Page::empty(), BlockId(1)),
                   UnfullPage::new(PageId(3), BlockId(1), BlockId(2)));
        assert_eq!(pmgr.get_page(PageId(1)), unf_p);
    }

    #[test]
    fn store_page() {
        let path = &create_test_dir("fs_page_manager/store_page");
        let mut pmgr = FsPageManager::new(&path.join("pages.bin"));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
    }

    #[test]
    fn delete_page() {
        let path = &create_test_dir("fs_page_manager/delete_page");
        let mut pmgr = FsPageManager::new(&path.join("pages.bin"));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(1));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(2));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(3));
        pmgr.delete_page(PageId(1));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(1));
    }

    #[test]
    fn get_page() {
        let path = &create_test_dir("fs_page_manager/get_page");
        let mut pmgr = FsPageManager::new(&path.join("pages.bin"));
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
        assert_eq!(pmgr.get_page(PageId(0)), Page::empty());
        let mut p = Page::empty();
        p[BlockId::first()] = Block([1; BLOCKSIZE]);
        assert_eq!(pmgr.store_full(p), PageId(1));
        assert_eq!(pmgr.get_page(PageId(1)), p);
    }

    #[test]
    fn combined() {
        let path = &create_test_dir("fs_page_manager/combined");
        let mut pmgr = FsPageManager::new(&path.join("pages.bin"));
        let mut p = Page::empty();
        p[BlockId::first()] = Block([1; BLOCKSIZE]);
        p[BlockId(1)] = Block([2; BLOCKSIZE]);
        assert_eq!(pmgr.store_full(Page::empty()), PageId(0));
        assert_eq!(pmgr.store_full(p), PageId(1));
        assert_eq!(pmgr.store_full(p), PageId(2));
        p[BlockId(2)] = Block([3; BLOCKSIZE]);
        assert_eq!(pmgr.store_full(p), PageId(3));
        assert_eq!(pmgr.get_page(PageId(3)), p);
        assert!(pmgr.get_page(PageId(1)) != p);
        pmgr.delete_page(PageId(1));
        assert_eq!(pmgr.store_full(p), PageId(1));
        assert_eq!(pmgr.get_page(PageId(1)), p);
    }
}
<|endoftext|>


<commit_before>//! A module to match and filter Request and Response events

use log_parser::*;

#[derive(Clone)]
#[derive(Debug)]
pub struct RequestResponsePair {
    pub request: log_events::Request,
    pub response: log_events::Response,
}

fn get_matching_response<'a>(request: &log_events::Request, responses: &'a Vec<log_events::Response>) -> Option<&'a log_events::Response> {
    match responses.binary_search_by_key(&request.id, |r| r.id) {
        Ok(index) => Some(&responses[index]),
        Err(_) => None
    }
}

pub fn extract_matching_request_response_pairs(requests: &mut Vec<log_events::Request>, responses: &mut Vec<log_events::Response>) -> Vec<RequestResponsePair> {
    let mut request_response_pairs: Vec<RequestResponsePair> = Vec::new();

    for request_index in 0..requests.len() {
        let matching_response_index: Option<usize> = responses.iter().position(|response| requests[request_index].id == response.id );

        if matching_response_index.is_some() {
            let request = requests.remove(request_index);
            let response = responses.remove(matching_response_index.unwrap());

            let pair = RequestResponsePair {
                request: request,
                response: response
            };

            request_response_pairs.push(pair);
        }
    }

    request_response_pairs
}

#[cfg(test)]
mod tests {
    use chrono::*;
    use http_status::*;
    use super::*;
    use log_parser;

    #[test]
    fn test_extract_matching_request_response_pairs() {
        let mut requests = vec![
            log_parser::log_events::Request {
                id: 1,
                time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200", "%d/%b/%Y:%H:%M:%S %z").unwrap(),
                url: "/some/path.html".to_string(),
                original_log_line: "whatever".to_string(),
            },
        ];

        let mut responses = vec![
            log_parser::log_events::Response {
                id: 1,
                time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200", "%d/%b/%Y:%H:%M:%S %z").unwrap(),
                mime_type: "text/html".to_string(),
                response_time: Duration::milliseconds(7),
                http_status: HttpStatus::OK,
                original_log_line: "whatever".to_string(),
            },
            log_parser::log_events::Response {
                id: 99,
                time: DateTime::parse_from_str("08/Apr/2016:09:58:47 +0200", "%d/%b/%Y:%H:%M:%S %z").unwrap(),
                mime_type: "text/html".to_string(),
                response_time: Duration::milliseconds(10),
                http_status: HttpStatus::OK,
                original_log_line: "whatever".to_string(),
            },
        ];

        let result = extract_matching_request_response_pairs(&mut requests, &mut responses);

        assert_eq!(result.len(), 1);
        assert_eq!(result[0].request.id, 1);
        assert_eq!(result[0].response.id, 1);

        assert_eq!(requests.len(), 0);
        assert_eq!(responses.len(), 1);
    }
}
<commit_msg>Remove unused private function<commit_after>//! A module to match and filter Request and Response events

use log_parser::*;

#[derive(Clone)]
#[derive(Debug)]
pub struct RequestResponsePair {
    pub request: log_events::Request,
    pub response: log_events::Response,
}

pub fn extract_matching_request_response_pairs(requests: &mut Vec<log_events::Request>, responses: &mut Vec<log_events::Response>) -> Vec<RequestResponsePair> {
    let mut request_response_pairs: Vec<RequestResponsePair> = Vec::new();

    for request_index in 0..requests.len() {
        let matching_response_index: Option<usize> = responses.iter().position(|response| requests[request_index].id == response.id );

        if matching_response_index.is_some() {
            let request = requests.remove(request_index);
            let response = responses.remove(matching_response_index.unwrap());

            let pair = RequestResponsePair {
                request: request,
                response: response
            };

            request_response_pairs.push(pair);
        }
    }

    request_response_pairs
}

#[cfg(test)]
mod tests {
    use chrono::*;
    use http_status::*;
    use super::*;
    use log_parser;

    #[test]
    fn test_extract_matching_request_response_pairs() {
        let mut requests = vec![
            log_parser::log_events::Request {
                id: 1,
                time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200", "%d/%b/%Y:%H:%M:%S %z").unwrap(),
                url: "/some/path.html".to_string(),
                original_log_line: "whatever".to_string(),
            },
        ];

        let mut responses = vec![
            log_parser::log_events::Response {
                id: 1,
                time: DateTime::parse_from_str("08/Apr/2016:09:57:47 +0200", "%d/%b/%Y:%H:%M:%S %z").unwrap(),
                mime_type: "text/html".to_string(),
                response_time: Duration::milliseconds(7),
                http_status: HttpStatus::OK,
                original_log_line: "whatever".to_string(),
            },
            log_parser::log_events::Response {
                id: 99,
                time: DateTime::parse_from_str("08/Apr/2016:09:58:47 +0200", "%d/%b/%Y:%H:%M:%S %z").unwrap(),
                mime_type: "text/html".to_string(),
                response_time: Duration::milliseconds(10),
                http_status: HttpStatus::OK,
                original_log_line: "whatever".to_string(),
            },
        ];

        let result = extract_matching_request_response_pairs(&mut requests, &mut responses);

        assert_eq!(result.len(), 1);
        assert_eq!(result[0].request.id, 1);
        assert_eq!(result[0].response.id, 1);

        assert_eq!(requests.len(), 0);
        assert_eq!(responses.len(), 1);
    }
}
<|endoftext|>


<commit_before>use super::camera::*;
use super::event_manager::*;
use GameContext;
use glium::glutin::{CursorState, ElementState, Event, MouseButton, VirtualKeyCode};
use base::math::*;
use base::world::*;
use std::rc::Rc;
use super::world_manager::*;
use game::*;

const VELOCITY: f32 = 1.5;
const LOW_VELOCITY: f32 = 1.0;
const FAST_VELOCITY: f32 = 3.0;
pub struct Player {
    cam: Camera,
    prev_mouse_pos: Option<(i32, i32)>,
    context: Rc<GameContext>,
    world_manager: WorldManager,
    walk_vel: Vector2<f32>,
    delta_z: f32,
    dz: f32,
    gravity: f32,
    tick_count: f32,
    is_jumping: bool,
    is_falling: bool,
    forward: bool,
    backward: bool,
    left: bool,
    right: bool,
    up: bool,
    down: bool,
    mouselock: bool,
    speed: bool,
}

impl Player {
    pub fn new(context: Rc<GameContext>, world_manager: WorldManager) -> Self {
        Player {
            cam: Camera {
                position: Point3::new(15.0, 10.0, 50.0),
                phi: -0.27,
                theta: 2.6,
            },
            world_manager: world_manager,
            prev_mouse_pos: None,
            context: context,
            delta_z: 0.0,
            gravity: 0.005,
            dz: 0.0,
            tick_count: 1.0,
            walk_vel: Vector2::new(0.0, 0.0),
            is_jumping: false,
            is_falling: false,
            forward: false,
            backward: false,
            left: false,
            right: false,
            up: false,
            down: false,
            mouselock: false,
            speed: false,
        }
    }
    pub fn get_height(&self) {
        let world = self.world_manager.get_world();

        let real_pos = Point2f::new(self.cam.position[0], self.cam.position[1]);

        let axial_pos = AxialPoint::from_real(real_pos);

        let argument = PillarIndex(axial_pos);
        let pillar = world.pillar_at(argument);
        // println!("--------- {:?}", pillar);

        //    let pillar = pillar.expect("chunk at player pos not loaded!");

        //   pillar.sections()[2]
        // let hexpil = world.pillar_at_axp(axial_pos);

        // println!("{:?}", hexpil);

        // let height = self.world_manager.
    }
    pub fn get_camera(&self) -> Camera {
        self.cam
    }
    pub fn jump(&mut self) {

        if self.is_jumping == false {
            self.dz = 0.11;
            self.is_jumping = true;
        }
    }

    pub fn update(&mut self, delta: f32) {

        // self.get_height();
        //  if !self.is_jumping && !self.is_falling {
        //     let height = self.get_height();
        //    self.cam.position[2] = 50.0;
        // }
        if self.cam.position[2] > 50.0 && !self.is_jumping {
            self.is_falling = true;
        }

        if self.is_jumping {
            println!("is falling {:?}", self.is_falling);
            self.is_falling = true;

            if self.delta_z > -100.0 {
                self.delta_z -= 1.0;
            }
        } else {
            self.cam.position[2] = 50.0;
        }

        if self.is_falling {
            self.dz = self.dz - self.gravity;

            if self.dz < -0.2 {
                self.dz = -0.2;
            }
            self.cam.move_up(self.dz);
        }

        if self.forward && !self.backward {
            if self.speed {
                if self.walk_vel[1] < FAST_VELOCITY {
                    self.walk_vel[1] += 0.1;
                }
            }
            if self.walk_vel[1] < VELOCITY {
                self.walk_vel[1] += 0.1;
            }
            self.cam.move_forward(self.walk_vel[1] * delta);
        }

        if self.backward && !self.forward {
            if self.walk_vel[1] < LOW_VELOCITY {
                self.walk_vel[1] += 0.1;
            }
            self.cam.move_backward(self.walk_vel[1] * delta);
        }

        if self.left && !self.right {
            if self.walk_vel[0] < LOW_VELOCITY {
                self.walk_vel[0] += 0.1;
            }
            self.cam.move_left(self.walk_vel[0] * delta);
        }

        if !self.left && self.right {
            if self.walk_vel[0] < LOW_VELOCITY {
                self.walk_vel[0] += 0.1;
            }
            self.cam.move_right(self.walk_vel[0] * delta);
        }

        if self.cam.position[2] <= 50.0 {
            self.is_falling = false;
            self.is_jumping = false;
        }
    }
}

impl EventHandler for Player {
    fn handle_event(&mut self, e: &Event) -> EventResponse {
        match *e {
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::W)) => {
                self.forward = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::W)) => {
                self.walk_vel[1] = 0.0;
                self.forward = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::S)) => {
                self.backward = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::S)) => {
                self.walk_vel[1] = 0.0;
                self.backward = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::A)) => {
                self.left = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::A)) => {
                self.walk_vel[0] = 0.0;
                self.left = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::D)) => {
                self.right = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::D)) => {
                self.walk_vel[0] = 0.0;
                self.right = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::Space)) => {
                self.jump();
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::Space)) => {
                self.up = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::LControl)) => {
                self.down = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::LControl)) => {
                self.down = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::LShift)) => {
                self.speed = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::LShift)) => {
                self.speed = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::F)) => {
                self.is_falling = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::F)) => {
                self.is_falling = false;
                EventResponse::Continue
            }
            Event::MouseInput(ElementState::Pressed, MouseButton::Left) => {
                info!("clicked");
                if let Some(window) = self.context.get_facade().get_window() {
                    let res = window.set_cursor_state(CursorState::Grab);
                    warn!("{:?}", res);
                } else {
                    warn!("Failed to obtain window from facade");
                }
                EventResponse::Continue
            }
            Event::MouseMoved(x, y) => {

                if let Some((prev_x, prev_y)) = self.prev_mouse_pos {
                    let x_diff = x - prev_x;
                    let y_diff = y - prev_y;
                    info!("x = {}, y = {}", x_diff, y_diff);
                    self.cam.change_dir(y_diff as f32 / 300.0, x_diff as f32 / 300.0);
                }

                self.prev_mouse_pos = Some((x, y));

                EventResponse::Continue
            }

            _ => EventResponse::NotHandled,
        }
    }
}
<commit_msg>Add player runing on Pillar (WIP)<commit_after>use super::camera::*;
use super::event_manager::*;
use GameContext;
use glium::glutin::{CursorState, ElementState, Event, MouseButton, VirtualKeyCode};
use base::math::*;
use base::world::*;
use std::rc::Rc;
use super::world_manager::*;
use game::*;

const VELOCITY: f32 = 1.5;
const LOW_VELOCITY: f32 = 1.0;
const FAST_VELOCITY: f32 = 3.0;
pub struct Player {
    cam: Camera,
    prev_mouse_pos: Option<(i32, i32)>,
    context: Rc<GameContext>,
    world_manager: WorldManager,
    walk_vel: Vector2<f32>,
    delta_z: f32,
    dz: f32,
    gravity: f32,
    tick_count: f32,
    is_jumping: bool,
    is_falling: bool,
    forward: bool,
    backward: bool,
    left: bool,
    right: bool,
    up: bool,
    down: bool,
    mouselock: bool,
    speed: bool,
}

impl Player {
    pub fn new(context: Rc<GameContext>, world_manager: WorldManager) -> Self {
        Player {
            cam: Camera {
                position: Point3::new(15.0, 10.0, 50.0),
                phi: -0.27,
                theta: 2.6,
            },
            world_manager: world_manager,
            prev_mouse_pos: None,
            context: context,
            delta_z: 0.0,
            gravity: 0.005,
            dz: 0.0,
            tick_count: 1.0,
            walk_vel: Vector2::new(0.0, 0.0),
            is_jumping: false,
            is_falling: false,
            forward: false,
            backward: false,
            left: false,
            right: false,
            up: false,
            down: false,
            mouselock: false,
            speed: false,
        }
    }
    pub fn get_height(&self) -> f32 {
        let world = self.world_manager.get_world();

        let real_pos = Point2f::new(self.cam.position[0], self.cam.position[1]);

        let axial_pos = AxialPoint::from_real(real_pos);

        let argument = PillarIndex(axial_pos);
        let pillar = world.pillar_at(argument);
        // let sect = pillar.unwrap_or_else(|| panic!("Pillar not found{:?}", pillar));
        // sect.sections()[0].top.to_real()

        if pillar.is_some() {
            let sect = pillar.unwrap().sections();
            sect[0].top.to_real()

        } else {
            0.0
        }
        // let pillar_section = pillar.unwrap().sections();
        //   let ref top = pillar_section[2];
        //    let pillar = pillar.expect("chunk at player pos not loaded!");

        //   pillar.sections()[2]
        // let hexpil = world.pillar_at_axp(axial_pos);

        //  println!("--------------------{:?}", pillar_section);

        // let height = self.world_manager.
    }
    pub fn get_camera(&self) -> Camera {
        self.cam
    }
    pub fn jump(&mut self) {

        if self.is_jumping == false {
            self.dz = 0.11;
            self.is_jumping = true;
        }
    }

    pub fn update(&mut self, delta: f32) {

        let height = self.get_height() + 1.0;
        println!("pil----------------- {}", height);
        println!("cam----------------- {}", self.cam.position[2]);

        if !self.is_jumping && !self.is_falling {
            if height < self.cam.position[2] {
                self.is_falling = true;
            } else {
                self.cam.position[2] = height;
            }

        }

        if self.cam.position[2] > height && !self.is_jumping {
            self.is_falling = true;
        }

        if self.is_jumping {
            println!("is falling {:?}", self.is_falling);
            self.is_falling = true;

            // Reduce vertical velocity smoothly
            if self.delta_z > -100.0 {
                self.delta_z -= 1.0;
            }
        }

        if self.is_falling {
            self.dz = self.dz - self.gravity;

            if self.dz < -0.2 {
                self.dz = -0.2;
            }
            self.cam.move_up(self.dz);
        }

        if self.forward && !self.backward {
            if self.speed {
                if self.walk_vel[1] < FAST_VELOCITY {
                    self.walk_vel[1] += 0.1;
                }
            }
            if self.walk_vel[1] < VELOCITY {
                self.walk_vel[1] += 0.1;
            }
            self.cam.move_forward(self.walk_vel[1] * delta);
        }

        if self.backward && !self.forward {
            if self.walk_vel[1] < LOW_VELOCITY {
                self.walk_vel[1] += 0.1;
            }
            self.cam.move_backward(self.walk_vel[1] * delta);
        }

        if self.left && !self.right {
            if self.walk_vel[0] < LOW_VELOCITY {
                self.walk_vel[0] += 0.1;
            }
            self.cam.move_left(self.walk_vel[0] * delta);
        }

        if !self.left && self.right {
            if self.walk_vel[0] < LOW_VELOCITY {
                self.walk_vel[0] += 0.1;
            }
            self.cam.move_right(self.walk_vel[0] * delta);
        }

        if self.cam.position[2] < height {
            self.is_falling = false;
            self.is_jumping = false;
            self.cam.position[2] = height;
        }
    }
}

impl EventHandler for Player {
    fn handle_event(&mut self, e: &Event) -> EventResponse {
        match *e {
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::W)) => {
                self.forward = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::W)) => {
                self.walk_vel[1] = 0.0;
                self.forward = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::S)) => {
                self.backward = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::S)) => {
                self.walk_vel[1] = 0.0;
                self.backward = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::A)) => {
                self.left = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::A)) => {
                self.walk_vel[0] = 0.0;
                self.left = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::D)) => {
                self.right = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::D)) => {
                self.walk_vel[0] = 0.0;
                self.right = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::Space)) => {
                self.jump();
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::Space)) => {
                self.up = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::LControl)) => {
                self.down = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::LControl)) => {
                self.down = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::LShift)) => {
                self.speed = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::LShift)) => {
                self.speed = false;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Pressed, _, Some(VirtualKeyCode::F)) => {
                self.is_falling = true;
                EventResponse::Continue
            }
            Event::KeyboardInput(ElementState::Released, _, Some(VirtualKeyCode::F)) => {
                self.is_falling = false;
                EventResponse::Continue
            }
            Event::MouseInput(ElementState::Pressed, MouseButton::Left) => {
                info!("clicked");
                if let Some(window) = self.context.get_facade().get_window() {
                    let res = window.set_cursor_state(CursorState::Grab);
                    warn!("{:?}", res);
                } else {
                    warn!("Failed to obtain window from facade");
                }
                EventResponse::Continue
            }
            Event::MouseMoved(x, y) => {

                if let Some((prev_x, prev_y)) = self.prev_mouse_pos {
                    let x_diff = x - prev_x;
                    let y_diff = y - prev_y;
                    info!("x = {}, y = {}", x_diff, y_diff);
                    self.cam.change_dir(y_diff as f32 / 300.0, x_diff as f32 / 300.0);
                }

                self.prev_mouse_pos = Some((x, y));

                EventResponse::Continue
            }

            _ => EventResponse::NotHandled,
        }
    }
}
<|endoftext|>