<commit_before>use std::process::exit;
use std::path::Path;
use std::env;

use clap;
use cargo_shim::{
    Profile,
    CargoPackage,
    CargoProject,
    CargoTarget,
    BuildType,
    BuildConfig,
    TargetKind,
    CargoResult,
    MessageFormat,
    target_to_build_target
};

use config::Config;
use emscripten::initialize_emscripten;
use error::Error;
use wasm;

pub struct BuildArgsMatcher< 'a > {
    pub matches: &'a clap::ArgMatches< 'a >,
    pub project: &'a CargoProject
}

impl< 'a > BuildArgsMatcher< 'a > {
    fn requested_build_type( &self ) -> BuildType {
        if self.matches.is_present( "release" ) {
            BuildType::Release
        } else {
            BuildType::Debug
        }
    }

    pub fn targeting_emscripten_asmjs( &self ) -> bool {
        !self.targeting_emscripten_wasm() && !self.targeting_native_wasm()
    }

    pub fn targeting_emscripten_wasm( &self ) -> bool {
        self.matches.is_present( "target-webasm-emscripten" )
    }

    pub fn targeting_native_wasm( &self ) -> bool {
        self.matches.is_present( "target-webasm" )
    }

    pub fn targeting_wasm( &self ) -> bool {
        self.targeting_emscripten_wasm() || self.targeting_native_wasm()
    }

    pub fn targeting_emscripten( &self ) -> bool {
        self.targeting_emscripten_wasm() || self.targeting_emscripten_asmjs()
    }

    fn use_system_emscripten( &self ) -> bool {
        self.matches.is_present( "use-system-emscripten" )
    }

    fn message_format( &self ) -> MessageFormat {
        if let Some( name ) = self.matches.value_of( "message-format" ) {
            match name {
                "human" => MessageFormat::Human,
                "json" => MessageFormat::Json,
                _ => unreachable!()
            }
        } else {
            MessageFormat::Human
        }
    }

    fn is_verbose( &self ) -> bool {
        self.matches.is_present( "verbose" )
    }

    fn build_type( &self ) -> BuildType {
        let build_type = self.requested_build_type();
        if self.targeting_native_wasm() && build_type == BuildType::Debug {
            // TODO: Remove this in the future.
            println_err!( "warning: debug builds on the wasm-unknown-unknown are currently totally broken" );
            println_err!( "         forcing a release build" );
            return BuildType::Release;
        }

        build_type
    }

    fn package( &self ) -> Result< Option< &CargoPackage >, Error > {
        if let Some( name ) = self.matches.value_of( "package" ) {
            match self.project.packages.iter().find( |package| package.name == name ) {
                None => Err( Error::ConfigurationError( format!( "package `{}` not found", name ) ) ),
                package => Ok( package )
            }
        } else {
            Ok( None )
        }
    }

    pub fn package_or_default( &self ) -> Result< &CargoPackage, Error > {
        Ok( self.package()?.unwrap_or_else( || self.project.default_package() ) )
    }

    fn target( &'a self, package: &'a CargoPackage ) -> Result< Option< &'a CargoTarget >, Error > {
        let targets = &package.targets;
        if self.matches.is_present( "lib" ) {
            match targets.iter().find( |target| target.kind == TargetKind::Lib ) {
                None => return Err( Error::ConfigurationError( format!( "no library targets found" ) ) ),
                target => Ok( target )
            }
        } else if let Some( name ) = self.matches.value_of( "bin" ) {
            match targets.iter().find( |target| target.kind == TargetKind::Bin && target.name == name ) {
                None => return Err( Error::ConfigurationError( format!( "no bin target named `{}`", name ) ) ),
                target => Ok( target )
            }
        } else if let Some( name ) = self.matches.value_of( "example" ) {
            match targets.iter().find( |target| target.kind == TargetKind::Example && target.name == name ) {
                None => return Err( Error::ConfigurationError( format!( "no example target named `{}`", name ) ) ),
                target => Ok( target )
            }
        } else if let Some( name ) = self.matches.value_of( "bench" ) {
            match targets.iter().find( |target| target.kind == TargetKind::Bench && target.name == name ) {
                None => return Err( Error::ConfigurationError( format!( "no bench target named `{}`", name ) ) ),
                target => Ok( target )
            }
        } else {
            Ok( None )
        }
    }

    pub fn target_or_select< F >( &'a self, package: &'a CargoPackage, filter: F ) -> Result< Vec< &'a CargoTarget >, Error >
        where for< 'r > F: Fn( &'r CargoTarget ) -> bool
    {
        Ok( self.target( package )?.map( |target| vec![ target ] ).unwrap_or_else( || {
            package.targets.iter().filter( |target| filter( target ) ).collect()
        }))
    }

    fn triplet_or_default( &self ) -> &str {
        if self.matches.is_present( "target-webasm") {
            "wasm32-unknown-unknown"
        } else if self.matches.is_present( "target-webasm-emscripten" ) {
            "wasm32-unknown-emscripten"
        } else {
            "asmjs-unknown-emscripten"
        }
    }

    fn features( &self ) -> Vec< &str > {
        if let Some( features ) = self.matches.value_of( "features" ) {
            features.split_whitespace().collect()
        } else {
            Vec::new()
        }
    }

    pub fn prepare_builder( &self, config: &Config, package: &CargoPackage, target: &CargoTarget, profile: Profile ) -> Builder {
        let mut extra_paths = Vec::new();
        let mut extra_rustflags = Vec::new();
        let mut extra_environment = Vec::new();

        if self.targeting_emscripten() {
            if let Some( emscripten ) = initialize_emscripten( self.use_system_emscripten(), self.targeting_wasm() ) {
                extra_paths.push( emscripten.emscripten_path.clone() );

                let emscripten_path = emscripten.emscripten_path.to_string_lossy().into_owned();
                let emscripten_llvm_path = emscripten.emscripten_llvm_path.to_string_lossy().into_owned();

                extra_environment.push( ("EMSCRIPTEN".to_owned(), emscripten_path) );
                extra_environment.push( ("EMSCRIPTEN_FASTCOMP".to_owned(), emscripten_llvm_path.clone()) );
                extra_environment.push( ("LLVM".to_owned(), emscripten_llvm_path) );
                if let Some( binaryen_path ) = emscripten.binaryen_path {
                    let binaryen_path = binaryen_path.to_string_lossy().into_owned();
                    extra_environment.push( ("BINARYEN".to_owned(), binaryen_path) );
                }
            }

            // When compiling tests we want the exit runtime,
            // when compiling for the Web we don't want it
            // since that's more efficient.
            let exit_runtime = profile == Profile::Main;

            extra_rustflags.push( "-C".to_owned() );
            extra_rustflags.push( "link-arg=-s".to_owned() );
            extra_rustflags.push( "-C".to_owned() );
            extra_rustflags.push( format!( "link-arg=NO_EXIT_RUNTIME={}", exit_runtime as u32 ) );

            // This will allow the initially preallocated chunk
            // of memory to grow. On asm.js this has a performance
            // impact which is why we don't turn it on by default there,
            // however according to the Emscripten documentation the WASM
            // backend doesn't have that problem, so we enable it there.
            //
            // See more here:
            //   https://kripken.github.io/emscripten-site/docs/optimizing/Optimizing-Code.html#memory-growth
            let allow_memory_growth = self.targeting_emscripten_wasm();

            extra_rustflags.push( "-C".to_owned() );
            extra_rustflags.push( "link-arg=-s".to_owned() );
            extra_rustflags.push( "-C".to_owned() );
            extra_rustflags.push( format!( "link-arg=ALLOW_MEMORY_GROWTH={}", allow_memory_growth as u32 ) );
        }

        if let Some( ref link_args ) = config.link_args {
            for arg in link_args {
                if arg.contains( " " ) {
                    // Not sure how to handle spaces, as `-C link-arg="{}"` doesn't work.
                    println_err!( "error: you have a space in one of the entries in `link-args` in your `Web.toml`;" );
                    println_err!( "       this is currently unsupported - aborting!" );
                    exit( 101 );
                }

                extra_rustflags.push( "-C".to_owned() );
                extra_rustflags.push( format!( "link-arg={}", arg ) );
            }
        }

        if self.targeting_native_wasm() && self.requested_build_type() == BuildType::Debug {
            extra_rustflags.push( "-C".to_owned() );
            extra_rustflags.push( "debuginfo=2".to_owned() );
        }

        if self.targeting_native_wasm() {
            // Incremental compilation currently doesn't work very well with
            // this target, so disable it.
            if env::var_os( "CARGO_INCREMENTAL" ).is_some() {
                extra_environment.push( ("CARGO_INCREMENTAL".to_owned(), "0".to_owned()) );
            }
        }

        Builder::new( BuildConfig {
            build_target: target_to_build_target( target, profile ),
            build_type: self.build_type(),
            triplet: Some( self.triplet_or_default().into() ),
            package: Some( package.name.clone() ),
            features: self.features().into_iter().map( |feature| feature.to_owned() ).collect(),
            no_default_features: self.matches.is_present( "no-default-features" ),
            enable_all_features: self.matches.is_present( "all-features" ),
            extra_paths,
            extra_rustflags,
            extra_environment,
            message_format: self.message_format(),
            is_verbose: self.is_verbose()
        })
    }
}

pub struct Builder( BuildConfig );

impl Builder {
    pub fn new( build_config: BuildConfig ) -> Self {
        Builder( build_config )
    }

    pub fn run( &self ) -> Result< CargoResult, Error > {
        let result = self.0.build( Some( |path: &Path| {
            if let Some( artifact ) = wasm::process_wasm_file( &self.0, path ) {
                vec![ artifact ]
            } else {
                Vec::new()
            }
        }));

        if result.is_ok() == false {
            return Err( Error::BuildError );
        }

        Ok( result )
    }
}
<commit_msg>Correct a typo<commit_after>use std::process::exit;
use std::path::Path;
use std::env;

use clap;
use cargo_shim::{
    Profile,
    CargoPackage,
    CargoProject,
    CargoTarget,
    BuildType,
    BuildConfig,
    TargetKind,
    CargoResult,
    MessageFormat,
    target_to_build_target
};

use config::Config;
use emscripten::initialize_emscripten;
use error::Error;
use wasm;

pub struct BuildArgsMatcher< 'a > {
    pub matches: &'a clap::ArgMatches< 'a >,
    pub project: &'a CargoProject
}

impl< 'a > BuildArgsMatcher< 'a > {
    fn requested_build_type( &self ) -> BuildType {
        if self.matches.is_present( "release" ) {
            BuildType::Release
        } else {
            BuildType::Debug
        }
    }

    pub fn targeting_emscripten_asmjs( &self ) -> bool {
        !self.targeting_emscripten_wasm() && !self.targeting_native_wasm()
    }

    pub fn targeting_emscripten_wasm( &self ) -> bool {
        self.matches.is_present( "target-webasm-emscripten" )
    }

    pub fn targeting_native_wasm( &self ) -> bool {
        self.matches.is_present( "target-webasm" )
    }

    pub fn targeting_wasm( &self ) -> bool {
        self.targeting_emscripten_wasm() || self.targeting_native_wasm()
    }

    pub fn targeting_emscripten( &self ) -> bool {
        self.targeting_emscripten_wasm() || self.targeting_emscripten_asmjs()
    }

    fn use_system_emscripten( &self ) -> bool {
        self.matches.is_present( "use-system-emscripten" )
    }

    fn message_format( &self ) -> MessageFormat {
        if let Some( name ) = self.matches.value_of( "message-format" ) {
            match name {
                "human" => MessageFormat::Human,
                "json" => MessageFormat::Json,
                _ => unreachable!()
            }
        } else {
            MessageFormat::Human
        }
    }

    fn is_verbose( &self ) -> bool {
        self.matches.is_present( "verbose" )
    }

    fn build_type( &self ) -> BuildType {
        let build_type = self.requested_build_type();
        if self.targeting_native_wasm() && build_type == BuildType::Debug {
            // TODO: Remove this in the future.
            println_err!( "warning: debug builds on the wasm32-unknown-unknown are currently totally broken" );
            println_err!( "         forcing a release build" );
            return BuildType::Release;
        }

        build_type
    }

    fn package( &self ) -> Result< Option< &CargoPackage >, Error > {
        if let Some( name ) = self.matches.value_of( "package" ) {
            match self.project.packages.iter().find( |package| package.name == name ) {
                None => Err( Error::ConfigurationError( format!( "package `{}` not found", name ) ) ),
                package => Ok( package )
            }
        } else {
            Ok( None )
        }
    }

    pub fn package_or_default( &self ) -> Result< &CargoPackage, Error > {
        Ok( self.package()?.unwrap_or_else( || self.project.default_package() ) )
    }

    fn target( &'a self, package: &'a CargoPackage ) -> Result< Option< &'a CargoTarget >, Error > {
        let targets = &package.targets;
        if self.matches.is_present( "lib" ) {
            match targets.iter().find( |target| target.kind == TargetKind::Lib ) {
                None => return Err( Error::ConfigurationError( format!( "no library targets found" ) ) ),
                target => Ok( target )
            }
        } else if let Some( name ) = self.matches.value_of( "bin" ) {
            match targets.iter().find( |target| target.kind == TargetKind::Bin && target.name == name ) {
                None => return Err( Error::ConfigurationError( format!( "no bin target named `{}`", name ) ) ),
                target => Ok( target )
            }
        } else if let Some( name ) = self.matches.value_of( "example" ) {
            match targets.iter().find( |target| target.kind == TargetKind::Example && target.name == name ) {
                None => return Err( Error::ConfigurationError( format!( "no example target named `{}`", name ) ) ),
                target => Ok( target )
            }
        } else if let Some( name ) = self.matches.value_of( "bench" ) {
            match targets.iter().find( |target| target.kind == TargetKind::Bench && target.name == name ) {
                None => return Err( Error::ConfigurationError( format!( "no bench target named `{}`", name ) ) ),
                target => Ok( target )
            }
        } else {
            Ok( None )
        }
    }

    pub fn target_or_select< F >( &'a self, package: &'a CargoPackage, filter: F ) -> Result< Vec< &'a CargoTarget >, Error >
        where for< 'r > F: Fn( &'r CargoTarget ) -> bool
    {
        Ok( self.target( package )?.map( |target| vec![ target ] ).unwrap_or_else( || {
            package.targets.iter().filter( |target| filter( target ) ).collect()
        }))
    }

    fn triplet_or_default( &self ) -> &str {
        if self.matches.is_present( "target-webasm") {
            "wasm32-unknown-unknown"
        } else if self.matches.is_present( "target-webasm-emscripten" ) {
            "wasm32-unknown-emscripten"
        } else {
            "asmjs-unknown-emscripten"
        }
    }

    fn features( &self ) -> Vec< &str > {
        if let Some( features ) = self.matches.value_of( "features" ) {
            features.split_whitespace().collect()
        } else {
            Vec::new()
        }
    }

    pub fn prepare_builder( &self, config: &Config, package: &CargoPackage, target: &CargoTarget, profile: Profile ) -> Builder {
        let mut extra_paths = Vec::new();
        let mut extra_rustflags = Vec::new();
        let mut extra_environment = Vec::new();

        if self.targeting_emscripten() {
            if let Some( emscripten ) = initialize_emscripten( self.use_system_emscripten(), self.targeting_wasm() ) {
                extra_paths.push( emscripten.emscripten_path.clone() );

                let emscripten_path = emscripten.emscripten_path.to_string_lossy().into_owned();
                let emscripten_llvm_path = emscripten.emscripten_llvm_path.to_string_lossy().into_owned();

                extra_environment.push( ("EMSCRIPTEN".to_owned(), emscripten_path) );
                extra_environment.push( ("EMSCRIPTEN_FASTCOMP".to_owned(), emscripten_llvm_path.clone()) );
                extra_environment.push( ("LLVM".to_owned(), emscripten_llvm_path) );
                if let Some( binaryen_path ) = emscripten.binaryen_path {
                    let binaryen_path = binaryen_path.to_string_lossy().into_owned();
                    extra_environment.push( ("BINARYEN".to_owned(), binaryen_path) );
                }
            }

            // When compiling tests we want the exit runtime,
            // when compiling for the Web we don't want it
            // since that's more efficient.
            let exit_runtime = profile == Profile::Main;

            extra_rustflags.push( "-C".to_owned() );
            extra_rustflags.push( "link-arg=-s".to_owned() );
            extra_rustflags.push( "-C".to_owned() );
            extra_rustflags.push( format!( "link-arg=NO_EXIT_RUNTIME={}", exit_runtime as u32 ) );

            // This will allow the initially preallocated chunk
            // of memory to grow. On asm.js this has a performance
            // impact which is why we don't turn it on by default there,
            // however according to the Emscripten documentation the WASM
            // backend doesn't have that problem, so we enable it there.
            //
            // See more here:
            //   https://kripken.github.io/emscripten-site/docs/optimizing/Optimizing-Code.html#memory-growth
            let allow_memory_growth = self.targeting_emscripten_wasm();

            extra_rustflags.push( "-C".to_owned() );
            extra_rustflags.push( "link-arg=-s".to_owned() );
            extra_rustflags.push( "-C".to_owned() );
            extra_rustflags.push( format!( "link-arg=ALLOW_MEMORY_GROWTH={}", allow_memory_growth as u32 ) );
        }

        if let Some( ref link_args ) = config.link_args {
            for arg in link_args {
                if arg.contains( " " ) {
                    // Not sure how to handle spaces, as `-C link-arg="{}"` doesn't work.
                    println_err!( "error: you have a space in one of the entries in `link-args` in your `Web.toml`;" );
                    println_err!( "       this is currently unsupported - aborting!" );
                    exit( 101 );
                }

                extra_rustflags.push( "-C".to_owned() );
                extra_rustflags.push( format!( "link-arg={}", arg ) );
            }
        }

        if self.targeting_native_wasm() && self.requested_build_type() == BuildType::Debug {
            extra_rustflags.push( "-C".to_owned() );
            extra_rustflags.push( "debuginfo=2".to_owned() );
        }

        if self.targeting_native_wasm() {
            // Incremental compilation currently doesn't work very well with
            // this target, so disable it.
            if env::var_os( "CARGO_INCREMENTAL" ).is_some() {
                extra_environment.push( ("CARGO_INCREMENTAL".to_owned(), "0".to_owned()) );
            }
        }

        Builder::new( BuildConfig {
            build_target: target_to_build_target( target, profile ),
            build_type: self.build_type(),
            triplet: Some( self.triplet_or_default().into() ),
            package: Some( package.name.clone() ),
            features: self.features().into_iter().map( |feature| feature.to_owned() ).collect(),
            no_default_features: self.matches.is_present( "no-default-features" ),
            enable_all_features: self.matches.is_present( "all-features" ),
            extra_paths,
            extra_rustflags,
            extra_environment,
            message_format: self.message_format(),
            is_verbose: self.is_verbose()
        })
    }
}

pub struct Builder( BuildConfig );

impl Builder {
    pub fn new( build_config: BuildConfig ) -> Self {
        Builder( build_config )
    }

    pub fn run( &self ) -> Result< CargoResult, Error > {
        let result = self.0.build( Some( |path: &Path| {
            if let Some( artifact ) = wasm::process_wasm_file( &self.0, path ) {
                vec![ artifact ]
            } else {
                Vec::new()
            }
        }));

        if result.is_ok() == false {
            return Err( Error::BuildError );
        }

        Ok( result )
    }
}
<|endoftext|>


<commit_before>use rusqlite::{SqliteConnection, SqliteTransaction, SqliteError};
use rusqlite::types::{ToSql, FromSql};
use rustc_serialize::json::{Json, ParserError};
use jmap::util::{FromJson, ParseError};
use jmap::contact::Contact;
use std::error::Error;
use std::convert::From;
use std::fmt;
use self::DbError::*;

use std::path::Path;

const VERSION: u32 = 1;

const CREATE_SQL: [&'static str; 6] = [
r###"
CREATE TABLE records (
    rowid       INTEGER PRIMARY KEY,
    id          TEXT NOT NULL,
    userid      INTEGER NOT NULL,
    type        INTEGER NOT NULL,
    modseq      INTEGER NOT NULL,
    json        TEXT NOT NULL,
    UNIQUE( id, userid )
);
"###,
r###"
CREATE INDEX idx_record_id_userid          ON records ( id, userid );
"###,
r###"
CREATE INDEX idx_record_userid_type        ON records ( userid, type );
"###,
r###"
CREATE INDEX idx_record_userid_type_modseq ON records ( userid, type, modseq );
"###,
r###"
CREATE TABLE modseq (
    userid      INTEGER NOT NULL,
    type        INTEGER NOT NULL,
    modseq      INTEGER NOT NULL,
    UNIQUE( userid, type )
);
"###,
r###"
CREATE INDEX idx_userid_type ON modseq ( userid, type );
"###,
];

/*
const UPGRADE_SQL: [&'static str; 1] = [
    // v1
    ""
];
*/

#[derive(Clone, PartialEq, Debug)]
pub enum DbError {
    InternalError(String),
}

impl Error for DbError {
    fn description(&self) -> &str {
        match *self {
            InternalError(_) => "internal database error",
        }
    }
}

impl fmt::Display for DbError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", match *self {
            InternalError(ref e) => format!("internal database error: {}", e),
        }.to_string())
    }
}

impl From<SqliteError> for DbError {
    fn from(e: SqliteError) -> DbError {
        InternalError(format!("sqlite: {}", e))
    }
}

impl From<ParserError> for DbError {
    fn from(e: ParserError) -> DbError {
        InternalError(format!("json: {}", e))
    }
}

impl From<ParseError> for DbError {
    fn from(e: ParseError) -> DbError {
        InternalError(format!("jmap: {}", e))
    }
}

pub type Transaction<'a> = SqliteTransaction<'a>;

#[derive(Debug)]
pub struct Db {
    conn: SqliteConnection,
}

impl Db {
    pub fn open() -> Result<Db,DbError> {
        //let conn = try!(SqliteConnection::open_in_memory());
        let conn = try!(SqliteConnection::open(&Path::new("db.sqlite")));
        let db = Db {
            conn: conn,
        };

        try!(db.upgrade());

        Ok(db)
    }

    fn transaction(&self) -> Result<Transaction,DbError> {
        match self.conn.transaction() {
            Ok(t) => Ok(t),
            Err(e) => Err(DbError::from(e)),
        }
    }

    fn exec(&self, sql: &str) -> Result<(),DbError> {
        let mut stmt = try!(self.conn.prepare(sql));
        try!(stmt.execute(&[]));
        Ok(())
    }

    fn exec_value<T>(&self, sql: &str, params: &[&ToSql]) -> Result<Option<T>,DbError> where T: FromSql {
        let mut stmt = try!(self.conn.prepare(sql));
        let mut res = try!(stmt.query(params));

        match res.next() {
            None       => Ok(None),
            Some(next) =>
                match next {
                    Err(e)   => Err(InternalError(format!("sqlite: {}", e))),
                    Ok(next) => {
                        let v: T = next.get(0);
                        Ok(Some(v))
                    },
            }
        }
    }

    fn version(&self) -> Result<u32,DbError> {
        let v = try!(self.exec_value::<i32>("PRAGMA user_version", &[])).unwrap();
        Ok(v as u32)
    }

    fn set_version(&self, v: u32) -> Result<(),DbError> {
        try!(self.exec(format!("PRAGMA user_version = {}", v as i32).as_ref()));
        Ok(())
    }

    fn upgrade(&self) -> Result<(),DbError> {
        let txn = try!(self.transaction());

        let ver = try!(self.version());
        if ver == VERSION { return Ok(()) }

        // new database
        if ver == 0 {
            for sql in CREATE_SQL.iter() {
                try!(self.exec(sql));
            }
        }

        /*
        // existing database, upgrade required
        else {
            // XXX
        }
        */

        try!(self.set_version(VERSION));

        try!(txn.commit());

        println!("upgraded db to version {}", VERSION);

        Ok(())
    }

    pub fn get_records(&self, userid: i64, ids: Option<&Vec<String>>) -> Result<Vec<Contact>,DbError> {
        let objtype = 1; // XXX contacts are type 1 for now

        let mut sql = "SELECT json FROM records WHERE userid = ? AND type = ?".to_string();
        let mut params: Vec<&ToSql> = vec!(&userid, &objtype);

        if let Some(ref ids) = ids {
            sql.push_str(" AND id IN ( ");

            let mut i = ids.iter();
            if let Some(id) = i.next() {
                sql.push_str("?");
                params.push(id);
            }
            for id in i {
                sql.push_str(",?");
                params.push(id);
            }

            sql.push_str(" )");
        }

        let mut stmt = try!(self.conn.prepare(sql.as_ref()));
        let res = try!(stmt.query(params.as_ref()));

        let mut records: Vec<Contact> = Vec::new();
        for row in res {
            if let Ok(ref r) = row {
                let json = try!(Json::from_str((r.get::<String>(0)).as_ref()));
                records.push(try!(Contact::from_json(&json)));
            }
        }

        Ok(records)
    }
}
<commit_msg>Allow callers to create transactions<commit_after>use rusqlite::{SqliteConnection, SqliteTransaction, SqliteError};
use rusqlite::types::{ToSql, FromSql};
use rustc_serialize::json::{Json, ParserError};
use jmap::util::{FromJson, ParseError};
use jmap::contact::Contact;
use std::error::Error;
use std::convert::From;
use std::fmt;
use self::DbError::*;

use std::path::Path;

const VERSION: u32 = 1;

const CREATE_SQL: [&'static str; 6] = [
r###"
CREATE TABLE records (
    rowid       INTEGER PRIMARY KEY,
    id          TEXT NOT NULL,
    userid      INTEGER NOT NULL,
    type        INTEGER NOT NULL,
    modseq      INTEGER NOT NULL,
    json        TEXT NOT NULL,
    UNIQUE( id, userid )
);
"###,
r###"
CREATE INDEX idx_record_id_userid          ON records ( id, userid );
"###,
r###"
CREATE INDEX idx_record_userid_type        ON records ( userid, type );
"###,
r###"
CREATE INDEX idx_record_userid_type_modseq ON records ( userid, type, modseq );
"###,
r###"
CREATE TABLE modseq (
    userid      INTEGER NOT NULL,
    type        INTEGER NOT NULL,
    modseq      INTEGER NOT NULL,
    UNIQUE( userid, type )
);
"###,
r###"
CREATE INDEX idx_userid_type ON modseq ( userid, type );
"###,
];

/*
const UPGRADE_SQL: [&'static str; 1] = [
    // v1
    ""
];
*/

#[derive(Clone, PartialEq, Debug)]
pub enum DbError {
    InternalError(String),
}

impl Error for DbError {
    fn description(&self) -> &str {
        match *self {
            InternalError(_) => "internal database error",
        }
    }
}

impl fmt::Display for DbError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}", match *self {
            InternalError(ref e) => format!("internal database error: {}", e),
        }.to_string())
    }
}

impl From<SqliteError> for DbError {
    fn from(e: SqliteError) -> DbError {
        InternalError(format!("sqlite: {}", e))
    }
}

impl From<ParserError> for DbError {
    fn from(e: ParserError) -> DbError {
        InternalError(format!("json: {}", e))
    }
}

impl From<ParseError> for DbError {
    fn from(e: ParseError) -> DbError {
        InternalError(format!("jmap: {}", e))
    }
}

pub type Transaction<'a> = SqliteTransaction<'a>;

#[derive(Debug)]
pub struct Db {
    conn: SqliteConnection,
}

impl Db {
    pub fn open() -> Result<Db,DbError> {
        //let conn = try!(SqliteConnection::open_in_memory());
        let conn = try!(SqliteConnection::open(&Path::new("db.sqlite")));
        let db = Db {
            conn: conn,
        };

        try!(db.upgrade());

        Ok(db)
    }

    pub fn transaction(&self) -> Result<Transaction,DbError> {
        match self.conn.transaction() {
            Ok(t) => Ok(t),
            Err(e) => Err(DbError::from(e)),
        }
    }

    fn exec(&self, sql: &str) -> Result<(),DbError> {
        let mut stmt = try!(self.conn.prepare(sql));
        try!(stmt.execute(&[]));
        Ok(())
    }

    fn exec_value<T>(&self, sql: &str, params: &[&ToSql]) -> Result<Option<T>,DbError> where T: FromSql {
        let mut stmt = try!(self.conn.prepare(sql));
        let mut res = try!(stmt.query(params));

        match res.next() {
            None       => Ok(None),
            Some(next) =>
                match next {
                    Err(e)   => Err(InternalError(format!("sqlite: {}", e))),
                    Ok(next) => {
                        let v: T = next.get(0);
                        Ok(Some(v))
                    },
            }
        }
    }

    fn version(&self) -> Result<u32,DbError> {
        let v = try!(self.exec_value::<i32>("PRAGMA user_version", &[])).unwrap();
        Ok(v as u32)
    }

    fn set_version(&self, v: u32) -> Result<(),DbError> {
        try!(self.exec(format!("PRAGMA user_version = {}", v as i32).as_ref()));
        Ok(())
    }

    fn upgrade(&self) -> Result<(),DbError> {
        let txn = try!(self.transaction());

        let ver = try!(self.version());
        if ver == VERSION { return Ok(()) }

        // new database
        if ver == 0 {
            for sql in CREATE_SQL.iter() {
                try!(self.exec(sql));
            }
        }

        /*
        // existing database, upgrade required
        else {
            // XXX
        }
        */

        try!(self.set_version(VERSION));

        try!(txn.commit());

        println!("upgraded db to version {}", VERSION);

        Ok(())
    }

    pub fn get_records(&self, userid: i64, ids: Option<&Vec<String>>) -> Result<Vec<Contact>,DbError> {
        let objtype = 1; // XXX contacts are type 1 for now

        let mut sql = "SELECT json FROM records WHERE userid = ? AND type = ?".to_string();
        let mut params: Vec<&ToSql> = vec!(&userid, &objtype);

        if let Some(ref ids) = ids {
            sql.push_str(" AND id IN ( ");

            let mut i = ids.iter();
            if let Some(id) = i.next() {
                sql.push_str("?");
                params.push(id);
            }
            for id in i {
                sql.push_str(",?");
                params.push(id);
            }

            sql.push_str(" )");
        }

        let mut stmt = try!(self.conn.prepare(sql.as_ref()));
        let res = try!(stmt.query(params.as_ref()));

        let mut records: Vec<Contact> = Vec::new();
        for row in res {
            if let Ok(ref r) = row {
                let json = try!(Json::from_str((r.get::<String>(0)).as_ref()));
                records.push(try!(Contact::from_json(&json)));
            }
        }

        Ok(records)
    }
}
<|endoftext|>


<commit_before>use std::ops::Deref;

use crate::*;

/// The `sled` embedded database! Implements
/// `Deref<Target = sled::Tree>` to refer to
/// a default keyspace.
#[derive(Clone)]
pub struct Db {
    #[doc(hidden)]
    pub context: Context,
    pub(crate) default: Tree,
    tenants: Arc<RwLock<FastMap8<IVec, Tree>>>,
}

/// Opens a `Db` with a default configuration at the
/// specified path. This will create a new storage
/// directory at the specified path if it does
/// not already exist. You can use the `Db::was_recovered`
/// method to determine if your database was recovered
/// from a previous instance. You can use `Config::create_new`
/// if you want to increase the chances that the database
/// will be freshly created.
pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Db> {
    Config::new().path(path).open()
}

#[allow(unsafe_code)]
unsafe impl Send for Db {}

#[allow(unsafe_code)]
unsafe impl Sync for Db {}

impl Deref for Db {
    type Target = Tree;

    fn deref(&self) -> &Tree {
        &self.default
    }
}

impl Debug for Db {
    fn fmt(
        &self,
        f: &mut fmt::Formatter<'_>,
    ) -> std::result::Result<(), fmt::Error> {
        let tenants = self.tenants.read();
        writeln!(f, "Db {{")?;
        for (raw_name, tree) in tenants.iter() {
            let name = std::str::from_utf8(raw_name)
                .ok()
                .map_or_else(|| format!("{:?}", raw_name), String::from);
            write!(f, "    Tree: {:?} contents: {:?}", name, tree)?;
        }
        write!(f, "}}")?;
        Ok(())
    }
}

impl Db {
    #[doc(hidden)]
    #[deprecated(since = "0.30.2", note = "replaced by `sled::open`")]
    pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
        Config::new().path(path).open()
    }

    pub(crate) fn start_inner(config: RunningConfig) -> Result<Self> {
        let _measure = Measure::new(&M.tree_start);

        let context = Context::start(config)?;

        #[cfg(all(
            not(miri),
            any(
                windows,
                target_os = "linux",
                target_os = "macos",
                target_os = "dragonfly",
                target_os = "freebsd",
                target_os = "openbsd",
                target_os = "netbsd",
            )
        ))]
        {
            let flusher_pagecache = context.pagecache.clone();
            let flusher = context.flush_every_ms.map(move |fem| {
                flusher::Flusher::new(
                    "log flusher".to_owned(),
                    flusher_pagecache,
                    fem,
                )
            });
            *context.flusher.lock() = flusher;
        }

        // create or open the default tree
        let guard = pin();
        let default =
            meta::open_tree(&context, DEFAULT_TREE_ID.to_vec(), &guard)?;

        let ret = Self {
            context: context.clone(),
            default,
            tenants: Arc::new(RwLock::new(FastMap8::default())),
        };

        let mut tenants = ret.tenants.write();

        for (id, root) in context.pagecache.get_meta(&guard)?.tenants() {
            let tree = Tree(Arc::new(TreeInner {
                tree_id: id.clone(),
                subscribers: Subscribers::default(),
                context: context.clone(),
                root: AtomicU64::new(root),
                merge_operator: RwLock::new(None),
            }));
            assert!(tenants.insert(id, tree).is_none());
        }

        drop(tenants);

        #[cfg(feature = "event_log")]
        {
            for (_name, tree) in ret.tenants.read().iter() {
                tree.verify_integrity().unwrap();
            }
            ret.context.event_log.verify();
        }

        Ok(ret)
    }

    /// Open or create a new disk-backed Tree with its own keyspace,
    /// accessible from the `Db` via the provided identifier.
    pub fn open_tree<V: AsRef<[u8]>>(&self, name: V) -> Result<Tree> {
        let name_ref = name.as_ref();
        let tenants = self.tenants.read();
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }
        drop(tenants);

        let guard = pin();

        let mut tenants = self.tenants.write();

        // we need to check this again in case another
        // thread opened it concurrently.
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }

        let tree = meta::open_tree(&self.context, name_ref.to_vec(), &guard)?;

        assert!(tenants.insert(name_ref.into(), tree.clone()).is_none());

        Ok(tree)
    }

    /// Remove a disk-backed collection.
    pub fn drop_tree<V: AsRef<[u8]>>(&self, name: V) -> Result<bool> {
        let name = name.as_ref();
        if name == DEFAULT_TREE_ID {
            return Err(Error::Unsupported(
                "cannot remove the core structures".into(),
            ));
        }
        trace!("dropping tree {:?}", name,);

        let mut tenants = self.tenants.write();

        let tree = if let Some(tree) = tenants.remove(&*name) {
            tree
        } else {
            return Ok(false);
        };

        let guard = pin();

        let mut root_id =
            Some(self.context.pagecache.meta_pid_for_name(name, &guard)?);

        let mut leftmost_chain: Vec<PageId> = vec![root_id.unwrap()];
        let mut cursor = root_id.unwrap();
        while let Some(view) = self.view_for_pid(cursor, &guard)? {
            if let Some(index) = view.data.index_ref() {
                let leftmost_child = index.pointers[0];
                leftmost_chain.push(leftmost_child);
                cursor = leftmost_child;
            } else {
                break;
            }
        }

        loop {
            let res = self
                .context
                .pagecache
                .cas_root_in_meta(name, root_id, None, &guard)?;

            if let Err(actual_root) = res {
                root_id = actual_root;
            } else {
                break;
            }
        }

        tree.root.store(u64::max_value(), SeqCst);

        // drop writer lock
        drop(tenants);

        tree.gc_pages(leftmost_chain)?;

        guard.flush();

        Ok(true)
    }

    /// Returns the trees names saved in this Db.
    pub fn tree_names(&self) -> Vec<IVec> {
        let tenants = self.tenants.read();
        tenants.iter().map(|(name, _)| name.clone()).collect()
    }

    /// Returns `true` if the database was
    /// recovered from a previous process.
    /// Note that database state is only
    /// guaranteed to be present up to the
    /// last call to `flush`! Otherwise state
    /// is synced to disk periodically if the
    /// `sync_every_ms` configuration option
    /// is set to `Some(number_of_ms_between_syncs)`
    /// or if the IO buffer gets filled to
    /// capacity before being rotated.
    pub fn was_recovered(&self) -> bool {
        self.context.was_recovered()
    }

    /// Generate a monotonic ID. Not guaranteed to be
    /// contiguous. Written to disk every `idgen_persist_interval`
    /// operations, followed by a blocking flush. During recovery, we
    /// take the last recovered generated ID and add 2x
    /// the `idgen_persist_interval` to it. While persisting, if the
    /// previous persisted counter wasn't synced to disk yet, we will do
    /// a blocking flush to fsync the latest counter, ensuring
    /// that we will never give out the same counter twice.
    pub fn generate_id(&self) -> Result<u64> {
        self.context.generate_id()
    }

    /// A database export method for all collections in the `Db`,
    /// for use in sled version upgrades. Can be used in combination
    /// with the `import` method below on a database running a later
    /// version.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the export.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn export(
        &self,
    ) -> Vec<(CollectionType, CollectionName, impl Iterator<Item = Vec<Vec<u8>>>)>
    {
        let tenants = self.tenants.read();

        let mut ret = vec![];

        for (name, tree) in tenants.iter() {
            ret.push((
                b"tree".to_vec(),
                name.to_vec(),
                tree.iter().map(|kv_opt| {
                    let kv = kv_opt.unwrap();
                    vec![kv.0.to_vec(), kv.1.to_vec()]
                }),
            ));
        }

        ret
    }

    /// Imports the collections from a previous database.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the import.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn import(
        &self,
        export: Vec<(
            CollectionType,
            CollectionName,
            impl Iterator<Item = Vec<Vec<u8>>>,
        )>,
    ) {
        for (collection_type, collection_name, collection_iter) in export {
            match collection_type {
                ref t if t == b"tree" => {
                    let tree = self
                        .open_tree(collection_name)
                        .expect("failed to open new tree during import");
                    for mut kv in collection_iter {
                        let v = kv
                            .pop()
                            .expect("failed to get value from tree export");
                        let k = kv
                            .pop()
                            .expect("failed to get key from tree export");
                        let old = tree.insert(k, v).expect(
                            "failed to insert value during tree import",
                        );
                        assert!(
                            old.is_none(),
                            "import is overwriting existing data"
                        );
                    }
                }
                other => panic!("unknown collection type {:?}", other),
            }
        }
    }

    /// Returns the CRC32 of all keys and values
    /// in this Db.
    ///
    /// This is O(N) and locks all underlying Trees
    /// for the duration of the entire scan.
    pub fn checksum(&self) -> Result<u32> {
        let tenants_mu = self.tenants.write();

        // we use a btreemap to ensure lexicographic
        // iteration over tree names to have consistent
        // checksums.
        let tenants: BTreeMap<_, _> = tenants_mu.iter().collect();

        let mut hasher = crc32fast::Hasher::new();
        let mut locks = vec![];

        locks.push(concurrency_control::write());

        for (name, tree) in &tenants {
            hasher.update(name);

            let mut iter = tree.iter();
            while let Some(kv_res) = iter.next_inner() {
                let (k, v) = kv_res?;
                hasher.update(&k);
                hasher.update(&v);
            }
        }

        Ok(hasher.finalize())
    }

    /// Returns the on-disk size of the storage files
    /// for this database.
    pub fn size_on_disk(&self) -> Result<u64> {
        self.context.pagecache.size_on_disk()
    }

    /// Traverses all files and calculates their total physical
    /// size, then traverses all pages and calculates their
    /// total logical size, then divides the physical size
    /// by the logical size.
    #[doc(hidden)]
    pub fn space_amplification(&self) -> Result<f64> {
        self.context.pagecache.space_amplification()
    }
}

/// These types provide the information that allows an entire
/// system to be exported and imported to facilitate
/// major upgrades. It is comprised entirely
/// of standard library types to be forward compatible.
/// NB this definitions are expensive to change, because
/// they impact the migration path.
type CollectionType = Vec<u8>;
type CollectionName = Vec<u8>;
<commit_msg>avoid shadowing name per clippy<commit_after>use std::ops::Deref;

use crate::*;

/// The `sled` embedded database! Implements
/// `Deref<Target = sled::Tree>` to refer to
/// a default keyspace.
#[derive(Clone)]
pub struct Db {
    #[doc(hidden)]
    pub context: Context,
    pub(crate) default: Tree,
    tenants: Arc<RwLock<FastMap8<IVec, Tree>>>,
}

/// Opens a `Db` with a default configuration at the
/// specified path. This will create a new storage
/// directory at the specified path if it does
/// not already exist. You can use the `Db::was_recovered`
/// method to determine if your database was recovered
/// from a previous instance. You can use `Config::create_new`
/// if you want to increase the chances that the database
/// will be freshly created.
pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Db> {
    Config::new().path(path).open()
}

#[allow(unsafe_code)]
unsafe impl Send for Db {}

#[allow(unsafe_code)]
unsafe impl Sync for Db {}

impl Deref for Db {
    type Target = Tree;

    fn deref(&self) -> &Tree {
        &self.default
    }
}

impl Debug for Db {
    fn fmt(
        &self,
        f: &mut fmt::Formatter<'_>,
    ) -> std::result::Result<(), fmt::Error> {
        let tenants = self.tenants.read();
        writeln!(f, "Db {{")?;
        for (raw_name, tree) in tenants.iter() {
            let name = std::str::from_utf8(raw_name)
                .ok()
                .map_or_else(|| format!("{:?}", raw_name), String::from);
            write!(f, "    Tree: {:?} contents: {:?}", name, tree)?;
        }
        write!(f, "}}")?;
        Ok(())
    }
}

impl Db {
    #[doc(hidden)]
    #[deprecated(since = "0.30.2", note = "replaced by `sled::open`")]
    pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
        Config::new().path(path).open()
    }

    pub(crate) fn start_inner(config: RunningConfig) -> Result<Self> {
        let _measure = Measure::new(&M.tree_start);

        let context = Context::start(config)?;

        #[cfg(all(
            not(miri),
            any(
                windows,
                target_os = "linux",
                target_os = "macos",
                target_os = "dragonfly",
                target_os = "freebsd",
                target_os = "openbsd",
                target_os = "netbsd",
            )
        ))]
        {
            let flusher_pagecache = context.pagecache.clone();
            let flusher = context.flush_every_ms.map(move |fem| {
                flusher::Flusher::new(
                    "log flusher".to_owned(),
                    flusher_pagecache,
                    fem,
                )
            });
            *context.flusher.lock() = flusher;
        }

        // create or open the default tree
        let guard = pin();
        let default =
            meta::open_tree(&context, DEFAULT_TREE_ID.to_vec(), &guard)?;

        let ret = Self {
            context: context.clone(),
            default,
            tenants: Arc::new(RwLock::new(FastMap8::default())),
        };

        let mut tenants = ret.tenants.write();

        for (id, root) in context.pagecache.get_meta(&guard)?.tenants() {
            let tree = Tree(Arc::new(TreeInner {
                tree_id: id.clone(),
                subscribers: Subscribers::default(),
                context: context.clone(),
                root: AtomicU64::new(root),
                merge_operator: RwLock::new(None),
            }));
            assert!(tenants.insert(id, tree).is_none());
        }

        drop(tenants);

        #[cfg(feature = "event_log")]
        {
            for (_name, tree) in ret.tenants.read().iter() {
                tree.verify_integrity().unwrap();
            }
            ret.context.event_log.verify();
        }

        Ok(ret)
    }

    /// Open or create a new disk-backed Tree with its own keyspace,
    /// accessible from the `Db` via the provided identifier.
    pub fn open_tree<V: AsRef<[u8]>>(&self, name: V) -> Result<Tree> {
        let name_ref = name.as_ref();
        let tenants = self.tenants.read();
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }
        drop(tenants);

        let guard = pin();

        let mut tenants = self.tenants.write();

        // we need to check this again in case another
        // thread opened it concurrently.
        if let Some(tree) = tenants.get(name_ref) {
            return Ok(tree.clone());
        }

        let tree = meta::open_tree(&self.context, name_ref.to_vec(), &guard)?;

        assert!(tenants.insert(name_ref.into(), tree.clone()).is_none());

        Ok(tree)
    }

    /// Remove a disk-backed collection.
    pub fn drop_tree<V: AsRef<[u8]>>(&self, name: V) -> Result<bool> {
        let name_ref = name.as_ref();
        if name_ref == DEFAULT_TREE_ID {
            return Err(Error::Unsupported(
                "cannot remove the core structures".into(),
            ));
        }
        trace!("dropping tree {:?}", name_ref,);

        let mut tenants = self.tenants.write();

        let tree = if let Some(tree) = tenants.remove(&*name_ref) {
            tree
        } else {
            return Ok(false);
        };

        let guard = pin();

        let mut root_id =
            Some(self.context.pagecache.meta_pid_for_name(name_ref, &guard)?);

        let mut leftmost_chain: Vec<PageId> = vec![root_id.unwrap()];
        let mut cursor = root_id.unwrap();
        while let Some(view) = self.view_for_pid(cursor, &guard)? {
            if let Some(index) = view.data.index_ref() {
                let leftmost_child = index.pointers[0];
                leftmost_chain.push(leftmost_child);
                cursor = leftmost_child;
            } else {
                break;
            }
        }

        loop {
            let res = self
                .context
                .pagecache
                .cas_root_in_meta(name_ref, root_id, None, &guard)?;

            if let Err(actual_root) = res {
                root_id = actual_root;
            } else {
                break;
            }
        }

        tree.root.store(u64::max_value(), SeqCst);

        // drop writer lock
        drop(tenants);

        tree.gc_pages(leftmost_chain)?;

        guard.flush();

        Ok(true)
    }

    /// Returns the trees names saved in this Db.
    pub fn tree_names(&self) -> Vec<IVec> {
        let tenants = self.tenants.read();
        tenants.iter().map(|(name, _)| name.clone()).collect()
    }

    /// Returns `true` if the database was
    /// recovered from a previous process.
    /// Note that database state is only
    /// guaranteed to be present up to the
    /// last call to `flush`! Otherwise state
    /// is synced to disk periodically if the
    /// `sync_every_ms` configuration option
    /// is set to `Some(number_of_ms_between_syncs)`
    /// or if the IO buffer gets filled to
    /// capacity before being rotated.
    pub fn was_recovered(&self) -> bool {
        self.context.was_recovered()
    }

    /// Generate a monotonic ID. Not guaranteed to be
    /// contiguous. Written to disk every `idgen_persist_interval`
    /// operations, followed by a blocking flush. During recovery, we
    /// take the last recovered generated ID and add 2x
    /// the `idgen_persist_interval` to it. While persisting, if the
    /// previous persisted counter wasn't synced to disk yet, we will do
    /// a blocking flush to fsync the latest counter, ensuring
    /// that we will never give out the same counter twice.
    pub fn generate_id(&self) -> Result<u64> {
        self.context.generate_id()
    }

    /// A database export method for all collections in the `Db`,
    /// for use in sled version upgrades. Can be used in combination
    /// with the `import` method below on a database running a later
    /// version.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the export.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn export(
        &self,
    ) -> Vec<(CollectionType, CollectionName, impl Iterator<Item = Vec<Vec<u8>>>)>
    {
        let tenants = self.tenants.read();

        let mut ret = vec![];

        for (name, tree) in tenants.iter() {
            ret.push((
                b"tree".to_vec(),
                name.to_vec(),
                tree.iter().map(|kv_opt| {
                    let kv = kv_opt.unwrap();
                    vec![kv.0.to_vec(), kv.1.to_vec()]
                }),
            ));
        }

        ret
    }

    /// Imports the collections from a previous database.
    ///
    /// # Panics
    ///
    /// Panics if any IO problems occur while trying
    /// to perform the import.
    ///
    /// # Examples
    ///
    /// If you want to migrate from one version of sled
    /// to another, you need to pull in both versions
    /// by using version renaming:
    ///
    /// `Cargo.toml`:
    ///
    /// ```toml
    /// [dependencies]
    /// sled = "0.32"
    /// old_sled = { version = "0.31", package = "sled" }
    /// ```
    ///
    /// and in your code, remember that old versions of
    /// sled might have a different way to open them
    /// than the current `sled::open` method:
    ///
    /// ```
    /// # use sled as old_sled;
    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
    /// let old = old_sled::open("my_old_db")?;
    ///
    /// // may be a different version of sled,
    /// // the export type is version agnostic.
    /// let new = sled::open("my_new_db")?;
    ///
    /// let export = old.export();
    /// new.import(export);
    ///
    /// assert_eq!(old.checksum()?, new.checksum()?);
    /// # Ok(()) }
    /// ```
    pub fn import(
        &self,
        export: Vec<(
            CollectionType,
            CollectionName,
            impl Iterator<Item = Vec<Vec<u8>>>,
        )>,
    ) {
        for (collection_type, collection_name, collection_iter) in export {
            match collection_type {
                ref t if t == b"tree" => {
                    let tree = self
                        .open_tree(collection_name)
                        .expect("failed to open new tree during import");
                    for mut kv in collection_iter {
                        let v = kv
                            .pop()
                            .expect("failed to get value from tree export");
                        let k = kv
                            .pop()
                            .expect("failed to get key from tree export");
                        let old = tree.insert(k, v).expect(
                            "failed to insert value during tree import",
                        );
                        assert!(
                            old.is_none(),
                            "import is overwriting existing data"
                        );
                    }
                }
                other => panic!("unknown collection type {:?}", other),
            }
        }
    }

    /// Returns the CRC32 of all keys and values
    /// in this Db.
    ///
    /// This is O(N) and locks all underlying Trees
    /// for the duration of the entire scan.
    pub fn checksum(&self) -> Result<u32> {
        let tenants_mu = self.tenants.write();

        // we use a btreemap to ensure lexicographic
        // iteration over tree names to have consistent
        // checksums.
        let tenants: BTreeMap<_, _> = tenants_mu.iter().collect();

        let mut hasher = crc32fast::Hasher::new();
        let mut locks = vec![];

        locks.push(concurrency_control::write());

        for (name, tree) in &tenants {
            hasher.update(name);

            let mut iter = tree.iter();
            while let Some(kv_res) = iter.next_inner() {
                let (k, v) = kv_res?;
                hasher.update(&k);
                hasher.update(&v);
            }
        }

        Ok(hasher.finalize())
    }

    /// Returns the on-disk size of the storage files
    /// for this database.
    pub fn size_on_disk(&self) -> Result<u64> {
        self.context.pagecache.size_on_disk()
    }

    /// Traverses all files and calculates their total physical
    /// size, then traverses all pages and calculates their
    /// total logical size, then divides the physical size
    /// by the logical size.
    #[doc(hidden)]
    pub fn space_amplification(&self) -> Result<f64> {
        self.context.pagecache.space_amplification()
    }
}

/// These types provide the information that allows an entire
/// system to be exported and imported to facilitate
/// major upgrades. It is comprised entirely
/// of standard library types to be forward compatible.
/// NB this definitions are expensive to change, because
/// they impact the migration path.
type CollectionType = Vec<u8>;
type CollectionName = Vec<u8>;
<|endoftext|>


<commit_before>use std::cmp::min;
use data::source::*;

/// Iterates over a series of shrunk pools. If we imagine that our buffer has
/// a sz of (1 << (log2sz-1)) < sz ≤ (1 << log2sz), then where:
/// f(x) = 1<<(log2sz-x) we want to cut out
/// chunks of:
/// ```
/// 0..f(0),
/// 0..f(1), f(1)..2f(1),
/// 0..f(2), f(2)..2f(2), 2f(2)..3f(2), 3f(2)..4f(2),
/// ```
///
/// In other words, we remove the whole lot, then first half, second half,
/// first quarter, second quarter, etc.
#[derive(Debug)]
struct DeltaDebuggingRemovalsShrinker {
    seed: InfoPool,
    log2sz: usize,
    level: usize,
    chunk: usize,
}

#[derive(Debug)]
#[cfg(test)]
struct DeltaDebugSegmentIterator {
    size: usize,
    log2sz: usize,
    level: usize,
    chunk: usize,
}

impl DeltaDebuggingRemovalsShrinker {
    fn new(seed: InfoPool) -> Self {
        let max_idx = seed.data.len().saturating_sub(1);
        let max_pow = 0usize.count_zeros();
        let pow = max_pow - max_idx.leading_zeros();
        DeltaDebuggingRemovalsShrinker {
            seed,
            log2sz: pow as usize,
            // Ranges from 0..self.log2sz
            level: 0,
            // Ranges from 0..(1<<self.level)
            chunk: 0,
        }
    }
}

impl Iterator for DeltaDebuggingRemovalsShrinker {
    type Item = InfoPool;
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            trace!("DeltaDebuggingRemovalsShrinker#next: {:?}", self);
            if self.level > self.log2sz {
                return None;
            }

            let granularity = self.log2sz - self.level;
            let width = 1 << granularity;
            let chunk = self.chunk;

            let start = chunk * width;
            let end = start + width;

            if start >= self.seed.data.len() {
                trace!(
                    "Out of slice ({},{}) >= {}",
                    start,
                    end,
                    self.seed.data.len()
                );
                self.chunk = 0;
                self.level += 1;
                continue;
            } else {
                self.chunk += 1;
            }
            let start = min(start, self.seed.data.len());
            let end = min(end, self.seed.data.len());

            let mut candidate = InfoPool::new();
            candidate.data.clear();
            candidate.data.extend(&self.seed.data[0..start]);
            candidate.data.extend(&self.seed.data[end..]);
            debug!("removed {},{}", start, end);
            trace!("candidate {:?}", candidate);

            return Some(candidate);
        }
    }
}

#[derive(Debug)]
struct ScalarShrinker {
    seed: InfoPool,
    pos: usize,
    bitoff: u32,
}

impl ScalarShrinker {
    fn new(pool: InfoPool) -> Self {
        ScalarShrinker {
            seed: pool,
            pos: 0,
            bitoff: 0,
        }
    }
}
impl Iterator for ScalarShrinker {
    type Item = InfoPool;
    fn next(&mut self) -> Option<Self::Item> {
        let mut candidate = self.seed.clone();
        while self.pos < self.seed.data.len() {
            trace!("ScalarShrinker#next: {:?}", self);
            let pos = self.pos;
            let bitoff = self.bitoff;

            let orig_val = candidate.data[pos];
            let new_val = orig_val - (orig_val.overflowing_shr(bitoff).0);

            let log2val = (0u8.leading_zeros()) - orig_val.leading_zeros();
            if self.bitoff >= log2val {
                self.pos += 1;
                self.bitoff = 0;
                continue;
            } else {
                self.bitoff += 1;
            }

            if orig_val != new_val {
                candidate.data[pos] = new_val;
                debug!(
                    "shrunk item -(bitoff:{}) {} {}->{}",
                    bitoff, pos, orig_val, new_val
                );
                trace!("candidate {:?}", candidate);

                return Some(candidate);
            }
        }
        return None;
    }
}

/// Try to find the smallest pool `p` such that the predicate `pred` returns
/// true. Given that our [generators](../generators/index.html) tend to
/// generate smaller outputs from smaller inputs, by minimizing the source
/// pool we can find the smallest value that provokes a failure.
///
/// If we do not find any smaller pool that satisifies `pred`; we return
/// `None`.
///
/// Currently, we have two heuristics for shrinking: removing slices, and
/// reducing individual values.
///
/// Removing slices tries to remove as much of the pool as it can whilst still
/// having the predicate hold. At present, we do this by removing each half
/// and testing, then each quarter, eighths, and so on. Eventually, we plan to
/// track the regions of data that each generator draws from, and use that to
/// optimise the shrinking process.
///
/// Reducing individual values basically goes through each position in the
/// pool, and then tries reducing it to zero, then half, thn three quarters,
/// seven eighths, and so on.
pub fn minimize<F: Fn(InfoRecorder<InfoReplay>) -> bool>(
    p: &InfoPool,
    pred: &F,
) -> Option<InfoPool> {
    let shrunk_pools = DeltaDebuggingRemovalsShrinker::new(p.clone()).chain(ScalarShrinker::new(p.clone()));

    debug!("Shrinking pool");
    let mut matching_shrinks = shrunk_pools.filter(|c| {
        let pool = InfoRecorder::new(c.replay());
        let test = pred(pool);
        trace!("test result: {:?} <= {:?}", test, c);
        test
    });

    if let Some(candidate) = matching_shrinks.next() {
        debug!("Re-Shrinking");
        trace!("candidate {:?}", candidate);
        let result = minimize(&candidate, pred).unwrap_or(candidate);
        debug!("Re-Shrinking done");
        Some(result)
    } else {
        debug!("Nothing smaller found");
        trace!("... than {:?}", p);
        None
    }
}

#[cfg(test)]
fn ulog2(val: usize) -> usize {
    let max_pow = 0usize.count_zeros() as usize;
    max_pow - val.leading_zeros() as usize
}

#[cfg(test)]
impl DeltaDebugSegmentIterator {
    fn new(size: usize) -> Self {
        let log2sz = ulog2(size.saturating_sub(1));
        let level = 0;
        let chunk = 0;
        DeltaDebugSegmentIterator {
            size,
            log2sz,
            level,
            chunk,
        }
    }
}

#[cfg(test)]
impl Iterator for DeltaDebugSegmentIterator {
    type Item = (usize, usize);
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            trace!("DeltaDebugSegmentIterator#next: {:?}", self);
            if self.level > self.log2sz {
                return None;
            }

            let granularity = self.log2sz - self.level;
            let width = 1 << granularity;
            let chunk = self.chunk;

            let start = chunk * width;
            let end = start + width;

            if start >= self.size {
                trace!("Out of slice ({},{}) >= {}", start, end, self.size);
                self.chunk = 0;
                self.level += 1;
                continue;
            } else {
                self.chunk += 1;
            }
            let start = min(start, self.size);
            let end = min(end, self.size);

            debug!("Yield start/end: {},{}", start, end);

            return Some((start, end));
        }
    }
}

#[cfg(test)]
mod tests {
    extern crate env_logger;
    use super::*;
    use std::collections::BTreeMap;

    #[test]
    fn minimiser_should_minimise_to_empty() {
        let p = InfoPool::of_vec(vec![1]);
        let min = minimize(&p, &|_| true);

        assert_eq!(min.as_ref().map(|p| p.buffer()), Some([].as_ref()))
    }

    fn take_n<I: InfoSource>(mut src: I, n: usize) -> Vec<u8> {
        let mut res = Vec::new();
        for _ in 0..n {
            res.push(src.draw_u8())
        }
        res
    }

    #[test]
    fn minimiser_should_minimise_to_minimum_given_size() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec(vec![1; 4]);
        let min = minimize(&p, &|t| {
            take_n(t, 16).into_iter().filter(|&v| v > 0).count() > 1
        }).expect("some smaller pool");

        assert_eq!(min.buffer(), &[1, 1])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| take_n(t, 16).into_iter().any(|v| v >= 3))
            .expect("some smaller pool");

        assert_eq!(min.buffer(), &[3])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_to_empty() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min =
            minimize(&p, &|t| take_n(t, 16).into_iter().any(|_| true)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[] as &[u8])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values_by_search() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| take_n(t, 16).into_iter().any(|v| v >= 13))
            .expect("some smaller pool");

        assert_eq!(min.buffer(), &[13])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_accounting_for_overflow() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| take_n(t, 16).into_iter().any(|v| v >= 251))
            .expect("some smaller pool");

        assert_eq!(min.buffer(), &[251])
    }

    #[test]
    fn shrink_by_delta_debug_removal_should_produce_somewhat_unique_outputs() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec((0..256usize).map(|v| v as u8).collect::<Vec<_>>());
        let mut counts = BTreeMap::new();
        for val in DeltaDebuggingRemovalsShrinker::new(p) {
            debug!("{:?}", val);
            *counts.entry(val).or_insert(0) += 1;
        }

        assert!(
            counts.values().all(|&val| val == 1),
            "Expect all items to be unique; non-unique entries {:?}",
            counts
                .iter()
                .filter(|&(_, &v)| v != 1)
                .collect::<BTreeMap<_, _>>()
        )
    }

    #[test]
    fn shrink_by_scalar_should_produce_somewhat_unique_outputs() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec((0..256usize).map(|v| v as u8).collect::<Vec<_>>());
        let mut counts = BTreeMap::new();
        for val in ScalarShrinker::new(p) {
            let ent = counts.entry(val.clone()).or_insert(0);
            *ent += 1;
            if *ent > 1 {
                debug!("Dup! {}: {:?}", *ent, val);
            }
        }

        assert!(
            counts.values().all(|&val| val == 1),
            "Expect all items to be unique; non-unique entries {:?}",
            counts
                .iter()
                .filter(|&(_, &v)| v != 1)
                .collect::<BTreeMap<_, _>>()
        )
    }

    #[test]
    fn delta_debug_segments_should_generate_segments_on_power_of_two_boundary() {
        use std::collections::BTreeSet;
        let iter = DeltaDebugSegmentIterator::new(7);
        let items = iter.collect::<BTreeSet<_>>();
        assert_eq!(
            items,
            vec![
                (0, 7),
                (0, 4),
                (4, 7),
                (0, 2),
                (2, 4),
                (4, 6),
                (6, 7),
                (0, 1),
                (2, 3),
                (4, 5),
                (6, 7),
                (1, 2),
                (3, 4),
                (5, 6),
            ].into_iter()
                .collect()
        );
    }

    #[test]
    fn delta_debug_segments_should_generate_segments_of_non_increasing_size() {
        let lengths = DeltaDebugSegmentIterator::new(7)
            .map(|(start, end)| end - start)
            .collect::<Vec<_>>();

        assert!(
            lengths
                .iter()
                .zip(lengths.iter().skip(1))
                .all(|(a, b)| a >= b),
            "Generated lengths {:?} are monotonically decreasing",
            lengths
        );
    }

}
<commit_msg>Add another removal (from iterator) shrinker.<commit_after>use std::cmp::min;
use data::source::*;

/// Iterates over a series of shrunk pools. If we imagine that our buffer has
/// a sz of (1 << (log2sz-1)) < sz ≤ (1 << log2sz), then where:
/// f(x) = 1<<(log2sz-x) we want to cut out
/// chunks of:
/// ```
/// 0..f(0),
/// 0..f(1), f(1)..2f(1),
/// 0..f(2), f(2)..2f(2), 2f(2)..3f(2), 3f(2)..4f(2),
/// ```
///
/// In other words, we remove the whole lot, then first half, second half,
/// first quarter, second quarter, etc.
#[derive(Debug)]
struct DeltaDebuggingRemovalsShrinker {
    seed: InfoPool,
    log2sz: usize,
    level: usize,
    chunk: usize,
}

#[cfg(test)]
#[derive(Debug)]
struct DeltaDebugSegmentIterator {
    size: usize,
    log2sz: usize,
    level: usize,
    chunk: usize,
}

#[cfg(test)]
#[derive(Debug)]
struct RemovalShrinker<I> {
    seed: InfoPool,
    segments: I,
}

impl DeltaDebuggingRemovalsShrinker {
    fn new(seed: InfoPool) -> Self {
        let max_idx = seed.data.len().saturating_sub(1);
        let max_pow = 0usize.count_zeros();
        let pow = max_pow - max_idx.leading_zeros();
        DeltaDebuggingRemovalsShrinker {
            seed,
            log2sz: pow as usize,
            // Ranges from 0..self.log2sz
            level: 0,
            // Ranges from 0..(1<<self.level)
            chunk: 0,
        }
    }
}

impl Iterator for DeltaDebuggingRemovalsShrinker {
    type Item = InfoPool;
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            trace!("DeltaDebuggingRemovalsShrinker#next: {:?}", self);
            if self.level > self.log2sz {
                return None;
            }

            let granularity = self.log2sz - self.level;
            let width = 1 << granularity;
            let chunk = self.chunk;

            let start = chunk * width;
            let end = start + width;

            if start >= self.seed.data.len() {
                trace!(
                    "Out of slice ({},{}) >= {}",
                    start,
                    end,
                    self.seed.data.len()
                );
                self.chunk = 0;
                self.level += 1;
                continue;
            } else {
                self.chunk += 1;
            }
            let start = min(start, self.seed.data.len());
            let end = min(end, self.seed.data.len());

            let mut candidate = InfoPool::new();
            candidate.data.clear();
            candidate.data.extend(&self.seed.data[0..start]);
            candidate.data.extend(&self.seed.data[end..]);
            debug!("removed {},{}", start, end);
            trace!("candidate {:?}", candidate);

            return Some(candidate);
        }
    }
}

#[derive(Debug)]
struct ScalarShrinker {
    seed: InfoPool,
    pos: usize,
    bitoff: u32,
}

impl ScalarShrinker {
    fn new(pool: InfoPool) -> Self {
        ScalarShrinker {
            seed: pool,
            pos: 0,
            bitoff: 0,
        }
    }
}
impl Iterator for ScalarShrinker {
    type Item = InfoPool;
    fn next(&mut self) -> Option<Self::Item> {
        let mut candidate = self.seed.clone();
        while self.pos < self.seed.data.len() {
            trace!("ScalarShrinker#next: {:?}", self);
            let pos = self.pos;
            let bitoff = self.bitoff;

            let orig_val = candidate.data[pos];
            let new_val = orig_val - (orig_val.overflowing_shr(bitoff).0);

            let log2val = (0u8.leading_zeros()) - orig_val.leading_zeros();
            if self.bitoff >= log2val {
                self.pos += 1;
                self.bitoff = 0;
                continue;
            } else {
                self.bitoff += 1;
            }

            if orig_val != new_val {
                candidate.data[pos] = new_val;
                debug!(
                    "shrunk item -(bitoff:{}) {} {}->{}",
                    bitoff, pos, orig_val, new_val
                );
                trace!("candidate {:?}", candidate);

                return Some(candidate);
            }
        }
        return None;
    }
}

/// Try to find the smallest pool `p` such that the predicate `pred` returns
/// true. Given that our [generators](../generators/index.html) tend to
/// generate smaller outputs from smaller inputs, by minimizing the source
/// pool we can find the smallest value that provokes a failure.
///
/// If we do not find any smaller pool that satisifies `pred`; we return
/// `None`.
///
/// Currently, we have two heuristics for shrinking: removing slices, and
/// reducing individual values.
///
/// Removing slices tries to remove as much of the pool as it can whilst still
/// having the predicate hold. At present, we do this by removing each half
/// and testing, then each quarter, eighths, and so on. Eventually, we plan to
/// track the regions of data that each generator draws from, and use that to
/// optimise the shrinking process.
///
/// Reducing individual values basically goes through each position in the
/// pool, and then tries reducing it to zero, then half, thn three quarters,
/// seven eighths, and so on.
pub fn minimize<F: Fn(InfoRecorder<InfoReplay>) -> bool>(
    p: &InfoPool,
    pred: &F,
) -> Option<InfoPool> {
    let shrunk_pools =
        DeltaDebuggingRemovalsShrinker::new(p.clone()).chain(ScalarShrinker::new(p.clone()));

    debug!("Shrinking pool");
    let mut matching_shrinks = shrunk_pools.filter(|c| {
        let pool = InfoRecorder::new(c.replay());
        let test = pred(pool);
        trace!("test result: {:?} <= {:?}", test, c);
        test
    });

    if let Some(candidate) = matching_shrinks.next() {
        debug!("Re-Shrinking");
        trace!("candidate {:?}", candidate);
        let result = minimize(&candidate, pred).unwrap_or(candidate);
        debug!("Re-Shrinking done");
        Some(result)
    } else {
        debug!("Nothing smaller found");
        trace!("... than {:?}", p);
        None
    }
}

#[cfg(test)]
fn ulog2(val: usize) -> usize {
    let max_pow = 0usize.count_zeros() as usize;
    max_pow - val.leading_zeros() as usize
}

#[cfg(test)]
impl DeltaDebugSegmentIterator {
    fn new(size: usize) -> Self {
        let log2sz = ulog2(size.saturating_sub(1));
        let level = 0;
        let chunk = 0;
        DeltaDebugSegmentIterator {
            size,
            log2sz,
            level,
            chunk,
        }
    }
}

#[cfg(test)]
impl Iterator for DeltaDebugSegmentIterator {
    type Item = (usize, usize);
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            trace!("DeltaDebugSegmentIterator#next: {:?}", self);
            if self.level > self.log2sz {
                return None;
            }

            let granularity = self.log2sz - self.level;
            let width = 1 << granularity;
            let chunk = self.chunk;

            let start = chunk * width;
            let end = start + width;

            if start >= self.size {
                trace!("Out of slice ({},{}) >= {}", start, end, self.size);
                self.chunk = 0;
                self.level += 1;
                continue;
            } else {
                self.chunk += 1;
            }
            let start = min(start, self.size);
            let end = min(end, self.size);

            debug!("Yield start/end: {},{}", start, end);

            return Some((start, end));
        }
    }
}

#[cfg(test)]
impl<I> RemovalShrinker<I> {
    fn new(seed: InfoPool, segments: I) -> Self {
        RemovalShrinker { seed, segments }
    }
}

#[cfg(test)]
impl<I: Iterator<Item = (usize, usize)>> Iterator for RemovalShrinker<I> {
    type Item = InfoPool;
    fn next(&mut self) -> Option<Self::Item> {
        self.segments.next().map(|(start, end)| {
            let start = min(start, self.seed.data.len());
            let end = min(end, self.seed.data.len());

            let mut candidate = InfoPool::new();
            candidate.data.clear();
            candidate.data.extend(&self.seed.data[0..start]);
            candidate.data.extend(&self.seed.data[end..]);
            debug!("removed {},{}", start, end);
            trace!("candidate {:?}", candidate);

            candidate
        })
    }
}
#[cfg(test)]
mod tests {
    extern crate env_logger;
    use super::*;
    use std::collections::BTreeMap;

    #[test]
    fn minimiser_should_minimise_to_empty() {
        let p = InfoPool::of_vec(vec![1]);
        let min = minimize(&p, &|_| true);

        assert_eq!(min.as_ref().map(|p| p.buffer()), Some([].as_ref()))
    }

    fn take_n<I: InfoSource>(mut src: I, n: usize) -> Vec<u8> {
        let mut res = Vec::new();
        for _ in 0..n {
            res.push(src.draw_u8())
        }
        res
    }

    #[test]
    fn minimiser_should_minimise_to_minimum_given_size() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec(vec![1; 4]);
        let min = minimize(&p, &|t| {
            take_n(t, 16).into_iter().filter(|&v| v > 0).count() > 1
        }).expect("some smaller pool");

        assert_eq!(min.buffer(), &[1, 1])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| take_n(t, 16).into_iter().any(|v| v >= 3))
            .expect("some smaller pool");

        assert_eq!(min.buffer(), &[3])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_to_empty() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min =
            minimize(&p, &|t| take_n(t, 16).into_iter().any(|_| true)).expect("some smaller pool");

        assert_eq!(min.buffer(), &[] as &[u8])
    }

    #[test]
    fn minimiser_should_minimise_scalar_values_by_search() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| take_n(t, 16).into_iter().any(|v| v >= 13))
            .expect("some smaller pool");

        assert_eq!(min.buffer(), &[13])
    }
    #[test]
    fn minimiser_should_minimise_scalar_values_accounting_for_overflow() {
        let p = InfoPool::of_vec(vec![255; 3]);
        let min = minimize(&p, &|t| take_n(t, 16).into_iter().any(|v| v >= 251))
            .expect("some smaller pool");

        assert_eq!(min.buffer(), &[251])
    }

    #[test]
    fn shrink_by_delta_debug_removal_should_produce_somewhat_unique_outputs() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec((0..256usize).map(|v| v as u8).collect::<Vec<_>>());
        let mut counts = BTreeMap::new();
        for val in DeltaDebuggingRemovalsShrinker::new(p) {
            debug!("{:?}", val);
            *counts.entry(val).or_insert(0) += 1;
        }

        assert!(
            counts.values().all(|&val| val == 1),
            "Expect all items to be unique; non-unique entries {:?}",
            counts
                .iter()
                .filter(|&(_, &v)| v != 1)
                .collect::<BTreeMap<_, _>>()
        )
    }

    #[test]
    fn shrink_by_scalar_should_produce_somewhat_unique_outputs() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec((0..256usize).map(|v| v as u8).collect::<Vec<_>>());
        let mut counts = BTreeMap::new();
        for val in ScalarShrinker::new(p) {
            let ent = counts.entry(val.clone()).or_insert(0);
            *ent += 1;
            if *ent > 1 {
                debug!("Dup! {}: {:?}", *ent, val);
            }
        }

        assert!(
            counts.values().all(|&val| val == 1),
            "Expect all items to be unique; non-unique entries {:?}",
            counts
                .iter()
                .filter(|&(_, &v)| v != 1)
                .collect::<BTreeMap<_, _>>()
        )
    }

    #[test]
    fn delta_debug_segments_should_generate_segments_on_power_of_two_boundary() {
        use std::collections::BTreeSet;
        let iter = DeltaDebugSegmentIterator::new(7);
        let items = iter.collect::<BTreeSet<_>>();
        assert_eq!(
            items,
            vec![
                (0, 7),
                (0, 4),
                (4, 7),
                (0, 2),
                (2, 4),
                (4, 6),
                (6, 7),
                (0, 1),
                (2, 3),
                (4, 5),
                (6, 7),
                (1, 2),
                (3, 4),
                (5, 6),
            ].into_iter()
                .collect()
        );
    }

    #[test]
    fn delta_debug_segments_should_generate_segments_of_non_increasing_size() {
        let lengths = DeltaDebugSegmentIterator::new(7)
            .map(|(start, end)| end - start)
            .collect::<Vec<_>>();

        assert!(
            lengths
                .iter()
                .zip(lengths.iter().skip(1))
                .all(|(a, b)| a >= b),
            "Generated lengths {:?} are monotonically decreasing",
            lengths
        );
    }

    #[test]
    fn shrink_by_removal_should_remove_stated_slices() {
        env_logger::init().unwrap_or(());
        let p = InfoPool::of_vec(vec![0, 1, 2, 3, 4]);
        let vals = RemovalShrinker::new(p, ::std::iter::once((2, 3))).collect::<Vec<_>>();

        assert_eq!(vals, vec![InfoPool::of_vec(vec![0, 1, 3, 4])]);
    }
}
<|endoftext|>


<commit_before>//! Netlink module
//!
//! The netlink module contains the netlink-based management capabilities of
//! the socketcan crate. Quoth wikipedia:
//!
//!
//! > Netlink socket family is a Linux kernel interface used for inter-process
//! > communication (IPC) between both the kernel and userspace processes, and
//! > between different userspace processes, in a way similar to the Unix
//! > domain sockets.
//!
//!
//! The netlink module currently relies on the
//! [netlink-rs](https://crates.io/crates/netlink-rs), which has some
//! deficiencies. If those are not fixed, a reimplmentation of netlink-rs'
//! functionality might be required.

use byte_conv::As as AsBytes;
use libc::{self, c_char, c_ushort, c_int, c_uint};
use netlink_rs::socket::{Msg as NetlinkMessage, Socket as NetlinkSocket, NetlinkAddr,
                         Payload as NetlinkPayload, NlMsgHeader};
use netlink_rs::Protocol as NetlinkProtocol;
use nix;
use nix::net::if_::if_nametoindex;
use std::{mem, io};

// linux/rtnetlink.h
const RTM_NEWLINK: u16 = 16;

// linux/socket.h
const AF_UNSPEC: c_char = 0;

// linux/if.h; netdevice(7)
const IFF_UP: c_uint = 1;

/// Mirrors the `struct ifinfomsg` (see rtnetlink(7))
#[repr(C)]
#[derive(Clone, Copy, Debug)]
struct IfInfoMsg {
    /// Address family, should always be `AF_UNSPEC`
    family: c_char,

    /// Padding bytes, should be set to 0.
    _pad: c_char, // must be 0

    /// Device type (FIXME: ?)
    dev_type: c_ushort,

    /// The interface index, can be retrieved using `if_nametoindex` from the
    /// `nix` crate.
    index: c_int,

    /// Device flags (FIXME: ?)
    flags: c_uint,

    /// Change mask
    change: c_uint,
}

impl IfInfoMsg {
    fn new(if_index: i32, flags: c_uint, change: c_uint) -> IfInfoMsg {
        IfInfoMsg {
            family: AF_UNSPEC,
            _pad: 0,

            // dev_type: ARPHRD_CAN,
            dev_type: 0,
            index: if_index,
            flags: flags,

            change: change,
        }
    }
}

/// Sends a netlink message down a netlink socket, and checks if an ACK was
/// properly received.
fn send_and_read_ack(sock: &mut NetlinkSocket,
                     msg: NetlinkMessage,
                     dest: &NetlinkAddr)
                     -> io::Result<()> {
    // FIXME: check return length
    let _ = sock.send(msg, dest)?;

    // receive all pending messages
    let (addr, msgs) = sock.recv()?;

    match msgs.into_iter().nth(0) {
        Some(msg) => {
            println!("Received Address: {:?}", addr);
            println!("Received Message: {:?}", msg);
            match *msg.payload() {
                NetlinkPayload::Ack(_) => (),
                NetlinkPayload::Err(errno, _) => {
                    return Err(io::Error::from_raw_os_error(-errno));
                }
                _ => panic!("Message received is not an ACK"),
            }
        }
        None => panic!("Expect ACK, but no ACK received"),
    }

    Ok(())

}

/// Opens a new netlink socket, bound to this process' PID
fn open_nl_route_socket() -> io::Result<NetlinkSocket> {
    let sock = NetlinkSocket::new(NetlinkProtocol::Route)?;

    // retrieve PID
    let pid = unsafe { libc::getpid() } as u32;
    let pid = 0;  // FIXME: is this necessary?

    // after opening the socket, bind it to be able to receive messages back
    // groups is set to 0, because we want no notifications
    let bind_addr = NetlinkAddr::new(pid, 0);
    sock.bind(bind_addr)?;

    Ok(sock)
}

/// Brings down a CAN interface
fn bring_down(if_index: c_uint) -> io::Result<()> {
    let mut nl = open_nl_route_socket()?;

    // prepare message
    let mut header = NlMsgHeader::user_defined(RTM_NEWLINK, mem::size_of::<IfInfoMsg>() as u32);
    header.ack();

    let info = IfInfoMsg::new(if_index as i32, 0, IFF_UP);
    let msg = NetlinkMessage::new(header, NetlinkPayload::Data(info.as_bytes()));

    let kernel_addr = NetlinkAddr::new(0, 0);

    // send the message
    send_and_read_ack(&mut nl, msg, &kernel_addr)
}

/// SocketCAN interface
///
/// Controlled through the kernel's netlink interface, CAN devices can be
/// brought up or down or configured through this.
pub struct CanInterface {
    if_index: c_uint,
}

impl CanInterface {
    /// Open CAN interface by name
    ///
    /// Similar to `open_if`, but looks up the device by name instead
    pub fn open(ifname: &str) -> Result<CanInterface, nix::Error> {
        let if_index = if_nametoindex(ifname)?;
        Ok(CanInterface::open_if(if_index))
    }

    /// Open CAN interface
    ///
    /// Creates a new `CanInterface` instance. No actual "opening" is necessary
    /// or performed when calling this function.
    pub fn open_if(if_index: c_uint) -> CanInterface {
        CanInterface { if_index: if_index }
    }
}
<commit_msg>Check send length.<commit_after>//! Netlink module
//!
//! The netlink module contains the netlink-based management capabilities of
//! the socketcan crate. Quoth wikipedia:
//!
//!
//! > Netlink socket family is a Linux kernel interface used for inter-process
//! > communication (IPC) between both the kernel and userspace processes, and
//! > between different userspace processes, in a way similar to the Unix
//! > domain sockets.
//!
//!
//! The netlink module currently relies on the
//! [netlink-rs](https://crates.io/crates/netlink-rs), which has some
//! deficiencies. If those are not fixed, a reimplmentation of netlink-rs'
//! functionality might be required.

use byte_conv::As as AsBytes;
use libc::{self, c_char, c_ushort, c_int, c_uint};
use netlink_rs::socket::{Msg as NetlinkMessage, Socket as NetlinkSocket, NetlinkAddr,
                         Payload as NetlinkPayload, NlMsgHeader};
use netlink_rs::Protocol as NetlinkProtocol;
use nix;
use nix::net::if_::if_nametoindex;
use std::{mem, io};

// linux/rtnetlink.h
const RTM_NEWLINK: u16 = 16;

// linux/socket.h
const AF_UNSPEC: c_char = 0;

// linux/if.h; netdevice(7)
const IFF_UP: c_uint = 1;

/// Mirrors the `struct ifinfomsg` (see rtnetlink(7))
#[repr(C)]
#[derive(Clone, Copy, Debug)]
struct IfInfoMsg {
    /// Address family, should always be `AF_UNSPEC`
    family: c_char,

    /// Padding bytes, should be set to 0.
    _pad: c_char, // must be 0

    /// Device type (FIXME: ?)
    dev_type: c_ushort,

    /// The interface index, can be retrieved using `if_nametoindex` from the
    /// `nix` crate.
    index: c_int,

    /// Device flags (FIXME: ?)
    flags: c_uint,

    /// Change mask
    change: c_uint,
}

impl IfInfoMsg {
    fn new(if_index: i32, flags: c_uint, change: c_uint) -> IfInfoMsg {
        IfInfoMsg {
            family: AF_UNSPEC,
            _pad: 0,

            // dev_type: ARPHRD_CAN,
            dev_type: 0,
            index: if_index,
            flags: flags,

            change: change,
        }
    }
}

/// Sends a netlink message down a netlink socket, and checks if an ACK was
/// properly received.
fn send_and_read_ack(sock: &mut NetlinkSocket,
                     msg: NetlinkMessage,
                     dest: &NetlinkAddr)
                     -> io::Result<()> {

    let msg_len = msg.header().msg_length() as usize;
    let bytes_sent = sock.send(msg, dest)?;
    if bytes_sent != msg_len {
        return Err(io::Error::new(io::ErrorKind::Other, "Incomplete write"));
    }

    // receive all pending messages
    let (addr, msgs) = sock.recv()?;

    match msgs.into_iter().nth(0) {
        Some(msg) => {
            println!("Received Address: {:?}", addr);
            println!("Received Message: {:?}", msg);
            match *msg.payload() {
                NetlinkPayload::Ack(_) => (),
                NetlinkPayload::Err(errno, _) => {
                    return Err(io::Error::from_raw_os_error(-errno));
                }
                _ => panic!("Message received is not an ACK"),
            }
        }
        None => panic!("Expect ACK, but no ACK received"),
    }

    Ok(())

}

/// Opens a new netlink socket, bound to this process' PID
fn open_nl_route_socket() -> io::Result<NetlinkSocket> {
    let sock = NetlinkSocket::new(NetlinkProtocol::Route)?;

    // retrieve PID
    let pid = unsafe { libc::getpid() } as u32;
    let pid = 0;  // FIXME: is this necessary?

    // after opening the socket, bind it to be able to receive messages back
    // groups is set to 0, because we want no notifications
    let bind_addr = NetlinkAddr::new(pid, 0);
    sock.bind(bind_addr)?;

    Ok(sock)
}

/// Brings down a CAN interface
fn bring_down(if_index: c_uint) -> io::Result<()> {
    let mut nl = open_nl_route_socket()?;

    // prepare message
    let mut header = NlMsgHeader::user_defined(RTM_NEWLINK, mem::size_of::<IfInfoMsg>() as u32);
    header.ack();

    let info = IfInfoMsg::new(if_index as i32, 0, IFF_UP);
    let msg = NetlinkMessage::new(header, NetlinkPayload::Data(info.as_bytes()));

    let kernel_addr = NetlinkAddr::new(0, 0);

    // send the message
    send_and_read_ack(&mut nl, msg, &kernel_addr)
}

/// SocketCAN interface
///
/// Controlled through the kernel's netlink interface, CAN devices can be
/// brought up or down or configured through this.
pub struct CanInterface {
    if_index: c_uint,
}

impl CanInterface {
    /// Open CAN interface by name
    ///
    /// Similar to `open_if`, but looks up the device by name instead
    pub fn open(ifname: &str) -> Result<CanInterface, nix::Error> {
        let if_index = if_nametoindex(ifname)?;
        Ok(CanInterface::open_if(if_index))
    }

    /// Open CAN interface
    ///
    /// Creates a new `CanInterface` instance. No actual "opening" is necessary
    /// or performed when calling this function.
    pub fn open_if(if_index: c_uint) -> CanInterface {
        CanInterface { if_index: if_index }
    }
}
<|endoftext|>